diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 52571d4..288205b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: - name: Setup Node uses: actions/setup-node@v4 with: - node-version: 22 + node-version: 18 cache: pnpm - name: Install dependencies diff --git a/AGENTS.md b/AGENTS.md index bd6202c..447048a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -38,6 +38,9 @@ pnpm. - Use `node:` specifiers for built-in modules. - Keep modules small and single-purpose; prefer focused helpers in `src/`. +- Prefer early returns to reduce nested control flow. +- Avoid `else if` branches when early returns or separate conditionals are clearer. +- Avoid type casts when a safe type guard or discriminated union can be used. - Place shared types in `src/types/` and import them via `import type`. - Use `index.ts` barrels for public entrypoints. diff --git a/build.config.ts b/build.config.ts index 583df29..7f493e8 100644 --- a/build.config.ts +++ b/build.config.ts @@ -1,16 +1,58 @@ +import path from "node:path"; import { defineBuildConfig } from "unbuild"; export default defineBuildConfig({ entries: [ { input: "src/cli/index", name: "cli" }, { input: "src/api", name: "api" }, - { input: "src/lock", name: "lock" }, + { input: "src/cache/lock", name: "lock" }, + { + builder: "mkdist", + input: "./src", + outDir: "./dist/esm", + }, ], declaration: true, clean: true, sourcemap: true, rollup: { emitCJS: false, + alias: { + entries: [ + { + find: /^#cache\/(.*)$/, + replacement: path.resolve("src/cache/$1"), + }, + { + find: /^#cli\/(.*)$/, + replacement: path.resolve("src/cli/$1"), + }, + { + find: /^#commands\/(.*)$/, + replacement: path.resolve("src/commands/$1"), + }, + { + find: /^#config\/(.*)$/, + replacement: path.resolve("src/config/$1"), + }, + { + find: "#config", + replacement: path.resolve("src/config/index"), + }, + { + find: /^#core\/(.*)$/, + replacement: path.resolve("src/$1"), + }, + { + find: /^#git\/(.*)$/, + replacement: path.resolve("src/git/$1"), + }, + { + find: /^#types\/(.*)$/, + replacement: path.resolve("src/types/$1"), + }, + ], + }, inlineDependencies: ["picocolors"], esbuild: { minify: true, diff --git a/docs.config.schema.json b/docs.config.schema.json index 7e96111..3293cd6 100644 --- a/docs.config.schema.json +++ b/docs.config.schema.json @@ -58,14 +58,6 @@ "ignoreHidden": { "type": "boolean" }, - "allowHosts": { - "minItems": 1, - "type": "array", - "items": { - "type": "string", - "minLength": 1 - } - }, "toc": { "anyOf": [ { @@ -79,6 +71,14 @@ }, "unwrapSingleRootDir": { "type": "boolean" + }, + "allowHosts": { + "minItems": 1, + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } } }, "additionalProperties": false @@ -88,22 +88,6 @@ "items": { "type": "object", "properties": { - "id": { - "type": "string", - "minLength": 1 - }, - "repo": { - "type": "string", - "minLength": 1 - }, - "targetDir": { - "type": "string", - "minLength": 1 - }, - "targetMode": { - "type": "string", - "enum": ["symlink", "copy"] - }, "ref": { "type": "string", "minLength": 1 @@ -113,6 +97,7 @@ "enum": ["materialize"] }, "include": { + "minItems": 1, "type": "array", "items": { "type": "string", @@ -126,6 +111,10 @@ "minLength": 1 } }, + "targetMode": { + "type": "string", + "enum": ["symlink", "copy"] + }, "required": { "type": "boolean" }, @@ -140,6 +129,32 @@ "ignoreHidden": { "type": "boolean" }, + "toc": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "enum": ["tree", "compressed"] + } + ] + }, + "unwrapSingleRootDir": { + "type": "boolean" + }, + "id": { + "type": "string", + "minLength": 1 + }, + "repo": { + "type": "string", + "minLength": 1 + }, + "targetDir": { + "type": "string", + "minLength": 1 + }, "integrity": { "type": "object", "properties": { @@ -160,20 +175,6 @@ }, "required": ["type", "value"], "additionalProperties": false - }, - "toc": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "string", - "enum": ["tree", "compressed"] - } - ] - }, - "unwrapSingleRootDir": { - "type": "boolean" } }, "required": ["id", "repo"], diff --git a/package.json b/package.json index fec98d7..035531e 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,8 @@ "files": [ "bin", "dist/cli.mjs", - "dist/chunks/*.mjs", + "dist/esm/**/*.mjs", + "dist/esm/**/*.d.ts", "dist/lock.mjs", "dist/shared/*.mjs", "README.md", @@ -49,32 +50,70 @@ "test": "pnpm build && node --test", "test:coverage": "pnpm build && c8 --include dist --exclude bin --reporter=text node --test", "bench": "pnpm build && node scripts/benchmarks/run.mjs", + "complexity": "node scripts/complexity/run.mjs", "schema:build": "node scripts/generate-schema.mjs", "size": "size-limit", "test:watch": "node --test --watch", "typecheck": "tsc --noEmit", "prepare": "simple-git-hooks" }, + "imports": { + "#cache/*": { + "types": "./dist/esm/cache/*.d.ts", + "default": "./dist/esm/cache/*.mjs" + }, + "#cli/*": { + "types": "./dist/esm/cli/*.d.ts", + "default": "./dist/esm/cli/*.mjs" + }, + "#commands/*": { + "types": "./dist/esm/commands/*.d.ts", + "default": "./dist/esm/commands/*.mjs" + }, + "#core/*": { + "types": "./dist/esm/*.d.ts", + "default": "./dist/esm/*.mjs" + }, + "#config": { + "types": "./dist/esm/config/index.d.ts", + "default": "./dist/esm/config/index.mjs" + }, + "#config/*": { + "types": "./dist/esm/config/*.d.ts", + "default": "./dist/esm/config/*.mjs" + }, + "#git/*": { + "types": "./dist/esm/git/*.d.ts", + "default": "./dist/esm/git/*.mjs" + }, + "#types/*": { + "types": "./dist/esm/types/*.d.ts", + "default": "./dist/esm/types/*.mjs" + } + }, "dependencies": { "@clack/prompts": "^1.0.0", "cac": "^6.7.14", + "cli-truncate": "^4.0.0", "execa": "^9.6.1", "fast-glob": "^3.3.2", + "log-update": "^7.0.2", "picocolors": "^1.1.1", - "picomatch": "^2.3.1", + "picomatch": "^4.0.3", "zod": "^4.3.6" }, "devDependencies": { - "@biomejs/biome": "^2.3.8", - "@size-limit/file": "^11.2.0", - "@types/node": "^24.2.1", + "@biomejs/biome": "^2.3.14", + "@size-limit/file": "^12.0.0", + "@types/node": "^25.2.0", "bumpp": "^10.3.2", "c8": "^10.1.3", "jiti": "^2.5.1", "lint-staged": "^16.2.7", "simple-git-hooks": "^2.13.1", - "size-limit": "^11.2.0", + "size-limit": "^12.0.0", "tinybench": "^6.0.0", + "ts-complex": "^1.0.0", "typescript": "^5.9.3", "unbuild": "^3.6.1" }, @@ -84,6 +123,11 @@ "limit": "10 kB" } ], + "complexity": { + "maxCyclomatic": 20, + "minMaintainability": 60, + "top": 10 + }, "simple-git-hooks": { "pre-commit": "pnpm lint-staged && pnpm typecheck" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 81da0e0..0160ac1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,31 +14,37 @@ importers: cac: specifier: ^6.7.14 version: 6.7.14 + cli-truncate: + specifier: ^4.0.0 + version: 4.0.0 execa: specifier: ^9.6.1 version: 9.6.1 fast-glob: specifier: ^3.3.2 version: 3.3.3 + log-update: + specifier: ^7.0.2 + version: 7.0.2 picocolors: specifier: ^1.1.1 version: 1.1.1 picomatch: - specifier: ^2.3.1 - version: 2.3.1 + specifier: ^4.0.3 + version: 4.0.3 zod: specifier: ^4.3.6 version: 4.3.6 devDependencies: '@biomejs/biome': - specifier: ^2.3.8 - version: 2.3.13 + specifier: ^2.3.14 + version: 2.3.14 '@size-limit/file': - specifier: ^11.2.0 - version: 11.2.0(size-limit@11.2.0) + specifier: ^12.0.0 + version: 12.0.0(size-limit@12.0.0(jiti@2.6.1)) '@types/node': - specifier: ^24.2.1 - version: 24.10.9 + specifier: ^25.2.0 + version: 25.2.0 bumpp: specifier: ^10.3.2 version: 10.4.0 @@ -55,11 +61,14 @@ importers: specifier: ^2.13.1 version: 2.13.1 size-limit: - specifier: ^11.2.0 - version: 11.2.0 + specifier: ^12.0.0 + version: 12.0.0(jiti@2.6.1) tinybench: specifier: ^6.0.0 version: 6.0.0 + ts-complex: + specifier: ^1.0.0 + version: 1.0.0 typescript: specifier: ^5.9.3 version: 5.9.3 @@ -81,55 +90,55 @@ packages: resolution: {integrity: sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==} engines: {node: '>=18'} - '@biomejs/biome@2.3.13': - resolution: {integrity: sha512-Fw7UsV0UAtWIBIm0M7g5CRerpu1eKyKAXIazzxhbXYUyMkwNrkX/KLkGI7b+uVDQ5cLUMfOC9vR60q9IDYDstA==} + '@biomejs/biome@2.3.14': + resolution: {integrity: sha512-QMT6QviX0WqXJCaiqVMiBUCr5WRQ1iFSjvOLoTk6auKukJMvnMzWucXpwZB0e8F00/1/BsS9DzcKgWH+CLqVuA==} engines: {node: '>=14.21.3'} hasBin: true - '@biomejs/cli-darwin-arm64@2.3.13': - resolution: {integrity: sha512-0OCwP0/BoKzyJHnFdaTk/i7hIP9JHH9oJJq6hrSCPmJPo8JWcJhprK4gQlhFzrwdTBAW4Bjt/RmCf3ZZe59gwQ==} + '@biomejs/cli-darwin-arm64@2.3.14': + resolution: {integrity: sha512-UJGPpvWJMkLxSRtpCAKfKh41Q4JJXisvxZL8ChN1eNW3m/WlPFJ6EFDCE7YfUb4XS8ZFi3C1dFpxUJ0Ety5n+A==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [darwin] - '@biomejs/cli-darwin-x64@2.3.13': - resolution: {integrity: sha512-AGr8OoemT/ejynbIu56qeil2+F2WLkIjn2d8jGK1JkchxnMUhYOfnqc9sVzcRxpG9Ycvw4weQ5sprRvtb7Yhcw==} + '@biomejs/cli-darwin-x64@2.3.14': + resolution: {integrity: sha512-PNkLNQG6RLo8lG7QoWe/hhnMxJIt1tEimoXpGQjwS/dkdNiKBLPv4RpeQl8o3s1OKI3ZOR5XPiYtmbGGHAOnLA==} engines: {node: '>=14.21.3'} cpu: [x64] os: [darwin] - '@biomejs/cli-linux-arm64-musl@2.3.13': - resolution: {integrity: sha512-TUdDCSY+Eo/EHjhJz7P2GnWwfqet+lFxBZzGHldrvULr59AgahamLs/N85SC4+bdF86EhqDuuw9rYLvLFWWlXA==} + '@biomejs/cli-linux-arm64-musl@2.3.14': + resolution: {integrity: sha512-LInRbXhYujtL3sH2TMCH/UBwJZsoGwfQjBrMfl84CD4hL/41C/EU5mldqf1yoFpsI0iPWuU83U+nB2TUUypWeg==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] - '@biomejs/cli-linux-arm64@2.3.13': - resolution: {integrity: sha512-xvOiFkrDNu607MPMBUQ6huHmBG1PZLOrqhtK6pXJW3GjfVqJg0Z/qpTdhXfcqWdSZHcT+Nct2fOgewZvytESkw==} + '@biomejs/cli-linux-arm64@2.3.14': + resolution: {integrity: sha512-KT67FKfzIw6DNnUNdYlBg+eU24Go3n75GWK6NwU4+yJmDYFe9i/MjiI+U/iEzKvo0g7G7MZqoyrhIYuND2w8QQ==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] - '@biomejs/cli-linux-x64-musl@2.3.13': - resolution: {integrity: sha512-0bdwFVSbbM//Sds6OjtnmQGp4eUjOTt6kHvR/1P0ieR9GcTUAlPNvPC3DiavTqq302W34Ae2T6u5VVNGuQtGlQ==} + '@biomejs/cli-linux-x64-musl@2.3.14': + resolution: {integrity: sha512-KQU7EkbBBuHPW3/rAcoiVmhlPtDSGOGRPv9js7qJVpYTzjQmVR+C9Rfcz+ti8YCH+zT1J52tuBybtP4IodjxZQ==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] - '@biomejs/cli-linux-x64@2.3.13': - resolution: {integrity: sha512-s+YsZlgiXNq8XkgHs6xdvKDFOj/bwTEevqEY6rC2I3cBHbxXYU1LOZstH3Ffw9hE5tE1sqT7U23C00MzkXztMw==} + '@biomejs/cli-linux-x64@2.3.14': + resolution: {integrity: sha512-ZsZzQsl9U+wxFrGGS4f6UxREUlgHwmEfu1IrXlgNFrNnd5Th6lIJr8KmSzu/+meSa9f4rzFrbEW9LBBA6ScoMA==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] - '@biomejs/cli-win32-arm64@2.3.13': - resolution: {integrity: sha512-QweDxY89fq0VvrxME+wS/BXKmqMrOTZlN9SqQ79kQSIc3FrEwvW/PvUegQF6XIVaekncDykB5dzPqjbwSKs9DA==} + '@biomejs/cli-win32-arm64@2.3.14': + resolution: {integrity: sha512-+IKYkj/pUBbnRf1G1+RlyA3LWiDgra1xpS7H2g4BuOzzRbRB+hmlw0yFsLprHhbbt7jUzbzAbAjK/Pn0FDnh1A==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [win32] - '@biomejs/cli-win32-x64@2.3.13': - resolution: {integrity: sha512-trDw2ogdM2lyav9WFQsdsfdVy1dvZALymRpgmWsvSez0BJzBjulhOT/t+wyKeh3pZWvwP3VMs1SoOKwO3wecMQ==} + '@biomejs/cli-win32-x64@2.3.14': + resolution: {integrity: sha512-oizCjdyQ3WJEswpb3Chdngeat56rIdSYK12JI3iI11Mt5T5EXcZ7WLuowzEaFPNJ3zmOQFliMN8QY1Pi+qsfdQ==} engines: {node: '>=14.21.3'} cpu: [x64] os: [win32] @@ -516,11 +525,11 @@ packages: resolution: {integrity: sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==} engines: {node: '>=18'} - '@size-limit/file@11.2.0': - resolution: {integrity: sha512-OZHE3putEkQ/fgzz3Tp/0hSmfVo3wyTpOJSRNm6AmcwX4Nm9YtTfbQQ/hZRwbBFR23S7x2Sd9EbqYzngKwbRoA==} - engines: {node: ^18.0.0 || >=20.0.0} + '@size-limit/file@12.0.0': + resolution: {integrity: sha512-OzKYpDzWJ2jo6cAIzVsaPuvzZTmMLDoVCViEvsctmImxpXzwJZcuBEpPohFKKdgVdZuNTU8WstmvywPq55Njdw==} + engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} peerDependencies: - size-limit: 11.2.0 + size-limit: 12.0.0 '@types/estree@1.0.8': resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} @@ -528,8 +537,8 @@ packages: '@types/istanbul-lib-coverage@2.0.6': resolution: {integrity: sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==} - '@types/node@24.10.9': - resolution: {integrity: sha512-ne4A0IpG3+2ETuREInjPNhUGis1SFjv1d5asp8MzEAGtOZeTeHVDOYqOgqfhvseqg/iXty2hjBf1zAOb7RNiNw==} + '@types/node@25.2.0': + resolution: {integrity: sha512-DZ8VwRFUNzuqJ5khrvwMXHmvPe+zGayJhr2CDNiKB1WBE1ST8Djl00D0IC4vvNmHMdj6DlbYRIaFE7WHjlDl5w==} '@types/resolve@1.20.2': resolution: {integrity: sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q==} @@ -632,10 +641,6 @@ packages: caniuse-lite@1.0.30001766: resolution: {integrity: sha512-4C0lfJ0/YPjJQHagaE9x2Elb69CIqEPZeG0anQt9SIvIoOH4a4uaRl73IavyO+0qZh6MDLH//DrXThEYKHkmYA==} - chokidar@4.0.3: - resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} - engines: {node: '>= 14.16.0'} - chokidar@5.0.0: resolution: {integrity: sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==} engines: {node: '>= 20.19.0'} @@ -650,6 +655,10 @@ packages: resolution: {integrity: sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==} engines: {node: '>=18'} + cli-truncate@4.0.0: + resolution: {integrity: sha512-nPdaFdQ0h/GEigbPClz11D0v/ZJEwxmeVZGeMo3Z5StPtUTkA9o1lD6QwoirYiSDzbcwn2XcjwmCp68W1IS4TA==} + engines: {node: '>=18'} + cli-truncate@5.1.1: resolution: {integrity: sha512-SroPvNHxUnk+vIW/dOSfNqdy1sPEFkrTk6TUtqLCnBlo3N7TNYYkzzN7uSD6+jVjrdO4+p8nH7JzH6cIvUem6A==} engines: {node: '>=20'} @@ -919,6 +928,10 @@ packages: resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} engines: {node: '>=8'} + is-fullwidth-code-point@4.0.0: + resolution: {integrity: sha512-O4L094N2/dZ7xqVdrXhh9r1KODPJpFms8B5sGdJLPy664AgvXsreZUyCQQNItZRDlYug4xStLjNp/sz3HvBowQ==} + engines: {node: '>=12'} + is-fullwidth-code-point@5.1.0: resolution: {integrity: sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ==} engines: {node: '>=18'} @@ -1007,10 +1020,17 @@ packages: lodash.uniq@4.5.0: resolution: {integrity: sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==} + lodash@4.17.23: + resolution: {integrity: sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==} + log-update@6.1.0: resolution: {integrity: sha512-9ie8ItPR6tjY5uYJh8K/Zrv/RMZ5VOlOWvtZdEHYSTFKZfIBPQa9tOAEeAWhd+AnIneLJ22w5fjOYtoutpWq5w==} engines: {node: '>=18'} + log-update@7.0.2: + resolution: {integrity: sha512-cSSF1K5w9juI2+JeSRAdaTUZJf6cJB0aWwWO1nQQkcWw44+bIfXmhZMwK2eEsv6tXvU3UfKX/kzcX6SP+1tLAw==} + engines: {node: '>=20'} + lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} @@ -1366,10 +1386,6 @@ packages: rc9@2.1.2: resolution: {integrity: sha512-btXCnMmRIBINM2LDZoEmOogIZU7Qe7zn4BpomSKZ/ykbLObuBdvG+mFq11DL6fjH1DRwHhrlgtYWG96bJiC7Cg==} - readdirp@4.1.2: - resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} - engines: {node: '>= 14.18.0'} - readdirp@5.0.0: resolution: {integrity: sha512-9u/XQ1pvrQtYyMpZe7DXKv2p5CNvyVwzUB6uhLAnQwHMSgKMBR62lc7AHljaeteeHXn11XTAaLLUVZYVZyuRBQ==} engines: {node: '>= 20.19.0'} @@ -1440,10 +1456,19 @@ packages: sisteransi@1.0.5: resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==} - size-limit@11.2.0: - resolution: {integrity: sha512-2kpQq2DD/pRpx3Tal/qRW1SYwcIeQ0iq8li5CJHQgOC+FtPn2BVmuDtzUCgNnpCrbgtfEHqh+iWzxK+Tq6C+RQ==} - engines: {node: ^18.0.0 || >=20.0.0} + size-limit@12.0.0: + resolution: {integrity: sha512-JBG8dioIs0m2kHOhs9jD6E/tZKD08vmbf2bfqj/rJyNWqJxk/ZcakixjhYtsqdbi+AKVbfPkt3g2RRZiKaizYA==} + engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} hasBin: true + peerDependencies: + jiti: ^2.0.0 + peerDependenciesMeta: + jiti: + optional: true + + slice-ansi@5.0.0: + resolution: {integrity: sha512-FC+lgizVPfie0kkhqUScwRu1O/lF6NOgJmlCgK+/LYxDCTk8sGelYaHDhFcDN+Sn3Cv+3VSa4Byeo+IMCzpMgQ==} + engines: {node: '>=12'} slice-ansi@7.1.2: resolution: {integrity: sha512-iOBWFgUX7caIZiuutICxVgX1SdxwAVFFKwt1EvMYYec/NWO5meOJ6K5uQxhrYBdQJne4KxiqZc+KptFOWFSI9w==} @@ -1524,6 +1549,22 @@ packages: resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} engines: {node: '>=8.0'} + ts-complex@1.0.0: + resolution: {integrity: sha512-LmtPgVIsiIyfNdsXpKRca0QdZozKLJv+MxzC6VlhoIA8C5UEDVAA/oMOGdg2YrrPSaJZMiZymoAlCb0zscTu2g==} + + tslib@1.14.1: + resolution: {integrity: sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==} + + tsutils@2.29.0: + resolution: {integrity: sha512-g5JVHCIJwzfISaXpXE1qvNalca5Jwob6FjI4AoPlqMusJ6ftFE7IkkFoMhVLRgK+4Kx3gkzb8UZK5t5yTTvEmA==} + peerDependencies: + typescript: '>=2.1.0 || >=2.1.0-dev || >=2.2.0-dev || >=2.3.0-dev || >=2.4.0-dev || >=2.5.0-dev || >=2.6.0-dev || >=2.7.0-dev || >=2.8.0-dev || >=2.9.0-dev || >= 3.0.0-dev || >= 3.1.0-dev' + + typescript@2.9.2: + resolution: {integrity: sha512-Gr4p6nFNaoufRIY4NMdpQRNmgxVIGMs4Fcu/ujdYk3nAZqk7supzBE9idmvfZIlH/Cuj//dvi+019qEue9lV0w==} + engines: {node: '>=4.2.0'} + hasBin: true + typescript@5.9.3: resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} engines: {node: '>=14.17'} @@ -1624,39 +1665,39 @@ snapshots: '@bcoe/v8-coverage@1.0.2': {} - '@biomejs/biome@2.3.13': + '@biomejs/biome@2.3.14': optionalDependencies: - '@biomejs/cli-darwin-arm64': 2.3.13 - '@biomejs/cli-darwin-x64': 2.3.13 - '@biomejs/cli-linux-arm64': 2.3.13 - '@biomejs/cli-linux-arm64-musl': 2.3.13 - '@biomejs/cli-linux-x64': 2.3.13 - '@biomejs/cli-linux-x64-musl': 2.3.13 - '@biomejs/cli-win32-arm64': 2.3.13 - '@biomejs/cli-win32-x64': 2.3.13 + '@biomejs/cli-darwin-arm64': 2.3.14 + '@biomejs/cli-darwin-x64': 2.3.14 + '@biomejs/cli-linux-arm64': 2.3.14 + '@biomejs/cli-linux-arm64-musl': 2.3.14 + '@biomejs/cli-linux-x64': 2.3.14 + '@biomejs/cli-linux-x64-musl': 2.3.14 + '@biomejs/cli-win32-arm64': 2.3.14 + '@biomejs/cli-win32-x64': 2.3.14 - '@biomejs/cli-darwin-arm64@2.3.13': + '@biomejs/cli-darwin-arm64@2.3.14': optional: true - '@biomejs/cli-darwin-x64@2.3.13': + '@biomejs/cli-darwin-x64@2.3.14': optional: true - '@biomejs/cli-linux-arm64-musl@2.3.13': + '@biomejs/cli-linux-arm64-musl@2.3.14': optional: true - '@biomejs/cli-linux-arm64@2.3.13': + '@biomejs/cli-linux-arm64@2.3.14': optional: true - '@biomejs/cli-linux-x64-musl@2.3.13': + '@biomejs/cli-linux-x64-musl@2.3.14': optional: true - '@biomejs/cli-linux-x64@2.3.13': + '@biomejs/cli-linux-x64@2.3.14': optional: true - '@biomejs/cli-win32-arm64@2.3.13': + '@biomejs/cli-win32-arm64@2.3.14': optional: true - '@biomejs/cli-win32-x64@2.3.13': + '@biomejs/cli-win32-x64@2.3.14': optional: true '@clack/core@1.0.0': @@ -1909,15 +1950,15 @@ snapshots: '@sindresorhus/merge-streams@4.0.0': {} - '@size-limit/file@11.2.0(size-limit@11.2.0)': + '@size-limit/file@12.0.0(size-limit@12.0.0(jiti@2.6.1))': dependencies: - size-limit: 11.2.0 + size-limit: 12.0.0(jiti@2.6.1) '@types/estree@1.0.8': {} '@types/istanbul-lib-coverage@2.0.6': {} - '@types/node@24.10.9': + '@types/node@25.2.0': dependencies: undici-types: 7.16.0 @@ -2032,10 +2073,6 @@ snapshots: caniuse-lite@1.0.30001766: {} - chokidar@4.0.3: - dependencies: - readdirp: 4.1.2 - chokidar@5.0.0: dependencies: readdirp: 5.0.0 @@ -2050,6 +2087,11 @@ snapshots: dependencies: restore-cursor: 5.1.0 + cli-truncate@4.0.0: + dependencies: + slice-ansi: 5.0.0 + string-width: 7.2.0 + cli-truncate@5.1.1: dependencies: slice-ansi: 7.1.2 @@ -2355,6 +2397,8 @@ snapshots: is-fullwidth-code-point@3.0.0: {} + is-fullwidth-code-point@4.0.0: {} + is-fullwidth-code-point@5.1.0: dependencies: get-east-asian-width: 1.4.0 @@ -2438,6 +2482,8 @@ snapshots: lodash.uniq@4.5.0: {} + lodash@4.17.23: {} + log-update@6.1.0: dependencies: ansi-escapes: 7.2.0 @@ -2446,6 +2492,14 @@ snapshots: strip-ansi: 7.1.2 wrap-ansi: 9.0.2 + log-update@7.0.2: + dependencies: + ansi-escapes: 7.2.0 + cli-cursor: 5.0.0 + slice-ansi: 7.1.2 + strip-ansi: 7.1.2 + wrap-ansi: 9.0.2 + lru-cache@10.4.3: {} magic-string@0.30.21: @@ -2764,8 +2818,6 @@ snapshots: defu: 6.1.4 destr: 2.0.5 - readdirp@4.1.2: {} - readdirp@5.0.0: {} require-directory@2.1.1: {} @@ -2846,15 +2898,20 @@ snapshots: sisteransi@1.0.5: {} - size-limit@11.2.0: + size-limit@12.0.0(jiti@2.6.1): dependencies: bytes-iec: 3.1.1 - chokidar: 4.0.3 - jiti: 2.6.1 lilconfig: 3.1.3 nanospinner: 1.2.2 picocolors: 1.1.1 tinyglobby: 0.2.15 + optionalDependencies: + jiti: 2.6.1 + + slice-ansi@5.0.0: + dependencies: + ansi-styles: 6.2.3 + is-fullwidth-code-point: 4.0.0 slice-ansi@7.1.2: dependencies: @@ -2939,6 +2996,21 @@ snapshots: dependencies: is-number: 7.0.0 + ts-complex@1.0.0: + dependencies: + lodash: 4.17.23 + tsutils: 2.29.0(typescript@2.9.2) + typescript: 2.9.2 + + tslib@1.14.1: {} + + tsutils@2.29.0(typescript@2.9.2): + dependencies: + tslib: 1.14.1 + typescript: 2.9.2 + + typescript@2.9.2: {} + typescript@5.9.3: {} ufo@1.6.3: {} diff --git a/scripts/complexity/run.mjs b/scripts/complexity/run.mjs new file mode 100644 index 0000000..bc2ae29 --- /dev/null +++ b/scripts/complexity/run.mjs @@ -0,0 +1,209 @@ +import { readFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import path from "node:path"; + +import fg from "fast-glob"; + +const require = createRequire(import.meta.url); +const tscomplex = require("ts-complex"); + +const CONFIG_PATH = path.resolve(process.cwd(), "package.json"); +const DEFAULTS = { + maxCyclomatic: 20, + minMaintainability: 60, + top: 10, +}; + +const loadConfig = async () => { + const raw = await readFile(CONFIG_PATH, "utf8"); + const pkg = JSON.parse(raw); + return { + ...DEFAULTS, + ...(pkg.complexity ?? {}), + }; +}; + +const summarize = (value) => + value.toLocaleString("en-US", { + minimumFractionDigits: 2, + maximumFractionDigits: 2, + }); + +const formatNumber = (value) => + Number.isFinite(value) ? summarize(value) : "n/a"; + +const parseLocation = (name) => { + if (!name || typeof name !== "string") return null; + const trimmed = name.trim(); + if (!trimmed.startsWith("{") || !trimmed.endsWith("}")) return null; + try { + const parsed = JSON.parse(trimmed); + if (typeof parsed?.pos === "number" && typeof parsed?.end === "number") { + return { pos: parsed.pos, end: parsed.end }; + } + } catch { + return null; + } + return null; +}; + +const lineOffsetsCache = new Map(); + +const getLineOffsets = async (filePath) => { + if (lineOffsetsCache.has(filePath)) { + return lineOffsetsCache.get(filePath); + } + const raw = await readFile(filePath, "utf8"); + const offsets = [0]; + for (let i = 0; i < raw.length; i += 1) { + if (raw[i] === "\n") { + offsets.push(i + 1); + } + } + lineOffsetsCache.set(filePath, offsets); + return offsets; +}; + +const offsetToLineColumn = async (filePath, offset) => { + const offsets = await getLineOffsets(filePath); + let low = 0; + let high = offsets.length - 1; + while (low <= high) { + const mid = Math.floor((low + high) / 2); + if (offsets[mid] <= offset) { + low = mid + 1; + } else { + high = mid - 1; + } + } + const lineIndex = Math.max(0, high); + const lineNumber = lineIndex + 1; + const columnNumber = offset - offsets[lineIndex] + 1; + return { line: lineNumber, column: columnNumber }; +}; + +const formatLabel = async (name, filePath) => { + const location = parseLocation(name); + if (location && filePath) { + const start = await offsetToLineColumn(filePath, location.pos); + return `@${start.line}:${start.column}`; + } + return name || ""; +}; + +const main = async () => { + const config = await loadConfig(); + const files = await fg(["src/**/*.ts"], { + ignore: ["**/*.d.ts"], + absolute: true, + }); + const metrics = []; + const functions = []; + + for (const file of files) { + const cyclomatic = tscomplex.calculateCyclomaticComplexity(file); + const maintainability = tscomplex.calculateMaintainability(file); + metrics.push({ file, maintainability }); + for (const [name, complexity] of Object.entries(cyclomatic)) { + functions.push({ file, name, complexity }); + } + } + + const maintainabilityValues = metrics + .map((entry) => entry.maintainability.minMaintainability) + .filter( + (value) => + typeof value === "number" && Number.isFinite(value) && value >= 0, + ); + const worstMaintainability = maintainabilityValues.length + ? Math.min(...maintainabilityValues) + : null; + const maxCyclomatic = functions.reduce( + (max, entry) => Math.max(max, entry.complexity), + 0, + ); + + process.stdout.write("\nComplexity\n"); + process.stdout.write(`files=${files.length}\n`); + process.stdout.write( + `maxCyclomatic=${formatNumber(maxCyclomatic)} limit=${config.maxCyclomatic}\n`, + ); + process.stdout.write( + `minMaintainability=${formatNumber(worstMaintainability)} limit=${config.minMaintainability}\n`, + ); + + const sorted = functions + .sort((left, right) => right.complexity - left.complexity) + .slice(0, config.top); + if (sorted.length > 0) { + process.stdout.write("\nTop functions\n"); + process.stdout.write("Location CC File\n"); + process.stdout.write( + "-------------------------------- ---- -------------------------\n", + ); + for (const entry of sorted) { + const rel = path.relative(process.cwd(), entry.file); + const name = (await formatLabel(entry.name, entry.file)) + .slice(0, 32) + .padEnd(32); + const cc = entry.complexity.toString().padStart(4); + process.stdout.write(`${name} ${cc} ${rel}\n`); + } + } + + const maintainabilitySorted = metrics + .map((entry) => ({ + file: entry.file, + min: entry.maintainability.minMaintainability, + avg: entry.maintainability.averageMaintainability, + })) + .filter( + (entry) => + typeof entry.min === "number" && + Number.isFinite(entry.min) && + entry.min >= 0, + ) + .sort((left, right) => (left.min ?? 0) - (right.min ?? 0)) + .slice(0, config.top); + if (maintainabilitySorted.length > 0) { + process.stdout.write("\nLowest maintainability\n"); + process.stdout.write("Min Avg File\n"); + process.stdout.write("----- ----- -------------------------\n"); + for (const entry of maintainabilitySorted) { + const rel = path.relative(process.cwd(), entry.file); + const min = formatNumber(entry.min).padStart(5); + const avg = formatNumber(entry.avg).padStart(5); + process.stdout.write(`${min} ${avg} ${rel}\n`); + } + } + + const failures = []; + for (const entry of functions) { + if (entry.complexity > config.maxCyclomatic) { + const location = await formatLabel(entry.name, entry.file); + const rel = path.relative(process.cwd(), entry.file); + const label = location.startsWith("@") + ? `${rel}:${location.slice(1)}` + : `${location} in ${rel}`; + failures.push(`${label} (${entry.complexity})`); + } + } + if ( + typeof worstMaintainability === "number" && + worstMaintainability < config.minMaintainability + ) { + failures.push( + `Maintainability below threshold: ${summarize(worstMaintainability)}`, + ); + } + + if (failures.length > 0) { + process.stderr.write("\nComplexity limits exceeded:\n"); + for (const failure of failures) { + process.stderr.write(`- ${failure}\n`); + } + process.exitCode = 1; + } +}; + +await main(); diff --git a/src/add.ts b/src/add.ts deleted file mode 100644 index 5c89cc1..0000000 --- a/src/add.ts +++ /dev/null @@ -1,157 +0,0 @@ -import { access, readFile, writeFile } from "node:fs/promises"; -import path from "node:path"; -import { - DEFAULT_CACHE_DIR, - DEFAULT_CONFIG, - type DocsCacheConfig, - resolveConfigPath, - stripDefaultConfigValues, - validateConfig, - writeConfig, -} from "./config"; -import { ensureGitignoreEntry } from "./gitignore"; -import { resolveTargetDir } from "./paths"; -import { resolveRepoInput } from "./resolve-repo"; -import { assertSafeSourceId } from "./source-id"; - -const exists = async (target: string) => { - try { - await access(target); - return true; - } catch { - return false; - } -}; - -const PACKAGE_JSON = "package.json"; - -const loadPackageConfig = async (configPath: string) => { - const raw = await readFile(configPath, "utf8"); - const parsed = JSON.parse(raw) as Record; - const config = parsed["docs-cache"]; - if (!config) { - return { parsed, config: null }; - } - return { - parsed, - config: validateConfig(config), - }; -}; - -const resolveConfigTarget = async (configPath?: string) => { - if (configPath) { - const resolvedPath = resolveConfigPath(configPath); - return { - resolvedPath, - mode: path.basename(resolvedPath) === PACKAGE_JSON ? "package" : "config", - }; - } - const defaultPath = resolveConfigPath(); - if (await exists(defaultPath)) { - return { resolvedPath: defaultPath, mode: "config" }; - } - const packagePath = path.resolve(process.cwd(), PACKAGE_JSON); - if (await exists(packagePath)) { - const pkg = await loadPackageConfig(packagePath); - if (pkg.config) { - return { resolvedPath: packagePath, mode: "package" }; - } - } - return { resolvedPath: defaultPath, mode: "config" }; -}; - -export const addSources = async (params: { - configPath?: string; - entries: Array<{ id?: string; repo: string; targetDir?: string }>; -}) => { - const target = await resolveConfigTarget(params.configPath); - const resolvedPath = target.resolvedPath; - let config = DEFAULT_CONFIG; - let rawConfig: DocsCacheConfig | null = null; - let rawPackage: Record | null = null; - let hadDocsCacheConfig = false; - if (await exists(resolvedPath)) { - if (target.mode === "package") { - const pkg = await loadPackageConfig(resolvedPath); - rawPackage = pkg.parsed; - rawConfig = pkg.config; - config = rawConfig ?? DEFAULT_CONFIG; - hadDocsCacheConfig = Boolean(rawConfig); - } else { - const raw = await readFile(resolvedPath, "utf8"); - rawConfig = JSON.parse(raw.toString()); - config = validateConfig(rawConfig); - hadDocsCacheConfig = true; - } - } - - const schema = - "https://raw.githubusercontent.com/fbosch/docs-cache/main/docs.config.schema.json"; - const existingIds = new Set(config.sources.map((source) => source.id)); - const skipped: string[] = []; - const newSources = params.entries - .map((entry) => { - const resolved = resolveRepoInput(entry.repo); - const sourceId = entry.id || resolved.inferredId; - if (!sourceId) { - throw new Error("Unable to infer id. Provide an explicit id."); - } - const safeId = assertSafeSourceId(sourceId, "source id"); - if (existingIds.has(safeId)) { - skipped.push(safeId); - return null; - } - existingIds.add(safeId); - if (entry.targetDir) { - resolveTargetDir(resolvedPath, entry.targetDir); - } - return { - id: safeId, - repo: resolved.repoUrl, - ...(entry.targetDir ? { targetDir: entry.targetDir } : {}), - ...(resolved.ref ? { ref: resolved.ref } : {}), - }; - }) - .filter(Boolean) as Array<{ - id: string; - repo: string; - targetDir?: string; - ref?: string; - }>; - if (newSources.length === 0) { - throw new Error("All sources already exist in config."); - } - const nextConfig: DocsCacheConfig = { - $schema: schema, - sources: [...config.sources, ...newSources], - }; - if (rawConfig?.cacheDir) { - nextConfig.cacheDir = rawConfig.cacheDir; - } - if (rawConfig?.defaults) { - nextConfig.defaults = rawConfig.defaults; - } - - if (target.mode === "package") { - const pkg = rawPackage ?? {}; - pkg["docs-cache"] = stripDefaultConfigValues(nextConfig); - await writeFile(resolvedPath, `${JSON.stringify(pkg, null, 2)}\n`, "utf8"); - } else { - await writeConfig(resolvedPath, nextConfig); - } - const gitignoreResult = !hadDocsCacheConfig - ? await ensureGitignoreEntry( - path.dirname(resolvedPath), - rawConfig?.cacheDir ?? DEFAULT_CACHE_DIR, - ) - : null; - - return { - configPath: resolvedPath, - sources: newSources, - skipped, - created: true, - gitignoreUpdated: gitignoreResult?.updated ?? false, - gitignorePath: gitignoreResult?.gitignorePath ?? null, - }; -}; diff --git a/src/api.ts b/src/api.ts index a1748dd..eaab882 100644 --- a/src/api.ts +++ b/src/api.ts @@ -1,14 +1,14 @@ -export { cleanCache } from "./clean"; -export { cleanGitCache } from "./clean-git-cache"; -export { parseArgs } from "./cli/parse-args"; -export { loadConfig } from "./config"; -export { redactRepoUrl } from "./git/redact"; -export { enforceHostAllowlist, parseLsRemote } from "./git/resolve-remote"; -export { initConfig } from "./init"; -export { DEFAULT_LOCK_FILENAME } from "./lock"; -export { pruneCache } from "./prune"; -export { removeSources } from "./remove"; -export { resolveRepoInput } from "./resolve-repo"; -export { printSyncPlan, runSync } from "./sync"; -export { applyTargetDir } from "./targets"; -export { verifyCache } from "./verify"; +export { DEFAULT_LOCK_FILENAME } from "#cache/lock"; +export { applyTargetDir } from "#cache/targets"; +export { parseArgs } from "#cli/parse-args"; +export { cleanCache } from "#commands/clean"; +export { cleanGitCache } from "#commands/clean-git-cache"; +export { initConfig } from "#commands/init"; +export { pruneCache } from "#commands/prune"; +export { removeSources } from "#commands/remove"; +export { printSyncPlan, runSync } from "#commands/sync"; +export { verifyCache } from "#commands/verify"; +export { loadConfig } from "#config"; +export { redactRepoUrl } from "#git/redact"; +export { enforceHostAllowlist, parseLsRemote } from "#git/resolve-remote"; +export { resolveRepoInput } from "#git/resolve-repo"; diff --git a/src/cache-layout.ts b/src/cache/cache-layout.ts similarity index 89% rename from src/cache-layout.ts rename to src/cache/cache-layout.ts index 1c29ab8..6355706 100644 --- a/src/cache-layout.ts +++ b/src/cache/cache-layout.ts @@ -1,6 +1,6 @@ import { mkdir } from "node:fs/promises"; -import { getCacheLayout } from "./paths"; +import { getCacheLayout } from "#core/paths"; export const ensureCacheLayout = async ( cacheDir: string, diff --git a/src/lock.ts b/src/cache/lock.ts similarity index 100% rename from src/lock.ts rename to src/cache/lock.ts diff --git a/src/manifest.ts b/src/cache/manifest.ts similarity index 100% rename from src/manifest.ts rename to src/cache/manifest.ts diff --git a/src/materialize.ts b/src/cache/materialize.ts similarity index 92% rename from src/materialize.ts rename to src/cache/materialize.ts index e261c42..fb0b0e2 100644 --- a/src/materialize.ts +++ b/src/cache/materialize.ts @@ -14,11 +14,11 @@ import os from "node:os"; import path from "node:path"; import { pipeline } from "node:stream/promises"; import fg from "fast-glob"; -import { symbols, ui } from "./cli/ui"; -import { getErrnoCode } from "./errors"; -import { MANIFEST_FILENAME } from "./manifest"; -import { getCacheLayout, toPosixPath } from "./paths"; -import { assertSafeSourceId } from "./source-id"; +import { MANIFEST_FILENAME } from "#cache/manifest"; +import { symbols, ui } from "#cli/ui"; +import { getErrnoCode } from "#core/errors"; +import { getCacheLayout, toPosixPath } from "#core/paths"; +import { assertSafeSourceId } from "#core/source-id"; type MaterializeParams = { sourceId: string; @@ -31,6 +31,8 @@ type MaterializeParams = { ignoreHidden?: boolean; unwrapSingleRootDir?: boolean; json?: boolean; + progressLogger?: (message: string) => void; + progressThrottleMs?: number; }; type ResolvedMaterializeParams = { @@ -44,6 +46,8 @@ type ResolvedMaterializeParams = { ignoreHidden: boolean; unwrapSingleRootDir: boolean; json: boolean; + progressLogger?: (message: string) => void; + progressThrottleMs: number; }; type ManifestStats = { @@ -173,6 +177,7 @@ const resolveMaterializeParams = ( ignoreHidden: params.ignoreHidden ?? false, unwrapSingleRootDir: params.unwrapSingleRootDir ?? false, json: params.json ?? false, + progressThrottleMs: params.progressThrottleMs ?? 120, }); const acquireLock = async (lockPath: string, timeoutMs = 5000) => { @@ -254,6 +259,7 @@ export const materializeSource = async (params: MaterializeParams) => { normalized: normalizePath(relativePath), })) .sort((left, right) => left.normalized.localeCompare(right.normalized)); + const totalEntries = entries.length; const unwrapPrefix = resolveUnwrapPrefix( entries, resolved.unwrapSingleRootDir, @@ -272,6 +278,7 @@ export const materializeSource = async (params: MaterializeParams) => { ); let bytes = 0; let fileCount = 0; + let lastProgressAt = 0; const concurrency = Math.max( 1, Math.min( @@ -369,6 +376,22 @@ export const materializeSource = async (params: MaterializeParams) => { await writeManifestLine(line); fileCount += 1; } + if (resolved.progressLogger && totalEntries > 0) { + const now = Date.now(); + const shouldEmit = + now - lastProgressAt >= resolved.progressThrottleMs || + fileCount === totalEntries; + if (shouldEmit) { + lastProgressAt = now; + const percent = Math.min( + 100, + Math.round((fileCount / totalEntries) * 100), + ); + resolved.progressLogger( + `materializing ${fileCount}/${totalEntries} (${percent}%)`, + ); + } + } } await new Promise((resolve, reject) => { manifestStream.end(() => resolve()); diff --git a/src/targets.ts b/src/cache/targets.ts similarity index 94% rename from src/targets.ts rename to src/cache/targets.ts index 5bc91cd..1dd56b4 100644 --- a/src/targets.ts +++ b/src/cache/targets.ts @@ -1,8 +1,8 @@ import { cp, mkdir, readdir, rm, symlink } from "node:fs/promises"; import path from "node:path"; -import { getErrnoCode } from "./errors"; -import { MANIFEST_FILENAME } from "./manifest"; -import { DEFAULT_TOC_FILENAME } from "./paths"; +import { MANIFEST_FILENAME } from "#cache/manifest"; +import { getErrnoCode } from "#core/errors"; +import { DEFAULT_TOC_FILENAME } from "#core/paths"; type TargetDeps = { cp: typeof cp; diff --git a/src/toc.ts b/src/cache/toc.ts similarity index 94% rename from src/toc.ts rename to src/cache/toc.ts index 2503e97..41753c8 100644 --- a/src/toc.ts +++ b/src/cache/toc.ts @@ -1,9 +1,13 @@ import { access, readFile, rm, writeFile } from "node:fs/promises"; import path from "node:path"; -import { symbols, ui } from "./cli/ui"; -import type { DocsCacheResolvedSource, TocFormat } from "./config"; -import type { DocsCacheLock } from "./lock"; -import { DEFAULT_TOC_FILENAME, resolveTargetDir, toPosixPath } from "./paths"; +import type { DocsCacheLock } from "#cache/lock"; +import { symbols, ui } from "#cli/ui"; +import type { DocsCacheResolvedSource, TocFormat } from "#config"; +import { + DEFAULT_TOC_FILENAME, + resolveTargetDir, + toPosixPath, +} from "#core/paths"; type TocEntry = { id: string; @@ -225,14 +229,6 @@ export const writeToc = async (params: { if (tocEnabled) { const result = resultsById.get(id); - if (result?.status === "up-to-date") { - try { - await access(sourceTocPath); - continue; - } catch { - // Missing TOC; regenerate below. - } - } let existingContent: string | null = null; try { existingContent = await readFile(sourceTocPath, "utf8"); @@ -240,6 +236,12 @@ export const writeToc = async (params: { existingContent = null; } const sourceTocContent = generateSourceToc(entry, tocFormat); + if ( + result?.status === "up-to-date" && + existingContent === sourceTocContent + ) { + continue; + } if (existingContent !== null && existingContent !== sourceTocContent) { ui.line( `${symbols.warn} Overwriting existing ${DEFAULT_TOC_FILENAME} for ${id}`, diff --git a/src/cli/exit-code.ts b/src/cli/exit-code.ts index c434d7a..3192354 100644 --- a/src/cli/exit-code.ts +++ b/src/cli/exit-code.ts @@ -3,8 +3,10 @@ * * @see https://nodejs.org/api/process.html#process_exit_codes */ -export enum ExitCode { - Success = 0, - FatalError = 1, - InvalidArgument = 9, -} +export const ExitCode = { + Success: 0, + FatalError: 1, + InvalidArgument: 9, +}; + +export type ExitCode = (typeof ExitCode)[keyof typeof ExitCode]; diff --git a/src/cli/index.ts b/src/cli/index.ts index 9fae9d5..597f853 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -1,10 +1,10 @@ import path from "node:path"; import process from "node:process"; import pc from "picocolors"; -import { ExitCode } from "./exit-code"; -import { parseArgs } from "./parse-args"; -import type { CliCommand } from "./types"; -import { setSilentMode, setVerboseMode, symbols, ui } from "./ui"; +import { ExitCode } from "#cli/exit-code"; +import { parseArgs } from "#cli/parse-args"; +import type { CliCommand } from "#cli/types"; +import { setSilentMode, setVerboseMode, symbols, ui } from "#cli/ui"; export const CLI_NAME = "docs-cache"; @@ -49,249 +49,296 @@ const printError = (message: string) => { process.stderr.write(`${symbols.error} ${message}\n`); }; -const runCommand = async (parsed: CliCommand) => { - const command = parsed.command; +const runAdd = async (parsed: Extract) => { const options = parsed.options; - if (command === "add") { - const { addSources } = await import("../add"); - const { runSync } = await import("../sync"); - if (parsed.entries.length === 0) { - throw new Error( - "Usage: docs-cache add [--source --target ] ", - ); + const { addSources } = await import("#commands/add"); + const { runSync } = await import("#commands/sync"); + if (parsed.entries.length === 0) { + throw new Error( + "Usage: docs-cache add [--source --target ] ", + ); + } + const result = await addSources({ + configPath: options.config, + entries: parsed.entries, + }); + if (options.offline) { + if (!options.json) { + ui.line(`${symbols.warn} Offline: skipped sync`); } - const result = await addSources({ + } else { + await runSync({ configPath: options.config, - entries: parsed.entries, + cacheDirOverride: options.cacheDir, + json: options.json, + lockOnly: options.lockOnly, + offline: options.offline, + failOnMiss: options.failOnMiss, + sourceFilter: result.sources.map((source) => source.id), + timeoutMs: options.timeoutMs, + verbose: options.verbose, }); - if (!options.offline) { - await runSync({ - configPath: options.config, - cacheDirOverride: options.cacheDir, - json: options.json, - lockOnly: options.lockOnly, - offline: options.offline, - failOnMiss: options.failOnMiss, - sourceFilter: result.sources.map((source) => source.id), - timeoutMs: options.timeoutMs, - verbose: options.verbose, - }); - } else if (!options.json) { - ui.line(`${symbols.warn} Offline: skipped sync`); - } - if (options.json) { - process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); - } else { - for (const source of result.sources) { - const repoLabel = source.repo - .replace(/^https?:\/\//, "") - .replace(/\.git$/, ""); - const targetLabel = source.targetDir - ? ` ${pc.dim("->")} ${pc.magenta(source.targetDir)}` - : ""; - ui.item( - symbols.success, - source.id, - `${pc.blue(repoLabel)}${targetLabel}`, - ); - } - if (result.skipped?.length) { - ui.line( - `${symbols.warn} Skipped ${result.skipped.length} existing source${result.skipped.length === 1 ? "" : "s"}: ${result.skipped.join(", ")}`, - ); - } - ui.line( - `${symbols.info} Updated ${pc.gray(path.relative(process.cwd(), result.configPath) || "docs.config.json")}`, - ); - if (result.gitignoreUpdated && result.gitignorePath) { - ui.line( - `${symbols.info} Updated ${pc.gray(ui.path(result.gitignorePath))}`, - ); - } - } + } + if (options.json) { + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); return; } - if (command === "remove") { - const { removeSources } = await import("../remove"); - const { pruneCache } = await import("../prune"); - if (parsed.ids.length === 0) { - throw new Error("Usage: docs-cache remove "); - } - const result = await removeSources({ - configPath: options.config, - ids: parsed.ids, - }); - if (options.json) { - process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); - } else { - if (result.removed.length > 0) { - ui.line( - `${symbols.success} Removed ${result.removed.length} source${result.removed.length === 1 ? "" : "s"}: ${result.removed.join(", ")}`, - ); - } - if (result.missing.length > 0) { - ui.line( - `${symbols.warn} Missing ${result.missing.length} source${result.missing.length === 1 ? "" : "s"}: ${result.missing.join(", ")}`, - ); - } - if (result.targetsRemoved.length > 0) { - const targetLabels = result.targetsRemoved - .map((entry) => `${entry.id} -> ${ui.path(entry.targetDir)}`) - .join(", "); - ui.line( - `${symbols.success} Removed ${result.targetsRemoved.length} target${result.targetsRemoved.length === 1 ? "" : "s"}: ${targetLabels}`, - ); - } - ui.line( - `${symbols.info} Updated ${pc.gray(path.relative(process.cwd(), result.configPath) || "docs.config.json")}`, - ); - } - if (options.prune) { - await pruneCache({ - configPath: options.config, - cacheDirOverride: options.cacheDir, - json: options.json, - }); - } + for (const source of result.sources) { + const repoLabel = source.repo + .replace(/^https?:\/\//, "") + .replace(/\.git$/, ""); + const targetLabel = source.targetDir + ? ` ${pc.dim("->")} ${pc.magenta(source.targetDir)}` + : ""; + ui.item(symbols.success, source.id, `${pc.blue(repoLabel)}${targetLabel}`); + } + if (result.skipped?.length) { + ui.line( + `${symbols.warn} Skipped ${result.skipped.length} existing source${result.skipped.length === 1 ? "" : "s"}: ${result.skipped.join(", ")}`, + ); + } + ui.line( + `${symbols.info} Updated ${pc.gray(path.relative(process.cwd(), result.configPath) || "docs.config.json")}`, + ); + if (result.gitignoreUpdated && result.gitignorePath) { + ui.line( + `${symbols.info} Updated ${pc.gray(ui.path(result.gitignorePath))}`, + ); + } +}; + +const runRemove = async ( + parsed: Extract, +) => { + const options = parsed.options; + const { removeSources } = await import("#commands/remove"); + const { pruneCache } = await import("#commands/prune"); + if (parsed.ids.length === 0) { + throw new Error("Usage: docs-cache remove "); + } + const result = await removeSources({ + configPath: options.config, + ids: parsed.ids, + }); + if (options.json) { + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); return; } - if (command === "status") { - const { getStatus, printStatus } = await import("../status"); - const status = await getStatus({ + if (result.removed.length > 0) { + ui.line( + `${symbols.success} Removed ${result.removed.length} source${result.removed.length === 1 ? "" : "s"}: ${result.removed.join(", ")}`, + ); + } + if (result.missing.length > 0) { + ui.line( + `${symbols.warn} Missing ${result.missing.length} source${result.missing.length === 1 ? "" : "s"}: ${result.missing.join(", ")}`, + ); + } + if (result.targetsRemoved.length > 0) { + const targetLabels = result.targetsRemoved + .map((entry) => `${entry.id} -> ${ui.path(entry.targetDir)}`) + .join(", "); + ui.line( + `${symbols.success} Removed ${result.targetsRemoved.length} target${result.targetsRemoved.length === 1 ? "" : "s"}: ${targetLabels}`, + ); + } + ui.line( + `${symbols.info} Updated ${pc.gray(path.relative(process.cwd(), result.configPath) || "docs.config.json")}`, + ); + if (options.prune) { + await pruneCache({ configPath: options.config, cacheDirOverride: options.cacheDir, json: options.json, }); - if (options.json) { - process.stdout.write(`${JSON.stringify(status, null, 2)}\n`); - } else { - printStatus(status); - } + } +}; + +const runStatus = async ( + parsed: Extract, +) => { + const options = parsed.options; + const { getStatus, printStatus } = await import("#commands/status"); + const status = await getStatus({ + configPath: options.config, + cacheDirOverride: options.cacheDir, + json: options.json, + }); + if (options.json) { + process.stdout.write(`${JSON.stringify(status, null, 2)}\n`); return; } - if (command === "clean") { - const { cleanCache } = await import("../clean"); - const result = await cleanCache({ - configPath: options.config, - cacheDirOverride: options.cacheDir, - json: options.json, - }); - if (options.json) { - process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); - } else if (result.removed) { - ui.line( - `${symbols.success} Removed cache at ${ui.path(result.cacheDir)}`, - ); - } else { - ui.line( - `${symbols.info} Cache already missing at ${ui.path(result.cacheDir)}`, - ); - } + printStatus(status); +}; + +const runClean = async (parsed: Extract) => { + const options = parsed.options; + const { cleanCache } = await import("#commands/clean"); + const result = await cleanCache({ + configPath: options.config, + cacheDirOverride: options.cacheDir, + json: options.json, + }); + if (options.json) { + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); return; } - if (command === "clean-cache") { - const { cleanGitCache } = await import("../clean-git-cache"); - const result = await cleanGitCache(); - if (options.json) { - process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); - } else if (result.removed) { - const sizeInMB = - result.bytesFreed !== undefined - ? `${(result.bytesFreed / 1024 / 1024).toFixed(2)} MB` - : "unknown size"; - const repoLabel = - result.repoCount !== undefined - ? ` (${result.repoCount} cached repositor${result.repoCount === 1 ? "y" : "ies"})` - : ""; - ui.line( - `${symbols.success} Cleared global git cache${repoLabel}: ${sizeInMB} freed`, - ); - ui.line(`${symbols.info} Cache location: ${ui.path(result.cacheDir)}`); - } else { - ui.line( - `${symbols.info} Global git cache already empty at ${ui.path(result.cacheDir)}`, - ); - } + if (result.removed) { + ui.line(`${symbols.success} Removed cache at ${ui.path(result.cacheDir)}`); return; } - if (command === "prune") { - const { pruneCache } = await import("../prune"); - const result = await pruneCache({ - configPath: options.config, - cacheDirOverride: options.cacheDir, - json: options.json, - }); - if (options.json) { - process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); - } else if (result.removed.length === 0) { - ui.line(`${symbols.info} No cache entries to prune.`); - } else { - ui.line( - `${symbols.success} Pruned ${result.removed.length} cache entr${result.removed.length === 1 ? "y" : "ies"}: ${result.removed.join(", ")}`, - ); - } + ui.line( + `${symbols.info} Cache already missing at ${ui.path(result.cacheDir)}`, + ); +}; + +const runCleanCache = async ( + parsed: Extract, +) => { + const options = parsed.options; + const { cleanGitCache } = await import("#commands/clean-git-cache"); + const result = await cleanGitCache(); + if (options.json) { + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); return; } - if (command === "sync") { - const { printSyncPlan, runSync } = await import("../sync"); - const plan = await runSync({ - configPath: options.config, - cacheDirOverride: options.cacheDir, - json: options.json, - lockOnly: options.lockOnly, - offline: options.offline, - failOnMiss: options.failOnMiss, - timeoutMs: options.timeoutMs, - verbose: options.verbose, - }); - if (options.json) { - process.stdout.write(`${JSON.stringify(plan, null, 2)}\n`); - } else { - printSyncPlan(plan); - } + if (!result.removed) { + ui.line( + `${symbols.info} Global git cache already empty at ${ui.path(result.cacheDir)}`, + ); return; } - if (command === "verify") { - const { printVerify, verifyCache } = await import("../verify"); - const report = await verifyCache({ - configPath: options.config, - cacheDirOverride: options.cacheDir, - json: options.json, - }); - if (options.json) { - process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); - } else { - printVerify(report); - } - if (report.results.some((result) => !result.ok)) { - process.exit(ExitCode.FatalError); - } + const sizeInMB = + result.bytesFreed !== undefined + ? `${(result.bytesFreed / 1024 / 1024).toFixed(2)} MB` + : "unknown size"; + const repoLabel = + result.repoCount !== undefined + ? ` (${result.repoCount} cached repositor${result.repoCount === 1 ? "y" : "ies"})` + : ""; + ui.line( + `${symbols.success} Cleared global git cache${repoLabel}: ${sizeInMB} freed`, + ); + ui.line(`${symbols.info} Cache location: ${ui.path(result.cacheDir)}`); +}; + +const runPrune = async (parsed: Extract) => { + const options = parsed.options; + const { pruneCache } = await import("#commands/prune"); + const result = await pruneCache({ + configPath: options.config, + cacheDirOverride: options.cacheDir, + json: options.json, + }); + if (options.json) { + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); return; } - if (command === "init") { - const { initConfig } = await import("../init"); - if (options.config) { - throw new Error("Init does not accept --config. Use the project root."); - } - const result = await initConfig({ - cacheDirOverride: options.cacheDir, - json: options.json, - }); - if (options.json) { - process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); - } else { - ui.line( - `${symbols.success} Wrote ${pc.gray(ui.path(result.configPath))}`, - ); - if (result.gitignoreUpdated && result.gitignorePath) { - ui.line( - `${symbols.info} Updated ${pc.gray(ui.path(result.gitignorePath))}`, - ); - } - } + if (result.removed.length === 0) { + ui.line(`${symbols.info} No cache entries to prune.`); + return; + } + ui.line( + `${symbols.success} Pruned ${result.removed.length} cache entr${result.removed.length === 1 ? "y" : "ies"}: ${result.removed.join(", ")}`, + ); +}; + +const runSyncCommand = async ( + parsed: Extract, +) => { + const options = parsed.options; + const { printSyncPlan, runSync } = await import("#commands/sync"); + const plan = await runSync({ + configPath: options.config, + cacheDirOverride: options.cacheDir, + json: options.json, + lockOnly: options.lockOnly, + offline: options.offline, + failOnMiss: options.failOnMiss, + timeoutMs: options.timeoutMs, + verbose: options.verbose, + }); + if (options.json) { + process.stdout.write(`${JSON.stringify(plan, null, 2)}\n`); + return; + } + printSyncPlan(plan); +}; + +const runVerify = async ( + parsed: Extract, +) => { + const options = parsed.options; + const { printVerify, verifyCache } = await import("#commands/verify"); + const report = await verifyCache({ + configPath: options.config, + cacheDirOverride: options.cacheDir, + json: options.json, + }); + if (options.json) { + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + printVerify(report); + } + if (report.results.some((result) => !result.ok)) { + process.exit(ExitCode.FatalError); + } +}; + +const runInit = async (parsed: Extract) => { + const options = parsed.options; + const { initConfig } = await import("#commands/init"); + if (options.config) { + throw new Error("Init does not accept --config. Use the project root."); + } + const result = await initConfig({ + cacheDirOverride: options.cacheDir, + json: options.json, + }); + if (options.json) { + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); return; } - ui.line(`${CLI_NAME} ${command}: not implemented yet.`); + ui.line(`${symbols.success} Wrote ${pc.gray(ui.path(result.configPath))}`); + if (result.gitignoreUpdated && result.gitignorePath) { + ui.line( + `${symbols.info} Updated ${pc.gray(ui.path(result.gitignorePath))}`, + ); + } +}; + +const runCommand = async (parsed: CliCommand) => { + switch (parsed.command) { + case "add": + await runAdd(parsed); + return; + case "remove": + await runRemove(parsed); + return; + case "status": + await runStatus(parsed); + return; + case "clean": + await runClean(parsed); + return; + case "clean-cache": + await runCleanCache(parsed); + return; + case "prune": + await runPrune(parsed); + return; + case "sync": + await runSyncCommand(parsed); + return; + case "verify": + await runVerify(parsed); + return; + case "init": + await runInit(parsed); + return; + default: + ui.line(`${CLI_NAME} ${parsed.command}: not implemented yet.`); + } }; /** @@ -330,12 +377,12 @@ export async function main(): Promise { await runCommand(parsed.parsed); } catch (error) { - errorHandler(error as Error); + errorHandler(error); } } -function errorHandler(error: Error): void { - const message = error.message || String(error); +function errorHandler(error: unknown): void { + const message = error instanceof Error ? error.message : String(error); printError(message); process.exit(ExitCode.FatalError); } diff --git a/src/cli/live-output.ts b/src/cli/live-output.ts new file mode 100644 index 0000000..d652b44 --- /dev/null +++ b/src/cli/live-output.ts @@ -0,0 +1,53 @@ +import cliTruncate from "cli-truncate"; +import { createLogUpdate } from "log-update"; + +type LiveOutputOptions = { + stdout?: NodeJS.WriteStream; + maxWidth?: number; +}; + +export type LiveOutput = { + render: (lines: string[]) => void; + persist: (lines: string[]) => void; + clear: () => void; + stop: () => void; +}; + +const normalizeLines = (lines: string[]) => + lines.map((line) => (line.length === 0 ? line : line)); + +export const createLiveOutput = ( + options: LiveOutputOptions = {}, +): LiveOutput => { + const stdout = options.stdout ?? process.stdout; + const updater = createLogUpdate(stdout); + const maxWidth = options.maxWidth ?? Math.max(20, (stdout.columns ?? 80) - 2); + const truncate = (line: string) => + cliTruncate(line, maxWidth, { position: "end" }); + + const render = (lines: string[]) => { + const output = normalizeLines(lines).map(truncate).join("\n"); + updater(output); + }; + + const persist = (lines: string[]) => { + const output = normalizeLines(lines).map(truncate).join("\n"); + updater(output); + updater.done(); + }; + + const clear = () => { + updater.clear(); + }; + + const stop = () => { + updater.done(); + }; + + return { + render, + persist, + clear, + stop, + }; +}; diff --git a/src/cli/parse-args.ts b/src/cli/parse-args.ts index a26fd6b..f26a0ac 100644 --- a/src/cli/parse-args.ts +++ b/src/cli/parse-args.ts @@ -1,7 +1,7 @@ import process from "node:process"; import cac from "cac"; -import { ExitCode } from "./exit-code"; +import { ExitCode } from "#cli/exit-code"; import type { AddEntry, CliCommand, CliOptions } from "./types"; const COMMANDS = [ @@ -38,94 +38,153 @@ const POSITIONAL_SKIP_OPTIONS = new Set([ "--concurrency", "--timeout-ms", ]); +const ADD_ONLY_OPTIONS_WITH_VALUES = new Set([ + "--id", + "--source", + "--target", + "--target-dir", +]); + +const ADD_ENTRY_SKIP_OPTIONS = new Set([ + "--config", + "--cache-dir", + "--concurrency", + "--timeout-ms", +]); + +const VALUE_FLAGS = new Set([ + ...POSITIONAL_SKIP_OPTIONS, + ...ADD_ONLY_OPTIONS_WITH_VALUES, +]); + +type AddParseState = { + entries: AddEntry[]; + lastIndex: number; + pendingId: string | null; + lastWasRepoAdded: boolean; +}; + +const getArgValue = (arg: string, next: string | undefined, flag: string) => { + const rawValue = arg === flag ? next : arg.slice(flag.length + 1); + if (!rawValue || rawValue.startsWith("-")) { + throw new Error(`${flag} expects a value.`); + } + return rawValue; +}; + +const addEntry = (state: AddParseState, repo: string) => { + state.entries.push({ + repo, + ...(state.pendingId ? { id: state.pendingId } : {}), + }); + state.lastIndex = state.entries.length - 1; + state.pendingId = null; + state.lastWasRepoAdded = true; +}; + +const applyPendingId = (state: AddParseState, value: string) => { + const canApply = + state.lastWasRepoAdded && + state.lastIndex !== -1 && + state.entries[state.lastIndex]?.id === undefined && + state.pendingId === null; + if (!canApply) { + if (state.pendingId !== null) { + throw new Error("--id must be followed by a source."); + } + state.pendingId = value; + state.lastWasRepoAdded = false; + return; + } + state.entries[state.lastIndex].id = value; + state.lastWasRepoAdded = false; +}; + +const setTarget = (state: AddParseState, targetDir: string) => { + if (state.lastIndex === -1) { + throw new Error("--target must follow a --source entry."); + } + state.entries[state.lastIndex].targetDir = targetDir; + state.lastWasRepoAdded = false; +}; + +const findCommandIndex = (rawArgs: string[]) => { + for (let index = 0; index < rawArgs.length; index += 1) { + const arg = rawArgs[index]; + if (arg.startsWith("--")) { + const [flag] = arg.split("="); + if (VALUE_FLAGS.has(flag) && !arg.includes("=")) { + index += 1; + } + continue; + } + return index; + } + return -1; +}; const parseAddEntries = (rawArgs: string[]): AddEntry[] => { - const commandIndex = rawArgs.findIndex((arg) => !arg.startsWith("-")); + const commandIndex = findCommandIndex(rawArgs); const tail = commandIndex === -1 ? [] : rawArgs.slice(commandIndex + 1); - const entries: AddEntry[] = []; - let lastIndex = -1; - let pendingId: string | null = null; - let lastWasRepoAdded = false; - const skipNextFor = new Set([ - "--config", - "--cache-dir", - "--concurrency", - "--timeout-ms", - ]); + const state: AddParseState = { + entries: [], + lastIndex: -1, + pendingId: null, + lastWasRepoAdded: false, + }; for (let index = 0; index < tail.length; index += 1) { const arg = tail[index]; if (arg === "--id" || arg.startsWith("--id=")) { - const rawValue = arg === "--id" ? tail[index + 1] : arg.slice(5); - if (!rawValue || rawValue.startsWith("-")) { - throw new Error("--id expects a value."); - } + const value = getArgValue(arg, tail[index + 1], "--id"); if (arg === "--id") { index += 1; } - if ( - lastWasRepoAdded && - lastIndex !== -1 && - entries[lastIndex]?.id === undefined && - pendingId === null - ) { - entries[lastIndex].id = rawValue; - lastWasRepoAdded = false; - continue; - } - if (pendingId !== null) { - throw new Error("--id must be followed by a source."); - } - pendingId = rawValue; - lastWasRepoAdded = false; + applyPendingId(state, value); continue; } - if (arg === "--source") { - const next = tail[index + 1]; - if (!next || next.startsWith("-")) { - throw new Error("--source expects a value."); + if (arg === "--source" || arg.startsWith("--source=")) { + const value = getArgValue(arg, tail[index + 1], "--source"); + addEntry(state, value); + if (arg === "--source") { + index += 1; } - entries.push({ repo: next, ...(pendingId ? { id: pendingId } : {}) }); - lastIndex = entries.length - 1; - pendingId = null; - lastWasRepoAdded = true; - index += 1; continue; } - if (arg === "--target" || arg === "--target-dir") { - const next = tail[index + 1]; - if (!next || next.startsWith("-")) { - throw new Error("--target expects a value."); + if (arg === "--target" || arg.startsWith("--target=")) { + const value = getArgValue(arg, tail[index + 1], "--target"); + setTarget(state, value); + if (arg === "--target") { + index += 1; } - if (lastIndex === -1) { - throw new Error("--target must follow a --source entry."); + continue; + } + if (arg === "--target-dir" || arg.startsWith("--target-dir=")) { + const value = getArgValue(arg, tail[index + 1], "--target-dir"); + setTarget(state, value); + if (arg === "--target-dir") { + index += 1; } - entries[lastIndex].targetDir = next; - index += 1; - lastWasRepoAdded = false; continue; } - if (skipNextFor.has(arg)) { + if (ADD_ENTRY_SKIP_OPTIONS.has(arg)) { index += 1; - lastWasRepoAdded = false; + state.lastWasRepoAdded = false; continue; } if (arg.startsWith("--")) { - lastWasRepoAdded = false; + state.lastWasRepoAdded = false; continue; } - entries.push({ repo: arg, ...(pendingId ? { id: pendingId } : {}) }); - lastIndex = entries.length - 1; - pendingId = null; - lastWasRepoAdded = true; + addEntry(state, arg); } - if (pendingId !== null) { + if (state.pendingId !== null) { throw new Error("--id must be followed by a source."); } - return entries; + return state.entries; }; const parsePositionals = (rawArgs: string[]) => { - const commandIndex = rawArgs.findIndex((arg) => !arg.startsWith("-")); + const commandIndex = findCommandIndex(rawArgs); const tail = commandIndex === -1 ? [] : rawArgs.slice(commandIndex + 1); const positionals: string[] = []; for (let index = 0; index < tail.length; index += 1) { @@ -150,14 +209,107 @@ const assertAddOnlyOptions = (command: Command | null, rawArgs: string[]) => { if (ADD_ONLY_OPTIONS.has(arg)) { throw new Error(`${arg} is only valid for add.`); } - if ( - arg.startsWith("--id=") || - arg.startsWith("--source=") || - arg.startsWith("--target=") || - arg.startsWith("--target-dir=") - ) { - throw new Error(`${arg.split("=")[0]} is only valid for add.`); + if (!arg.startsWith("--")) { + continue; } + const [flag] = arg.split("="); + if (ADD_ONLY_OPTIONS_WITH_VALUES.has(flag)) { + throw new Error(`${flag} is only valid for add.`); + } + } +}; + +const buildOptions = (result: ReturnType["parse"]>) => { + const options: CliOptions = { + config: result.options.config, + cacheDir: result.options.cacheDir, + offline: Boolean(result.options.offline), + failOnMiss: Boolean(result.options.failOnMiss), + lockOnly: Boolean(result.options.lockOnly), + prune: Boolean(result.options.prune), + concurrency: result.options.concurrency + ? Number(result.options.concurrency) + : undefined, + json: Boolean(result.options.json), + timeoutMs: result.options.timeoutMs + ? Number(result.options.timeoutMs) + : undefined, + silent: Boolean(result.options.silent), + verbose: Boolean(result.options.verbose), + }; + + if (options.concurrency !== undefined && options.concurrency < 1) { + throw new Error("--concurrency must be a positive number."); + } + if ( + options.concurrency !== undefined && + !Number.isFinite(options.concurrency) + ) { + throw new Error("--concurrency must be a positive number."); + } + if (options.timeoutMs !== undefined && options.timeoutMs < 1) { + throw new Error("--timeout-ms must be a positive number."); + } + if (options.timeoutMs !== undefined && !Number.isFinite(options.timeoutMs)) { + throw new Error("--timeout-ms must be a positive number."); + } + + return options; +}; + +const getCommandFromArgs = (rawArgs: string[]) => { + const commandIndex = findCommandIndex(rawArgs); + const command = + commandIndex === -1 ? undefined : (rawArgs[commandIndex] as Command); + if (command && !COMMANDS.includes(command)) { + throw new Error(`Unknown command '${command}'.`); + } + return command ?? null; +}; + +const getPositionals = ( + command: Command | null, + rawArgs: string[], + entries: AddEntry[] | null, +) => { + if (command === "add") { + const addEntries = entries ?? parseAddEntries(rawArgs); + return { positionals: addEntries.map((entry) => entry.repo), addEntries }; + } + return { positionals: parsePositionals(rawArgs), addEntries: entries }; +}; + +const buildParsedCommand = ( + command: Command | null, + options: CliOptions, + positionals: string[], + addEntries: AddEntry[] | null, +): CliCommand => { + switch (command) { + case "add": + return { + command: "add", + entries: addEntries ?? [], + options, + }; + case "remove": + return { command: "remove", ids: positionals, options }; + case "sync": + return { command: "sync", options }; + case "status": + return { command: "status", options }; + case "clean": + return { command: "clean", options }; + case "clean-cache": + return { command: "clean-cache", options }; + case "prune": + return { command: "prune", options }; + case "verify": + return { command: "verify", options }; + case "init": + return { command: "init", options }; + default: + return { command: null, options }; } }; @@ -197,90 +349,18 @@ export const parseArgs = (argv = process.argv): ParsedArgs => { const result = cli.parse(argv, { run: false }); const rawArgs = argv.slice(2); - const commandIndex = rawArgs.findIndex((arg) => !arg.startsWith("-")); - const command = - commandIndex === -1 ? undefined : (rawArgs[commandIndex] as Command); - if (command && !COMMANDS.includes(command)) { - throw new Error(`Unknown command '${command}'.`); - } - - const options: CliOptions = { - config: result.options.config, - cacheDir: result.options.cacheDir, - offline: Boolean(result.options.offline), - failOnMiss: Boolean(result.options.failOnMiss), - lockOnly: Boolean(result.options.lockOnly), - prune: Boolean(result.options.prune), - concurrency: result.options.concurrency - ? Number(result.options.concurrency) - : undefined, - json: Boolean(result.options.json), - timeoutMs: result.options.timeoutMs - ? Number(result.options.timeoutMs) - : undefined, - silent: Boolean(result.options.silent), - verbose: Boolean(result.options.verbose), - }; - - if (options.concurrency !== undefined && options.concurrency < 1) { - throw new Error("--concurrency must be a positive number."); - } - if (options.timeoutMs !== undefined && options.timeoutMs < 1) { - throw new Error("--timeout-ms must be a positive number."); - } - - assertAddOnlyOptions(command ?? null, rawArgs); - let addEntries: AddEntry[] | null = null; - const positionals = (() => { - switch (command ?? null) { - case "add": - addEntries = parseAddEntries(rawArgs); - return addEntries.map((entry) => entry.repo); - case "remove": - return parsePositionals(rawArgs); - default: - return parsePositionals(rawArgs); - } - })(); - let parsed: CliCommand; - switch (command ?? null) { - case "add": - parsed = { - command: "add", - entries: addEntries ?? parseAddEntries(rawArgs), - options, - }; - break; - case "remove": - parsed = { command: "remove", ids: positionals, options }; - break; - case "sync": - parsed = { command: "sync", options }; - break; - case "status": - parsed = { command: "status", options }; - break; - case "clean": - parsed = { command: "clean", options }; - break; - case "clean-cache": - parsed = { command: "clean-cache", options }; - break; - case "prune": - parsed = { command: "prune", options }; - break; - case "verify": - parsed = { command: "verify", options }; - break; - case "init": - parsed = { command: "init", options }; - break; - default: - parsed = { command: null, options }; - break; - } + const command = getCommandFromArgs(rawArgs); + const options = buildOptions(result); + assertAddOnlyOptions(command, rawArgs); + const { positionals, addEntries } = getPositionals(command, rawArgs, null); + const parsed = buildParsedCommand( + command, + options, + positionals, + addEntries, + ); return { - command: command ?? null, + command, options, positionals, rawArgs, diff --git a/src/cli/run.ts b/src/cli/run.ts index efc922b..f1e4c05 100644 --- a/src/cli/run.ts +++ b/src/cli/run.ts @@ -1,3 +1,3 @@ -import { main } from "./index"; +import { main } from "#cli/index"; main(); diff --git a/src/cli/task-reporter.ts b/src/cli/task-reporter.ts new file mode 100644 index 0000000..9dd1352 --- /dev/null +++ b/src/cli/task-reporter.ts @@ -0,0 +1,156 @@ +import pc from "picocolors"; +import { createLiveOutput, type LiveOutput } from "#cli/live-output"; +import { symbols } from "#cli/ui"; + +type TaskState = "pending" | "running" | "success" | "warn" | "error"; + +const formatDuration = (ms: number) => { + const seconds = Math.max(0, ms / 1000); + if (seconds < 60) { + return `${seconds.toFixed(1)}s`; + } + const minutes = Math.floor(seconds / 60); + const remainder = seconds % 60; + return `${minutes}m ${remainder.toFixed(1)}s`; +}; + +export type TaskReporterOptions = { + maxLiveLines?: number; + output?: LiveOutput; +}; + +export class TaskReporter { + private readonly output: LiveOutput; + private readonly maxLiveLines: number; + private readonly startTime = Date.now(); + private readonly tasks = new Map(); + private readonly results: string[] = []; + private readonly liveLines: string[] = []; + private readonly hasTty = Boolean(process.stdout.isTTY); + private timer: NodeJS.Timeout | null = null; + private warnings = 0; + private errors = 0; + + constructor(options: TaskReporterOptions = {}) { + this.output = options.output ?? createLiveOutput(); + this.maxLiveLines = options.maxLiveLines ?? 4; + this.startTimer(); + } + + start(label: string) { + this.tasks.set(label, "running"); + this.render(); + } + + info(label: string, details?: string) { + this.results.push(this.formatLine(symbols.info, label, details)); + this.render(); + } + + warn(label: string, details?: string) { + this.warnings += 1; + this.results.push(this.formatLine(symbols.warn, label, details)); + this.render(); + } + + error(label: string, details?: string) { + this.errors += 1; + this.results.push(this.formatLine(symbols.error, label, details)); + this.render(); + } + + success(label: string, details?: string, icon: string = symbols.success) { + this.tasks.set(label, "success"); + this.results.push(this.formatLine(icon, label, details)); + this.liveLines.length = 0; + this.render(); + } + + debug(text: string) { + this.liveLines.push(pc.dim(text)); + if (this.liveLines.length > this.maxLiveLines) { + this.liveLines.splice(0, this.liveLines.length - this.maxLiveLines); + } + this.render(); + } + + finish(summary?: string) { + this.liveLines.length = 0; + const durationMs = Date.now() - this.startTime; + const parts = [ + `Completed in ${formatDuration(durationMs)}`, + this.warnings + ? `${this.warnings} warning${this.warnings === 1 ? "" : "s"}` + : null, + this.errors + ? `${this.errors} error${this.errors === 1 ? "" : "s"}` + : null, + ].filter(Boolean) as string[]; + const suffix = parts.length ? ` · ${parts.join(" · ")}` : ""; + const message = summary + ? `${summary}${suffix}` + : `${symbols.info}${suffix}`; + this.output.persist(this.composeView([message])); + this.stopTimer(); + } + + stop() { + this.output.stop(); + this.stopTimer(); + } + + private render() { + if (!this.hasTty) return; + this.output.render(this.composeView()); + } + + private startTimer() { + if (!this.hasTty) return; + this.timer = setInterval(() => { + if (this.hasRunningTasks()) { + this.render(); + } + }, 100); + this.timer.unref?.(); + } + + private stopTimer() { + if (!this.timer) return; + clearInterval(this.timer); + this.timer = null; + } + + private composeView(extraFooter?: string[]) { + const running = Array.from(this.tasks.entries()) + .filter(([, state]) => state === "running") + .map(([label]) => `${pc.cyan("→")} ${label}`); + const runningCount = running.length; + const completedCount = this.results.length; + const elapsed = this.hasRunningTasks() + ? pc.dim( + `elapsed: ${formatDuration(Date.now() - this.startTime)} · ${runningCount} running · ${completedCount} completed`, + ) + : ""; + const lines = [ + ...this.results, + ...running, + ...this.liveLines, + elapsed, + ...(extraFooter ?? []), + ].filter((line) => line.length > 0); + return lines.length > 0 ? lines : [" "]; + } + + private hasRunningTasks() { + for (const state of this.tasks.values()) { + if (state === "running") return true; + } + return false; + } + + private formatLine(icon: string, label: string, details?: string) { + const partLabel = pc.bold(label); + const partDetails = details ? pc.gray(details) : ""; + return ` ${icon} ${partLabel} ${partDetails}`.trimEnd(); + } +} diff --git a/src/cli/ui.ts b/src/cli/ui.ts index 76a09e5..3882e1e 100644 --- a/src/cli/ui.ts +++ b/src/cli/ui.ts @@ -1,10 +1,11 @@ import path from "node:path"; import pc from "picocolors"; -import { toPosixPath } from "../paths"; +import { toPosixPath } from "#core/paths"; export const symbols = { error: pc.red("✖"), success: pc.green("✔"), + synced: pc.green("●"), info: pc.blue("ℹ"), warn: pc.yellow("⚠"), }; @@ -20,6 +21,10 @@ export const setVerboseMode = (verbose: boolean) => { _verboseMode = verbose; }; +export const isSilentMode = () => _silentMode; + +export const isVerboseMode = () => _verboseMode; + export const ui = { // Formatters path: (value: string) => { diff --git a/src/commands/add.ts b/src/commands/add.ts new file mode 100644 index 0000000..e09e678 --- /dev/null +++ b/src/commands/add.ts @@ -0,0 +1,104 @@ +import path from "node:path"; +import { DEFAULT_CACHE_DIR, type DocsCacheConfig } from "#config"; +import { + mergeConfigBase, + readConfigAtPath, + resolveConfigTarget, + writeConfigFile, +} from "#config/io"; +import { ensureGitignoreEntry } from "#core/gitignore"; +import { resolveTargetDir } from "#core/paths"; +import { assertSafeSourceId } from "#core/source-id"; +import { resolveRepoInput } from "#git/resolve-repo"; + +const buildNewSources = ( + entries: Array<{ id?: string; repo: string; targetDir?: string }>, + config: DocsCacheConfig, + resolvedPath: string, +) => { + const existingIds = new Set(config.sources.map((source) => source.id)); + const skipped: string[] = []; + const newSources = entries + .map((entry) => { + const resolved = resolveRepoInput(entry.repo); + const sourceId = entry.id || resolved.inferredId; + if (!sourceId) { + throw new Error("Unable to infer id. Provide an explicit id."); + } + const safeId = assertSafeSourceId(sourceId, "source id"); + if (existingIds.has(safeId)) { + skipped.push(safeId); + return null; + } + existingIds.add(safeId); + if (entry.targetDir) { + resolveTargetDir(resolvedPath, entry.targetDir); + } + return { + id: safeId, + repo: resolved.repoUrl, + ...(entry.targetDir ? { targetDir: entry.targetDir } : {}), + ...(resolved.ref ? { ref: resolved.ref } : {}), + }; + }) + .filter(Boolean) as Array<{ + id: string; + repo: string; + targetDir?: string; + ref?: string; + }>; + return { newSources, skipped }; +}; + +const ensureGitignore = async ( + resolvedPath: string, + cacheDir: string, + shouldWrite: boolean, +) => { + if (!shouldWrite) { + return null; + } + return ensureGitignoreEntry(path.dirname(resolvedPath), cacheDir); +}; + +export const addSources = async (params: { + configPath?: string; + entries: Array<{ id?: string; repo: string; targetDir?: string }>; +}) => { + const target = await resolveConfigTarget(params.configPath); + const resolvedPath = target.resolvedPath; + const { config, rawConfig, rawPackage, hadDocsCacheConfig } = + await readConfigAtPath(target, { allowMissing: true }); + const { newSources, skipped } = buildNewSources( + params.entries, + config, + resolvedPath, + ); + if (newSources.length === 0) { + throw new Error("All sources already exist in config."); + } + const nextConfig = mergeConfigBase(rawConfig ?? config, [ + ...config.sources, + ...newSources, + ]); + await writeConfigFile({ + mode: target.mode, + resolvedPath, + config: nextConfig, + rawPackage, + }); + const gitignoreResult = await ensureGitignore( + resolvedPath, + rawConfig?.cacheDir ?? DEFAULT_CACHE_DIR, + !hadDocsCacheConfig, + ); + + return { + configPath: resolvedPath, + sources: newSources, + skipped, + created: true, + gitignoreUpdated: gitignoreResult?.updated ?? false, + gitignorePath: gitignoreResult?.gitignorePath ?? null, + }; +}; diff --git a/src/clean-git-cache.ts b/src/commands/clean-git-cache.ts similarity index 95% rename from src/clean-git-cache.ts rename to src/commands/clean-git-cache.ts index 382883b..8e86754 100644 --- a/src/clean-git-cache.ts +++ b/src/commands/clean-git-cache.ts @@ -1,7 +1,7 @@ import { readdir, rm, stat } from "node:fs/promises"; import path from "node:path"; -import { exists, resolveGitCacheDir } from "./git/cache-dir"; +import { exists, resolveGitCacheDir } from "#git/cache-dir"; const getDirSize = async (dirPath: string): Promise => { try { diff --git a/src/clean.ts b/src/commands/clean.ts similarity index 87% rename from src/clean.ts rename to src/commands/clean.ts index c823519..fa3e12a 100644 --- a/src/clean.ts +++ b/src/commands/clean.ts @@ -1,6 +1,6 @@ import { access, rm } from "node:fs/promises"; -import { DEFAULT_CACHE_DIR, loadConfig } from "./config"; -import { resolveCacheDir } from "./paths"; +import { DEFAULT_CACHE_DIR, loadConfig } from "#config"; +import { resolveCacheDir } from "#core/paths"; type CleanOptions = { configPath?: string; diff --git a/src/commands/init.ts b/src/commands/init.ts new file mode 100644 index 0000000..f4e5c76 --- /dev/null +++ b/src/commands/init.ts @@ -0,0 +1,230 @@ +import { access, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { + confirm as clackConfirm, + isCancel as clackIsCancel, + select as clackSelect, + text as clackText, +} from "@clack/prompts"; +import { + DEFAULT_CACHE_DIR, + DEFAULT_CONFIG_FILENAME, + type DocsCacheConfig, + stripDefaultConfigValues, + writeConfig, +} from "#config"; +import { ensureGitignoreEntry, getGitignoreStatus } from "#core/gitignore"; + +type InitOptions = { + cacheDirOverride?: string; + json: boolean; + cwd?: string; +}; + +type PromptDeps = { + confirm?: typeof clackConfirm; + isCancel?: typeof clackIsCancel; + select?: typeof clackSelect; + text?: typeof clackText; +}; + +const exists = async (target: string) => { + try { + await access(target); + return true; + } catch { + return false; + } +}; + +const readJson = async (filePath: string) => { + const raw = await readFile(filePath, "utf8"); + return JSON.parse(raw) as Record; +}; + +const findExistingConfigPaths = async (cwd: string) => { + const defaultConfigPath = path.resolve(cwd, DEFAULT_CONFIG_FILENAME); + const packagePath = path.resolve(cwd, "package.json"); + const existingConfigPaths: string[] = []; + if (await exists(defaultConfigPath)) { + existingConfigPaths.push(defaultConfigPath); + } + if (await exists(packagePath)) { + const parsed = await readJson(packagePath); + if (parsed["docs-cache"]) { + existingConfigPaths.push(packagePath); + } + } + return { existingConfigPaths, defaultConfigPath, packagePath }; +}; + +const selectConfigPath = async ( + packagePath: string, + defaultConfigPath: string, + select: typeof clackSelect, + isCancel: typeof clackIsCancel, +) => { + if (!(await exists(packagePath))) { + return defaultConfigPath; + } + const parsed = await readJson(packagePath); + if (parsed["docs-cache"]) { + return defaultConfigPath; + } + const locationAnswer = await select({ + message: "Config location", + options: [ + { value: "config", label: "docs.config.json" }, + { value: "package", label: "package.json" }, + ], + initialValue: "config", + }); + if (isCancel(locationAnswer)) { + throw new Error("Init cancelled."); + } + return locationAnswer === "package" ? packagePath : defaultConfigPath; +}; + +const promptInitAnswers = async ( + cacheDir: string, + cwd: string, + confirm: typeof clackConfirm, + text: typeof clackText, + isCancel: typeof clackIsCancel, +) => { + const cacheDirAnswer = await text({ + message: "Cache directory", + initialValue: cacheDir, + }); + if (isCancel(cacheDirAnswer)) { + throw new Error("Init cancelled."); + } + const cacheDirValue = cacheDirAnswer || DEFAULT_CACHE_DIR; + const tocAnswer = await confirm({ + message: + "Generate TOC.md (table of contents with links to all documentation)", + initialValue: true, + }); + if (isCancel(tocAnswer)) { + throw new Error("Init cancelled."); + } + const gitignoreStatus = await getGitignoreStatus(cwd, cacheDirValue); + let gitignoreAnswer = false; + if (gitignoreStatus.entry && !gitignoreStatus.hasEntry) { + const reply = await confirm({ + message: "Add cache directory to .gitignore", + initialValue: true, + }); + if (isCancel(reply)) { + throw new Error("Init cancelled."); + } + gitignoreAnswer = reply; + } + return { + cacheDir: cacheDirValue, + toc: tocAnswer, + gitignore: gitignoreAnswer, + }; +}; + +const buildBaseConfig = (cacheDir: string, toc: boolean): DocsCacheConfig => { + const config: DocsCacheConfig = { + $schema: + "https://raw.githubusercontent.com/fbosch/docs-cache/main/docs.config.schema.json", + sources: [], + }; + if (cacheDir !== DEFAULT_CACHE_DIR) { + config.cacheDir = cacheDir; + } + if (!toc) { + config.defaults = { toc: false }; + } + return config; +}; + +const writePackageConfig = async ( + configPath: string, + config: DocsCacheConfig, + gitignore: boolean, +) => { + const raw = await readFile(configPath, "utf8"); + const pkg = JSON.parse(raw) as Record; + if (pkg["docs-cache"]) { + throw new Error(`docs-cache config already exists in ${configPath}.`); + } + pkg["docs-cache"] = stripDefaultConfigValues(config); + await writeFile(configPath, `${JSON.stringify(pkg, null, 2)}\n`, "utf8"); + const gitignoreResult = gitignore + ? await ensureGitignoreEntry( + path.dirname(configPath), + config.cacheDir ?? DEFAULT_CACHE_DIR, + ) + : null; + return { + configPath, + created: true, + gitignoreUpdated: gitignoreResult?.updated ?? false, + gitignorePath: gitignoreResult?.gitignorePath ?? null, + }; +}; + +const writeStandaloneConfig = async ( + configPath: string, + config: DocsCacheConfig, + gitignore: boolean, +) => { + await writeConfig(configPath, config); + const gitignoreResult = gitignore + ? await ensureGitignoreEntry( + path.dirname(configPath), + config.cacheDir ?? DEFAULT_CACHE_DIR, + ) + : null; + return { + configPath, + created: true, + gitignoreUpdated: gitignoreResult?.updated ?? false, + gitignorePath: gitignoreResult?.gitignorePath ?? null, + }; +}; + +export const initConfig = async ( + options: InitOptions, + deps: PromptDeps = {}, +) => { + const confirm = deps.confirm ?? clackConfirm; + const isCancel = deps.isCancel ?? clackIsCancel; + const select = deps.select ?? clackSelect; + const text = deps.text ?? clackText; + const cwd = options.cwd ?? process.cwd(); + const { existingConfigPaths, defaultConfigPath, packagePath } = + await findExistingConfigPaths(cwd); + if (existingConfigPaths.length > 0) { + throw new Error( + `Config already exists at ${existingConfigPaths.join(", ")}. Init aborted.`, + ); + } + const configPath = await selectConfigPath( + packagePath, + defaultConfigPath, + select, + isCancel, + ); + const cacheDir = options.cacheDirOverride ?? DEFAULT_CACHE_DIR; + const answers = await promptInitAnswers( + cacheDir, + cwd, + confirm, + text, + isCancel, + ); + const resolvedConfigPath = path.resolve(cwd, configPath); + const config = buildBaseConfig(answers.cacheDir, answers.toc); + if (path.basename(resolvedConfigPath) === "package.json") { + return writePackageConfig(resolvedConfigPath, config, answers.gitignore); + } + if (await exists(resolvedConfigPath)) { + throw new Error(`Config already exists at ${resolvedConfigPath}.`); + } + return writeStandaloneConfig(resolvedConfigPath, config, answers.gitignore); +}; diff --git a/src/prune.ts b/src/commands/prune.ts similarity index 92% rename from src/prune.ts rename to src/commands/prune.ts index b200107..bdbc986 100644 --- a/src/prune.ts +++ b/src/commands/prune.ts @@ -1,7 +1,7 @@ import { access, readdir, rm } from "node:fs/promises"; import path from "node:path"; -import { DEFAULT_CACHE_DIR, loadConfig } from "./config"; -import { resolveCacheDir } from "./paths"; +import { DEFAULT_CACHE_DIR, loadConfig } from "#config"; +import { resolveCacheDir } from "#core/paths"; type PruneOptions = { configPath?: string; diff --git a/src/commands/remove.ts b/src/commands/remove.ts new file mode 100644 index 0000000..56706e8 --- /dev/null +++ b/src/commands/remove.ts @@ -0,0 +1,102 @@ +import { rm } from "node:fs/promises"; +import type { DocsCacheConfig } from "#config"; +import { + mergeConfigBase, + readConfigAtPath, + resolveConfigTarget, + writeConfigFile, +} from "#config/io"; +import { resolveTargetDir } from "#core/paths"; +import { resolveRepoInput } from "#git/resolve-repo"; + +const resolveIdsToRemove = (ids: string[], config: DocsCacheConfig) => { + const sourcesById = new Map( + config.sources.map((source) => [source.id, source]), + ); + const sourcesByRepo = new Map( + config.sources.map((source) => [source.repo, source]), + ); + const idsToRemove = new Set(); + const missing: string[] = []; + for (const token of ids) { + if (sourcesById.has(token)) { + idsToRemove.add(token); + continue; + } + const resolved = resolveRepoInput(token); + if (resolved.repoUrl && sourcesByRepo.has(resolved.repoUrl)) { + const source = sourcesByRepo.get(resolved.repoUrl); + if (source) { + idsToRemove.add(source.id); + } + continue; + } + if (resolved.inferredId && sourcesById.has(resolved.inferredId)) { + idsToRemove.add(resolved.inferredId); + continue; + } + missing.push(token); + } + return { idsToRemove, missing }; +}; + +const removeTargets = async ( + resolvedPath: string, + removedSources: DocsCacheConfig["sources"], +) => { + const targetRemovals: Array<{ id: string; targetDir: string }> = []; + for (const source of removedSources) { + if (!source.targetDir) { + continue; + } + const targetDir = resolveTargetDir(resolvedPath, source.targetDir); + await rm(targetDir, { recursive: true, force: true }); + targetRemovals.push({ id: source.id, targetDir }); + } + return targetRemovals; +}; + +export const removeSources = async (params: { + configPath?: string; + ids: string[]; +}) => { + if (params.ids.length === 0) { + throw new Error("No sources specified to remove."); + } + const target = await resolveConfigTarget(params.configPath); + const resolvedPath = target.resolvedPath; + const { config, rawConfig, rawPackage } = await readConfigAtPath(target); + if (target.mode === "package" && !rawConfig) { + throw new Error(`Missing docs-cache config in ${resolvedPath}.`); + } + const { idsToRemove, missing } = resolveIdsToRemove(params.ids, config); + const remaining = config.sources.filter( + (source) => !idsToRemove.has(source.id), + ); + const removed = config.sources + .filter((source) => idsToRemove.has(source.id)) + .map((source) => source.id); + const removedSources = config.sources.filter((source) => + idsToRemove.has(source.id), + ); + + if (removed.length === 0) { + throw new Error("No matching sources found to remove."); + } + + const nextConfig = mergeConfigBase(rawConfig ?? config, remaining); + await writeConfigFile({ + mode: target.mode, + resolvedPath, + config: nextConfig, + rawPackage, + }); + const targetRemovals = await removeTargets(resolvedPath, removedSources); + + return { + configPath: resolvedPath, + removed, + missing, + targetsRemoved: targetRemovals, + }; +}; diff --git a/src/status.ts b/src/commands/status.ts similarity index 93% rename from src/status.ts rename to src/commands/status.ts index 48113d2..40f3efc 100644 --- a/src/status.ts +++ b/src/commands/status.ts @@ -1,9 +1,9 @@ import { access } from "node:fs/promises"; import pc from "picocolors"; -import { symbols, ui } from "./cli/ui"; -import { DEFAULT_CACHE_DIR, loadConfig } from "./config"; -import { DEFAULT_LOCK_FILENAME, readLock, resolveLockPath } from "./lock"; -import { getCacheLayout, resolveCacheDir } from "./paths"; +import { DEFAULT_LOCK_FILENAME, readLock, resolveLockPath } from "#cache/lock"; +import { symbols, ui } from "#cli/ui"; +import { DEFAULT_CACHE_DIR, loadConfig } from "#config"; +import { getCacheLayout, resolveCacheDir } from "#core/paths"; type StatusOptions = { configPath?: string; diff --git a/src/commands/sync.ts b/src/commands/sync.ts new file mode 100644 index 0000000..49b17a0 --- /dev/null +++ b/src/commands/sync.ts @@ -0,0 +1,940 @@ +import { createHash } from "node:crypto"; +import { access, mkdir, readFile } from "node:fs/promises"; +import path from "node:path"; +import pc from "picocolors"; +import type { DocsCacheLock } from "#cache/lock"; +import { readLock, resolveLockPath, writeLock } from "#cache/lock"; +import { MANIFEST_FILENAME } from "#cache/manifest"; +import { computeManifestHash, materializeSource } from "#cache/materialize"; +import { applyTargetDir } from "#cache/targets"; +import { writeToc } from "#cache/toc"; +import { TaskReporter } from "#cli/task-reporter"; +import { isSilentMode, symbols, ui } from "#cli/ui"; +import { verifyCache } from "#commands/verify"; +import { + DEFAULT_CACHE_DIR, + DEFAULT_CONFIG, + type DocsCacheDefaults, + type DocsCacheResolvedSource, + loadConfig, +} from "#config"; +import { resolveCacheDir, resolveTargetDir } from "#core/paths"; +import { fetchSource } from "#git/fetch-source"; +import { resolveRemoteCommit } from "#git/resolve-remote"; +import type { SyncOptions, SyncResult } from "#types/sync"; + +type SyncDeps = { + resolveRemoteCommit?: typeof resolveRemoteCommit; + fetchSource?: typeof fetchSource; + materializeSource?: typeof materializeSource; +}; + +const formatBytes = (value: number) => { + if (value < 1024) { + return `${value} B`; + } + const units = ["KB", "MB", "GB", "TB"]; + let size = value; + let index = -1; + while (size >= 1024 && index < units.length - 1) { + size /= 1024; + index += 1; + } + return `${size.toFixed(1)} ${units[index]}`; +}; + +const exists = async (target: string) => { + try { + await access(target); + return true; + } catch { + return false; + } +}; + +const hasDocs = async (cacheDir: string, sourceId: string) => { + const sourceDir = path.join(cacheDir, sourceId); + if (!(await exists(sourceDir))) { + return false; + } + return await exists(path.join(sourceDir, MANIFEST_FILENAME)); +}; + +const normalizePatterns = (patterns?: string[]) => { + if (!patterns || patterns.length === 0) { + return []; + } + const normalized = patterns + .map((pattern) => pattern.trim()) + .filter((pattern) => pattern.length > 0); + return Array.from(new Set(normalized)).sort(); +}; + +const RULES_HASH_BLACKLIST = [ + "id", + "repo", + "ref", + "targetDir", + "targetMode", + "required", + "integrity", + "toc", +] as const; + +type RulesHashBlacklistKey = (typeof RULES_HASH_BLACKLIST)[number]; +type RulesHashKey = Exclude< + keyof DocsCacheResolvedSource, + RulesHashBlacklistKey +>; + +const RULES_HASH_KEYS = [ + "mode", + "include", + "exclude", + "maxBytes", + "maxFiles", + "ignoreHidden", + "unwrapSingleRootDir", +] as const satisfies ReadonlyArray; + +const normalizeRulesValue = ( + key: RulesHashKey, + value: DocsCacheResolvedSource[RulesHashKey], +) => { + if (key === "include" && Array.isArray(value)) { + return normalizePatterns(value); + } + if (key === "exclude" && Array.isArray(value)) { + return normalizePatterns(value); + } + return value; +}; + +const computeRulesHash = (source: DocsCacheResolvedSource) => { + const entries = RULES_HASH_KEYS.map((key) => [ + key, + normalizeRulesValue(key, source[key]), + ]) as Array<[string, unknown]>; + entries.sort(([left]: [string, unknown], [right]: [string, unknown]) => + left.localeCompare(right), + ); + const payload = Object.fromEntries(entries); + const hash = createHash("sha256"); + hash.update(JSON.stringify(payload)); + return hash.digest("hex"); +}; + +export const getSyncPlan = async ( + options: SyncOptions, + deps: SyncDeps = {}, +) => { + const { config, resolvedPath, sources } = await loadConfig( + options.configPath, + ); + const defaults = (config.defaults ?? + DEFAULT_CONFIG.defaults) as DocsCacheDefaults; + const resolvedCacheDir = resolveCacheDir( + resolvedPath, + config.cacheDir ?? DEFAULT_CACHE_DIR, + options.cacheDirOverride, + ); + const lockPath = resolveLockPath(resolvedPath); + const lockExists = await exists(lockPath); + + let lockData: Awaited> | null = null; + if (lockExists) { + lockData = await readLock(lockPath); + } + + const resolveCommit = deps.resolveRemoteCommit ?? resolveRemoteCommit; + const filteredSources = options.sourceFilter?.length + ? sources.filter((source) => options.sourceFilter?.includes(source.id)) + : sources; + const results: SyncResult[] = await Promise.all( + filteredSources.map(async (source) => { + const lockEntry = lockData?.sources?.[source.id]; + const rulesSha256 = computeRulesSha(source, defaults); + if (options.offline) { + return buildOfflineResult({ + source, + lockEntry, + defaults, + resolvedCacheDir, + rulesSha256, + }); + } + return buildOnlineResult({ + source, + lockEntry, + defaults, + options, + resolveCommit, + rulesSha256, + }); + }), + ); + + return { + config, + configPath: resolvedPath, + cacheDir: resolvedCacheDir, + lockPath, + lockExists, + lockData, + results, + sources: filteredSources, + defaults, + }; +}; + +const loadToolVersion = async () => { + const cwdPath = path.resolve(process.cwd(), "package.json"); + try { + const raw = await readFile(cwdPath, "utf8"); + const pkg = JSON.parse(raw.toString()); + return typeof pkg.version === "string" ? pkg.version : "0.0.0"; + } catch { + // fallback to bundle-relative location + } + try { + const raw = await readFile( + new URL("../package.json", import.meta.url), + "utf8", + ); + const pkg = JSON.parse(raw.toString()); + return typeof pkg.version === "string" ? pkg.version : "0.0.0"; + } catch { + // fallback to dist/chunks relative location + } + try { + const raw = await readFile( + new URL("../../package.json", import.meta.url), + "utf8", + ); + const pkg = JSON.parse(raw.toString()); + return typeof pkg.version === "string" ? pkg.version : "0.0.0"; + } catch { + return "0.0.0"; + } +}; + +const buildLockSource = ( + result: SyncResult, + prior: DocsCacheLock["sources"][string] | undefined, + now: string, +) => ({ + repo: result.repo, + ref: result.ref, + resolvedCommit: result.resolvedCommit, + bytes: result.bytes ?? prior?.bytes ?? 0, + fileCount: result.fileCount ?? prior?.fileCount ?? 0, + manifestSha256: + result.manifestSha256 ?? prior?.manifestSha256 ?? result.resolvedCommit, + rulesSha256: result.rulesSha256 ?? prior?.rulesSha256, + updatedAt: now, +}); + +const buildLock = async ( + plan: Awaited>, + previous: Awaited> | null, +) => { + const toolVersion = await loadToolVersion(); + const now = new Date().toISOString(); + const sources = { ...(previous?.sources ?? {}) }; + for (const result of plan.results) { + const prior = sources[result.id]; + sources[result.id] = buildLockSource(result, prior, now); + } + return { + version: 1 as const, + generatedAt: now, + toolVersion, + sources, + }; +}; + +type SyncPlan = Awaited>; +type SyncJob = { + result: SyncResult; + source: SyncPlan["sources"][number]; +}; + +const buildSyncResultBase = (params: { + source: DocsCacheResolvedSource; + lockEntry: DocsCacheLock["sources"][string] | undefined; + defaults: DocsCacheDefaults; + resolvedCommit: string; + rulesSha256: string; + repo?: string; + ref?: string; +}) => { + const { + source, + lockEntry, + defaults, + resolvedCommit, + rulesSha256, + repo, + ref, + } = params; + return { + id: source.id, + repo: repo ?? lockEntry?.repo ?? source.repo, + ref: ref ?? lockEntry?.ref ?? source.ref ?? defaults.ref, + resolvedCommit, + lockCommit: lockEntry?.resolvedCommit ?? null, + lockRulesSha256: lockEntry?.rulesSha256, + bytes: lockEntry?.bytes, + fileCount: lockEntry?.fileCount, + manifestSha256: lockEntry?.manifestSha256, + rulesSha256, + }; +}; + +const computeRulesSha = ( + source: DocsCacheResolvedSource, + defaults: DocsCacheDefaults, +) => { + const include = source.include ?? defaults.include; + const exclude = source.exclude ?? defaults.exclude; + return computeRulesHash({ + ...source, + include, + exclude, + }); +}; + +const buildOfflineResult = async (params: { + source: DocsCacheResolvedSource; + lockEntry: DocsCacheLock["sources"][string] | undefined; + defaults: DocsCacheDefaults; + resolvedCacheDir: string; + rulesSha256: string; +}): Promise => { + const { source, lockEntry, defaults, resolvedCacheDir, rulesSha256 } = params; + const docsPresent = await hasDocs(resolvedCacheDir, source.id); + const resolvedCommit = lockEntry?.resolvedCommit ?? "offline"; + const base = buildSyncResultBase({ + source, + lockEntry, + defaults, + resolvedCommit, + rulesSha256, + }); + return { + ...base, + status: lockEntry && docsPresent ? "up-to-date" : "missing", + }; +}; + +const buildOnlineResult = async (params: { + source: DocsCacheResolvedSource; + lockEntry: DocsCacheLock["sources"][string] | undefined; + defaults: DocsCacheDefaults; + options: SyncOptions; + resolveCommit: typeof resolveRemoteCommit; + rulesSha256: string; +}): Promise => { + const { source, lockEntry, defaults, options, resolveCommit, rulesSha256 } = + params; + const resolved = await resolveCommit({ + repo: source.repo, + ref: source.ref, + allowHosts: defaults.allowHosts, + timeoutMs: options.timeoutMs, + logger: options.verbose && !options.json ? ui.debug : undefined, + }); + const upToDate = + lockEntry?.resolvedCommit === resolved.resolvedCommit && + lockEntry?.rulesSha256 === rulesSha256; + let status: SyncResult["status"] = "missing"; + if (lockEntry) { + status = upToDate ? "up-to-date" : "changed"; + } + const base = buildSyncResultBase({ + source, + lockEntry, + defaults, + resolvedCommit: resolved.resolvedCommit, + rulesSha256, + repo: resolved.repo, + ref: resolved.ref, + }); + return { ...base, status }; +}; + +const logFetchStatus = ( + reporter: TaskReporter | null, + options: SyncOptions, + sourceId: string, + fromCache: boolean, +) => { + if (reporter) { + reporter.debug( + `${sourceId}: ${fromCache ? "restored from cache" : "downloaded"}`, + ); + return; + } + if (!options.json) { + ui.step(fromCache ? "Restoring from cache" : "Downloading repo", sourceId); + } +}; + +const logMaterializeStart = ( + reporter: TaskReporter | null, + options: SyncOptions, + sourceId: string, +) => { + if (reporter) { + reporter.debug(`${sourceId}: materializing`); + return; + } + if (!options.json) { + ui.step("Materializing", sourceId); + } +}; + +const reportNoChanges = ( + reporter: TaskReporter | null, + options: SyncOptions, + sourceId: string, +) => { + if (reporter) { + reporter.success(sourceId, "no content changes"); + return; + } + if (!options.json) { + ui.item(symbols.success, sourceId, "no content changes"); + } +}; + +const reportSynced = ( + reporter: TaskReporter | null, + options: SyncOptions, + sourceId: string, + fileCount: number, +) => { + if (reporter) { + reporter.success(sourceId, `synced ${fileCount} files`, symbols.synced); + return; + } + if (!options.json) { + ui.item(symbols.synced, sourceId, `synced ${fileCount} files`); + } +}; + +const createLoggers = ( + reporter: TaskReporter | null, + options: SyncOptions, + sourceId: string, +) => { + const logDebug = + options.verbose && !options.json + ? reporter + ? (msg: string) => reporter.debug(msg) + : ui.debug + : undefined; + const logProgress = reporter + ? (msg: string) => reporter.debug(`${sourceId}: ${msg}`) + : undefined; + return { logDebug, logProgress }; +}; + +const applyTargetIfNeeded = async ( + plan: SyncPlan, + defaults: DocsCacheDefaults, + source: SyncPlan["sources"][number], +) => { + if (!source.targetDir) { + return; + } + const resolvedTarget = resolveTargetDir(plan.configPath, source.targetDir); + await applyTargetDir({ + sourceDir: path.join(plan.cacheDir, source.id), + targetDir: resolvedTarget, + mode: source.targetMode ?? defaults.targetMode, + explicitTargetMode: source.targetMode !== undefined, + unwrapSingleRootDir: source.unwrapSingleRootDir, + }); +}; + +const materializeJob = async (params: { + plan: SyncPlan; + options: SyncOptions; + defaults: DocsCacheDefaults; + reporter: TaskReporter | null; + source: SyncPlan["sources"][number]; + fetch: Awaited>; + runMaterialize: typeof materializeSource; + result: SyncResult; +}) => { + const { + plan, + options, + defaults, + reporter, + source, + fetch, + runMaterialize, + result, + } = params; + logMaterializeStart(reporter, options, source.id); + const stats = await runMaterialize({ + sourceId: source.id, + repoDir: fetch.repoDir, + cacheDir: plan.cacheDir, + include: source.include ?? defaults.include, + exclude: source.exclude, + maxBytes: source.maxBytes ?? defaults.maxBytes, + maxFiles: source.maxFiles ?? defaults.maxFiles, + ignoreHidden: source.ignoreHidden ?? defaults.ignoreHidden, + unwrapSingleRootDir: source.unwrapSingleRootDir, + json: options.json, + progressLogger: reporter + ? (msg: string) => reporter.debug(`${source.id}: ${msg}`) + : undefined, + }); + await applyTargetIfNeeded(plan, defaults, source); + result.bytes = stats.bytes; + result.fileCount = stats.fileCount; + result.manifestSha256 = stats.manifestSha256; + result.status = "up-to-date"; + reportSynced(reporter, options, source.id, stats.fileCount); +}; + +const verifyAndRepairCache = async (params: { + plan: SyncPlan; + options: SyncOptions; + docsPresence: Map; + defaults: DocsCacheDefaults; + reporter: TaskReporter | null; + runJobs: (jobs: SyncJob[]) => Promise; +}) => { + const { plan, options, docsPresence, defaults, reporter, runJobs } = params; + if (options.offline) { + return 0; + } + const shouldVerify = !options.json || plan.results.length > 0; + if (!shouldVerify) { + return 0; + } + const verifyReport = await verifyCache({ + configPath: plan.configPath, + cacheDirOverride: plan.cacheDir, + json: true, + }); + const failed = verifyReport.results.filter((result) => !result.ok); + if (failed.length === 0) { + return 0; + } + const retryJobs = await buildJobs( + plan, + options, + docsPresence, + failed.map((result) => result.id), + true, + ); + if (retryJobs.length > 0) { + await runJobs(retryJobs); + await ensureTargets(plan, defaults); + } + const retryReport = await verifyCache({ + configPath: plan.configPath, + cacheDirOverride: plan.cacheDir, + json: true, + }); + const stillFailed = retryReport.results.filter((result) => !result.ok); + if (stillFailed.length === 0) { + return 0; + } + reportVerifyFailures(reporter, options, stillFailed); + return 1; +}; + +const tryReuseManifest = async (params: { + result: SyncResult; + source: SyncPlan["sources"][number]; + lockEntry: DocsCacheLock["sources"][string] | undefined; + plan: SyncPlan; + defaults: DocsCacheDefaults; + fetch: Awaited>; + reporter: TaskReporter | null; + options: SyncOptions; +}) => { + const { + result, + source, + lockEntry, + plan, + defaults, + fetch, + reporter, + options, + } = params; + if (result.status === "up-to-date") { + return false; + } + if (!lockEntry?.manifestSha256) { + return false; + } + if (lockEntry.rulesSha256 !== result.rulesSha256) { + return false; + } + const manifestPath = path.join(plan.cacheDir, source.id, MANIFEST_FILENAME); + if (!(await exists(manifestPath))) { + return false; + } + const computed = await computeManifestHash({ + sourceId: source.id, + repoDir: fetch.repoDir, + cacheDir: plan.cacheDir, + include: source.include ?? defaults.include, + exclude: source.exclude, + maxBytes: source.maxBytes ?? defaults.maxBytes, + maxFiles: source.maxFiles ?? defaults.maxFiles, + ignoreHidden: source.ignoreHidden ?? defaults.ignoreHidden, + }); + if (computed.manifestSha256 !== lockEntry.manifestSha256) { + return false; + } + result.bytes = computed.bytes; + result.fileCount = computed.fileCount; + result.manifestSha256 = computed.manifestSha256; + result.status = "up-to-date"; + reportNoChanges(reporter, options, source.id); + return true; +}; + +const buildJobs = async ( + plan: SyncPlan, + options: SyncOptions, + docsPresence: Map, + ids?: string[], + force?: boolean, +): Promise => { + const pick = ids?.length + ? plan.results.filter((result) => ids.includes(result.id)) + : plan.results; + const jobs = await Promise.all( + pick.map(async (result) => { + const source = plan.sources.find((entry) => entry.id === result.id); + if (!source) { + return null; + } + if (options.offline) { + const lockEntry = plan.lockData?.sources?.[result.id]; + if (!lockEntry?.resolvedCommit) { + return null; + } + } + if (force) { + return { result, source }; + } + let docsPresent = docsPresence.get(result.id); + if (docsPresent === undefined) { + docsPresent = await hasDocs(plan.cacheDir, result.id); + docsPresence.set(result.id, docsPresent); + } + const needsMaterialize = result.status !== "up-to-date" || !docsPresent; + if (!needsMaterialize) { + return null; + } + return { result, source }; + }), + ); + return jobs.filter(Boolean) as SyncJob[]; +}; + +const ensureTargets = async (plan: SyncPlan, defaults: DocsCacheDefaults) => { + await Promise.all( + plan.sources.map(async (source) => { + if (!source.targetDir) { + return; + } + const resolvedTarget = resolveTargetDir( + plan.configPath, + source.targetDir, + ); + if (await exists(resolvedTarget)) { + return; + } + await applyTargetDir({ + sourceDir: path.join(plan.cacheDir, source.id), + targetDir: resolvedTarget, + mode: source.targetMode ?? defaults.targetMode, + explicitTargetMode: source.targetMode !== undefined, + unwrapSingleRootDir: source.unwrapSingleRootDir, + }); + }), + ); +}; + +const summarizePlan = (plan: SyncPlan) => { + const totalBytes = plan.results.reduce( + (sum, result) => sum + (result.bytes ?? 0), + 0, + ); + const totalFiles = plan.results.reduce( + (sum, result) => sum + (result.fileCount ?? 0), + 0, + ); + return { totalBytes, totalFiles }; +}; + +const reportVerifyFailures = ( + reporter: TaskReporter | null, + options: SyncOptions, + stillFailed: Array<{ id: string; issues: string[] }>, +) => { + if (stillFailed.length === 0) { + return; + } + if (reporter) { + for (const failed of stillFailed) { + reporter.warn(failed.id, failed.issues.join("; ")); + } + return; + } + if (!options.json) { + const details = stillFailed + .map((result) => `${result.id} (${result.issues.join("; ")})`) + .join(", "); + ui.line( + `${symbols.warn} Verify failed for ${stillFailed.length} source(s): ${details}`, + ); + } +}; + +const finalizeSync = async (params: { + plan: SyncPlan; + previous: Awaited> | null; + reporter: TaskReporter | null; + options: SyncOptions; + startTime: bigint; + warningCount: number; +}) => { + const { plan, previous, reporter, options, startTime, warningCount } = params; + const lock = await buildLock(plan, previous); + await writeLock(plan.lockPath, lock); + const { totalBytes, totalFiles } = summarizePlan(plan); + if (reporter) { + const summary = `${symbols.info} ${formatBytes(totalBytes)} · ${totalFiles} files`; + reporter.finish(summary); + } + if (!reporter && !options.json) { + const elapsedMs = Number(process.hrtime.bigint() - startTime) / 1_000_000; + ui.line( + `${symbols.info} Completed in ${elapsedMs.toFixed(0)}ms · ${formatBytes(totalBytes)} · ${totalFiles} files${warningCount ? ` · ${warningCount} warning${warningCount === 1 ? "" : "s"}` : ""}`, + ); + } + await writeToc({ + cacheDir: plan.cacheDir, + configPath: plan.configPath, + lock, + sources: plan.sources, + results: plan.results, + }); + plan.lockExists = true; + return plan; +}; + +const createJobRunner = (params: { + plan: SyncPlan; + options: SyncOptions; + defaults: DocsCacheDefaults; + reporter: TaskReporter | null; + runFetch: typeof fetchSource; + runMaterialize: typeof materializeSource; +}) => { + const { plan, options, defaults, reporter, runFetch, runMaterialize } = + params; + return async (jobs: SyncJob[]) => { + const concurrencyRaw = options.concurrency ?? 4; + const concurrency = Math.floor(concurrencyRaw); + if (!Number.isFinite(concurrencyRaw) || concurrency < 1) { + throw new TypeError( + "Invalid options.concurrency; must be a positive number.", + ); + } + let index = 0; + const runNext = async () => { + const job = jobs[index]; + if (!job || !job.source) { + return; + } + index += 1; + const { result, source } = job; + const lockEntry = plan.lockData?.sources?.[source.id]; + const { logDebug, logProgress } = createLoggers( + reporter, + options, + source.id, + ); + + if (reporter) { + reporter.start(source.id); + } + + const fetch = await runFetch({ + sourceId: source.id, + repo: source.repo, + ref: source.ref, + resolvedCommit: result.resolvedCommit, + cacheDir: plan.cacheDir, + include: source.include ?? defaults.include, + timeoutMs: options.timeoutMs, + logger: logDebug, + progressLogger: logProgress, + offline: options.offline, + }); + logFetchStatus(reporter, options, source.id, fetch.fromCache); + try { + const reusedManifest = await tryReuseManifest({ + result, + source, + lockEntry, + plan, + defaults, + fetch, + reporter, + options, + }); + if (reusedManifest) { + await runNext(); + return; + } + await materializeJob({ + plan, + options, + defaults, + reporter, + source, + fetch, + runMaterialize, + result, + }); + } finally { + await fetch.cleanup(); + } + await runNext(); + }; + + await Promise.all( + Array.from({ length: Math.min(concurrency, jobs.length) }, runNext), + ); + }; +}; + +export const runSync = async (options: SyncOptions, deps: SyncDeps = {}) => { + const startTime = process.hrtime.bigint(); + let warningCount = 0; + const plan = await getSyncPlan(options, deps); + await mkdir(plan.cacheDir, { recursive: true }); + + const isTestRunner = process.argv.includes("--test"); + const useLiveOutput = + !options.json && !isSilentMode() && process.stdout.isTTY && !isTestRunner; + const reporter = useLiveOutput ? new TaskReporter() : null; + const previous = plan.lockData; + const requiredMissing = plan.results.filter((result) => { + const source = plan.sources.find((entry) => entry.id === result.id); + return result.status === "missing" && (source?.required ?? true); + }); + if (options.failOnMiss && requiredMissing.length > 0) { + throw new Error( + `Missing required source(s): ${requiredMissing.map((result) => result.id).join(", ")}.`, + ); + } + if (!options.lockOnly) { + const defaults = plan.defaults; + const runFetch = deps.fetchSource ?? fetchSource; + const runMaterialize = deps.materializeSource ?? materializeSource; + const docsPresence = new Map(); + const runJobs = createJobRunner({ + plan, + options, + defaults, + reporter, + runFetch, + runMaterialize, + }); + + const initialJobs = await buildJobs(plan, options, docsPresence); + await runJobs(initialJobs); + await ensureTargets(plan, defaults); + warningCount += await verifyAndRepairCache({ + plan, + options, + docsPresence, + defaults, + reporter, + runJobs, + }); + } + return finalizeSync({ + plan, + previous, + reporter, + options, + startTime, + warningCount, + }); +}; + +export const printSyncPlan = ( + plan: Awaited>, +) => { + const summary = { + upToDate: plan.results.filter((r) => r.status === "up-to-date").length, + changed: plan.results.filter((r) => r.status === "changed").length, + missing: plan.results.filter((r) => r.status === "missing").length, + }; + + if (plan.results.length === 0) { + ui.line(`${symbols.info} No sources to sync.`); + return; + } + + ui.line( + `${symbols.info} ${plan.results.length} sources (${summary.upToDate} up-to-date, ${summary.changed} changed, ${summary.missing} missing)`, + ); + + for (const result of plan.results) { + const shortResolved = ui.hash(result.resolvedCommit); + const shortLock = ui.hash(result.lockCommit); + const rulesChanged = + Boolean(result.lockRulesSha256) && + Boolean(result.rulesSha256) && + result.lockRulesSha256 !== result.rulesSha256; + + if (result.status === "up-to-date") { + ui.item( + symbols.success, + result.id, + `${pc.dim("up-to-date")} ${pc.gray(shortResolved)}`, + ); + continue; + } + if (result.status === "changed") { + if (result.lockCommit === result.resolvedCommit && rulesChanged) { + ui.item( + symbols.warn, + result.id, + `${pc.dim("rules changed")} ${pc.gray(shortResolved)}`, + ); + continue; + } + ui.item( + symbols.warn, + result.id, + `${pc.dim("changed")} ${pc.gray(shortLock)} ${pc.dim("->")} ${pc.gray(shortResolved)}`, + ); + continue; + } + ui.item( + symbols.warn, + result.id, + `${pc.dim("missing")} ${pc.gray(shortResolved)}`, + ); + } +}; diff --git a/src/verify.ts b/src/commands/verify.ts similarity index 92% rename from src/verify.ts rename to src/commands/verify.ts index 168b66e..ad4d0da 100644 --- a/src/verify.ts +++ b/src/commands/verify.ts @@ -1,10 +1,10 @@ import { access, stat } from "node:fs/promises"; import path from "node:path"; -import { symbols, ui } from "./cli/ui"; -import { DEFAULT_CACHE_DIR, loadConfig } from "./config"; -import { getErrnoCode } from "./errors"; -import { streamManifestEntries } from "./manifest"; -import { resolveCacheDir, resolveTargetDir } from "./paths"; +import { streamManifestEntries } from "#cache/manifest"; +import { symbols, ui } from "#cli/ui"; +import { DEFAULT_CACHE_DIR, loadConfig } from "#config"; +import { getErrnoCode } from "#core/errors"; +import { resolveCacheDir, resolveTargetDir } from "#core/paths"; type VerifyOptions = { configPath?: string; diff --git a/src/config-schema.ts b/src/config-schema.ts deleted file mode 100644 index f3f9535..0000000 --- a/src/config-schema.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { z } from "zod"; - -export const TargetModeSchema = z.enum(["symlink", "copy"]); -export const CacheModeSchema = z.enum(["materialize"]); -export const TocFormatSchema = z.enum(["tree", "compressed"]); -export const IntegritySchema = z - .object({ - type: z.enum(["commit", "manifest"]), - value: z.string().nullable(), - }) - .strict(); - -export const DefaultsSchema = z - .object({ - ref: z.string().min(1), - mode: CacheModeSchema, - include: z.array(z.string().min(1)).min(1), - exclude: z.array(z.string().min(1)).optional(), - targetMode: TargetModeSchema.optional(), - required: z.boolean(), - maxBytes: z.number().min(1), - maxFiles: z.number().min(1).optional(), - ignoreHidden: z.boolean(), - allowHosts: z.array(z.string().min(1)).min(1), - toc: z.union([z.boolean(), TocFormatSchema]).optional(), - unwrapSingleRootDir: z.boolean().optional(), - }) - .strict(); - -export const SourceSchema = z - .object({ - id: z.string().min(1), - repo: z.string().min(1), - targetDir: z.string().min(1).optional(), - targetMode: TargetModeSchema.optional(), - ref: z.string().min(1).optional(), - mode: CacheModeSchema.optional(), - include: z.array(z.string().min(1)).optional(), - exclude: z.array(z.string().min(1)).optional(), - required: z.boolean().optional(), - maxBytes: z.number().min(1).optional(), - maxFiles: z.number().min(1).optional(), - ignoreHidden: z.boolean().optional(), - integrity: IntegritySchema.optional(), - toc: z.union([z.boolean(), TocFormatSchema]).optional(), - unwrapSingleRootDir: z.boolean().optional(), - }) - .strict(); - -export const ConfigSchema = z - .object({ - $schema: z.string().min(1).optional(), - cacheDir: z.string().min(1).optional(), - targetMode: TargetModeSchema.optional(), - defaults: DefaultsSchema.partial().optional(), - sources: z.array(SourceSchema), - }) - .strict(); diff --git a/src/config.ts b/src/config.ts deleted file mode 100644 index cf5b789..0000000 --- a/src/config.ts +++ /dev/null @@ -1,555 +0,0 @@ -import { access, readFile, writeFile } from "node:fs/promises"; -import path from "node:path"; -import { ConfigSchema } from "./config-schema"; -import { resolveTargetDir } from "./paths"; -import { assertSafeSourceId } from "./source-id"; - -export type CacheMode = "materialize"; - -export type TocFormat = "tree" | "compressed"; - -export type IntegrityType = "commit" | "manifest"; - -export interface DocsCacheIntegrity { - type: IntegrityType; - value: string | null; -} - -export interface DocsCacheDefaults { - ref: string; - mode: CacheMode; - include: string[]; - exclude?: string[]; - targetMode?: "symlink" | "copy"; - required: boolean; - maxBytes: number; - maxFiles?: number; - ignoreHidden: boolean; - allowHosts: string[]; - toc?: boolean | TocFormat; - unwrapSingleRootDir?: boolean; -} - -export interface DocsCacheSource { - id: string; - repo: string; - targetDir?: string; - targetMode?: "symlink" | "copy"; - ref?: string; - mode?: CacheMode; - include?: string[]; - exclude?: string[]; - required?: boolean; - maxBytes?: number; - maxFiles?: number; - ignoreHidden?: boolean; - integrity?: DocsCacheIntegrity; - toc?: boolean | TocFormat; - unwrapSingleRootDir?: boolean; -} - -export interface DocsCacheConfig { - $schema?: string; - cacheDir?: string; - targetMode?: "symlink" | "copy"; - defaults?: Partial; - sources: DocsCacheSource[]; -} - -export interface DocsCacheResolvedSource { - id: string; - repo: string; - targetDir?: string; - targetMode?: "symlink" | "copy"; - ref: string; - mode: CacheMode; - include?: string[]; - exclude?: string[]; - required: boolean; - maxBytes: number; - maxFiles?: number; - ignoreHidden: boolean; - integrity?: DocsCacheIntegrity; - toc?: boolean | TocFormat; - unwrapSingleRootDir?: boolean; -} - -export const DEFAULT_CONFIG_FILENAME = "docs.config.json"; -export const DEFAULT_CACHE_DIR = ".docs"; -const PACKAGE_JSON_FILENAME = "package.json"; -const DEFAULT_TARGET_MODE = process.platform === "win32" ? "copy" : "symlink"; -export const DEFAULT_CONFIG: DocsCacheConfig = { - cacheDir: DEFAULT_CACHE_DIR, - defaults: { - ref: "HEAD", - mode: "materialize", - include: ["**/*.{md,mdx,markdown,mkd,txt,rst,adoc,asciidoc}"], - exclude: [], - targetMode: DEFAULT_TARGET_MODE, - required: true, - maxBytes: 200000000, - ignoreHidden: false, - allowHosts: ["github.com", "gitlab.com", "visualstudio.com"], - toc: true, - unwrapSingleRootDir: false, - }, - sources: [], -}; - -const isEqualStringArray = (left?: string[], right?: string[]) => { - if (!left || !right) { - return left === right; - } - if (left.length !== right.length) { - return false; - } - return left.every((value, index) => value === right[index]); -}; - -const isObject = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value); - -const pruneDefaults = ( - value: Record, - baseline: Record, -): Record => { - const result: Record = {}; - for (const [key, entry] of Object.entries(value)) { - const base = baseline[key]; - if (Array.isArray(entry) && Array.isArray(base)) { - if (!isEqualStringArray(entry, base)) { - result[key] = entry; - } - continue; - } - if (isObject(entry) && isObject(base)) { - const pruned = pruneDefaults(entry, base); - if (Object.keys(pruned).length > 0) { - result[key] = pruned; - } - continue; - } - if (entry !== base) { - result[key] = entry; - } - } - return result; -}; - -export const stripDefaultConfigValues = ( - config: DocsCacheConfig, -): DocsCacheConfig => { - const baseline: DocsCacheConfig = { - ...DEFAULT_CONFIG, - $schema: config.$schema, - defaults: { - ...DEFAULT_CONFIG.defaults, - ...(config.targetMode ? { targetMode: config.targetMode } : undefined), - }, - }; - const pruned = pruneDefaults( - config as unknown as Record, - baseline as unknown as Record, - ); - const next: DocsCacheConfig = { - $schema: pruned.$schema as DocsCacheConfig["$schema"], - cacheDir: pruned.cacheDir as DocsCacheConfig["cacheDir"], - targetMode: pruned.targetMode as DocsCacheConfig["targetMode"], - defaults: pruned.defaults as DocsCacheConfig["defaults"], - sources: config.sources, - }; - if (!next.defaults || Object.keys(next.defaults).length === 0) { - delete next.defaults; - } - return next; -}; - -const isRecord = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value); - -const assertString = (value: unknown, label: string): string => { - if (typeof value !== "string" || value.length === 0) { - throw new Error(`${label} must be a non-empty string.`); - } - return value; -}; - -const assertBoolean = (value: unknown, label: string): boolean => { - if (typeof value !== "boolean") { - throw new Error(`${label} must be a boolean.`); - } - return value; -}; - -const assertNumber = (value: unknown, label: string): number => { - if (typeof value !== "number" || Number.isNaN(value)) { - throw new Error(`${label} must be a number.`); - } - return value; -}; - -const assertPositiveNumber = (value: unknown, label: string): number => { - const numberValue = assertNumber(value, label); - if (numberValue < 1) { - throw new Error(`${label} must be greater than zero.`); - } - return numberValue; -}; - -const assertStringArray = (value: unknown, label: string): string[] => { - if (!Array.isArray(value) || value.length === 0) { - throw new Error(`${label} must be a non-empty array of strings.`); - } - for (const entry of value) { - if (typeof entry !== "string" || entry.length === 0) { - throw new Error(`${label} must contain non-empty strings.`); - } - } - return value as string[]; -}; - -const assertTargetMode = ( - value: unknown, - label: string, -): "symlink" | "copy" => { - const mode = assertString(value, label) as "symlink" | "copy"; - if (mode !== "symlink" && mode !== "copy") { - throw new Error(`${label} must be "symlink" or "copy".`); - } - return mode; -}; - -const assertMode = (value: unknown, label: string): CacheMode => { - if (value !== "materialize") { - throw new Error(`${label} must be "materialize".`); - } - return value; -}; - -const assertIntegrity = (value: unknown, label: string): DocsCacheIntegrity => { - if (!isRecord(value)) { - throw new Error(`${label} must be an object.`); - } - const type = value.type; - if (type !== "commit" && type !== "manifest") { - throw new Error(`${label}.type must be "commit" or "manifest".`); - } - const integrityValue = value.value; - if (typeof integrityValue !== "string" && integrityValue !== null) { - throw new Error(`${label}.value must be a string or null.`); - } - return { type, value: integrityValue }; -}; - -export const validateConfig = (input: unknown): DocsCacheConfig => { - if (!isRecord(input)) { - throw new Error("Config must be a JSON object."); - } - const parsed = ConfigSchema.safeParse(input); - if (!parsed.success) { - const details = parsed.error.issues - .map((issue) => `${issue.path.join(".") || "config"} ${issue.message}`) - .join("; "); - throw new Error(`Config does not match schema: ${details}.`); - } - const configInput = parsed.data; - - const cacheDir = configInput.cacheDir - ? assertString(configInput.cacheDir, "cacheDir") - : DEFAULT_CACHE_DIR; - - const defaultsInput = configInput.defaults; - const targetModeOverride = - configInput.targetMode !== undefined - ? assertTargetMode(configInput.targetMode, "targetMode") - : undefined; - const defaultValues = DEFAULT_CONFIG.defaults as DocsCacheDefaults; - let defaults: DocsCacheDefaults = defaultValues; - if (defaultsInput !== undefined) { - if (!isRecord(defaultsInput)) { - throw new Error("defaults must be an object."); - } - - defaults = { - ref: - defaultsInput.ref !== undefined - ? assertString(defaultsInput.ref, "defaults.ref") - : defaultValues.ref, - mode: - defaultsInput.mode !== undefined - ? assertMode(defaultsInput.mode, "defaults.mode") - : defaultValues.mode, - include: - defaultsInput.include !== undefined - ? assertStringArray(defaultsInput.include, "defaults.include") - : defaultValues.include, - exclude: - defaultsInput.exclude !== undefined - ? assertStringArray(defaultsInput.exclude, "defaults.exclude") - : defaultValues.exclude, - targetMode: - defaultsInput.targetMode !== undefined - ? assertTargetMode(defaultsInput.targetMode, "defaults.targetMode") - : (targetModeOverride ?? defaultValues.targetMode), - required: - defaultsInput.required !== undefined - ? assertBoolean(defaultsInput.required, "defaults.required") - : defaultValues.required, - maxBytes: - defaultsInput.maxBytes !== undefined - ? assertPositiveNumber(defaultsInput.maxBytes, "defaults.maxBytes") - : defaultValues.maxBytes, - maxFiles: - defaultsInput.maxFiles !== undefined - ? assertPositiveNumber(defaultsInput.maxFiles, "defaults.maxFiles") - : defaultValues.maxFiles, - ignoreHidden: - defaultsInput.ignoreHidden !== undefined - ? assertBoolean(defaultsInput.ignoreHidden, "defaults.ignoreHidden") - : defaultValues.ignoreHidden, - allowHosts: - defaultsInput.allowHosts !== undefined - ? assertStringArray(defaultsInput.allowHosts, "defaults.allowHosts") - : defaultValues.allowHosts, - toc: - defaultsInput.toc !== undefined - ? (defaultsInput.toc as boolean | TocFormat) - : defaultValues.toc, - unwrapSingleRootDir: - defaultsInput.unwrapSingleRootDir !== undefined - ? assertBoolean( - defaultsInput.unwrapSingleRootDir, - "defaults.unwrapSingleRootDir", - ) - : defaultValues.unwrapSingleRootDir, - }; - } else if (targetModeOverride !== undefined) { - defaults = { - ...defaultValues, - targetMode: targetModeOverride, - }; - } - - const sources = configInput.sources.map((entry, index) => { - if (!isRecord(entry)) { - throw new Error(`sources[${index}] must be an object.`); - } - const source: DocsCacheSource = { - id: assertSafeSourceId(entry.id, `sources[${index}].id`), - repo: assertString(entry.repo, `sources[${index}].repo`), - }; - if (entry.targetDir !== undefined) { - source.targetDir = assertString( - entry.targetDir, - `sources[${index}].targetDir`, - ); - } - if (entry.targetMode !== undefined) { - const targetMode = assertString( - entry.targetMode, - `sources[${index}].targetMode`, - ); - if (targetMode !== "symlink" && targetMode !== "copy") { - throw new Error( - `sources[${index}].targetMode must be "symlink" or "copy".`, - ); - } - source.targetMode = targetMode; - } - if (entry.ref !== undefined) { - source.ref = assertString(entry.ref, `sources[${index}].ref`); - } - if (entry.mode !== undefined) { - source.mode = assertMode(entry.mode, `sources[${index}].mode`); - } - if (entry.include !== undefined) { - source.include = assertStringArray( - entry.include, - `sources[${index}].include`, - ); - } - if (entry.exclude !== undefined) { - source.exclude = assertStringArray( - entry.exclude, - `sources[${index}].exclude`, - ); - } - if (entry.required !== undefined) { - source.required = assertBoolean( - entry.required, - `sources[${index}].required`, - ); - } - if (entry.maxBytes !== undefined) { - source.maxBytes = assertPositiveNumber( - entry.maxBytes, - `sources[${index}].maxBytes`, - ); - } - if (entry.maxFiles !== undefined) { - source.maxFiles = assertPositiveNumber( - entry.maxFiles, - `sources[${index}].maxFiles`, - ); - } - if (entry.ignoreHidden !== undefined) { - source.ignoreHidden = assertBoolean( - entry.ignoreHidden, - `sources[${index}].ignoreHidden`, - ); - } - if (entry.integrity !== undefined) { - source.integrity = assertIntegrity( - entry.integrity, - `sources[${index}].integrity`, - ); - } - - if (entry.toc !== undefined) { - source.toc = entry.toc as boolean | TocFormat; - } - if (entry.unwrapSingleRootDir !== undefined) { - source.unwrapSingleRootDir = assertBoolean( - entry.unwrapSingleRootDir, - `sources[${index}].unwrapSingleRootDir`, - ); - } - - return source; - }); - - // Validate unique source IDs - const idSet = new Set(); - const duplicates: string[] = []; - for (const source of sources) { - if (idSet.has(source.id)) { - duplicates.push(source.id); - } - idSet.add(source.id); - } - if (duplicates.length > 0) { - throw new Error( - `Duplicate source IDs found: ${duplicates.join(", ")}. Each source must have a unique ID.`, - ); - } - - return { - cacheDir, - targetMode: targetModeOverride, - defaults, - sources, - }; -}; - -export const resolveSources = ( - config: DocsCacheConfig, -): DocsCacheResolvedSource[] => { - const defaults = (config.defaults ?? - DEFAULT_CONFIG.defaults) as DocsCacheDefaults; - return config.sources.map((source) => ({ - id: source.id, - repo: source.repo, - targetDir: source.targetDir, - targetMode: source.targetMode ?? defaults.targetMode, - ref: source.ref ?? defaults.ref, - mode: source.mode ?? defaults.mode, - include: source.include ?? defaults.include, - exclude: source.exclude ?? defaults.exclude, - required: source.required ?? defaults.required, - maxBytes: source.maxBytes ?? defaults.maxBytes, - maxFiles: source.maxFiles ?? defaults.maxFiles, - ignoreHidden: source.ignoreHidden ?? defaults.ignoreHidden, - integrity: source.integrity, - toc: source.toc ?? defaults.toc, - unwrapSingleRootDir: - source.unwrapSingleRootDir ?? defaults.unwrapSingleRootDir, - })); -}; - -export const resolveConfigPath = (configPath?: string) => - configPath - ? path.resolve(configPath) - : path.resolve(process.cwd(), DEFAULT_CONFIG_FILENAME); - -const resolvePackagePath = () => - path.resolve(process.cwd(), PACKAGE_JSON_FILENAME); - -const exists = async (target: string) => { - try { - await access(target); - return true; - } catch { - return false; - } -}; - -const loadConfigFromFile = async ( - filePath: string, - mode: "config" | "package", -) => { - let raw: string; - try { - raw = await readFile(filePath, "utf8"); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - throw new Error(`Failed to read config at ${filePath}: ${message}`); - } - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - throw new Error(`Invalid JSON in ${filePath}: ${message}`); - } - const configInput = - mode === "package" - ? (parsed as Record)?.["docs-cache"] - : parsed; - if (mode === "package" && configInput === undefined) { - throw new Error(`Missing docs-cache config in ${filePath}.`); - } - const config = validateConfig(configInput); - for (const source of config.sources) { - if (source.targetDir) { - resolveTargetDir(filePath, source.targetDir); - } - } - return { - config, - resolvedPath: filePath, - sources: resolveSources(config), - }; -}; - -export const writeConfig = async ( - configPath: string, - config: DocsCacheConfig, -) => { - const data = `${JSON.stringify(config, null, 2)}\n`; - await writeFile(configPath, data, "utf8"); -}; - -export const loadConfig = async (configPath?: string) => { - const resolvedPath = resolveConfigPath(configPath); - const isPackageConfig = path.basename(resolvedPath) === PACKAGE_JSON_FILENAME; - if (configPath) { - return loadConfigFromFile( - resolvedPath, - isPackageConfig ? "package" : "config", - ); - } - if (await exists(resolvedPath)) { - return loadConfigFromFile(resolvedPath, "config"); - } - const packagePath = resolvePackagePath(); - if (await exists(packagePath)) { - try { - return await loadConfigFromFile(packagePath, "package"); - } catch { - // fall through to error below - } - } - throw new Error( - `No docs.config.json found at ${resolvedPath} and no docs-cache config in ${packagePath}.`, - ); -}; diff --git a/src/config/index.ts b/src/config/index.ts new file mode 100644 index 0000000..5615149 --- /dev/null +++ b/src/config/index.ts @@ -0,0 +1,260 @@ +import { access, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { + CacheMode, + DocsCacheConfig, + DocsCacheDefaults, + DocsCacheIntegrity, + DocsCacheResolvedSource, + DocsCacheSource, + TocFormat, +} from "#config/schema"; +import { ConfigSchema } from "#config/schema"; +import { resolveTargetDir } from "#core/paths"; + +export type { + CacheMode, + DocsCacheConfig, + DocsCacheDefaults, + DocsCacheIntegrity, + DocsCacheResolvedSource, + DocsCacheSource, + TocFormat, +}; + +export const DEFAULT_CONFIG_FILENAME = "docs.config.json"; +export const DEFAULT_CACHE_DIR = ".docs"; +const PACKAGE_JSON_FILENAME = "package.json"; +const DEFAULT_TARGET_MODE = process.platform === "win32" ? "copy" : "symlink"; +export const DEFAULT_CONFIG: DocsCacheConfig = { + cacheDir: DEFAULT_CACHE_DIR, + defaults: { + ref: "HEAD", + mode: "materialize", + include: ["**/*.{md,mdx,markdown,mkd,txt,rst,adoc,asciidoc}"], + exclude: [], + targetMode: DEFAULT_TARGET_MODE, + required: true, + maxBytes: 200000000, + ignoreHidden: false, + allowHosts: ["github.com", "gitlab.com", "visualstudio.com"], + toc: true, + unwrapSingleRootDir: false, + }, + sources: [], +} as const; + +const isEqualStringArray = (left?: string[], right?: string[]) => { + if (!left || !right) { + return left === right; + } + if (left.length !== right.length) { + return false; + } + return left.every((value, index) => value === right[index]); +}; + +const isObject = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +const pruneDefaults = ( + value: Record, + baseline: Record, +): Record => { + const result: Record = {}; + for (const [key, entry] of Object.entries(value)) { + const base = baseline[key]; + if (Array.isArray(entry) && Array.isArray(base)) { + if (!isEqualStringArray(entry, base)) { + result[key] = entry; + } + continue; + } + if (isObject(entry) && isObject(base)) { + const pruned = pruneDefaults(entry, base); + if (Object.keys(pruned).length > 0) { + result[key] = pruned; + } + continue; + } + if (entry !== base) { + result[key] = entry; + } + } + return result; +}; + +export const stripDefaultConfigValues = ( + config: DocsCacheConfig, +): DocsCacheConfig => { + const baseline: DocsCacheConfig = { + ...DEFAULT_CONFIG, + $schema: config.$schema, + defaults: { + ...DEFAULT_CONFIG.defaults, + ...(config.targetMode ? { targetMode: config.targetMode } : undefined), + }, + }; + const pruned = pruneDefaults( + config as unknown as Record, + baseline as unknown as Record, + ); + const next: DocsCacheConfig = { + $schema: pruned.$schema as DocsCacheConfig["$schema"], + cacheDir: pruned.cacheDir as DocsCacheConfig["cacheDir"], + targetMode: pruned.targetMode as DocsCacheConfig["targetMode"], + defaults: pruned.defaults as DocsCacheConfig["defaults"], + sources: config.sources, + }; + if (!next.defaults || Object.keys(next.defaults).length === 0) { + delete next.defaults; + } + return next; +}; + +export const validateConfig = (input: unknown): DocsCacheConfig => { + if (typeof input !== "object" || input === null || Array.isArray(input)) { + throw new Error("Config must be a JSON object."); + } + const parsed = ConfigSchema.safeParse(input); + if (!parsed.success) { + const details = parsed.error.issues + .map((issue) => `${issue.path.join(".") || "config"} ${issue.message}`) + .join("; "); + throw new Error(`Config does not match schema: ${details}.`); + } + const configInput = parsed.data; + const cacheDir = configInput.cacheDir ?? DEFAULT_CACHE_DIR; + const defaultValues = DEFAULT_CONFIG.defaults as DocsCacheDefaults; + const targetModeOverride = configInput.targetMode; + const defaultsInput = configInput.defaults; + const defaults: DocsCacheDefaults = { + ...defaultValues, + ...(defaultsInput ?? {}), + targetMode: + defaultsInput?.targetMode ?? + targetModeOverride ?? + defaultValues.targetMode, + }; + + return { + cacheDir, + targetMode: targetModeOverride, + defaults, + sources: configInput.sources as DocsCacheSource[], + }; +}; + +export const resolveSources = ( + config: DocsCacheConfig, +): DocsCacheResolvedSource[] => { + const defaults = (config.defaults ?? + DEFAULT_CONFIG.defaults) as DocsCacheDefaults; + const { allowHosts: _allowHosts, ...defaultValues } = defaults; + return config.sources.map((source) => ({ + ...defaultValues, + ...source, + targetMode: source.targetMode ?? defaultValues.targetMode, + include: source.include ?? defaultValues.include, + exclude: source.exclude ?? defaultValues.exclude, + maxFiles: source.maxFiles ?? defaultValues.maxFiles, + toc: source.toc ?? defaultValues.toc, + unwrapSingleRootDir: + source.unwrapSingleRootDir ?? defaultValues.unwrapSingleRootDir, + })); +}; + +export const resolveConfigPath = (configPath?: string) => + configPath + ? path.resolve(configPath) + : path.resolve(process.cwd(), DEFAULT_CONFIG_FILENAME); + +const resolvePackagePath = () => + path.resolve(process.cwd(), PACKAGE_JSON_FILENAME); + +const exists = async (target: string) => { + try { + await access(target); + return true; + } catch { + return false; + } +}; + +const loadConfigFromFile = async ( + filePath: string, + mode: "config" | "package", +) => { + let raw: string; + try { + raw = await readFile(filePath, "utf8"); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Failed to read config at ${filePath}: ${message}`); + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Invalid JSON in ${filePath}: ${message}`); + } + const configInput = + mode === "package" + ? (parsed as Record)?.["docs-cache"] + : parsed; + if (mode === "package" && configInput === undefined) { + throw new Error(`Missing docs-cache config in ${filePath}.`); + } + const config = validateConfig(configInput); + for (const source of config.sources) { + if (source.targetDir) { + resolveTargetDir(filePath, source.targetDir); + } + } + return { + config, + resolvedPath: filePath, + sources: resolveSources(config), + }; +}; + +export const writeConfig = async ( + configPath: string, + config: DocsCacheConfig, +) => { + const data = `${JSON.stringify(config, null, 2)}\n`; + await writeFile(configPath, data, "utf8"); +}; + +export const loadConfig = async (configPath?: string) => { + const resolvedPath = resolveConfigPath(configPath); + const isPackageConfig = path.basename(resolvedPath) === PACKAGE_JSON_FILENAME; + if (configPath) { + return loadConfigFromFile( + resolvedPath, + isPackageConfig ? "package" : "config", + ); + } + if (await exists(resolvedPath)) { + return loadConfigFromFile(resolvedPath, "config"); + } + const packagePath = resolvePackagePath(); + if (await exists(packagePath)) { + try { + return await loadConfigFromFile(packagePath, "package"); + } catch (error) { + if ( + error instanceof Error && + error.message.includes("Missing docs-cache config") + ) { + // fall through to error below + } else { + throw error; + } + } + } + throw new Error( + `No docs.config.json found at ${resolvedPath} and no docs-cache config in ${packagePath}.`, + ); +}; diff --git a/src/config/io.ts b/src/config/io.ts new file mode 100644 index 0000000..813733b --- /dev/null +++ b/src/config/io.ts @@ -0,0 +1,139 @@ +import { access, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; + +import { + DEFAULT_CONFIG, + type DocsCacheConfig, + resolveConfigPath, + stripDefaultConfigValues, + validateConfig, + writeConfig, +} from "#config"; + +const PACKAGE_JSON = "package.json"; +const SCHEMA_URL = + "https://raw.githubusercontent.com/fbosch/docs-cache/main/docs.config.schema.json"; + +const exists = async (target: string) => { + try { + await access(target); + return true; + } catch { + return false; + } +}; + +export type ConfigTarget = { + resolvedPath: string; + mode: "package" | "config"; +}; + +export const loadPackageConfig = async (configPath: string) => { + const raw = await readFile(configPath, "utf8"); + const parsed = JSON.parse(raw) as Record; + const config = parsed["docs-cache"]; + if (!config) { + return { parsed, config: null }; + } + return { + parsed, + config: validateConfig(config), + }; +}; + +export const resolveConfigTarget = async ( + configPath?: string, +): Promise => { + if (configPath) { + const resolvedPath = resolveConfigPath(configPath); + return { + resolvedPath, + mode: path.basename(resolvedPath) === PACKAGE_JSON ? "package" : "config", + }; + } + const defaultPath = resolveConfigPath(); + if (await exists(defaultPath)) { + return { resolvedPath: defaultPath, mode: "config" }; + } + const packagePath = path.resolve(process.cwd(), PACKAGE_JSON); + if (await exists(packagePath)) { + const pkg = await loadPackageConfig(packagePath); + if (pkg.config) { + return { resolvedPath: packagePath, mode: "package" }; + } + } + return { resolvedPath: defaultPath, mode: "config" }; +}; + +export const readConfigAtPath = async ( + target: ConfigTarget, + options?: { allowMissing?: boolean }, +) => { + if (!(await exists(target.resolvedPath))) { + if (!options?.allowMissing) { + throw new Error(`Config not found at ${target.resolvedPath}.`); + } + if (target.mode === "package") { + throw new Error(`package.json not found at ${target.resolvedPath}.`); + } + return { + config: DEFAULT_CONFIG, + rawConfig: null, + rawPackage: null, + hadDocsCacheConfig: false, + }; + } + if (target.mode === "package") { + const pkg = await loadPackageConfig(target.resolvedPath); + return { + config: pkg.config ?? DEFAULT_CONFIG, + rawConfig: pkg.config, + rawPackage: pkg.parsed, + hadDocsCacheConfig: Boolean(pkg.config), + }; + } + const raw = await readFile(target.resolvedPath, "utf8"); + const rawConfig = JSON.parse(raw.toString()); + return { + config: validateConfig(rawConfig), + rawConfig, + rawPackage: null, + hadDocsCacheConfig: true, + }; +}; + +export const mergeConfigBase = ( + config: DocsCacheConfig, + sources: DocsCacheConfig["sources"], +): DocsCacheConfig => { + const nextConfig: DocsCacheConfig = { + $schema: config.$schema ?? SCHEMA_URL, + sources, + }; + if (config.cacheDir) { + nextConfig.cacheDir = config.cacheDir; + } + if (config.defaults) { + nextConfig.defaults = config.defaults; + } + if (config.targetMode) { + nextConfig.targetMode = config.targetMode; + } + return nextConfig; +}; + +export const writeConfigFile = async (params: { + mode: "package" | "config"; + resolvedPath: string; + config: DocsCacheConfig; + rawPackage: Record | null; +}) => { + const { mode, resolvedPath, config, rawPackage } = params; + if (mode === "package") { + const pkg = rawPackage ?? {}; + pkg["docs-cache"] = stripDefaultConfigValues(config); + await writeFile(resolvedPath, `${JSON.stringify(pkg, null, 2)}\n`, "utf8"); + return; + } + await writeConfig(resolvedPath, config); +}; diff --git a/src/config/schema.ts b/src/config/schema.ts new file mode 100644 index 0000000..dffa2e9 --- /dev/null +++ b/src/config/schema.ts @@ -0,0 +1,98 @@ +import * as z from "zod"; +import { assertSafeSourceId } from "#core/source-id"; + +export const TargetModeSchema = z.enum(["symlink", "copy"]); +export const CacheModeSchema = z.enum(["materialize"]); +export const TocFormatSchema = z.enum(["tree", "compressed"]); +export const IntegritySchema = z + .object({ + type: z.enum(["commit", "manifest"]), + value: z.string().nullable(), + }) + .strict(); + +const CommonOptionsSchema = z.object({ + ref: z.string().min(1), + mode: CacheModeSchema, + include: z.array(z.string().min(1)).min(1), + exclude: z.array(z.string().min(1)).optional(), + targetMode: TargetModeSchema.optional(), + required: z.boolean(), + maxBytes: z.number().min(1), + maxFiles: z.number().min(1).optional(), + ignoreHidden: z.boolean(), + toc: z.union([z.boolean(), TocFormatSchema]).optional(), + unwrapSingleRootDir: z.boolean().optional(), +}); + +export const DefaultsSchema = CommonOptionsSchema.extend({ + allowHosts: z.array(z.string().min(1)).min(1), +}).strict(); + +export const SourceSchema = CommonOptionsSchema.partial() + .extend({ + id: z + .string() + .min(1) + .superRefine((value, ctx) => { + try { + assertSafeSourceId(value, "id"); + } catch (error) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + error instanceof Error ? error.message : "Invalid source id.", + }); + } + }), + repo: z.string().min(1), + targetDir: z.string().min(1).optional(), + integrity: IntegritySchema.optional(), + }) + .extend({ + include: z + .array(z.string().min(1)) + .min(1, { message: "include must be a non-empty array" }) + .optional(), + }) + .strict(); + +export const ResolvedSourceSchema = SourceSchema.extend( + CommonOptionsSchema.shape, +).strict(); + +export const ConfigSchema = z + .object({ + $schema: z.string().min(1).optional(), + cacheDir: z.string().min(1).optional(), + targetMode: TargetModeSchema.optional(), + defaults: DefaultsSchema.partial().optional(), + sources: z.array(SourceSchema), + }) + .strict() + .superRefine((value, ctx) => { + const seen = new Set(); + const duplicates = new Set(); + value.sources.forEach((source) => { + if (seen.has(source.id)) { + duplicates.add(source.id); + } else { + seen.add(source.id); + } + }); + if (duplicates.size > 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["sources"], + message: `Duplicate source IDs found: ${Array.from(duplicates).join(", ")}.`, + }); + } + }); + +export type DocsCacheDefaults = z.infer; +export type DocsCacheSource = z.infer; +export type DocsCacheResolvedSource = z.infer; +export type DocsCacheConfig = z.infer; +export type DocsCacheIntegrity = z.infer; +export type CacheMode = z.infer; +export type TocFormat = z.infer; diff --git a/src/git/fetch-source.ts b/src/git/fetch-source.ts index 29d9acd..7bb4fed 100644 --- a/src/git/fetch-source.ts +++ b/src/git/fetch-source.ts @@ -6,15 +6,121 @@ import { pathToFileURL } from "node:url"; import { execa } from "execa"; -import { getErrnoCode } from "../errors"; -import { assertSafeSourceId } from "../source-id"; -import { exists, resolveGitCacheDir } from "./cache-dir"; +import { getErrnoCode } from "#core/errors"; +import { assertSafeSourceId } from "#core/source-id"; +import { exists, resolveGitCacheDir } from "#git/cache-dir"; const DEFAULT_TIMEOUT_MS = 120000; // 120 seconds (2 minutes) const DEFAULT_GIT_DEPTH = 1; const DEFAULT_RM_RETRIES = 3; const DEFAULT_RM_BACKOFF_MS = 100; +const buildGitEnv = () => { + const pathValue = process.env.PATH ?? process.env.Path; + const pathExtValue = + process.env.PATHEXT ?? + (process.platform === "win32" ? ".COM;.EXE;.BAT;.CMD" : undefined); + return { + ...process.env, + ...(pathValue ? { PATH: pathValue, Path: pathValue } : {}), + ...(pathExtValue ? { PATHEXT: pathExtValue } : {}), + HOME: process.env.HOME, + USER: process.env.USER, + USERPROFILE: process.env.USERPROFILE, + TMPDIR: process.env.TMPDIR, + TMP: process.env.TMP, + TEMP: process.env.TEMP, + SYSTEMROOT: process.env.SYSTEMROOT, + WINDIR: process.env.WINDIR, + SSH_AUTH_SOCK: process.env.SSH_AUTH_SOCK, + SSH_AGENT_PID: process.env.SSH_AGENT_PID, + HTTP_PROXY: process.env.HTTP_PROXY, + HTTPS_PROXY: process.env.HTTPS_PROXY, + NO_PROXY: process.env.NO_PROXY, + GIT_TERMINAL_PROMPT: "0", + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_NOGLOBAL: "1", + ...(process.platform === "win32" ? {} : { GIT_ASKPASS: "/bin/false" }), + }; +}; + +const buildGitConfigs = (allowFileProtocol?: boolean) => [ + "-c", + "core.hooksPath=/dev/null", + "-c", + "submodule.recurse=false", + "-c", + "protocol.ext.allow=never", + "-c", + `protocol.file.allow=${allowFileProtocol ? "always" : "never"}`, +]; + +const buildCommandArgs = ( + args: string[], + allowFileProtocol?: boolean, + forceProgress?: boolean, +) => { + const configs = buildGitConfigs(allowFileProtocol); + const commandArgs = [...configs, ...args]; + if (forceProgress) { + commandArgs.push("--progress"); + } + return commandArgs; +}; + +const isProgressLine = (line: string) => + line.includes("Receiving objects") || + line.includes("Resolving deltas") || + line.includes("Compressing objects") || + line.includes("Updating files") || + line.includes("Counting objects"); + +const shouldEmitProgress = ( + line: string, + now: number, + lastProgressAt: number, + throttleMs: number, +) => + now - lastProgressAt >= throttleMs || + line.includes("100%") || + line.includes("done"); + +const attachLoggers = ( + subprocess: ReturnType, + commandLabel: string, + options?: { + logger?: (message: string) => void; + progressLogger?: (message: string) => void; + progressThrottleMs?: number; + }, +) => { + if (!options?.logger && !options?.progressLogger) { + return; + } + let lastProgressAt = 0; + const forward = (stream: NodeJS.ReadableStream | null) => { + if (!stream) return; + stream.on("data", (chunk) => { + const text = + chunk instanceof Buffer ? chunk.toString("utf8") : String(chunk); + for (const line of text.split(/\r?\n/)) { + if (!line) continue; + options.logger?.(`${commandLabel} | ${line}`); + if (!options?.progressLogger) continue; + if (!isProgressLine(line)) continue; + const now = Date.now(); + const throttleMs = options.progressThrottleMs ?? 120; + if (shouldEmitProgress(line, now, lastProgressAt, throttleMs)) { + lastProgressAt = now; + options.progressLogger(line); + } + } + }); + }; + forward(subprocess.stdout); + forward(subprocess.stderr); +}; + const git = async ( args: string[], options?: { @@ -22,78 +128,27 @@ const git = async ( timeoutMs?: number; allowFileProtocol?: boolean; logger?: (message: string) => void; + progressLogger?: (message: string) => void; + progressThrottleMs?: number; + forceProgress?: boolean; }, ) => { - const pathValue = process.env.PATH ?? process.env.Path; - const pathExtValue = - process.env.PATHEXT ?? - (process.platform === "win32" ? ".COM;.EXE;.BAT;.CMD" : undefined); - - const configs = [ - "-c", - "core.hooksPath=/dev/null", - "-c", - "submodule.recurse=false", - "-c", - "protocol.ext.allow=never", - ]; - - // Configure file protocol access - if (options?.allowFileProtocol) { - // Explicitly allow file protocol for local cache clones - configs.push("-c", "protocol.file.allow=always"); - } else { - // Disallow file protocol by default (when false or undefined) - configs.push("-c", "protocol.file.allow=never"); - } - - const commandArgs = [...configs, ...args]; + const commandArgs = buildCommandArgs( + args, + options?.allowFileProtocol, + options?.forceProgress, + ); const commandLabel = `git ${commandArgs.join(" ")}`; options?.logger?.(commandLabel); const subprocess = execa("git", commandArgs, { cwd: options?.cwd, timeout: options?.timeoutMs ?? DEFAULT_TIMEOUT_MS, - maxBuffer: 10 * 1024 * 1024, // 10MB buffer for large repos + maxBuffer: 10 * 1024 * 1024, stdout: "pipe", stderr: "pipe", - env: { - ...process.env, - ...(pathValue ? { PATH: pathValue, Path: pathValue } : {}), - ...(pathExtValue ? { PATHEXT: pathExtValue } : {}), - HOME: process.env.HOME, - USER: process.env.USER, - USERPROFILE: process.env.USERPROFILE, - TMPDIR: process.env.TMPDIR, - TMP: process.env.TMP, - TEMP: process.env.TEMP, - SYSTEMROOT: process.env.SYSTEMROOT, - WINDIR: process.env.WINDIR, - SSH_AUTH_SOCK: process.env.SSH_AUTH_SOCK, - SSH_AGENT_PID: process.env.SSH_AGENT_PID, - HTTP_PROXY: process.env.HTTP_PROXY, - HTTPS_PROXY: process.env.HTTPS_PROXY, - NO_PROXY: process.env.NO_PROXY, - GIT_TERMINAL_PROMPT: "0", - GIT_CONFIG_NOSYSTEM: "1", - GIT_CONFIG_NOGLOBAL: "1", - ...(process.platform === "win32" ? {} : { GIT_ASKPASS: "/bin/false" }), - }, + env: buildGitEnv(), }); - if (options?.logger) { - const forward = (stream: NodeJS.ReadableStream | null) => { - if (!stream) return; - stream.on("data", (chunk) => { - const text = - chunk instanceof Buffer ? chunk.toString("utf8") : String(chunk); - for (const line of text.split(/\r?\n/)) { - if (!line) continue; - options.logger?.(`${commandLabel} | ${line}`); - } - }); - }; - forward(subprocess.stdout); - forward(subprocess.stderr); - } + attachLoggers(subprocess, commandLabel, options); await subprocess; }; @@ -153,6 +208,27 @@ const isPartialClone = async (repoPath: string) => { } }; +const hasCommitInRepo = async ( + repoPath: string, + commit: string, + options?: { + timeoutMs?: number; + allowFileProtocol?: boolean; + logger?: (message: string) => void; + }, +): Promise => { + try { + await git(["-C", repoPath, "cat-file", "-e", `${commit}^{commit}`], { + timeoutMs: options?.timeoutMs, + allowFileProtocol: options?.allowFileProtocol, + logger: options?.logger, + }); + return true; + } catch { + return false; + } +}; + const ensureCommitAvailable = async ( repoPath: string, commit: string, @@ -160,6 +236,7 @@ const ensureCommitAvailable = async ( timeoutMs?: number; allowFileProtocol?: boolean; logger?: (message: string) => void; + offline?: boolean; }, ) => { try { @@ -172,6 +249,9 @@ const ensureCommitAvailable = async ( } catch { // commit not present, fetch it } + if (options?.offline && !options?.allowFileProtocol) { + throw new Error(`Commit ${commit} not found in cache (offline).`); + } await git(["-C", repoPath, "fetch", "origin", commit], { timeoutMs: options?.timeoutMs, allowFileProtocol: options?.allowFileProtocol, @@ -188,6 +268,8 @@ type FetchParams = { include?: string[]; timeoutMs?: number; logger?: (message: string) => void; + progressLogger?: (message: string) => void; + offline?: boolean; }; type FetchResult = { @@ -196,6 +278,11 @@ type FetchResult = { fromCache: boolean; }; +type CloneResult = { + usedCache: boolean; + cleanup: () => Promise; +}; + const isSparseEligible = (include?: string[]) => { if (!include || include.length === 0) { return false; @@ -222,6 +309,9 @@ const extractSparsePaths = (include?: string[]) => { }; const cloneRepo = async (params: FetchParams, outDir: string) => { + if (params.offline) { + throw new Error(`Cannot clone ${params.repo} while offline.`); + } const isCommitRef = /^[0-9a-f]{7,40}$/i.test(params.ref); const useSparse = isSparseEligible(params.include); const buildCloneArgs = () => { @@ -246,10 +336,16 @@ const cloneRepo = async (params: FetchParams, outDir: string) => { } } cloneArgs.push(params.repo, outDir); - await git(cloneArgs, { timeoutMs: params.timeoutMs, logger: params.logger }); + await git(cloneArgs, { + timeoutMs: params.timeoutMs, + logger: params.logger, + progressLogger: params.progressLogger, + forceProgress: Boolean(params.progressLogger), + }); await ensureCommitAvailable(outDir, params.resolvedCommit, { timeoutMs: params.timeoutMs, logger: params.logger, + offline: params.offline, }); if (useSparse) { const sparsePaths = extractSparsePaths(params.include); @@ -269,59 +365,179 @@ const cloneRepo = async (params: FetchParams, outDir: string) => { ); }; +const addWorktreeFromCache = async ( + params: FetchParams, + cachePath: string, + outDir: string, +): Promise => { + await git( + [ + "-C", + cachePath, + "worktree", + "add", + "--detach", + outDir, + params.resolvedCommit, + ], + { + timeoutMs: params.timeoutMs, + logger: params.logger, + allowFileProtocol: true, + }, + ); + await git( + ["-C", outDir, "checkout", "--quiet", "--detach", params.resolvedCommit], + { + timeoutMs: params.timeoutMs, + logger: params.logger, + allowFileProtocol: true, + }, + ); + const sparsePaths = isSparseEligible(params.include) + ? extractSparsePaths(params.include) + : []; + if (sparsePaths.length > 0) { + await git(["-C", outDir, "sparse-checkout", "set", ...sparsePaths], { + timeoutMs: params.timeoutMs, + logger: params.logger, + allowFileProtocol: true, + }); + } + return { + usedCache: true, + cleanup: async () => { + try { + await git(["-C", cachePath, "worktree", "remove", "--force", outDir], { + timeoutMs: params.timeoutMs, + logger: params.logger, + allowFileProtocol: true, + }); + } catch { + // fall back to removing the directory directly + } + }, + }; +}; + +const buildFetchArgs = (ref: string, isCommitRef: boolean) => { + const fetchArgs = ["fetch", "origin"]; + if (!isCommitRef) { + const refSpec = + ref === "HEAD" ? "HEAD" : `${ref}:refs/remotes/origin/${ref}`; + fetchArgs.push(refSpec, "--depth", String(DEFAULT_GIT_DEPTH)); + return fetchArgs; + } + fetchArgs.push("--depth", String(DEFAULT_GIT_DEPTH)); + return fetchArgs; +}; + +const fetchCommitFromOrigin = async ( + params: FetchParams, + cachePath: string, + isCommitRef: boolean, +) => { + const fetchArgs = buildFetchArgs(params.ref, isCommitRef); + await git(["-C", cachePath, ...fetchArgs], { + timeoutMs: params.timeoutMs, + logger: params.logger, + progressLogger: params.progressLogger, + forceProgress: Boolean(params.progressLogger), + allowFileProtocol: true, + }); + await ensureCommitAvailable(cachePath, params.resolvedCommit, { + timeoutMs: params.timeoutMs, + logger: params.logger, + offline: params.offline, + }); +}; + +const handleValidCache = async ( + params: FetchParams, + cachePath: string, + isCommitRef: boolean, +): Promise<{ usedCache: boolean; worktreeUsed: boolean }> => { + if (await isPartialClone(cachePath)) { + if (params.offline) { + throw new Error(`Cache for ${params.repo} is partial (offline).`); + } + await removeDir(cachePath); + await cloneRepo(params, cachePath); + return { usedCache: false, worktreeUsed: false }; + } + try { + const commitExists = await hasCommitInRepo( + cachePath, + params.resolvedCommit, + { + timeoutMs: params.timeoutMs, + logger: params.logger, + }, + ); + if (commitExists) { + return { usedCache: true, worktreeUsed: true }; + } + if (params.offline) { + throw new Error( + `Commit ${params.resolvedCommit} not found in cache (offline).`, + ); + } + await fetchCommitFromOrigin(params, cachePath, isCommitRef); + return { usedCache: true, worktreeUsed: false }; + } catch (_error) { + if (params.offline) { + throw new Error(`Cache for ${params.repo} is unavailable (offline).`); + } + await removeDir(cachePath); + await cloneRepo(params, cachePath); + return { usedCache: false, worktreeUsed: false }; + } +}; + +const handleMissingCache = async ( + params: FetchParams, + cachePath: string, + cacheExists: boolean, +): Promise<{ usedCache: boolean; worktreeUsed: boolean }> => { + if (cacheExists) { + await removeDir(cachePath); + } + if (params.offline) { + throw new Error(`Cache for ${params.repo} is missing (offline).`); + } + await cloneRepo(params, cachePath); + return { usedCache: false, worktreeUsed: false }; +}; + // Clone or update a repository using persistent cache const cloneOrUpdateRepo = async ( params: FetchParams, outDir: string, -): Promise<{ usedCache: boolean }> => { +): Promise => { const cachePath = getPersistentCachePath(params.repo); const cacheExists = await exists(cachePath); const cacheValid = cacheExists && (await isValidGitRepo(cachePath)); const isCommitRef = /^[0-9a-f]{7,40}$/i.test(params.ref); const useSparse = isSparseEligible(params.include); let usedCache = cacheValid; + let worktreeUsed = false; const cacheRoot = resolveGitCacheDir(); await mkdir(cacheRoot, { recursive: true }); if (cacheValid) { - if (await isPartialClone(cachePath)) { - await removeDir(cachePath); - await cloneRepo(params, cachePath); - usedCache = false; - } else { - try { - const fetchArgs = ["fetch", "origin"]; - if (!isCommitRef) { - const refSpec = - params.ref === "HEAD" - ? "HEAD" - : `${params.ref}:refs/remotes/origin/${params.ref}`; - fetchArgs.push(refSpec, "--depth", String(DEFAULT_GIT_DEPTH)); - } else { - fetchArgs.push("--depth", String(DEFAULT_GIT_DEPTH)); - } + const result = await handleValidCache(params, cachePath, isCommitRef); + usedCache = result.usedCache; + worktreeUsed = result.worktreeUsed; + } + if (!cacheValid) { + const result = await handleMissingCache(params, cachePath, cacheExists); + usedCache = result.usedCache; + worktreeUsed = result.worktreeUsed; + } - await git(["-C", cachePath, ...fetchArgs], { - timeoutMs: params.timeoutMs, - logger: params.logger, - }); - await ensureCommitAvailable(cachePath, params.resolvedCommit, { - timeoutMs: params.timeoutMs, - logger: params.logger, - }); - } catch (_error) { - await removeDir(cachePath); - await cloneRepo(params, cachePath); - usedCache = false; - } - } - } else { - if (cacheExists) { - await removeDir(cachePath); - } - await cloneRepo(params, cachePath); - usedCache = false; + if (worktreeUsed && cacheValid) { + return addWorktreeFromCache(params, cachePath, outDir); } await mkdir(outDir, { recursive: true }); @@ -355,6 +571,8 @@ const cloneOrUpdateRepo = async ( timeoutMs: params.timeoutMs, allowFileProtocol: true, logger: params.logger, + progressLogger: params.progressLogger, + forceProgress: Boolean(params.progressLogger), }); if (useSparse) { @@ -372,6 +590,7 @@ const cloneOrUpdateRepo = async ( timeoutMs: params.timeoutMs, allowFileProtocol: true, logger: params.logger, + offline: params.offline, }); await git( @@ -383,27 +602,29 @@ const cloneOrUpdateRepo = async ( }, ); - return { usedCache }; + return { usedCache, cleanup: async () => undefined }; }; export const fetchSource = async ( params: FetchParams, ): Promise => { assertSafeSourceId(params.sourceId, "sourceId"); - const tempDir = await mkdtemp( + const tempRoot = await mkdtemp( path.join(tmpdir(), `docs-cache-${params.sourceId}-`), ); + const tempDir = path.join(tempRoot, "repo"); try { - const { usedCache } = await cloneOrUpdateRepo(params, tempDir); + const { usedCache, cleanup } = await cloneOrUpdateRepo(params, tempDir); return { repoDir: tempDir, cleanup: async () => { - await removeDir(tempDir); + await cleanup(); + await removeDir(tempRoot); }, fromCache: usedCache, }; } catch (error) { - await removeDir(tempDir); + await removeDir(tempRoot); throw error; } }; diff --git a/src/git/resolve-remote.ts b/src/git/resolve-remote.ts index f49fbcb..0d46544 100644 --- a/src/git/resolve-remote.ts +++ b/src/git/resolve-remote.ts @@ -1,7 +1,7 @@ import { execFile } from "node:child_process"; import { promisify } from "node:util"; -import { redactRepoUrl } from "./redact"; +import { redactRepoUrl } from "#git/redact"; const execFileAsync = promisify(execFile); diff --git a/src/resolve-repo.ts b/src/git/resolve-repo.ts similarity index 100% rename from src/resolve-repo.ts rename to src/git/resolve-repo.ts diff --git a/src/gitignore.ts b/src/gitignore.ts index 95b8489..8639180 100644 --- a/src/gitignore.ts +++ b/src/gitignore.ts @@ -1,6 +1,6 @@ import { access, readFile, writeFile } from "node:fs/promises"; import path from "node:path"; -import { toPosixPath } from "./paths"; +import { toPosixPath } from "#core/paths"; const exists = async (target: string) => { try { diff --git a/src/init.ts b/src/init.ts deleted file mode 100644 index 084c095..0000000 --- a/src/init.ts +++ /dev/null @@ -1,200 +0,0 @@ -import { access, readFile, writeFile } from "node:fs/promises"; -import path from "node:path"; -import { - confirm as clackConfirm, - isCancel as clackIsCancel, - select as clackSelect, - text as clackText, -} from "@clack/prompts"; -import { - DEFAULT_CACHE_DIR, - DEFAULT_CONFIG_FILENAME, - type DocsCacheConfig, - stripDefaultConfigValues, - writeConfig, -} from "./config"; -import { ensureGitignoreEntry, getGitignoreStatus } from "./gitignore"; - -type InitOptions = { - cacheDirOverride?: string; - json: boolean; - cwd?: string; -}; - -type PromptDeps = { - confirm?: typeof clackConfirm; - isCancel?: typeof clackIsCancel; - select?: typeof clackSelect; - text?: typeof clackText; -}; - -const exists = async (target: string) => { - try { - await access(target); - return true; - } catch { - return false; - } -}; - -export const initConfig = async ( - options: InitOptions, - deps: PromptDeps = {}, -) => { - const confirm = deps.confirm ?? clackConfirm; - const isCancel = deps.isCancel ?? clackIsCancel; - const select = deps.select ?? clackSelect; - const text = deps.text ?? clackText; - const cwd = options.cwd ?? process.cwd(); - const defaultConfigPath = path.resolve(cwd, DEFAULT_CONFIG_FILENAME); - const packagePath = path.resolve(cwd, "package.json"); - const existingConfigPaths: string[] = []; - if (await exists(defaultConfigPath)) { - existingConfigPaths.push(defaultConfigPath); - } - if (await exists(packagePath)) { - const raw = await readFile(packagePath, "utf8"); - const parsed = JSON.parse(raw) as Record; - if (parsed["docs-cache"]) { - existingConfigPaths.push(packagePath); - } - } - if (existingConfigPaths.length > 0) { - throw new Error( - `Config already exists at ${existingConfigPaths.join(", ")}. Init aborted.`, - ); - } - let usePackageConfig = false; - if (await exists(packagePath)) { - const raw = await readFile(packagePath, "utf8"); - const parsed = JSON.parse(raw) as Record; - if (!parsed["docs-cache"]) { - const locationAnswer = await select({ - message: "Config location", - options: [ - { value: "config", label: "docs.config.json" }, - { value: "package", label: "package.json" }, - ], - initialValue: "config", - }); - if (isCancel(locationAnswer)) { - throw new Error("Init cancelled."); - } - usePackageConfig = locationAnswer === "package"; - } - } - const configPath = usePackageConfig ? packagePath : defaultConfigPath; - const cacheDir = options.cacheDirOverride ?? DEFAULT_CACHE_DIR; - const cacheDirAnswer = await text({ - message: "Cache directory", - initialValue: cacheDir, - }); - if (isCancel(cacheDirAnswer)) { - throw new Error("Init cancelled."); - } - const cacheDirValue = cacheDirAnswer || DEFAULT_CACHE_DIR; - const tocAnswer = await confirm({ - message: - "Generate TOC.md (table of contents with links to all documentation)", - initialValue: true, - }); - if (isCancel(tocAnswer)) { - throw new Error("Init cancelled."); - } - const gitignoreStatus = await getGitignoreStatus(cwd, cacheDirValue); - let gitignoreAnswer = false; - if (gitignoreStatus.entry && !gitignoreStatus.hasEntry) { - const reply = await confirm({ - message: "Add cache directory to .gitignore", - initialValue: true, - }); - if (isCancel(reply)) { - throw new Error("Init cancelled."); - } - gitignoreAnswer = reply; - } - - const answers = { - configPath, - cacheDir: cacheDirAnswer, - toc: tocAnswer, - gitignore: gitignoreAnswer, - } as { - configPath: string; - cacheDir: string; - toc: boolean; - gitignore: boolean; - }; - - const resolvedConfigPath = path.resolve(cwd, answers.configPath); - if (path.basename(resolvedConfigPath) === "package.json") { - const raw = await readFile(resolvedConfigPath, "utf8"); - const pkg = JSON.parse(raw) as Record; - if (pkg["docs-cache"]) { - throw new Error( - `docs-cache config already exists in ${resolvedConfigPath}.`, - ); - } - const baseConfig: DocsCacheConfig = { - $schema: - "https://raw.githubusercontent.com/fbosch/docs-cache/main/docs.config.schema.json", - sources: [], - }; - const resolvedCacheDir = answers.cacheDir || DEFAULT_CACHE_DIR; - if (resolvedCacheDir !== DEFAULT_CACHE_DIR) { - baseConfig.cacheDir = resolvedCacheDir; - } - // Since TOC defaults to true, only set it explicitly if user chose false - if (!answers.toc) { - baseConfig.defaults = { toc: false }; - } - pkg["docs-cache"] = stripDefaultConfigValues(baseConfig); - await writeFile( - resolvedConfigPath, - `${JSON.stringify(pkg, null, 2)}\n`, - "utf8", - ); - const gitignoreResult = answers.gitignore - ? await ensureGitignoreEntry( - path.dirname(resolvedConfigPath), - resolvedCacheDir, - ) - : null; - return { - configPath: resolvedConfigPath, - created: true, - gitignoreUpdated: gitignoreResult?.updated ?? false, - gitignorePath: gitignoreResult?.gitignorePath ?? null, - }; - } - if (await exists(resolvedConfigPath)) { - throw new Error(`Config already exists at ${resolvedConfigPath}.`); - } - const config: DocsCacheConfig = { - $schema: - "https://raw.githubusercontent.com/fbosch/docs-cache/main/docs.config.schema.json", - sources: [], - }; - const resolvedCacheDir = answers.cacheDir || DEFAULT_CACHE_DIR; - if (resolvedCacheDir !== DEFAULT_CACHE_DIR) { - config.cacheDir = resolvedCacheDir; - } - // Since TOC defaults to true, only set it explicitly if user chose false - if (!answers.toc) { - config.defaults = { toc: false }; - } - - await writeConfig(resolvedConfigPath, config); - const gitignoreResult = answers.gitignore - ? await ensureGitignoreEntry( - path.dirname(resolvedConfigPath), - resolvedCacheDir, - ) - : null; - return { - configPath: resolvedConfigPath, - created: true, - gitignoreUpdated: gitignoreResult?.updated ?? false, - gitignorePath: gitignoreResult?.gitignorePath ?? null, - }; -}; diff --git a/src/remove.ts b/src/remove.ts deleted file mode 100644 index f5a9526..0000000 --- a/src/remove.ts +++ /dev/null @@ -1,171 +0,0 @@ -import { access, readFile, rm, writeFile } from "node:fs/promises"; -import path from "node:path"; -import { - DEFAULT_CONFIG, - type DocsCacheConfig, - resolveConfigPath, - stripDefaultConfigValues, - validateConfig, - writeConfig, -} from "./config"; -import { resolveTargetDir } from "./paths"; -import { resolveRepoInput } from "./resolve-repo"; - -const exists = async (target: string) => { - try { - await access(target); - return true; - } catch { - return false; - } -}; - -const PACKAGE_JSON = "package.json"; - -const loadPackageConfig = async (configPath: string) => { - const raw = await readFile(configPath, "utf8"); - const parsed = JSON.parse(raw) as Record; - const config = parsed["docs-cache"]; - if (!config) { - return { parsed, config: null }; - } - return { - parsed, - config: validateConfig(config), - }; -}; - -const resolveConfigTarget = async (configPath?: string) => { - if (configPath) { - const resolvedPath = resolveConfigPath(configPath); - return { - resolvedPath, - mode: path.basename(resolvedPath) === PACKAGE_JSON ? "package" : "config", - }; - } - const defaultPath = resolveConfigPath(); - if (await exists(defaultPath)) { - return { resolvedPath: defaultPath, mode: "config" }; - } - const packagePath = path.resolve(process.cwd(), PACKAGE_JSON); - if (await exists(packagePath)) { - const pkg = await loadPackageConfig(packagePath); - if (pkg.config) { - return { resolvedPath: packagePath, mode: "package" }; - } - } - return { resolvedPath: defaultPath, mode: "config" }; -}; - -export const removeSources = async (params: { - configPath?: string; - ids: string[]; -}) => { - if (params.ids.length === 0) { - throw new Error("No sources specified to remove."); - } - const target = await resolveConfigTarget(params.configPath); - const resolvedPath = target.resolvedPath; - let config = DEFAULT_CONFIG; - let rawConfig: DocsCacheConfig | null = null; - let rawPackage: Record | null = null; - if (await exists(resolvedPath)) { - if (target.mode === "package") { - const pkg = await loadPackageConfig(resolvedPath); - rawPackage = pkg.parsed; - rawConfig = pkg.config; - if (!rawConfig) { - throw new Error(`Missing docs-cache config in ${resolvedPath}.`); - } - config = rawConfig; - } else { - const raw = await readFile(resolvedPath, "utf8"); - rawConfig = JSON.parse(raw.toString()); - config = validateConfig(rawConfig); - } - } else { - throw new Error(`Config not found at ${resolvedPath}.`); - } - - const sourcesById = new Map( - config.sources.map((source) => [source.id, source]), - ); - const sourcesByRepo = new Map( - config.sources.map((source) => [source.repo, source]), - ); - const idsToRemove = new Set(); - const missing: string[] = []; - for (const token of params.ids) { - if (sourcesById.has(token)) { - idsToRemove.add(token); - continue; - } - const resolved = resolveRepoInput(token); - if (resolved.repoUrl && sourcesByRepo.has(resolved.repoUrl)) { - const source = sourcesByRepo.get(resolved.repoUrl); - if (source) { - idsToRemove.add(source.id); - } - continue; - } - if (resolved.inferredId && sourcesById.has(resolved.inferredId)) { - idsToRemove.add(resolved.inferredId); - continue; - } - missing.push(token); - } - const remaining = config.sources.filter( - (source) => !idsToRemove.has(source.id), - ); - const removed = config.sources - .filter((source) => idsToRemove.has(source.id)) - .map((source) => source.id); - const removedSources = config.sources.filter((source) => - idsToRemove.has(source.id), - ); - - if (removed.length === 0) { - throw new Error("No matching sources found to remove."); - } - - const nextConfig: DocsCacheConfig = { - $schema: - rawConfig?.$schema ?? - "https://raw.githubusercontent.com/fbosch/docs-cache/main/docs.config.schema.json", - sources: remaining, - }; - if (rawConfig?.cacheDir) { - nextConfig.cacheDir = rawConfig.cacheDir; - } - if (rawConfig?.defaults) { - nextConfig.defaults = rawConfig.defaults; - } - if (rawConfig?.targetMode) { - nextConfig.targetMode = rawConfig.targetMode; - } - - if (target.mode === "package") { - const pkg = rawPackage ?? {}; - pkg["docs-cache"] = stripDefaultConfigValues(nextConfig); - await writeFile(resolvedPath, `${JSON.stringify(pkg, null, 2)}\n`, "utf8"); - } else { - await writeConfig(resolvedPath, nextConfig); - } - - const targetRemovals: Array<{ id: string; targetDir: string }> = []; - for (const source of removedSources) { - if (!source.targetDir) { - continue; - } - const targetDir = resolveTargetDir(resolvedPath, source.targetDir); - await rm(targetDir, { recursive: true, force: true }); - targetRemovals.push({ id: source.id, targetDir }); - } - - return { - configPath: resolvedPath, - removed, - missing, - targetsRemoved: targetRemovals, - }; -}; diff --git a/src/sync.ts b/src/sync.ts deleted file mode 100644 index 60d7807..0000000 --- a/src/sync.ts +++ /dev/null @@ -1,622 +0,0 @@ -import { createHash } from "node:crypto"; -import { access, mkdir, readFile } from "node:fs/promises"; -import path from "node:path"; -import pc from "picocolors"; -import { symbols, ui } from "./cli/ui"; -import { - DEFAULT_CACHE_DIR, - DEFAULT_CONFIG, - type DocsCacheDefaults, - type DocsCacheResolvedSource, - loadConfig, -} from "./config"; -import { fetchSource } from "./git/fetch-source"; -import { resolveRemoteCommit } from "./git/resolve-remote"; -import { readLock, resolveLockPath, writeLock } from "./lock"; -import { MANIFEST_FILENAME } from "./manifest"; -import { computeManifestHash, materializeSource } from "./materialize"; -import { resolveCacheDir, resolveTargetDir } from "./paths"; -import { applyTargetDir } from "./targets"; -import { writeToc } from "./toc"; -import { verifyCache } from "./verify"; - -type SyncOptions = { - configPath?: string; - cacheDirOverride?: string; - json: boolean; - lockOnly: boolean; - offline: boolean; - failOnMiss: boolean; - verbose?: boolean; - concurrency?: number; - sourceFilter?: string[]; - timeoutMs?: number; -}; - -type SyncDeps = { - resolveRemoteCommit?: typeof resolveRemoteCommit; - fetchSource?: typeof fetchSource; - materializeSource?: typeof materializeSource; -}; - -type SyncResult = { - id: string; - repo: string; - ref: string; - resolvedCommit: string; - lockCommit: string | null; - lockRulesSha256?: string; - status: "up-to-date" | "changed" | "missing"; - bytes?: number; - fileCount?: number; - manifestSha256?: string; - rulesSha256?: string; -}; - -const formatBytes = (value: number) => { - if (value < 1024) { - return `${value} B`; - } - const units = ["KB", "MB", "GB", "TB"]; - let size = value; - let index = -1; - while (size >= 1024 && index < units.length - 1) { - size /= 1024; - index += 1; - } - return `${size.toFixed(1)} ${units[index]}`; -}; - -const exists = async (target: string) => { - try { - await access(target); - return true; - } catch { - return false; - } -}; - -const hasDocs = async (cacheDir: string, sourceId: string) => { - const sourceDir = path.join(cacheDir, sourceId); - if (!(await exists(sourceDir))) { - return false; - } - return await exists(path.join(sourceDir, MANIFEST_FILENAME)); -}; - -const normalizePatterns = (patterns?: string[]) => { - if (!patterns || patterns.length === 0) { - return []; - } - const normalized = patterns - .map((pattern) => pattern.trim()) - .filter((pattern) => pattern.length > 0); - return Array.from(new Set(normalized)).sort(); -}; - -const RULES_HASH_BLACKLIST = [ - "id", - "repo", - "ref", - "targetDir", - "targetMode", - "required", - "integrity", - "toc", -] as const; - -type RulesHashBlacklistKey = (typeof RULES_HASH_BLACKLIST)[number]; -type RulesHashKey = Exclude< - keyof DocsCacheResolvedSource, - RulesHashBlacklistKey ->; - -const RULES_HASH_KEYS = [ - "mode", - "include", - "exclude", - "maxBytes", - "maxFiles", - "ignoreHidden", - "unwrapSingleRootDir", -] as const satisfies ReadonlyArray; - -const normalizeRulesValue = ( - key: RulesHashKey, - value: DocsCacheResolvedSource[RulesHashKey], -) => { - if (key === "include" && Array.isArray(value)) { - return normalizePatterns(value); - } - if (key === "exclude" && Array.isArray(value)) { - return normalizePatterns(value); - } - return value; -}; - -const computeRulesHash = (source: DocsCacheResolvedSource) => { - const entries = RULES_HASH_KEYS.map((key) => [ - key, - normalizeRulesValue(key, source[key]), - ]) as Array<[string, unknown]>; - entries.sort(([left]: [string, unknown], [right]: [string, unknown]) => - left.localeCompare(right), - ); - const payload = Object.fromEntries(entries); - const hash = createHash("sha256"); - hash.update(JSON.stringify(payload)); - return hash.digest("hex"); -}; - -export const getSyncPlan = async ( - options: SyncOptions, - deps: SyncDeps = {}, -) => { - const { config, resolvedPath, sources } = await loadConfig( - options.configPath, - ); - const defaults = (config.defaults ?? - DEFAULT_CONFIG.defaults) as DocsCacheDefaults; - const resolvedCacheDir = resolveCacheDir( - resolvedPath, - config.cacheDir ?? DEFAULT_CACHE_DIR, - options.cacheDirOverride, - ); - const lockPath = resolveLockPath(resolvedPath); - const lockExists = await exists(lockPath); - - let lockData: Awaited> | null = null; - if (lockExists) { - lockData = await readLock(lockPath); - } - - const resolveCommit = deps.resolveRemoteCommit ?? resolveRemoteCommit; - const filteredSources = options.sourceFilter?.length - ? sources.filter((source) => options.sourceFilter?.includes(source.id)) - : sources; - const results: SyncResult[] = await Promise.all( - filteredSources.map(async (source) => { - const lockEntry = lockData?.sources?.[source.id]; - const include = source.include ?? defaults.include; - const exclude = source.exclude ?? defaults.exclude; - const rulesSha256 = computeRulesHash({ - ...source, - include, - exclude, - }); - if (options.offline) { - const docsPresent = await hasDocs(resolvedCacheDir, source.id); - return { - id: source.id, - repo: lockEntry?.repo ?? source.repo, - ref: lockEntry?.ref ?? source.ref ?? defaults.ref, - resolvedCommit: lockEntry?.resolvedCommit ?? "offline", - lockCommit: lockEntry?.resolvedCommit ?? null, - lockRulesSha256: lockEntry?.rulesSha256, - status: lockEntry && docsPresent ? "up-to-date" : "missing", - bytes: lockEntry?.bytes, - fileCount: lockEntry?.fileCount, - manifestSha256: lockEntry?.manifestSha256, - rulesSha256, - }; - } - const resolved = await resolveCommit({ - repo: source.repo, - ref: source.ref, - allowHosts: defaults.allowHosts, - timeoutMs: options.timeoutMs, - logger: options.verbose && !options.json ? ui.debug : undefined, - }); - const upToDate = - lockEntry?.resolvedCommit === resolved.resolvedCommit && - lockEntry?.rulesSha256 === rulesSha256; - const status = lockEntry - ? upToDate - ? "up-to-date" - : "changed" - : "missing"; - return { - id: source.id, - repo: resolved.repo, - ref: resolved.ref, - resolvedCommit: resolved.resolvedCommit, - lockCommit: lockEntry?.resolvedCommit ?? null, - lockRulesSha256: lockEntry?.rulesSha256, - status, - bytes: lockEntry?.bytes, - fileCount: lockEntry?.fileCount, - manifestSha256: lockEntry?.manifestSha256, - rulesSha256, - }; - }), - ); - - return { - config, - configPath: resolvedPath, - cacheDir: resolvedCacheDir, - lockPath, - lockExists, - lockData, - results, - sources: filteredSources, - defaults, - }; -}; - -const loadToolVersion = async () => { - const cwdPath = path.resolve(process.cwd(), "package.json"); - try { - const raw = await readFile(cwdPath, "utf8"); - const pkg = JSON.parse(raw.toString()); - return typeof pkg.version === "string" ? pkg.version : "0.0.0"; - } catch { - // fallback to bundle-relative location - } - try { - const raw = await readFile( - new URL("../package.json", import.meta.url), - "utf8", - ); - const pkg = JSON.parse(raw.toString()); - return typeof pkg.version === "string" ? pkg.version : "0.0.0"; - } catch { - // fallback to dist/chunks relative location - } - try { - const raw = await readFile( - new URL("../../package.json", import.meta.url), - "utf8", - ); - const pkg = JSON.parse(raw.toString()); - return typeof pkg.version === "string" ? pkg.version : "0.0.0"; - } catch { - return "0.0.0"; - } -}; - -const buildLock = async ( - plan: Awaited>, - previous: Awaited> | null, -) => { - const toolVersion = await loadToolVersion(); - const now = new Date().toISOString(); - const sources = { ...(previous?.sources ?? {}) }; - for (const result of plan.results) { - const prior = sources[result.id]; - sources[result.id] = { - repo: result.repo, - ref: result.ref, - resolvedCommit: result.resolvedCommit, - bytes: result.bytes ?? prior?.bytes ?? 0, - fileCount: result.fileCount ?? prior?.fileCount ?? 0, - manifestSha256: - result.manifestSha256 ?? prior?.manifestSha256 ?? result.resolvedCommit, - rulesSha256: result.rulesSha256 ?? prior?.rulesSha256, - updatedAt: now, - }; - } - return { - version: 1 as const, - generatedAt: now, - toolVersion, - sources, - }; -}; - -export const runSync = async (options: SyncOptions, deps: SyncDeps = {}) => { - const startTime = process.hrtime.bigint(); - let warningCount = 0; - const plan = await getSyncPlan(options, deps); - await mkdir(plan.cacheDir, { recursive: true }); - const previous = plan.lockData; - const requiredMissing = plan.results.filter((result) => { - const source = plan.sources.find((entry) => entry.id === result.id); - return result.status === "missing" && (source?.required ?? true); - }); - if (options.failOnMiss && requiredMissing.length > 0) { - throw new Error( - `Missing required source(s): ${requiredMissing.map((result) => result.id).join(", ")}.`, - ); - } - if (!options.lockOnly) { - const defaults = plan.defaults; - const runFetch = deps.fetchSource ?? fetchSource; - const runMaterialize = deps.materializeSource ?? materializeSource; - const docsPresence = new Map(); - const buildJobs = async (ids?: string[], force?: boolean) => { - const pick = ids?.length - ? plan.results.filter((result) => ids.includes(result.id)) - : plan.results; - const jobs = await Promise.all( - pick.map(async (result) => { - const source = plan.sources.find((entry) => entry.id === result.id); - if (!source) { - return null; - } - if (force) { - return { result, source }; - } - let docsPresent = docsPresence.get(result.id); - if (docsPresent === undefined) { - docsPresent = await hasDocs(plan.cacheDir, result.id); - docsPresence.set(result.id, docsPresent); - } - const needsMaterialize = - result.status !== "up-to-date" || !docsPresent; - return needsMaterialize ? { result, source } : null; - }), - ); - return jobs.filter(Boolean) as Array<{ - result: SyncResult; - source: (typeof plan.sources)[number]; - }>; - }; - - const ensureTargets = async () => { - await Promise.all( - plan.sources.map(async (source) => { - if (!source.targetDir) { - return; - } - const resolvedTarget = resolveTargetDir( - plan.configPath, - source.targetDir, - ); - if (await exists(resolvedTarget)) { - return; - } - await applyTargetDir({ - sourceDir: path.join(plan.cacheDir, source.id), - targetDir: resolvedTarget, - mode: source.targetMode ?? defaults.targetMode, - explicitTargetMode: source.targetMode !== undefined, - unwrapSingleRootDir: source.unwrapSingleRootDir, - }); - }), - ); - }; - - const runJobs = async ( - jobs: Array<{ - result: SyncResult; - source: (typeof plan.sources)[number]; - }>, - ) => { - const concurrency = options.concurrency ?? 4; - let index = 0; - const runNext = async () => { - const job = jobs[index]; - if (!job || !job.source) { - return; - } - index += 1; - const { result, source } = job; - const lockEntry = plan.lockData?.sources?.[source.id]; - const fetch = await runFetch({ - sourceId: source.id, - repo: source.repo, - ref: source.ref, - resolvedCommit: result.resolvedCommit, - cacheDir: plan.cacheDir, - include: source.include ?? defaults.include, - timeoutMs: options.timeoutMs, - logger: options.verbose && !options.json ? ui.debug : undefined, - }); - if (!options.json) { - ui.step( - fetch.fromCache ? "Restoring from cache" : "Downloading repo", - source.id, - ); - } - try { - const manifestPath = path.join( - plan.cacheDir, - source.id, - MANIFEST_FILENAME, - ); - if ( - result.status !== "up-to-date" && - lockEntry?.manifestSha256 && - lockEntry?.rulesSha256 === result.rulesSha256 && - (await exists(manifestPath)) - ) { - const computed = await computeManifestHash({ - sourceId: source.id, - repoDir: fetch.repoDir, - cacheDir: plan.cacheDir, - include: source.include ?? defaults.include, - exclude: source.exclude, - maxBytes: source.maxBytes ?? defaults.maxBytes, - maxFiles: source.maxFiles ?? defaults.maxFiles, - ignoreHidden: source.ignoreHidden ?? defaults.ignoreHidden, - }); - if (computed.manifestSha256 === lockEntry.manifestSha256) { - result.bytes = computed.bytes; - result.fileCount = computed.fileCount; - result.manifestSha256 = computed.manifestSha256; - result.status = "up-to-date"; - if (!options.json) { - ui.item(symbols.success, source.id, "no content changes"); - } - await runNext(); - return; - } - } - if (!options.json) { - ui.step("Materializing", source.id); - } - const stats = await runMaterialize({ - sourceId: source.id, - repoDir: fetch.repoDir, - cacheDir: plan.cacheDir, - include: source.include ?? defaults.include, - exclude: source.exclude, - maxBytes: source.maxBytes ?? defaults.maxBytes, - maxFiles: source.maxFiles ?? defaults.maxFiles, - ignoreHidden: source.ignoreHidden ?? defaults.ignoreHidden, - unwrapSingleRootDir: source.unwrapSingleRootDir, - json: options.json, - }); - if (source.targetDir) { - const resolvedTarget = resolveTargetDir( - plan.configPath, - source.targetDir, - ); - await applyTargetDir({ - sourceDir: path.join(plan.cacheDir, source.id), - targetDir: resolvedTarget, - mode: source.targetMode ?? defaults.targetMode, - explicitTargetMode: source.targetMode !== undefined, - unwrapSingleRootDir: source.unwrapSingleRootDir, - }); - } - result.bytes = stats.bytes; - result.fileCount = stats.fileCount; - result.manifestSha256 = stats.manifestSha256; - if (!options.json) { - ui.item( - symbols.success, - source.id, - `synced ${stats.fileCount} files`, - ); - } - } finally { - await fetch.cleanup(); - } - await runNext(); - }; - - await Promise.all( - Array.from({ length: Math.min(concurrency, jobs.length) }, runNext), - ); - }; - - if (options.offline) { - await ensureTargets(); - } else { - const initialJobs = await buildJobs(); - await runJobs(initialJobs); - await ensureTargets(); - } - if (!options.offline) { - const verifyReport = await verifyCache({ - configPath: plan.configPath, - cacheDirOverride: plan.cacheDir, - json: true, - }); - const failed = verifyReport.results.filter((result) => !result.ok); - if (failed.length > 0) { - const retryJobs = await buildJobs( - failed.map((result) => result.id), - true, - ); - if (retryJobs.length > 0) { - await runJobs(retryJobs); - await ensureTargets(); - } - const retryReport = await verifyCache({ - configPath: plan.configPath, - cacheDirOverride: plan.cacheDir, - json: true, - }); - const stillFailed = retryReport.results.filter((result) => !result.ok); - if (stillFailed.length > 0) { - warningCount += 1; - if (!options.json) { - const details = stillFailed - .map((result) => `${result.id} (${result.issues.join("; ")})`) - .join(", "); - ui.line( - `${symbols.warn} Verify failed for ${stillFailed.length} source(s): ${details}`, - ); - } - } - } - } - } - const lock = await buildLock(plan, previous); - await writeLock(plan.lockPath, lock); - if (!options.json) { - const elapsedMs = Number(process.hrtime.bigint() - startTime) / 1_000_000; - const totalBytes = plan.results.reduce( - (sum, result) => sum + (result.bytes ?? 0), - 0, - ); - const totalFiles = plan.results.reduce( - (sum, result) => sum + (result.fileCount ?? 0), - 0, - ); - ui.line( - `${symbols.info} Completed in ${elapsedMs.toFixed(0)}ms · ${formatBytes(totalBytes)} · ${totalFiles} files${warningCount ? ` · ${warningCount} warning${warningCount === 1 ? "" : "s"}` : ""}`, - ); - } - // Always call writeToc to handle both generation and cleanup - await writeToc({ - cacheDir: plan.cacheDir, - configPath: plan.configPath, - lock, - sources: plan.sources, - results: plan.results, - }); - plan.lockExists = true; - return plan; -}; - -export const printSyncPlan = ( - plan: Awaited>, -) => { - const summary = { - upToDate: plan.results.filter((r) => r.status === "up-to-date").length, - changed: plan.results.filter((r) => r.status === "changed").length, - missing: plan.results.filter((r) => r.status === "missing").length, - }; - - if (plan.results.length === 0) { - ui.line(`${symbols.info} No sources to sync.`); - return; - } - - ui.line( - `${symbols.info} ${plan.results.length} sources (${summary.upToDate} up-to-date, ${summary.changed} changed, ${summary.missing} missing)`, - ); - - for (const result of plan.results) { - const shortResolved = ui.hash(result.resolvedCommit); - const shortLock = ui.hash(result.lockCommit); - const rulesChanged = - Boolean(result.lockRulesSha256) && - Boolean(result.rulesSha256) && - result.lockRulesSha256 !== result.rulesSha256; - - if (result.status === "up-to-date") { - ui.item( - symbols.success, - result.id, - `${pc.dim("up-to-date")} ${pc.gray(shortResolved)}`, - ); - continue; - } - if (result.status === "changed") { - if (result.lockCommit === result.resolvedCommit && rulesChanged) { - ui.item( - symbols.warn, - result.id, - `${pc.dim("rules changed")} ${pc.gray(shortResolved)}`, - ); - continue; - } - ui.item( - symbols.warn, - result.id, - `${pc.dim("changed")} ${pc.gray(shortLock)} ${pc.dim("->")} ${pc.gray(shortResolved)}`, - ); - continue; - } - ui.item( - symbols.warn, - result.id, - `${pc.dim("missing")} ${pc.gray(shortResolved)}`, - ); - } -}; diff --git a/src/types/sync.ts b/src/types/sync.ts new file mode 100644 index 0000000..d037db5 --- /dev/null +++ b/src/types/sync.ts @@ -0,0 +1,26 @@ +export type SyncOptions = { + configPath?: string; + cacheDirOverride?: string; + json: boolean; + lockOnly: boolean; + offline: boolean; + failOnMiss: boolean; + verbose?: boolean; + concurrency?: number; + sourceFilter?: string[]; + timeoutMs?: number; +}; + +export type SyncResult = { + id: string; + repo: string; + ref: string; + resolvedCommit: string; + lockCommit: string | null; + lockRulesSha256?: string; + status: "up-to-date" | "changed" | "missing"; + bytes?: number; + fileCount?: number; + manifestSha256?: string; + rulesSha256?: string; +}; diff --git a/tests/cli-remove.test.js b/tests/cli-remove.test.js index e9d431e..39085d9 100644 --- a/tests/cli-remove.test.js +++ b/tests/cli-remove.test.js @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { execFile } from "node:child_process"; -import { access, readFile } from "node:fs/promises"; +import { access, mkdir, readFile, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { test } from "node:test"; @@ -131,3 +131,22 @@ test("remove accepts repo shorthands", async () => { const config = JSON.parse(raw); assert.equal(config.sources.length, 0); }); + +test("remove errors when package.json lacks docs-cache", async () => { + const tmpRoot = path.join(tmpdir(), `docs-cache-pkg-${Date.now()}`); + const packagePath = path.join(tmpRoot, "package.json"); + await mkdir(tmpRoot, { recursive: true }); + await writeFile(packagePath, JSON.stringify({ name: "docs-cache" }), "utf8"); + await assert.rejects( + execFileAsync("node", [ + "bin/docs-cache.mjs", + "remove", + "--config", + packagePath, + "nixos", + ]), + (error) => + error instanceof Error && + error.message.includes("Missing docs-cache config"), + ); +}); diff --git a/tests/sync-materialize.test.js b/tests/sync-materialize.test.js index 74d3417..cf235f7 100644 --- a/tests/sync-materialize.test.js +++ b/tests/sync-materialize.test.js @@ -292,6 +292,95 @@ test("sync re-materializes when docs missing even if commit unchanged", async () assert.equal(materialized, true); }); +test("sync offline materializes from cache when lock exists", async () => { + const tmpRoot = path.join( + tmpdir(), + `docs-cache-offline-materialize-${Date.now().toString(36)}`, + ); + await mkdir(tmpRoot, { recursive: true }); + const cacheDir = path.join(tmpRoot, ".docs"); + const repoDir = path.join(tmpRoot, "repo"); + const configPath = path.join(tmpRoot, "docs.config.json"); + + await mkdir(repoDir, { recursive: true }); + await writeFile(path.join(repoDir, "README.md"), "hello", "utf8"); + + const config = { + $schema: + "https://raw.githubusercontent.com/fbosch/docs-cache/main/docs.config.schema.json", + sources: [ + { + id: "local", + repo: "https://example.com/repo.git", + }, + ], + }; + await writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, "utf8"); + await writeFile( + path.join(tmpRoot, DEFAULT_LOCK_FILENAME), + JSON.stringify({ + version: 1, + generatedAt: new Date().toISOString(), + toolVersion: "0.1.0", + sources: { + local: { + repo: "https://example.com/repo.git", + ref: "HEAD", + resolvedCommit: "abc123", + bytes: 0, + fileCount: 0, + manifestSha256: "abc123", + updatedAt: new Date().toISOString(), + }, + }, + }), + ); + + let resolveCalled = false; + let fetchOffline = false; + let materialized = false; + + await runSync( + { + configPath, + cacheDirOverride: cacheDir, + json: false, + lockOnly: false, + offline: true, + failOnMiss: false, + }, + { + resolveRemoteCommit: async () => { + resolveCalled = true; + throw new Error("Should not resolve while offline"); + }, + fetchSource: async (params) => { + fetchOffline = params.offline === true; + return { + repoDir, + cleanup: async () => undefined, + }; + }, + materializeSource: async ({ cacheDir: cacheRoot, sourceId }) => { + materialized = true; + const outDir = path.join(cacheRoot, sourceId); + await mkdir(outDir, { recursive: true }); + await writeFile( + path.join(outDir, ".manifest.jsonl"), + `${JSON.stringify({ path: "README.md", size: 5 })}\n`, + ); + await writeFile(path.join(outDir, "README.md"), "hello", "utf8"); + return { bytes: 5, fileCount: 1 }; + }, + }, + ); + + assert.equal(resolveCalled, false); + assert.equal(fetchOffline, true); + assert.equal(materialized, true); + assert.equal(await exists(path.join(cacheDir, "local", "README.md")), true); +}); + test("sync offline fails when required source missing", async () => { const tmpRoot = path.join( tmpdir(), diff --git a/tsconfig.json b/tsconfig.json index e7b45a4..89f9291 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -5,10 +5,22 @@ "moduleResolution": "Bundler", "lib": ["ES2022"], "types": ["node"], + "baseUrl": ".", + "paths": { + "#cache/*": ["src/cache/*.ts"], + "#cli/*": ["src/cli/*.ts"], + "#commands/*": ["src/commands/*.ts"], + "#core/*": ["src/*.ts"], + "#config": ["src/config/index.ts"], + "#config/*": ["src/config/*.ts"], + "#git/*": ["src/git/*.ts"], + "#types/*": ["src/types/*.ts"] + }, "strict": true, "declaration": true, "declarationMap": true, "sourceMap": true, + "rootDir": ".", "outDir": "dist", "resolveJsonModule": true, "skipLibCheck": true