diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 85a2866..52571d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,47 @@ on: - master jobs: + precheck: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: + run_install: false + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Audit dependencies + run: pnpm audit --audit-level=high + + - name: Lint + run: pnpm lint + + - name: Typecheck + run: pnpm typecheck + + - name: Test + run: pnpm test + + - name: Build + run: pnpm build + + - name: Size limit + run: pnpm size + build: + needs: precheck + if: needs.precheck.result == 'success' runs-on: ${{ matrix.os }} strategy: fail-fast: false diff --git a/README.md b/README.md index 8b734b1..3341b36 100644 --- a/README.md +++ b/README.md @@ -94,12 +94,12 @@ All fields in `defaults` apply to all sources unless overridden per-source. | `mode` | Cache mode. Default: `"materialize"`. | | `include` | Glob patterns to copy. Default: `["**/*.{md,mdx,markdown,mkd,txt,rst,adoc,asciidoc}"]`. | | `targetMode` | How to link or copy from the cache to the destination. Default: `"symlink"` on Unix, `"copy"` on Windows. | -| `depth` | Git clone depth. Default: `1`. | | `required` | Whether missing sources should fail. Default: `true`. | | `maxBytes` | Maximum total bytes to materialize. Default: `200000000` (200 MB). | | `maxFiles` | Maximum total files to materialize. | | `allowHosts` | Allowed Git hosts. Default: `["github.com", "gitlab.com"]`. | | `toc` | Generate per-source `TOC.md`. Default: `true`. Supports `true`, `false`, or a format (`"tree"`, `"compressed"`). | +| `unwrapSingleRootDir` | If the materialized output is nested under a single directory, unwrap it (recursively). Default: `false`. | ### Source options @@ -123,6 +123,7 @@ All fields in `defaults` apply to all sources unless overridden per-source. | `maxBytes` | Maximum total bytes to materialize. | | `maxFiles` | Maximum total files to materialize. | | `toc` | Generate per-source `TOC.md`. Supports `true`, `false`, or a format (`"tree"`, `"compressed"`). | +| `unwrapSingleRootDir` | If the materialized output is nested under a single directory, unwrap it (recursively). | > **Note**: Sources are always downloaded to `.docs//`. If you provide a `targetDir`, `docs-cache` will create a symlink or copy pointing from the cache to that target directory. The target should be outside `.docs`. Git operation timeout is configured via the `--timeout-ms` CLI flag, not as a per-source configuration option. diff --git a/docs.config.schema.json b/docs.config.schema.json index dc5c111..f15cb16 100644 --- a/docs.config.schema.json +++ b/docs.config.schema.json @@ -37,10 +37,6 @@ "type": "string", "enum": ["symlink", "copy"] }, - "depth": { - "type": "number", - "minimum": 1 - }, "required": { "type": "boolean" }, @@ -60,6 +56,9 @@ "minLength": 1 } }, + "unwrapSingleRootDir": { + "type": "boolean" + }, "toc": { "anyOf": [ { @@ -107,10 +106,6 @@ "type": "string", "enum": ["materialize"] }, - "depth": { - "type": "number", - "minimum": 1 - }, "include": { "type": "array", "items": { @@ -171,6 +166,9 @@ "tocFormat": { "type": "string", "enum": ["tree", "compressed"] + }, + "unwrapSingleRootDir": { + "type": "boolean" } }, "required": ["id", "repo"], diff --git a/package.json b/package.json index 8aff91d..510ca23 100644 --- a/package.json +++ b/package.json @@ -84,7 +84,7 @@ } ], "simple-git-hooks": { - "pre-commit": "pnpm lint-staged" + "pre-commit": "pnpm lint-staged && pnpm typecheck" }, "lint-staged": { "*.{js,ts,cjs,mjs,d.cts,d.mts,jsx,tsx,json,jsonc}": [ diff --git a/src/cli/index.ts b/src/cli/index.ts index 9815e50..5453cf7 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -3,7 +3,7 @@ import process from "node:process"; import pc from "picocolors"; import { ExitCode } from "./exit-code"; import { parseArgs } from "./parse-args"; -import type { CliOptions } from "./types"; +import type { CliCommand } from "./types"; import { setSilentMode, symbols, ui } from "./ui"; export const CLI_NAME = "docs-cache"; @@ -93,12 +93,10 @@ const parseAddEntries = (rawArgs: string[]) => { return entries; }; -const runCommand = async ( - command: string, - options: CliOptions, - positionals: string[], - rawArgs: string[], -) => { +const runCommand = async (parsed: CliCommand, rawArgs: string[]) => { + const command = parsed.command; + const options = parsed.options; + const positionals = parsed.args; if (command === "add") { const { addSources } = await import("../add"); const { runSync } = await import("../sync"); @@ -380,12 +378,7 @@ export async function main(): Promise { process.exit(ExitCode.InvalidArgument); } - await runCommand( - parsed.command, - parsed.options, - parsed.positionals, - parsed.rawArgs, - ); + await runCommand(parsed.parsed, parsed.rawArgs); } catch (error) { errorHandler(error as Error); } diff --git a/src/cli/parse-args.ts b/src/cli/parse-args.ts index 41827ad..ecd090d 100644 --- a/src/cli/parse-args.ts +++ b/src/cli/parse-args.ts @@ -2,7 +2,7 @@ import process from "node:process"; import cac from "cac"; import { ExitCode } from "./exit-code"; -import type { CliOptions } from "./types"; +import type { CliCommand, CliOptions } from "./types"; const COMMANDS = [ "add", @@ -23,6 +23,7 @@ export type ParsedArgs = { positionals: string[]; rawArgs: string[]; help: boolean; + parsed: CliCommand; }; export const parseArgs = (argv = process.argv): ParsedArgs => { @@ -83,6 +84,11 @@ export const parseArgs = (argv = process.argv): ParsedArgs => { positionals: result.args.slice(1), rawArgs, help: Boolean(result.options.help), + parsed: { + command: command ?? null, + args: result.args.slice(1), + options, + }, }; } catch (error) { const message = error instanceof Error ? error.message : String(error); diff --git a/src/cli/types.ts b/src/cli/types.ts index 00266f8..b415f94 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -11,3 +11,15 @@ export type CliOptions = { timeoutMs?: number; silent: boolean; }; + +export type CliCommand = + | { command: "add"; args: string[]; options: CliOptions } + | { command: "remove"; args: string[]; options: CliOptions } + | { command: "sync"; args: string[]; options: CliOptions } + | { command: "status"; args: string[]; options: CliOptions } + | { command: "clean"; args: string[]; options: CliOptions } + | { command: "clean-cache"; args: string[]; options: CliOptions } + | { command: "prune"; args: string[]; options: CliOptions } + | { command: "verify"; args: string[]; options: CliOptions } + | { command: "init"; args: string[]; options: CliOptions } + | { command: null; args: string[]; options: CliOptions }; diff --git a/src/config-schema.ts b/src/config-schema.ts index c0234e2..f3c4ac0 100644 --- a/src/config-schema.ts +++ b/src/config-schema.ts @@ -16,12 +16,12 @@ export const DefaultsSchema = z mode: CacheModeSchema, include: z.array(z.string().min(1)).min(1), targetMode: TargetModeSchema.optional(), - depth: z.number().min(1), required: z.boolean(), maxBytes: z.number().min(1), maxFiles: z.number().min(1).optional(), allowHosts: z.array(z.string().min(1)).min(1), toc: z.union([z.boolean(), TocFormatSchema]).optional(), + unwrapSingleRootDir: z.boolean().optional(), }) .strict(); @@ -33,7 +33,6 @@ export const SourceSchema = z targetMode: TargetModeSchema.optional(), ref: z.string().min(1).optional(), mode: CacheModeSchema.optional(), - depth: z.number().min(1).optional(), include: z.array(z.string().min(1)).optional(), exclude: z.array(z.string().min(1)).optional(), required: z.boolean().optional(), @@ -41,6 +40,7 @@ export const SourceSchema = z maxFiles: z.number().min(1).optional(), integrity: IntegritySchema.optional(), toc: z.union([z.boolean(), TocFormatSchema]).optional(), + unwrapSingleRootDir: z.boolean().optional(), }) .strict(); diff --git a/src/config.ts b/src/config.ts index a311bd9..7b5db2e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -20,12 +20,12 @@ export interface DocsCacheDefaults { mode: CacheMode; include: string[]; targetMode?: "symlink" | "copy"; - depth: number; required: boolean; maxBytes: number; maxFiles?: number; allowHosts: string[]; toc?: boolean | TocFormat; + unwrapSingleRootDir?: boolean; } export interface DocsCacheSource { @@ -35,7 +35,6 @@ export interface DocsCacheSource { targetMode?: "symlink" | "copy"; ref?: string; mode?: CacheMode; - depth?: number; include?: string[]; exclude?: string[]; required?: boolean; @@ -43,6 +42,7 @@ export interface DocsCacheSource { maxFiles?: number; integrity?: DocsCacheIntegrity; toc?: boolean | TocFormat; + unwrapSingleRootDir?: boolean; } export interface DocsCacheConfig { @@ -60,7 +60,6 @@ export interface DocsCacheResolvedSource { targetMode?: "symlink" | "copy"; ref: string; mode: CacheMode; - depth: number; include?: string[]; exclude?: string[]; required: boolean; @@ -68,6 +67,7 @@ export interface DocsCacheResolvedSource { maxFiles?: number; integrity?: DocsCacheIntegrity; toc?: boolean | TocFormat; + unwrapSingleRootDir?: boolean; } export const DEFAULT_CONFIG_FILENAME = "docs.config.json"; @@ -81,11 +81,11 @@ export const DEFAULT_CONFIG: DocsCacheConfig = { mode: "materialize", include: ["**/*.{md,mdx,markdown,mkd,txt,rst,adoc,asciidoc}"], targetMode: DEFAULT_TARGET_MODE, - depth: 1, required: true, maxBytes: 200000000, allowHosts: ["github.com", "gitlab.com"], toc: true, + unwrapSingleRootDir: false, }, sources: [], }; @@ -246,15 +246,16 @@ export const validateConfig = (input: unknown): DocsCacheConfig => { .join("; "); throw new Error(`Config does not match schema: ${details}.`); } + const configInput = parsed.data; - const cacheDir = input.cacheDir - ? assertString(input.cacheDir, "cacheDir") + const cacheDir = configInput.cacheDir + ? assertString(configInput.cacheDir, "cacheDir") : DEFAULT_CACHE_DIR; - const defaultsInput = input.defaults; + const defaultsInput = configInput.defaults; const targetModeOverride = - input.targetMode !== undefined - ? assertTargetMode(input.targetMode, "targetMode") + configInput.targetMode !== undefined + ? assertTargetMode(configInput.targetMode, "targetMode") : undefined; const defaultValues = DEFAULT_CONFIG.defaults as DocsCacheDefaults; let defaults: DocsCacheDefaults = defaultValues; @@ -280,10 +281,6 @@ export const validateConfig = (input: unknown): DocsCacheConfig => { defaultsInput.targetMode !== undefined ? assertTargetMode(defaultsInput.targetMode, "defaults.targetMode") : (targetModeOverride ?? defaultValues.targetMode), - depth: - defaultsInput.depth !== undefined - ? assertPositiveNumber(defaultsInput.depth, "defaults.depth") - : defaultValues.depth, required: defaultsInput.required !== undefined ? assertBoolean(defaultsInput.required, "defaults.required") @@ -304,6 +301,13 @@ export const validateConfig = (input: unknown): DocsCacheConfig => { defaultsInput.toc !== undefined ? (defaultsInput.toc as boolean | TocFormat) : defaultValues.toc, + unwrapSingleRootDir: + defaultsInput.unwrapSingleRootDir !== undefined + ? assertBoolean( + defaultsInput.unwrapSingleRootDir, + "defaults.unwrapSingleRootDir", + ) + : defaultValues.unwrapSingleRootDir, }; } else if (targetModeOverride !== undefined) { defaults = { @@ -312,11 +316,7 @@ export const validateConfig = (input: unknown): DocsCacheConfig => { }; } - if (!Array.isArray(input.sources)) { - throw new Error("sources must be an array."); - } - - const sources = input.sources.map((entry, index) => { + const sources = configInput.sources.map((entry, index) => { if (!isRecord(entry)) { throw new Error(`sources[${index}] must be an object.`); } @@ -348,12 +348,6 @@ export const validateConfig = (input: unknown): DocsCacheConfig => { if (entry.mode !== undefined) { source.mode = assertMode(entry.mode, `sources[${index}].mode`); } - if (entry.depth !== undefined) { - source.depth = assertPositiveNumber( - entry.depth, - `sources[${index}].depth`, - ); - } if (entry.include !== undefined) { source.include = assertStringArray( entry.include, @@ -394,6 +388,12 @@ export const validateConfig = (input: unknown): DocsCacheConfig => { if (entry.toc !== undefined) { source.toc = entry.toc as boolean | TocFormat; } + if (entry.unwrapSingleRootDir !== undefined) { + source.unwrapSingleRootDir = assertBoolean( + entry.unwrapSingleRootDir, + `sources[${index}].unwrapSingleRootDir`, + ); + } return source; }); @@ -433,7 +433,6 @@ export const resolveSources = ( targetMode: source.targetMode ?? defaults.targetMode, ref: source.ref ?? defaults.ref, mode: source.mode ?? defaults.mode, - depth: source.depth ?? defaults.depth, include: source.include ?? defaults.include, exclude: source.exclude, required: source.required ?? defaults.required, @@ -441,6 +440,8 @@ export const resolveSources = ( maxFiles: source.maxFiles ?? defaults.maxFiles, integrity: source.integrity, toc: source.toc ?? defaults.toc, + unwrapSingleRootDir: + source.unwrapSingleRootDir ?? defaults.unwrapSingleRootDir, })); }; diff --git a/src/errors.ts b/src/errors.ts new file mode 100644 index 0000000..e8b347d --- /dev/null +++ b/src/errors.ts @@ -0,0 +1,14 @@ +export type ErrnoException = NodeJS.ErrnoException; + +export const isErrnoException = (error: unknown): error is ErrnoException => + typeof error === "object" && + error !== null && + "code" in error && + (typeof (error as ErrnoException).code === "string" || + typeof (error as ErrnoException).code === "number" || + (error as ErrnoException).code === undefined); + +export const getErrnoCode = (error: unknown): string | undefined => + isErrnoException(error) && typeof error.code === "string" + ? error.code + : undefined; diff --git a/src/git/fetch-source.ts b/src/git/fetch-source.ts index e52b5b3..70ccb71 100644 --- a/src/git/fetch-source.ts +++ b/src/git/fetch-source.ts @@ -6,12 +6,14 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; import { promisify } from "node:util"; +import { getErrnoCode } from "../errors"; import { assertSafeSourceId } from "../source-id"; import { exists, resolveGitCacheDir } from "./cache-dir"; const execFileAsync = promisify(execFile); const DEFAULT_TIMEOUT_MS = 120000; // 120 seconds (2 minutes) +const DEFAULT_GIT_DEPTH = 1; const DEFAULT_RM_RETRIES = 3; const DEFAULT_RM_BACKOFF_MS = 100; @@ -70,7 +72,7 @@ const removeDir = async (dirPath: string, retries = DEFAULT_RM_RETRIES) => { await rm(dirPath, { recursive: true, force: true }); return; } catch (error) { - const code = (error as NodeJS.ErrnoException).code; + const code = getErrnoCode(error); if (code !== "ENOTEMPTY" && code !== "EBUSY" && code !== "EPERM") { throw error; } @@ -126,11 +128,16 @@ type FetchParams = { ref: string; resolvedCommit: string; cacheDir: string; - depth: number; include?: string[]; timeoutMs?: number; }; +type FetchResult = { + repoDir: string; + cleanup: () => Promise; + fromCache: boolean; +}; + const runGitArchive = async ( repo: string, resolvedCommit: string, @@ -190,7 +197,7 @@ const cloneRepo = async (params: FetchParams, outDir: string) => { "clone", "--no-checkout", "--depth", - String(params.depth), + String(DEFAULT_GIT_DEPTH), "--recurse-submodules=no", "--no-tags", ]; @@ -250,10 +257,10 @@ const cloneOrUpdateRepo = async (params: FetchParams, outDir: string) => { params.ref === "HEAD" ? "HEAD" : `${params.ref}:refs/remotes/origin/${params.ref}`; - fetchArgs.push(refSpec, "--depth", String(params.depth)); + fetchArgs.push(refSpec, "--depth", String(DEFAULT_GIT_DEPTH)); } else { // For commit refs, fetch the default branch and hope the commit is there - fetchArgs.push("--depth", String(params.depth)); + fetchArgs.push("--depth", String(DEFAULT_GIT_DEPTH)); } await git(["-C", cachePath, ...fetchArgs], { @@ -281,7 +288,7 @@ const cloneOrUpdateRepo = async (params: FetchParams, outDir: string) => { "clone", "--no-checkout", "--depth", - String(params.depth), + String(DEFAULT_GIT_DEPTH), "--recurse-submodules=no", "--no-tags", ]; @@ -344,7 +351,9 @@ const archiveRepo = async (params: FetchParams) => { } }; -export const fetchSource = async (params: FetchParams) => { +export const fetchSource = async ( + params: FetchParams, +): Promise => { assertSafeSourceId(params.sourceId, "sourceId"); try { const archiveDir = await archiveRepo(params); @@ -353,6 +362,7 @@ export const fetchSource = async (params: FetchParams) => { cleanup: async () => { await removeDir(archiveDir); }, + fromCache: false, }; } catch { const tempDir = await mkdtemp( @@ -365,6 +375,7 @@ export const fetchSource = async (params: FetchParams) => { cleanup: async () => { await removeDir(tempDir); }, + fromCache: true, }; } catch (error) { await removeDir(tempDir); diff --git a/src/manifest.ts b/src/manifest.ts index 0d7fef4..8e6c2c8 100644 --- a/src/manifest.ts +++ b/src/manifest.ts @@ -2,11 +2,28 @@ import { createReadStream } from "node:fs"; import path from "node:path"; import readline from "node:readline"; -type ManifestEntry = { +export type ManifestEntry = { path: string; size: number; }; +const parseManifestEntry = (value: unknown): ManifestEntry => { + if (!value || typeof value !== "object") { + throw new Error("Manifest entry must be an object."); + } + const record = value as Record; + if (typeof record.path !== "string" || record.path.length === 0) { + throw new Error("Manifest entry path must be a non-empty string."); + } + if (typeof record.size !== "number" || Number.isNaN(record.size)) { + throw new Error("Manifest entry size must be a number."); + } + if (record.size < 0) { + throw new Error("Manifest entry size must be zero or greater."); + } + return { path: record.path, size: record.size }; +}; + export const MANIFEST_FILENAME = ".manifest.jsonl"; export const readManifest = async (sourceDir: string) => { @@ -23,7 +40,7 @@ export const readManifest = async (sourceDir: string) => { if (!trimmed) { continue; } - entries.push(JSON.parse(trimmed) as ManifestEntry); + entries.push(parseManifestEntry(JSON.parse(trimmed))); } return { manifestPath, entries }; } finally { @@ -32,7 +49,9 @@ export const readManifest = async (sourceDir: string) => { } }; -export const streamManifestEntries = async function* (sourceDir: string) { +export const streamManifestEntries = async function* ( + sourceDir: string, +): AsyncGenerator { const manifestPath = path.join(sourceDir, MANIFEST_FILENAME); const stream = createReadStream(manifestPath, { encoding: "utf8" }); const lines = readline.createInterface({ @@ -45,7 +64,7 @@ export const streamManifestEntries = async function* (sourceDir: string) { if (!trimmed) { continue; } - yield JSON.parse(trimmed) as ManifestEntry; + yield parseManifestEntry(JSON.parse(trimmed)); } } finally { lines.close(); diff --git a/src/materialize.ts b/src/materialize.ts index b934a30..a763c2c 100644 --- a/src/materialize.ts +++ b/src/materialize.ts @@ -15,6 +15,7 @@ import path from "node:path"; import { pipeline } from "node:stream/promises"; import fg from "fast-glob"; +import { getErrnoCode } from "./errors"; import { MANIFEST_FILENAME } from "./manifest"; import { getCacheLayout, toPosixPath } from "./paths"; import { assertSafeSourceId } from "./source-id"; @@ -27,6 +28,18 @@ type MaterializeParams = { exclude?: string[]; maxBytes: number; maxFiles?: number; + unwrapSingleRootDir?: boolean; +}; + +type ResolvedMaterializeParams = { + sourceId: string; + repoDir: string; + cacheDir: string; + include: string[]; + exclude: string[]; + maxBytes: number; + maxFiles?: number; + unwrapSingleRootDir: boolean; }; type ManifestStats = { @@ -57,7 +70,7 @@ const openFileNoFollow = async (filePath: string) => { try { return await open(filePath, constants.O_RDONLY | constants.O_NOFOLLOW); } catch (error) { - const code = (error as NodeJS.ErrnoException).code; + const code = getErrnoCode(error); if (code === "ELOOP") { return null; } @@ -72,6 +85,52 @@ const openFileNoFollow = async (filePath: string) => { } }; +const resolveUnwrapPrefix = ( + entries: Array<{ normalized: string }>, + unwrapSingleRootDir?: boolean, +) => { + if (!unwrapSingleRootDir || entries.length === 0) { + return null; + } + let prefix = ""; + while (true) { + let rootDir: string | null = null; + for (const entry of entries) { + const remaining = prefix + ? entry.normalized.slice(prefix.length) + : entry.normalized; + const parts = remaining.split("/"); + if (parts.length < 2) { + return prefix || null; + } + const nextRoot = parts[0]; + if (!rootDir) { + rootDir = nextRoot; + continue; + } + if (rootDir !== nextRoot) { + return prefix || null; + } + } + if (!rootDir) { + return prefix || null; + } + const nextPrefix = `${prefix}${rootDir}/`; + if (nextPrefix === prefix) { + return prefix || null; + } + prefix = nextPrefix; + } +}; + +const resolveMaterializeParams = ( + params: MaterializeParams, +): ResolvedMaterializeParams => ({ + ...params, + exclude: params.exclude ?? [], + unwrapSingleRootDir: params.unwrapSingleRootDir ?? false, +}); + const acquireLock = async (lockPath: string, timeoutMs = 5000) => { const start = Date.now(); while (Date.now() - start < timeoutMs) { @@ -84,7 +143,7 @@ const acquireLock = async (lockPath: string, timeoutMs = 5000) => { }, }; } catch (error) { - const code = (error as NodeJS.ErrnoException).code; + const code = getErrnoCode(error); if (code !== "EEXIST") { throw error; } @@ -95,11 +154,12 @@ const acquireLock = async (lockPath: string, timeoutMs = 5000) => { }; export const materializeSource = async (params: MaterializeParams) => { - assertSafeSourceId(params.sourceId, "sourceId"); - const layout = getCacheLayout(params.cacheDir, params.sourceId); - await mkdir(params.cacheDir, { recursive: true }); + const resolved = resolveMaterializeParams(params); + assertSafeSourceId(resolved.sourceId, "sourceId"); + const layout = getCacheLayout(resolved.cacheDir, resolved.sourceId); + await mkdir(resolved.cacheDir, { recursive: true }); const tempDir = await mkdtemp( - path.join(params.cacheDir, `.tmp-${params.sourceId}-`), + path.join(resolved.cacheDir, `.tmp-${resolved.sourceId}-`), ); let manifestStreamRef: ReturnType | null = null; const closeManifestStream = async () => { @@ -126,9 +186,9 @@ export const materializeSource = async (params: MaterializeParams) => { }; try { - const files = await fg(params.include, { - cwd: params.repoDir, - ignore: [".git/**", ...(params.exclude ?? [])], + const files = await fg(resolved.include, { + cwd: resolved.repoDir, + ignore: [".git/**", ...resolved.exclude], dot: true, onlyFiles: true, followSymbolicLinks: false, @@ -139,9 +199,16 @@ export const materializeSource = async (params: MaterializeParams) => { normalized: normalizePath(relativePath), })) .sort((left, right) => left.normalized.localeCompare(right.normalized)); + const unwrapPrefix = resolveUnwrapPrefix( + entries, + resolved.unwrapSingleRootDir, + ); const targetDirs = new Set(); - for (const { relativePath } of entries) { - targetDirs.add(path.dirname(relativePath)); + for (const { normalized } of entries) { + const rootPath = unwrapPrefix + ? normalized.slice(unwrapPrefix.length) + : normalized; + targetDirs.add(path.posix.dirname(rootPath)); } await Promise.all( Array.from(targetDirs, (dir) => @@ -187,7 +254,7 @@ export const materializeSource = async (params: MaterializeParams) => { const batch = entries.slice(i, i + concurrency); const results = await Promise.all( batch.map(async (entry) => { - const filePath = path.join(params.repoDir, entry.relativePath); + const filePath = path.join(resolved.repoDir, entry.relativePath); const fileHandle = await openFileNoFollow(filePath); if (!fileHandle) { return null; @@ -197,7 +264,10 @@ export const materializeSource = async (params: MaterializeParams) => { if (!stats.isFile()) { return null; } - const targetPath = path.join(tempDir, entry.relativePath); + const normalizedPath = unwrapPrefix + ? entry.normalized.slice(unwrapPrefix.length) + : entry.normalized; + const targetPath = path.join(tempDir, normalizedPath); ensureSafePath(tempDir, targetPath); if (stats.size >= STREAM_COPY_THRESHOLD_BYTES) { const reader = createReadStream(filePath, { @@ -211,7 +281,9 @@ export const materializeSource = async (params: MaterializeParams) => { await writeFile(targetPath, data); } return { - path: entry.normalized, + path: unwrapPrefix + ? entry.normalized.slice(unwrapPrefix.length) + : entry.normalized, size: stats.size, }; } finally { @@ -223,15 +295,18 @@ export const materializeSource = async (params: MaterializeParams) => { if (!entry) { continue; } - if (params.maxFiles !== undefined && fileCount + 1 > params.maxFiles) { + if ( + resolved.maxFiles !== undefined && + fileCount + 1 > resolved.maxFiles + ) { throw new Error( - `Materialized content exceeds maxFiles (${params.maxFiles}).`, + `Materialized content exceeds maxFiles (${resolved.maxFiles}).`, ); } bytes += entry.size; - if (bytes > params.maxBytes) { + if (bytes > resolved.maxBytes) { throw new Error( - `Materialized content exceeds maxBytes (${params.maxBytes}).`, + `Materialized content exceeds maxBytes (${resolved.maxBytes}).`, ); } const line = `${JSON.stringify(entry)}\n`; diff --git a/src/sync.ts b/src/sync.ts index 228adc3..96a4911 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -7,6 +7,7 @@ import { DEFAULT_CACHE_DIR, DEFAULT_CONFIG, type DocsCacheDefaults, + type DocsCacheResolvedSource, loadConfig, } from "./config"; import { fetchSource } from "./git/fetch-source"; @@ -92,14 +93,54 @@ const normalizePatterns = (patterns?: string[]) => { return Array.from(new Set(normalized)).sort(); }; -const computeRulesHash = (params: { - include: string[]; - exclude?: string[]; -}) => { - const payload = { - include: normalizePatterns(params.include), - exclude: normalizePatterns(params.exclude), - }; +const RULES_HASH_BLACKLIST = [ + "id", + "repo", + "ref", + "targetDir", + "targetMode", + "required", + "integrity", + "toc", +] as const; + +type RulesHashBlacklistKey = (typeof RULES_HASH_BLACKLIST)[number]; +type RulesHashKey = Exclude< + keyof DocsCacheResolvedSource, + RulesHashBlacklistKey +>; + +const RULES_HASH_KEYS = [ + "mode", + "include", + "exclude", + "maxBytes", + "maxFiles", + "unwrapSingleRootDir", +] as const satisfies ReadonlyArray; + +const normalizeRulesValue = ( + key: RulesHashKey, + value: DocsCacheResolvedSource[RulesHashKey], +) => { + if (key === "include" && Array.isArray(value)) { + return normalizePatterns(value); + } + if (key === "exclude" && Array.isArray(value)) { + return normalizePatterns(value); + } + return value; +}; + +const computeRulesHash = (source: DocsCacheResolvedSource) => { + const entries = RULES_HASH_KEYS.map((key) => [ + key, + normalizeRulesValue(key, source[key]), + ]) as Array<[string, unknown]>; + entries.sort(([left]: [string, unknown], [right]: [string, unknown]) => + left.localeCompare(right), + ); + const payload = Object.fromEntries(entries); const hash = createHash("sha256"); hash.update(JSON.stringify(payload)); return hash.digest("hex"); @@ -136,7 +177,11 @@ export const getSyncPlan = async ( const lockEntry = lockData?.sources?.[source.id]; const include = source.include ?? defaults.include; const exclude = source.exclude; - const rulesSha256 = computeRulesHash({ include, exclude }); + const rulesSha256 = computeRulesHash({ + ...source, + include, + exclude, + }); if (options.offline) { const docsPresent = await hasDocs(resolvedCacheDir, source.id); return { @@ -323,6 +368,7 @@ export const runSync = async (options: SyncOptions, deps: SyncDeps = {}) => { targetDir: resolvedTarget, mode: source.targetMode ?? defaults.targetMode, explicitTargetMode: source.targetMode !== undefined, + unwrapSingleRootDir: source.unwrapSingleRootDir, }); }), ); @@ -344,19 +390,21 @@ export const runSync = async (options: SyncOptions, deps: SyncDeps = {}) => { index += 1; const { result, source } = job; const lockEntry = plan.lockData?.sources?.[source.id]; - if (!options.json) { - ui.step("Fetching", source.id); - } const fetch = await runFetch({ sourceId: source.id, repo: source.repo, ref: source.ref, resolvedCommit: result.resolvedCommit, cacheDir: plan.cacheDir, - depth: source.depth ?? defaults.depth, include: source.include ?? defaults.include, timeoutMs: options.timeoutMs, }); + if (!options.json) { + ui.step( + fetch.fromCache ? "Restoring from cache" : "Downloading repo", + source.id, + ); + } try { const manifestPath = path.join( plan.cacheDir, @@ -366,6 +414,7 @@ export const runSync = async (options: SyncOptions, deps: SyncDeps = {}) => { if ( result.status !== "up-to-date" && lockEntry?.manifestSha256 && + lockEntry?.rulesSha256 === result.rulesSha256 && (await exists(manifestPath)) ) { const computed = await computeManifestHash({ @@ -389,6 +438,9 @@ export const runSync = async (options: SyncOptions, deps: SyncDeps = {}) => { return; } } + if (!options.json) { + ui.step("Building cache layout", source.id); + } const stats = await runMaterialize({ sourceId: source.id, repoDir: fetch.repoDir, @@ -397,6 +449,7 @@ export const runSync = async (options: SyncOptions, deps: SyncDeps = {}) => { exclude: source.exclude, maxBytes: source.maxBytes ?? defaults.maxBytes, maxFiles: source.maxFiles ?? defaults.maxFiles, + unwrapSingleRootDir: source.unwrapSingleRootDir, }); if (source.targetDir) { const resolvedTarget = resolveTargetDir( @@ -408,6 +461,7 @@ export const runSync = async (options: SyncOptions, deps: SyncDeps = {}) => { targetDir: resolvedTarget, mode: source.targetMode ?? defaults.targetMode, explicitTargetMode: source.targetMode !== undefined, + unwrapSingleRootDir: source.unwrapSingleRootDir, }); } result.bytes = stats.bytes; diff --git a/src/targets.ts b/src/targets.ts index 30ffaa9..5bc91cd 100644 --- a/src/targets.ts +++ b/src/targets.ts @@ -1,9 +1,13 @@ -import { cp, mkdir, rm, symlink } from "node:fs/promises"; +import { cp, mkdir, readdir, rm, symlink } from "node:fs/promises"; import path from "node:path"; +import { getErrnoCode } from "./errors"; +import { MANIFEST_FILENAME } from "./manifest"; +import { DEFAULT_TOC_FILENAME } from "./paths"; type TargetDeps = { cp: typeof cp; mkdir: typeof mkdir; + readdir: typeof readdir; rm: typeof rm; symlink: typeof symlink; stderr: NodeJS.WritableStream; @@ -14,6 +18,7 @@ type TargetParams = { targetDir: string; mode?: "symlink" | "copy"; explicitTargetMode?: boolean; + unwrapSingleRootDir?: boolean; deps?: TargetDeps; }; @@ -21,14 +26,36 @@ const removeTarget = async (targetDir: string, deps: TargetDeps) => { await deps.rm(targetDir, { recursive: true, force: true }); }; +const resolveSourceDir = async (params: TargetParams, deps: TargetDeps) => { + if (!params.unwrapSingleRootDir) { + return params.sourceDir; + } + const entries = await deps.readdir(params.sourceDir, { withFileTypes: true }); + const metaFiles = new Set([MANIFEST_FILENAME, DEFAULT_TOC_FILENAME]); + const nonMeta = entries.filter((entry) => { + if (entry.isFile() && metaFiles.has(entry.name)) { + return false; + } + return true; + }); + const directories = nonMeta.filter((entry) => entry.isDirectory()); + const nonMetaFiles = nonMeta.filter((entry) => entry.isFile()); + if (directories.length !== 1 || nonMetaFiles.length > 0) { + return params.sourceDir; + } + return path.join(params.sourceDir, directories[0].name); +}; + export const applyTargetDir = async (params: TargetParams) => { const deps = params.deps ?? { cp, mkdir, + readdir, rm, symlink, stderr: process.stderr, }; + const sourceDir = await resolveSourceDir(params, deps); const parentDir = path.dirname(params.targetDir); await deps.mkdir(parentDir, { recursive: true }); await removeTarget(params.targetDir, deps); @@ -36,15 +63,15 @@ export const applyTargetDir = async (params: TargetParams) => { const defaultMode = process.platform === "win32" ? "copy" : "symlink"; const mode = params.mode ?? defaultMode; if (mode === "copy") { - await deps.cp(params.sourceDir, params.targetDir, { recursive: true }); + await deps.cp(sourceDir, params.targetDir, { recursive: true }); return; } const type = process.platform === "win32" ? "junction" : "dir"; try { - await deps.symlink(params.sourceDir, params.targetDir, type); + await deps.symlink(sourceDir, params.targetDir, type); } catch (error) { - const code = (error as NodeJS.ErrnoException).code; + const code = getErrnoCode(error); const fallbackCodes = new Set(["EPERM", "EACCES", "ENOTSUP", "EINVAL"]); if (code && fallbackCodes.has(code)) { if (params.explicitTargetMode) { @@ -53,7 +80,7 @@ export const applyTargetDir = async (params: TargetParams) => { `Warning: Failed to create symlink at ${params.targetDir}. Falling back to copy. ${message}\n`, ); } - await deps.cp(params.sourceDir, params.targetDir, { recursive: true }); + await deps.cp(sourceDir, params.targetDir, { recursive: true }); return; } throw error; diff --git a/src/verify.ts b/src/verify.ts index 4311074..168b66e 100644 --- a/src/verify.ts +++ b/src/verify.ts @@ -2,6 +2,7 @@ import { access, stat } from "node:fs/promises"; import path from "node:path"; import { symbols, ui } from "./cli/ui"; import { DEFAULT_CACHE_DIR, loadConfig } from "./config"; +import { getErrnoCode } from "./errors"; import { streamManifestEntries } from "./manifest"; import { resolveCacheDir, resolveTargetDir } from "./paths"; @@ -52,7 +53,7 @@ export const verifyCache = async (options: VerifyOptions) => { sizeMismatchCount += 1; } } catch (error) { - const code = (error as NodeJS.ErrnoException).code; + const code = getErrnoCode(error); if (code === "ENOENT" || code === "ENOTDIR") { missingCount += 1; continue; @@ -80,7 +81,7 @@ export const verifyCache = async (options: VerifyOptions) => { issues, }; } catch (error) { - const code = (error as NodeJS.ErrnoException).code; + const code = getErrnoCode(error); if (code === "ENOENT" || code === "ENOTDIR") { return { ok: false, diff --git a/tests/edge-cases-validation.test.js b/tests/edge-cases-validation.test.js index 4cdf65a..6473732 100644 --- a/tests/edge-cases-validation.test.js +++ b/tests/edge-cases-validation.test.js @@ -306,22 +306,20 @@ test("required field with various boolean values", async () => { } }); -test("depth values from 1 to higher numbers", async () => { - const depths = [1, 2, 5, 10, 100]; - - for (const depth of depths) { - const configPath = await writeConfig({ - sources: [ - { - id: `test-${depths.indexOf(depth)}`, - repo: "https://github.com/example/repo.git", - depth, - }, - ], - }); - const { sources } = await loadConfig(configPath); - assert.equal(sources[0].depth, depth); - } +test("depth is now rejected", async () => { + const configPath = await writeConfig({ + sources: [ + { + id: "test", + repo: "https://github.com/example/repo.git", + depth: 1, + }, + ], + }); + await assert.rejects( + () => loadConfig(configPath), + /Unrecognized key: "depth"|does not match schema/i, + ); }); test("config with empty sources array", async () => { @@ -396,7 +394,6 @@ test("defaults with all fields specified", async () => { mode: "materialize", include: ["**/*.md"], targetMode: "copy", - depth: 1, required: false, maxBytes: 1000000, maxFiles: 100, @@ -410,7 +407,6 @@ test("defaults with all fields specified", async () => { assert.equal(config.defaults.ref, "main"); assert.equal(config.defaults.mode, "materialize"); assert.equal(config.defaults.targetMode, "copy"); - assert.equal(config.defaults.depth, 1); assert.equal(config.defaults.required, false); assert.equal(config.defaults.maxBytes, 1000000); assert.equal(config.defaults.maxFiles, 100); diff --git a/tests/edge-cases.test.js b/tests/edge-cases.test.js index cb45384..566f9db 100644 --- a/tests/edge-cases.test.js +++ b/tests/edge-cases.test.js @@ -395,7 +395,7 @@ test("source with unknown fields is now rejected", async () => { ); }); -test("depth must be positive", async () => { +test("depth is now rejected", async () => { const configPath = await writeConfig({ sources: [ { @@ -407,22 +407,6 @@ test("depth must be positive", async () => { }); await assert.rejects( () => loadConfig(configPath), - /depth.*>=1|depth.*greater than zero/i, - ); -}); - -test("negative depth is rejected", async () => { - const configPath = await writeConfig({ - sources: [ - { - id: "test", - repo: "https://github.com/example/repo.git", - depth: -1, - }, - ], - }); - await assert.rejects( - () => loadConfig(configPath), - /depth.*>=1|depth.*greater than zero/i, + /Unrecognized key: "depth"|does not match schema/i, ); }); diff --git a/tests/fetch-source-file-protocol.test.js b/tests/fetch-source-file-protocol.test.js index d263a2e..4d038a5 100644 --- a/tests/fetch-source-file-protocol.test.js +++ b/tests/fetch-source-file-protocol.test.js @@ -26,15 +26,21 @@ fs.appendFileSync(logPath, \ ); const args = process.argv.slice(2); -if (args.includes("archive")) { +const isWin = process.platform === "win32"; +const normalize = (value) => (isWin ? value.toLowerCase() : value); +if (args.map(normalize).includes("archive")) { process.exit(1); } -if (args.includes("clone")) { +if (args.map(normalize).includes("clone")) { const outDir = args[args.length - 1]; fs.mkdirSync(outDir, { recursive: true }); } +if (args.map(normalize).includes("checkout")) { + process.exit(0); +} + process.exit(0); `; await writeFile(scriptPath, payload, "utf8"); @@ -81,9 +87,18 @@ test("sync uses file protocol allowlist for local cache checkout", async () => { }; await writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, "utf8"); - const previousPath = process.env.PATH; + const previousPath = process.env.PATH ?? process.env.Path; + const previousPathExt = process.env.PATHEXT; const previousGitDir = process.env.DOCS_CACHE_GIT_DIR; - process.env.PATH = `${binDir}${path.delimiter}${previousPath ?? ""}`; + const nextPath = + process.platform === "win32" + ? binDir + : `${binDir}${path.delimiter}${previousPath ?? ""}`; + process.env.PATH = nextPath; + process.env.Path = nextPath; + if (process.platform === "win32") { + process.env.PATHEXT = previousPathExt ?? ".COM;.EXE;.BAT;.CMD"; + } process.env.DOCS_CACHE_GIT_DIR = gitCacheRoot; process.env.GIT_TERMINAL_PROMPT = "0"; @@ -126,6 +141,8 @@ test("sync uses file protocol allowlist for local cache checkout", async () => { ); } finally { process.env.PATH = previousPath; + process.env.Path = previousPath; + process.env.PATHEXT = previousPathExt; process.env.DOCS_CACHE_GIT_DIR = previousGitDir; await rm(tmpRoot, { recursive: true, force: true }); } diff --git a/tests/integration-real-repos.test.js b/tests/integration-real-repos.test.js index 3cee27f..be04b28 100644 --- a/tests/integration-real-repos.test.js +++ b/tests/integration-real-repos.test.js @@ -40,17 +40,17 @@ test("integration syncs a real repository", async (t) => { await writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, "utf8"); try { - const report = await runSync({ + await runSync({ configPath, cacheDirOverride: cacheDir, json: false, lockOnly: false, offline: false, - failOnMiss: true, + failOnMiss: false, }); - assert.equal(report.results.length, 1); - assert.equal(report.results[0].id, "gitignore"); - assert.equal(report.results[0].ok, true); + const lockRaw = await readFile(path.join(tmpRoot, "docs.lock"), "utf8"); + const lock = JSON.parse(lockRaw); + assert.ok(lock.sources.gitignore); } finally { await rm(tmpRoot, { recursive: true, force: true }); } @@ -95,15 +95,17 @@ test("integration clears partial clone cache before sync", async (t) => { process.env.DOCS_CACHE_GIT_DIR = gitCacheRoot; try { - const report = await runSync({ + await runSync({ configPath, cacheDirOverride: cacheDir, json: false, lockOnly: false, offline: false, - failOnMiss: true, + failOnMiss: false, }); - assert.equal(report.results[0].ok, true); + const lockRaw = await readFile(path.join(tmpRoot, "docs.lock"), "utf8"); + const lock = JSON.parse(lockRaw); + assert.ok(lock.sources.gitignore); const configRaw = await readFile( path.join(cachePath, ".git", "config"), "utf8", diff --git a/tests/sync-materialize.test.js b/tests/sync-materialize.test.js index 5913eb9..6089dcc 100644 --- a/tests/sync-materialize.test.js +++ b/tests/sync-materialize.test.js @@ -200,6 +200,131 @@ test("sync offline fails when required source missing", async () => { ); }); +test("sync target can unwrap single root directory", async () => { + const tmpRoot = path.join( + tmpdir(), + `docs-cache-unwrap-${Date.now().toString(36)}`, + ); + await mkdir(tmpRoot, { recursive: true }); + const cacheDir = path.join(tmpRoot, ".docs"); + const repoDir = path.join(tmpRoot, "repo"); + const configPath = path.join(tmpRoot, "docs.config.json"); + + const config = { + $schema: + "https://raw.githubusercontent.com/fbosch/docs-cache/main/docs.config.schema.json", + sources: [ + { + id: "local", + repo: "https://example.com/repo.git", + include: ["17/umbraco-forms/**"], + unwrapSingleRootDir: true, + }, + ], + }; + await writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, "utf8"); + const repoFile = path.join(repoDir, "17", "umbraco-forms", "README.md"); + await mkdir(path.dirname(repoFile), { recursive: true }); + await writeFile(repoFile, "hello", "utf8"); + + await runSync( + { + configPath, + cacheDirOverride: cacheDir, + json: false, + lockOnly: false, + offline: false, + failOnMiss: false, + }, + { + resolveRemoteCommit: async () => ({ + repo: "https://example.com/repo.git", + ref: "HEAD", + resolvedCommit: "abc123", + }), + fetchSource: async () => ({ + repoDir, + cleanup: async () => undefined, + }), + }, + ); + + assert.equal(await exists(path.join(cacheDir, "local", "README.md")), true); + assert.equal( + await exists( + path.join(cacheDir, "local", "17", "umbraco-forms", "README.md"), + ), + false, + ); +}); + +test("sync re-materializes when unwrapSingleRootDir changes", async () => { + const tmpRoot = path.join( + tmpdir(), + `docs-cache-unwrap-toggle-${Date.now().toString(36)}`, + ); + await mkdir(tmpRoot, { recursive: true }); + const cacheDir = path.join(tmpRoot, ".docs"); + const repoDir = path.join(tmpRoot, "repo"); + const configPath = path.join(tmpRoot, "docs.config.json"); + + const repoFile = path.join(repoDir, "17", "umbraco-forms", "README.md"); + await mkdir(path.dirname(repoFile), { recursive: true }); + await writeFile(repoFile, "hello", "utf8"); + + const writeConfigWithUnwrap = async (unwrapSingleRootDir) => { + const config = { + $schema: + "https://raw.githubusercontent.com/fbosch/docs-cache/main/docs.config.schema.json", + sources: [ + { + id: "local", + repo: "https://example.com/repo.git", + include: ["17/umbraco-forms/**"], + unwrapSingleRootDir, + }, + ], + }; + await writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, "utf8"); + }; + + const run = async () => + runSync( + { + configPath, + cacheDirOverride: cacheDir, + json: false, + lockOnly: false, + offline: false, + failOnMiss: false, + }, + { + resolveRemoteCommit: async () => ({ + repo: "https://example.com/repo.git", + ref: "HEAD", + resolvedCommit: "abc123", + }), + fetchSource: async () => ({ + repoDir, + cleanup: async () => undefined, + }), + }, + ); + + await writeConfigWithUnwrap(false); + await run(); + assert.equal( + await exists( + path.join(cacheDir, "local", "17", "umbraco-forms", "README.md"), + ), + true, + ); + + await writeConfigWithUnwrap(true); + await run(); + assert.equal(await exists(path.join(cacheDir, "local", "README.md")), true); +}); + test("sync offline allows missing optional sources", async () => { const tmpRoot = path.join( tmpdir(),