Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,47 @@ on:
- master

jobs:
precheck:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4

- name: Setup pnpm
uses: pnpm/action-setup@v4
with:
run_install: false

- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: 22
cache: pnpm

- name: Install dependencies
run: pnpm install --frozen-lockfile

- name: Audit dependencies
run: pnpm audit --audit-level=high

- name: Lint
run: pnpm lint

- name: Typecheck
run: pnpm typecheck

- name: Test
run: pnpm test

- name: Build
run: pnpm build

- name: Size limit
run: pnpm size

build:
needs: precheck
if: needs.precheck.result == 'success'
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
Expand Down
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -94,12 +94,12 @@ All fields in `defaults` apply to all sources unless overridden per-source.
| `mode` | Cache mode. Default: `"materialize"`. |
| `include` | Glob patterns to copy. Default: `["**/*.{md,mdx,markdown,mkd,txt,rst,adoc,asciidoc}"]`. |
| `targetMode` | How to link or copy from the cache to the destination. Default: `"symlink"` on Unix, `"copy"` on Windows. |
| `depth` | Git clone depth. Default: `1`. |
| `required` | Whether missing sources should fail. Default: `true`. |
| `maxBytes` | Maximum total bytes to materialize. Default: `200000000` (200 MB). |
| `maxFiles` | Maximum total files to materialize. |
| `allowHosts` | Allowed Git hosts. Default: `["github.com", "gitlab.com"]`. |
| `toc` | Generate per-source `TOC.md`. Default: `true`. Supports `true`, `false`, or a format (`"tree"`, `"compressed"`). |
| `unwrapSingleRootDir` | If the materialized output is nested under a single directory, unwrap it (recursively). Default: `false`. |
Comment thread
fbosch marked this conversation as resolved.
Comment thread
fbosch marked this conversation as resolved.

### Source options

Expand All @@ -123,6 +123,7 @@ All fields in `defaults` apply to all sources unless overridden per-source.
| `maxBytes` | Maximum total bytes to materialize. |
| `maxFiles` | Maximum total files to materialize. |
| `toc` | Generate per-source `TOC.md`. Supports `true`, `false`, or a format (`"tree"`, `"compressed"`). |
| `unwrapSingleRootDir` | If the materialized output is nested under a single directory, unwrap it (recursively). |

> **Note**: Sources are always downloaded to `.docs/<id>/`. If you provide a `targetDir`, `docs-cache` will create a symlink or copy pointing from the cache to that target directory. The target should be outside `.docs`. Git operation timeout is configured via the `--timeout-ms` CLI flag, not as a per-source configuration option.

Expand Down
11 changes: 3 additions & 8 deletions docs.config.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,6 @@
"type": "string",
"enum": ["symlink", "copy"]
},
"depth": {
"type": "number",
"minimum": 1
},
"required": {
"type": "boolean"
},
Expand Down Expand Up @@ -107,10 +103,6 @@
"type": "string",
"enum": ["materialize"]
},
"depth": {
"type": "number",
"minimum": 1
},
"include": {
"type": "array",
"items": {
Expand Down Expand Up @@ -171,6 +163,9 @@
"tocFormat": {
"type": "string",
"enum": ["tree", "compressed"]
},
"unwrapSingleRootDir": {
"type": "boolean"
}
},
"required": ["id", "repo"],
Expand Down
3 changes: 3 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,9 @@
"lint-staged": {
"*.{js,ts,cjs,mjs,d.cts,d.mts,jsx,tsx,json,jsonc}": [
"biome check --write --no-errors-on-unmatched"
],
"*.ts": [
"pnpm typecheck"
]
}
}
3 changes: 1 addition & 2 deletions src/config-schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ export const DefaultsSchema = z
mode: CacheModeSchema,
include: z.array(z.string().min(1)).min(1),
targetMode: TargetModeSchema.optional(),
depth: z.number().min(1),
required: z.boolean(),
maxBytes: z.number().min(1),
maxFiles: z.number().min(1).optional(),
Expand All @@ -33,14 +32,14 @@ export const SourceSchema = z
targetMode: TargetModeSchema.optional(),
ref: z.string().min(1).optional(),
mode: CacheModeSchema.optional(),
depth: z.number().min(1).optional(),
include: z.array(z.string().min(1)).optional(),
exclude: z.array(z.string().min(1)).optional(),
required: z.boolean().optional(),
maxBytes: z.number().min(1).optional(),
maxFiles: z.number().min(1).optional(),
integrity: IntegritySchema.optional(),
toc: z.union([z.boolean(), TocFormatSchema]).optional(),
unwrapSingleRootDir: z.boolean().optional(),
})
.strict();

Expand Down
24 changes: 9 additions & 15 deletions src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ export interface DocsCacheDefaults {
mode: CacheMode;
include: string[];
targetMode?: "symlink" | "copy";
depth: number;
required: boolean;
maxBytes: number;
maxFiles?: number;
Expand All @@ -35,14 +34,14 @@ export interface DocsCacheSource {
targetMode?: "symlink" | "copy";
ref?: string;
mode?: CacheMode;
depth?: number;
include?: string[];
exclude?: string[];
required?: boolean;
maxBytes?: number;
maxFiles?: number;
integrity?: DocsCacheIntegrity;
toc?: boolean | TocFormat;
unwrapSingleRootDir?: boolean;
}

export interface DocsCacheConfig {
Expand All @@ -60,14 +59,14 @@ export interface DocsCacheResolvedSource {
targetMode?: "symlink" | "copy";
ref: string;
mode: CacheMode;
depth: number;
include?: string[];
exclude?: string[];
required: boolean;
maxBytes: number;
maxFiles?: number;
integrity?: DocsCacheIntegrity;
toc?: boolean | TocFormat;
unwrapSingleRootDir?: boolean;
}

export const DEFAULT_CONFIG_FILENAME = "docs.config.json";
Expand All @@ -81,7 +80,6 @@ export const DEFAULT_CONFIG: DocsCacheConfig = {
mode: "materialize",
include: ["**/*.{md,mdx,markdown,mkd,txt,rst,adoc,asciidoc}"],
targetMode: DEFAULT_TARGET_MODE,
depth: 1,
required: true,
maxBytes: 200000000,
allowHosts: ["github.com", "gitlab.com"],
Expand Down Expand Up @@ -280,10 +278,6 @@ export const validateConfig = (input: unknown): DocsCacheConfig => {
defaultsInput.targetMode !== undefined
? assertTargetMode(defaultsInput.targetMode, "defaults.targetMode")
: (targetModeOverride ?? defaultValues.targetMode),
depth:
defaultsInput.depth !== undefined
? assertPositiveNumber(defaultsInput.depth, "defaults.depth")
: defaultValues.depth,
required:
defaultsInput.required !== undefined
? assertBoolean(defaultsInput.required, "defaults.required")
Expand Down Expand Up @@ -348,12 +342,6 @@ export const validateConfig = (input: unknown): DocsCacheConfig => {
if (entry.mode !== undefined) {
source.mode = assertMode(entry.mode, `sources[${index}].mode`);
}
if (entry.depth !== undefined) {
source.depth = assertPositiveNumber(
entry.depth,
`sources[${index}].depth`,
);
}
if (entry.include !== undefined) {
source.include = assertStringArray(
entry.include,
Expand Down Expand Up @@ -394,6 +382,12 @@ export const validateConfig = (input: unknown): DocsCacheConfig => {
if (entry.toc !== undefined) {
source.toc = entry.toc as boolean | TocFormat;
}
if (entry.unwrapSingleRootDir !== undefined) {
source.unwrapSingleRootDir = assertBoolean(
entry.unwrapSingleRootDir,
`sources[${index}].unwrapSingleRootDir`,
);
}

return source;
});
Expand Down Expand Up @@ -433,14 +427,14 @@ export const resolveSources = (
targetMode: source.targetMode ?? defaults.targetMode,
ref: source.ref ?? defaults.ref,
mode: source.mode ?? defaults.mode,
depth: source.depth ?? defaults.depth,
include: source.include ?? defaults.include,
exclude: source.exclude,
required: source.required ?? defaults.required,
maxBytes: source.maxBytes ?? defaults.maxBytes,
maxFiles: source.maxFiles ?? defaults.maxFiles,
integrity: source.integrity,
toc: source.toc ?? defaults.toc,
unwrapSingleRootDir: source.unwrapSingleRootDir,
Comment thread
fbosch marked this conversation as resolved.
Outdated
}));
};

Expand Down
22 changes: 16 additions & 6 deletions src/git/fetch-source.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import { exists, resolveGitCacheDir } from "./cache-dir";
const execFileAsync = promisify(execFile);

const DEFAULT_TIMEOUT_MS = 120000; // 120 seconds (2 minutes)
const DEFAULT_GIT_DEPTH = 1;
const DEFAULT_RM_RETRIES = 3;
const DEFAULT_RM_BACKOFF_MS = 100;

Expand Down Expand Up @@ -126,11 +127,16 @@ type FetchParams = {
ref: string;
resolvedCommit: string;
cacheDir: string;
depth: number;
include?: string[];
timeoutMs?: number;
};

type FetchResult = {
repoDir: string;
cleanup: () => Promise<void>;
fromCache: boolean;
};

const runGitArchive = async (
repo: string,
resolvedCommit: string,
Expand Down Expand Up @@ -190,7 +196,7 @@ const cloneRepo = async (params: FetchParams, outDir: string) => {
"clone",
"--no-checkout",
"--depth",
String(params.depth),
String(DEFAULT_GIT_DEPTH),
"--recurse-submodules=no",
"--no-tags",
];
Expand Down Expand Up @@ -250,10 +256,10 @@ const cloneOrUpdateRepo = async (params: FetchParams, outDir: string) => {
params.ref === "HEAD"
? "HEAD"
: `${params.ref}:refs/remotes/origin/${params.ref}`;
fetchArgs.push(refSpec, "--depth", String(params.depth));
fetchArgs.push(refSpec, "--depth", String(DEFAULT_GIT_DEPTH));
} else {
// For commit refs, fetch the default branch and hope the commit is there
fetchArgs.push("--depth", String(params.depth));
fetchArgs.push("--depth", String(DEFAULT_GIT_DEPTH));
}

await git(["-C", cachePath, ...fetchArgs], {
Expand Down Expand Up @@ -281,7 +287,7 @@ const cloneOrUpdateRepo = async (params: FetchParams, outDir: string) => {
"clone",
"--no-checkout",
"--depth",
String(params.depth),
String(DEFAULT_GIT_DEPTH),
"--recurse-submodules=no",
"--no-tags",
];
Expand Down Expand Up @@ -344,7 +350,9 @@ const archiveRepo = async (params: FetchParams) => {
}
};

export const fetchSource = async (params: FetchParams) => {
export const fetchSource = async (
params: FetchParams,
): Promise<FetchResult> => {
assertSafeSourceId(params.sourceId, "sourceId");
try {
const archiveDir = await archiveRepo(params);
Expand All @@ -353,6 +361,7 @@ export const fetchSource = async (params: FetchParams) => {
cleanup: async () => {
await removeDir(archiveDir);
},
fromCache: false,
};
} catch {
const tempDir = await mkdtemp(
Expand All @@ -365,6 +374,7 @@ export const fetchSource = async (params: FetchParams) => {
cleanup: async () => {
await removeDir(tempDir);
},
fromCache: true,
};
} catch (error) {
await removeDir(tempDir);
Expand Down
Loading
Loading