Skip to content
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,7 @@ coverage
TODO.md
.docs/
benchmarks/

*.md
!AGENTS.md
!README.md
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -100,10 +100,12 @@ These fields can be set in `defaults` and are inherited by every source unless o
| `maxBytes` | Maximum total bytes to materialize. Default: `200000000` (200 MB). |
| `maxFiles` | Maximum total files to materialize. |
| `ignoreHidden` | Skip hidden files and directories (dotfiles). Default: `false`. |
| `allowHosts` | Allowed Git hosts. Default: `["github.com", "gitlab.com", "visualstudio.com"]`. |
| `allowHosts` | Allowed Git hosts. Default: `["github.com", "gitlab.com", "visualstudio.com"]`. |
| `toc` | Generate per-source `TOC.md`. Default: `true`. Supports `true`, `false`, or a format: `"tree"` (human readable), `"compressed"` |
| `unwrapSingleRootDir` | If the materialized output is nested under a single directory, unwrap it (recursively). Default: `true`. |

> Brace expansion in `include` supports comma-separated lists (including multiple groups) like `**/*.{md,mdx}` and is capped at 500 expanded patterns per include entry. It does not support nested braces or numeric ranges.

### Source options

#### Required
Expand Down
81 changes: 50 additions & 31 deletions src/git/fetch-source.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,40 +9,13 @@ import { execa } from "execa";
import { getErrnoCode } from "#core/errors";
import { assertSafeSourceId } from "#core/source-id";
import { exists, resolveGitCacheDir } from "#git/cache-dir";
import { buildGitEnv } from "#git/git-env";

const DEFAULT_TIMEOUT_MS = 120000; // 120 seconds (2 minutes)
const DEFAULT_GIT_DEPTH = 1;
const DEFAULT_RM_RETRIES = 3;
const DEFAULT_RM_BACKOFF_MS = 100;

const buildGitEnv = () => {
const pathValue = process.env.PATH ?? process.env.Path;
const pathExtValue =
process.env.PATHEXT ??
(process.platform === "win32" ? ".COM;.EXE;.BAT;.CMD" : undefined);
return {
...process.env,
...(pathValue ? { PATH: pathValue, Path: pathValue } : {}),
...(pathExtValue ? { PATHEXT: pathExtValue } : {}),
HOME: process.env.HOME,
USER: process.env.USER,
USERPROFILE: process.env.USERPROFILE,
TMPDIR: process.env.TMPDIR,
TMP: process.env.TMP,
TEMP: process.env.TEMP,
SYSTEMROOT: process.env.SYSTEMROOT,
WINDIR: process.env.WINDIR,
SSH_AUTH_SOCK: process.env.SSH_AUTH_SOCK,
SSH_AGENT_PID: process.env.SSH_AGENT_PID,
HTTP_PROXY: process.env.HTTP_PROXY,
HTTPS_PROXY: process.env.HTTPS_PROXY,
NO_PROXY: process.env.NO_PROXY,
GIT_TERMINAL_PROMPT: "0",
GIT_CONFIG_NOSYSTEM: "1",
GIT_CONFIG_NOGLOBAL: "1",
...(process.platform === "win32" ? {} : { GIT_ASKPASS: "/bin/false" }),
};
};
const MAX_BRACE_EXPANSIONS = 500;

const buildGitConfigs = (allowFileProtocol?: boolean) => [
"-c",
Expand Down Expand Up @@ -286,8 +259,54 @@ type CloneResult = {
const patternHasGlob = (pattern: string) =>
pattern.includes("*") || pattern.includes("?") || pattern.includes("[");

const normalizeSparsePatterns = (include?: string[]) =>
(include ?? []).map((pattern) => pattern.replace(/\\/g, "/")).filter(Boolean);
const expandBracePattern = (pattern: string): string[] => {
const results: string[] = [];
const expand = (value: string) => {
const braceMatch = value.match(/^(.*?){([^}]+)}(.*)$/);
if (!braceMatch) {
results.push(value);
if (results.length > MAX_BRACE_EXPANSIONS) {
throw new Error(
`Brace expansion exceeded ${MAX_BRACE_EXPANSIONS} patterns for '${pattern}'.`,
);
}
return;
}
const [, prefix, values, suffix] = braceMatch;
const valueList = values
.split(",")
.map((entry) => entry.trim())
.filter((entry) => entry.length > 0);
if (valueList.length === 0) {
results.push(value);
if (results.length > MAX_BRACE_EXPANSIONS) {
throw new Error(
`Brace expansion exceeded ${MAX_BRACE_EXPANSIONS} patterns for '${pattern}'.`,
);
}
return;
}
for (const entry of valueList) {
const expandedPattern = `${prefix}${entry}${suffix}`;
expand(expandedPattern);
}
};

expand(pattern);
return results;
};
Comment thread
fbosch marked this conversation as resolved.
Comment thread
coderabbitai[bot] marked this conversation as resolved.

const normalizeSparsePatterns = (include?: string[]) => {
const patterns = include ?? [];
const expanded: string[] = [];
for (const pattern of patterns) {
const normalized = pattern.replace(/\\/g, "/");
if (!normalized) continue;
// Expand brace patterns for git sparse-checkout compatibility
expanded.push(...expandBracePattern(normalized));
Comment thread
fbosch marked this conversation as resolved.
}
return expanded;
};

const isDirectoryLiteral = (pattern: string) => pattern.endsWith("/");

Expand Down
30 changes: 30 additions & 0 deletions src/git/git-env.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
const buildGitEnv = (): NodeJS.ProcessEnv => {
const pathValue = process.env.PATH ?? process.env.Path;
const pathExtValue =
process.env.PATHEXT ??
(process.platform === "win32" ? ".COM;.EXE;.BAT;.CMD" : undefined);
return {
...process.env,
...(pathValue ? { PATH: pathValue, Path: pathValue } : {}),
...(pathExtValue ? { PATHEXT: pathExtValue } : {}),
HOME: process.env.HOME,
USER: process.env.USER,
USERPROFILE: process.env.USERPROFILE,
TMPDIR: process.env.TMPDIR,
TMP: process.env.TMP,
TEMP: process.env.TEMP,
SYSTEMROOT: process.env.SYSTEMROOT,
WINDIR: process.env.WINDIR,
SSH_AUTH_SOCK: process.env.SSH_AUTH_SOCK,
SSH_AGENT_PID: process.env.SSH_AGENT_PID,
HTTP_PROXY: process.env.HTTP_PROXY,
HTTPS_PROXY: process.env.HTTPS_PROXY,
NO_PROXY: process.env.NO_PROXY,
GIT_TERMINAL_PROMPT: "0",
GIT_CONFIG_NOSYSTEM: "1",
GIT_CONFIG_NOGLOBAL: "1",
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Outdated
...(process.platform === "win32" ? {} : { GIT_ASKPASS: "/bin/false" }),
};
};

export { buildGitEnv };
3 changes: 2 additions & 1 deletion src/git/resolve-remote.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { execFile } from "node:child_process";
import { promisify } from "node:util";

import { buildGitEnv } from "#git/git-env";
import { redactRepoUrl } from "#git/redact";

const execFileAsync = promisify(execFile);
Expand Down Expand Up @@ -90,6 +90,7 @@ export const resolveRemoteCommit = async (params: ResolveRemoteParams) => {
{
timeout: params.timeoutMs ?? DEFAULT_TIMEOUT_MS,
maxBuffer: 1024 * 1024,
env: buildGitEnv(),
},
);

Expand Down
60 changes: 60 additions & 0 deletions tests/integration-real-repos.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -123,3 +123,63 @@ test("integration clears partial clone cache before sync", async (t) => {
await rm(tmpRoot, { recursive: true, force: true });
}
});

test("integration uses default include pattern without explicit config", async (t) => {
if (!shouldRun()) {
t.skip("Set DOCS_CACHE_INTEGRATION=1 to run integration tests");
return;
}
const tmpRoot = path.join(
tmpdir(),
`docs-cache-defaults-${Date.now().toString(36)}`,
);
const cacheDir = path.join(tmpRoot, ".docs");
const configPath = path.join(tmpRoot, "docs.config.json");
const repo = "https://github.com/glanceapp/glance.git";

await mkdir(tmpRoot, { recursive: true });
const config = {
$schema:
"https://raw.githubusercontent.com/fbosch/docs-cache/main/docs.config.schema.json",
sources: [
{
id: "glance",
repo,
// No include pattern specified - should use defaults
},
],
};
await writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, "utf8");

try {
await runSync({
configPath,
cacheDirOverride: cacheDir,
json: false,
lockOnly: false,
offline: false,
failOnMiss: false,
});
const lockRaw = await readFile(
path.join(tmpRoot, DEFAULT_LOCK_FILENAME),
"utf8",
);
const lock = JSON.parse(lockRaw);
assert.ok(lock.sources.glance);
// The default pattern includes **/*.{md,mdx,markdown,mkd,txt,rst,adoc,asciidoc}
// glanceapp/glance has markdown files, so fileCount should be > 0
assert.ok(
lock.sources.glance.fileCount > 0,
`Expected files to be synced with default include pattern, got ${lock.sources.glance.fileCount}`,
);
// Verify that actual .md files were synced
const readmePath = path.join(cacheDir, "glance", "README.md");
const readmeContent = await readFile(readmePath, "utf8");
assert.ok(
readmeContent.length > 0,
"Expected README.md to be synced and have content",
);
} finally {
await rm(tmpRoot, { recursive: true, force: true });
}
});
Loading
Loading