docs: add searchable Sourcey skill catalog

docs/sourcey-catalog/README.md

@@ -0,0 +1,43 @@
+# Sourcey catalog extractor
+
+This directory creates a 24-page documentation catalog from Runx skill sources
+at one immutable Git commit. The local checkout is never used as source content.
+
+## Inputs
+
+`catalog.json` identifies the canonical GitHub repository, the pinned commit,
+and five ordered groups. Every entry declares its skill name, output slug,
+group, and source path. The extractor accepts only the Runx repository, a
+40-character lowercase commit SHA, the exact canonical 24 names and five
+groups, slugs equal to names, and paths of the form `skills/<name>/SKILL.md`.
+
+## Run
+
+```sh
+node --test docs/sourcey-catalog/extract.test.mjs
+node docs/sourcey-catalog/extract.mjs
+```
+
+The extractor reads each file through GitHub's Contents API at the catalog's
+commit. Optional authentication comes only from `GITHUB_TOKEN`, falling back to
+`GH_TOKEN`, and is sent only in the Authorization header. Tokens are redacted
+from errors and output. The extractor retries transient failures up to three
+total attempts, caps fetch concurrency at four, and waits at most 30 seconds
+for a rate-limit reset before returning the reset time and token guidance.
+Secondary rate limits are recognized from GitHub's response message or
+`Retry-After` header and return the same bounded-wait and token guidance.
+
+Each response must contain canonical base64 for a file no larger than 1 MiB.
+The extractor verifies the returned Git blob SHA before rendering the page.
+
+Generated pages are written in catalog order to `pages/*.md`. Each one contains
+the group, immutable source URL, commit, source path, authored Markdown with
+only recognized leading YAML frontmatter removed. Frontmatter is validated with
+the project's `yaml` parser before removal; malformed YAML is rejected. The
+extractor normalizes output to UTF-8 LF endings and rejects a symlink
+`outputDir`. Within a real output directory, it atomically replaces expected
+page paths and safely unlinks stale Markdown files or symlinks without following
+symlink targets.
+
+`extractCatalog` returns a `sha256:` digest of each final page in its page
+metadata. The digest is not embedded in the generated Markdown body.

docs/sourcey-catalog/build.mjs

@@ -0,0 +1,139 @@
+import { lstat, readFile, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { spawn } from "node:child_process";
+import { fileURLToPath } from "node:url";
+
+const SOURCEY_VERSION = "3.6.5";
+const CONFIG_FILE = "sourcey.config.ts";
+
+export function renderSourceyConfig(catalog) {
+  if (!catalog || !Array.isArray(catalog.groups)) throw new TypeError("catalog groups must be an array");
+
+  const groups = [
+    { name: "Introduction", slugs: ["introduction"] },
+    ...catalog.groups.map((group) => {
+      if (typeof group?.name !== "string" || !Array.isArray(group.entries)) {
+        throw new TypeError("catalog groups must have a name and entries array");
+      }
+      return { name: group.name, slugs: group.entries.map(({ slug }) => slug) };
+    })
+  ];
+  const seen = new Set();
+  for (const { slugs } of groups.slice(1)) {
+    for (const slug of slugs) {
+      if (typeof slug !== "string" || !/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(slug)) {
+        throw new Error("catalog slugs must be lowercase kebab-case");
+      }
+      if (seen.has(slug)) throw new Error(`duplicate slug: ${slug}`);
+      seen.add(slug);
+    }
+  }
+
+  const renderedGroups = groups.map(({ name, slugs }) => [
+    "            {",
+    `              group: ${JSON.stringify(name)},`,
+    `              pages: [${slugs.map((slug) => JSON.stringify(`pages/${slug}`)).join(", ")}],`,
+    "            },"
+  ].join("\n")).join("\n");
+
+  return `export default {
+  name: "Runx Governed Skill Catalog",
+  siteUrl: "https://github.com",
+  baseUrl: "/runxhq/runx",
+  repo: "https://github.com/runxhq/runx",
+  editBranch: "main",
+  editBasePath: "docs/sourcey-catalog",
+  theme: {
+    preset: "default",
+    colors: { primary: "#0f766e", light: "#14b8a6", dark: "#134e4a" },
+  },
+  navigation: {
+    tabs: [
+      {
+        tab: "Skills",
+        slug: "",
+        groups: [
+${renderedGroups}
+        ],
+      },
+    ],
+  },
+};
+`;
+}
+
+export function buildCommand(catalogDir, options = {}) {
+  const root = path.resolve(catalogDir);
+  const outputDir = path.resolve(options.outputDir ?? path.join(root, "site"));
+  const expectedOutput = path.join(root, "site");
+  if (outputDir !== expectedOutput) {
+    throw new Error("outputDir must be the site directory inside the catalog directory");
+  }
+
+  const args = ["build", "-o", "site", "--quiet"];
+  const command = options.sourceyBin
+    ? [path.resolve(options.sourceyBin), ...args]
+    : ["npx", "-y", `sourcey@${SOURCEY_VERSION}`, ...args];
+  return { command, outputDir };
+}
+
+export async function buildCatalog({ catalogDir, outputDir, sourceyBin } = {}) {
+  const root = path.resolve(catalogDir ?? path.dirname(fileURLToPath(import.meta.url)));
+  const plan = buildCommand(root, { outputDir, sourceyBin });
+  const catalog = JSON.parse(await readFile(path.join(root, "catalog.json"), "utf8"));
+  const config = renderSourceyConfig(catalog);
+
+  await writeFile(path.join(root, CONFIG_FILE), config, "utf8");
+  await rm(plan.outputDir, { recursive: true, force: true });
+  await run(plan.command, root);
+  await verifyBuildArtifacts(plan.outputDir, catalog);
+
+  return {
+    page_count: catalog.groups.flatMap((group) => group.entries).length,
+    output_dir: plan.outputDir,
+    command: plan.command
+  };
+}
+
+async function verifyBuildArtifacts(outputDir, catalog) {
+  const required = [
+    "index.html",
+    "search-index.json",
+    "sourcey.css",
+    "sourcey.js",
+    "llms.txt",
+    "llms-full.txt",
+    "pages/introduction.html",
+    ...catalog.groups.flatMap((group) => group.entries.map(({ slug }) => `pages/${slug}.html`)),
+  ];
+
+  for (const relativePath of required) {
+    const artifact = path.join(outputDir, relativePath);
+    try {
+      const metadata = await lstat(artifact);
+      if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.size === 0) throw new Error();
+    } catch {
+      throw new Error(`missing or empty Sourcey build artifact: ${relativePath}`);
+    }
+  }
+}
+
+async function run(command, cwd) {
+  await new Promise((resolve, reject) => {
+    const child = spawn(command[0], command.slice(1), { cwd, stdio: "inherit" });
+    child.once("error", reject);
+    child.once("exit", (code, signal) => {
+      if (code === 0) resolve();
+      else reject(new Error(`Sourcey build failed with ${signal ? `signal ${signal}` : `exit code ${code}`}`));
+    });
+  });
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  buildCatalog().then((result) => {
+    process.stdout.write(`${JSON.stringify(result)}\n`);
+  }).catch((error) => {
+    process.stderr.write(`${error.message}\n`);
+    process.exitCode = 1;
+  });
+}

docs/sourcey-catalog/build.test.mjs

@@ -0,0 +1,124 @@
+import assert from "node:assert/strict";
+import { chmod, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+
+import { buildCatalog, buildCommand, renderSourceyConfig } from "./build.mjs";
+
+const catalog = {
+  source_repository: "https://github.com/runxhq/runx",
+  source_commit: "5afc25a83edf1c1320df7ac0d78c36f1523b5677",
+  groups: [
+    { name: "Operate", entries: [{ slug: "agency" }, { slug: "work-plan" }] },
+    { name: "Research and data", entries: [{ slug: "research" }] },
+    { name: "GitHub and delivery", entries: [{ slug: "issue-to-pr" }] },
+    { name: "Safety and review", entries: [{ slug: "cve-audit" }] },
+    { name: "Outbound and tooling", entries: [{ slug: "sourcey" }] }
+  ]
+};
+
+test("config exposes every manifest page once after introduction in catalog group order", () => {
+  const config = renderSourceyConfig(catalog);
+  const expectedGroups = ["Introduction", ...catalog.groups.map((group) => group.name)];
+
+  assert.match(config, /name: "Runx Governed Skill Catalog"/);
+  assert.match(config, /siteUrl: "https:\/\/github\.com"/);
+  assert.match(config, /baseUrl: "\/runxhq\/runx"/);
+  assert.match(config, /repo: "https:\/\/github\.com\/runxhq\/runx"/);
+  assert.match(config, /editBranch: "main"/);
+  assert.match(config, /editBasePath: "docs\/sourcey-catalog"/);
+  assert.doesNotMatch(config, /from "sourcey"/);
+  assert.match(config, /groups: \[/);
+  assert.deepEqual(
+    [...config.matchAll(/group: "([^"]+)"/g)].map((match) => match[1]),
+    expectedGroups
+  );
+  assert.match(config, /group: "Introduction",\n\s+pages: \["pages\/introduction"\]/);
+
+  for (const entry of catalog.groups.flatMap((group) => group.entries)) {
+    assert.equal(config.match(new RegExp(`pages/${entry.slug.replaceAll("-", "\\-")}`, "g"))?.length, 1);
+  }
+});
+
+test("build command pins Sourcey 3.6.5 and writes only inside catalog site", () => {
+  const plan = buildCommand("/repo/docs/sourcey-catalog");
+
+  assert.deepEqual(plan.command, ["npx", "-y", "sourcey@3.6.5", "build", "-o", "site", "--quiet"]);
+  assert.equal(plan.outputDir, "/repo/docs/sourcey-catalog/site");
+  assert.throws(
+    () => buildCommand("/repo/docs/sourcey-catalog", { outputDir: "/repo/docs/site" }),
+    /inside the catalog directory/
+  );
+});
+
+test("config splits the public URL into the Sourcey 3.6.5 origin and base path", () => {
+  const config = renderSourceyConfig(catalog);
+
+  assert.match(config, /siteUrl: "https:\/\/github\.com"/);
+  assert.match(config, /baseUrl: "\/runxhq\/runx"/);
+});
+
+test("build wrapper accepts an explicit Sourcey binary for offline execution", async (t) => {
+  const catalogDir = await mkdtemp(path.join(tmpdir(), "sourcey-build-"));
+  t.after(() => rm(catalogDir, { recursive: true, force: true }));
+  await writeFile(path.join(catalogDir, "catalog.json"), `${JSON.stringify(catalog)}\n`, "utf8");
+  const sourceyBin = path.join(catalogDir, "sourcey-fixture.mjs");
+  await writeFile(sourceyBin, [
+    "#!/usr/bin/env node",
+    "import { mkdir, writeFile } from 'node:fs/promises';",
+    "await writeFile('invocation.json', JSON.stringify({ cwd: process.cwd(), args: process.argv.slice(2) }));",
+    "await mkdir('site/pages', { recursive: true });",
+    `for (const file of ${JSON.stringify([
+      "index.html",
+      "search-index.json",
+      "sourcey.css",
+      "sourcey.js",
+      "llms.txt",
+      "llms-full.txt",
+      "pages/introduction.html",
+      ...catalog.groups.flatMap((group) => group.entries.map((entry) => `pages/${entry.slug}.html`)),
+    ])}) await writeFile('site/' + file, 'fixture');`,
+  ].join("\n"), "utf8");
+  await chmod(sourceyBin, 0o755);
+
+  const result = await buildCatalog({ catalogDir, sourceyBin });
+  const invocation = JSON.parse(await readFile(path.join(catalogDir, "invocation.json"), "utf8"));
+
+  assert.deepEqual(invocation, {
+    cwd: await realpath(catalogDir),
+    args: ["build", "-o", "site", "--quiet"]
+  });
+  assert.equal(result.page_count, 6);
+  assert.equal(result.output_dir, path.join(catalogDir, "site"));
+  assert.deepEqual(result.command, [sourceyBin, "build", "-o", "site", "--quiet"]);
+  assert.match(await readFile(path.join(catalogDir, "sourcey.config.ts"), "utf8"), /pages\/agency/);
+});
+
+test("build wrapper refuses a successful command with missing static artifacts", async (t) => {
+  const catalogDir = await mkdtemp(path.join(tmpdir(), "sourcey-build-missing-"));
+  t.after(() => rm(catalogDir, { recursive: true, force: true }));
+  await writeFile(path.join(catalogDir, "catalog.json"), `${JSON.stringify(catalog)}\n`, "utf8");
+  const sourceyBin = path.join(catalogDir, "sourcey-fixture.mjs");
+  await writeFile(sourceyBin, "#!/usr/bin/env node\n", "utf8");
+  await chmod(sourceyBin, 0o755);
+
+  await assert.rejects(
+    buildCatalog({ catalogDir, sourceyBin }),
+    /missing or empty Sourcey build artifact/,
+  );
+});
+
+test("build wrapper reports a nonzero Sourcey exit", async (t) => {
+  const catalogDir = await mkdtemp(path.join(tmpdir(), "sourcey-build-failed-"));
+  t.after(() => rm(catalogDir, { recursive: true, force: true }));
+  await writeFile(path.join(catalogDir, "catalog.json"), `${JSON.stringify(catalog)}\n`, "utf8");
+  const sourceyBin = path.join(catalogDir, "sourcey-fixture.mjs");
+  await writeFile(sourceyBin, "#!/usr/bin/env node\nprocess.exitCode = 7;\n", "utf8");
+  await chmod(sourceyBin, 0o755);
+
+  await assert.rejects(
+    buildCatalog({ catalogDir, sourceyBin }),
+    /exit code 7/,
+  );
+});

docs/sourcey-catalog/catalog.json

@@ -0,0 +1,55 @@
+{
+  "source_repository": "https://github.com/runxhq/runx",
+  "source_commit": "5afc25a83edf1c1320df7ac0d78c36f1523b5677",
+  "groups": [
+    {
+      "name": "Operate",
+      "entries": [
+        { "name": "agency", "slug": "agency", "group": "Operate", "path": "skills/agency/SKILL.md" },
+        { "name": "business-ops", "slug": "business-ops", "group": "Operate", "path": "skills/business-ops/SKILL.md" },
+        { "name": "operator-inbox", "slug": "operator-inbox", "group": "Operate", "path": "skills/operator-inbox/SKILL.md" },
+        { "name": "ops-desk", "slug": "ops-desk", "group": "Operate", "path": "skills/ops-desk/SKILL.md" },
+        { "name": "work-plan", "slug": "work-plan", "group": "Operate", "path": "skills/work-plan/SKILL.md" }
+      ]
+    },
+    {
+      "name": "Research and data",
+      "entries": [
+        { "name": "deep-research", "slug": "deep-research", "group": "Research and data", "path": "skills/deep-research/SKILL.md" },
+        { "name": "research", "slug": "research", "group": "Research and data", "path": "skills/research/SKILL.md" },
+        { "name": "data-store", "slug": "data-store", "group": "Research and data", "path": "skills/data-store/SKILL.md" },
+        { "name": "knowledge-router", "slug": "knowledge-router", "group": "Research and data", "path": "skills/knowledge-router/SKILL.md" },
+        { "name": "web-fetch", "slug": "web-fetch", "group": "Research and data", "path": "skills/web-fetch/SKILL.md" }
+      ]
+    },
+    {
+      "name": "GitHub and delivery",
+      "entries": [
+        { "name": "github-sync", "slug": "github-sync", "group": "GitHub and delivery", "path": "skills/github-sync/SKILL.md" },
+        { "name": "issue-intake", "slug": "issue-intake", "group": "GitHub and delivery", "path": "skills/issue-intake/SKILL.md" },
+        { "name": "issue-triage", "slug": "issue-triage", "group": "GitHub and delivery", "path": "skills/issue-triage/SKILL.md" },
+        { "name": "issue-to-pr", "slug": "issue-to-pr", "group": "GitHub and delivery", "path": "skills/issue-to-pr/SKILL.md" },
+        { "name": "release", "slug": "release", "group": "GitHub and delivery", "path": "skills/release/SKILL.md" }
+      ]
+    },
+    {
+      "name": "Safety and review",
+      "entries": [
+        { "name": "audit-receipt", "slug": "audit-receipt", "group": "Safety and review", "path": "skills/audit-receipt/SKILL.md" },
+        { "name": "cve-audit", "slug": "cve-audit", "group": "Safety and review", "path": "skills/cve-audit/SKILL.md" },
+        { "name": "least-privilege", "slug": "least-privilege", "group": "Safety and review", "path": "skills/least-privilege/SKILL.md" },
+        { "name": "policy-author", "slug": "policy-author", "group": "Safety and review", "path": "skills/policy-author/SKILL.md" },
+        { "name": "review-receipt", "slug": "review-receipt", "group": "Safety and review", "path": "skills/review-receipt/SKILL.md" },
+        { "name": "sandbox-harden", "slug": "sandbox-harden", "group": "Safety and review", "path": "skills/sandbox-harden/SKILL.md" }
+      ]
+    },
+    {
+      "name": "Outbound and tooling",
+      "entries": [
+        { "name": "governed-outbound", "slug": "governed-outbound", "group": "Outbound and tooling", "path": "skills/governed-outbound/SKILL.md" },
+        { "name": "run-history", "slug": "run-history", "group": "Outbound and tooling", "path": "skills/run-history/SKILL.md" },
+        { "name": "sourcey", "slug": "sourcey", "group": "Outbound and tooling", "path": "skills/sourcey/SKILL.md" }
+      ]
+    }
+  ]
+}