Add llms.txt

- Fixes #2873

Co-Authored-By: sketch <[email protected]>
Change-ID: s2e8abba011e787cbk

Author: pokey

packages/cursorless-org/README.md

@@ -20,3 +20,16 @@ To learn more about Next.js, take a look at the following resources:
 - [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
 
 You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome!
+
+
+## LLMs Training Data
+
+During the build process, an `llms.txt` file is automatically generated in the output directory. This file contains a concatenation of all markdown files from the documentation directory (`packages/cursorless-org-docs/src/docs/`) and is used for LLM training and reference.
+
+You can find this file at `out/llms.txt` after running the build process.
+
+To generate this file manually without running a full build, you can run:
+
+```sh
+pnpm generate-llms
+```

packages/cursorless-org/package.json

@@ -5,15 +5,17 @@
   "private": true,
   "scripts": {
     "dev": "next dev",
-    "build": "next build",
+    "build": "next build && pnpm generate-llms",
     "start": "http-server out -a 127.0.0.1 -p 8080",
     "lint": "next lint",
     "compile": "tsc --build",
     "watch": "tsc --build --watch",
-    "clean": "rm -rf ./out tsconfig.tsbuildinfo ./dist ./build"
+    "clean": "rm -rf ./out tsconfig.tsbuildinfo ./dist ./build",
+    "generate-llms": "my-ts-node ./src/scripts/generateLlmsTxt.ts"
   },
   "dependencies": {
     "@cursorless/cheatsheet": "workspace:*",
+    "@cursorless/common": "workspace:*",
     "@mdx-js/loader": "3.0.1",
     "@mdx-js/react": "3.0.1",
     "@next/mdx": "15.3.2",

packages/cursorless-org/src/scripts/generateLlmsTxt.ts

@@ -0,0 +1,96 @@
+#!/usr/bin/env node
+
+/**
+ * Script to generate llms.txt by concatenating all markdown files in the docs directory.
+ * This file is used for LLM training and reference.
+ */
+
+import fs from "fs/promises";
+import path from "path";
+import { fileURLToPath } from "url";
+
+// For CommonJS compatibility
+declare const __dirname: string;
+
+/**
+ * Recursively find all markdown files in a directory
+ * @param dir The directory to search
+ * @returns Array of file paths
+ */
+async function findMarkdownFiles(dir: string): Promise<string[]> {
+  const entries = await fs.readdir(dir, { withFileTypes: true });
+  const filesPromises = entries.map(async (entry) => {
+    const entryPath = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      return findMarkdownFiles(entryPath);
+    } else if (entry.isFile() && (entry.name.endsWith(".md") || entry.name.endsWith(".mdx"))) {
+      return [entryPath];
+    }
+    return [];
+  });
+  
+  const files = await Promise.all(filesPromises);
+  return files.flat();
+}
+
+/**
+ * Main function to generate llms.txt
+ */
+export async function generateLlmsTxt(): Promise<void> {
+  // Get the directory of the source file
+  let scriptDir: string;
+  try {
+    // ESM approach
+    const __filename = fileURLToPath(import.meta.url);
+    scriptDir = path.dirname(__filename);
+  } catch (e) {
+    // Fallback for CommonJS
+    scriptDir = __dirname;
+  }
+  
+  // Get repo root from environment variable or fallback to relative path
+  const repoRoot = process.env.CURSORLESS_REPO_ROOT || path.resolve(scriptDir, "../../../");
+  
+  // Path to docs directory and output file
+  const docsDir = path.resolve(repoRoot, "packages/cursorless-org-docs/src/docs");
+  const outputFile = path.resolve(scriptDir, "../../out/llms.txt");
+  
+  console.log(`Generating llms.txt from markdown files in ${docsDir}`);
+  
+  try {
+    // Create the output directory if it doesn't exist
+    try {
+      await fs.mkdir(path.dirname(outputFile), { recursive: true });
+    } catch (err: any) {
+      if (err.code !== "EEXIST") throw err;
+    }
+    
+    // Find all markdown files
+    const markdownFiles = await findMarkdownFiles(docsDir);
+    console.log(`Found ${markdownFiles.length} markdown files`);
+    
+    // Read and concatenate all markdown files
+    let llmsContent = "# Cursorless Documentation\n\nThis file is auto-generated from all Markdown files in the Cursorless documentation.\n";
+    
+    for (const filePath of markdownFiles) {
+      // Add file path as a comment for reference
+      const relativePath = path.relative(repoRoot, filePath);
+      llmsContent += `\n\n<!-- File: ${relativePath} -->\n\n`;
+      
+      // Read and append the file content
+      const fileContent = await fs.readFile(filePath, "utf8");
+      llmsContent += fileContent;
+    }
+    
+    // Write the concatenated content to llms.txt
+    await fs.writeFile(outputFile, llmsContent.trim());
+    
+    console.log(`Successfully generated llms.txt at ${outputFile}`);
+  } catch (error) {
+    console.error("Error generating llms.txt:", error);
+    process.exit(1);
+  }
+}
+
+// Run the main function directly
+generateLlmsTxt();