Merge pull request #12 from hinter-net/llms.txt

Create llms.txt and llms.txt-full during build based on the sidebar

Author: bbenligiray

.gitignore

@@ -7,6 +7,8 @@
 # Generated files
 .docusaurus
 .cache-loader
+/static/llms.txt
+/static/llms-full.txt
 
 # Misc
 .DS_Store

.prettierignore

@@ -7,6 +7,8 @@
 # Generated files
 .docusaurus
 .cache-loader
+/static/llms.txt
+/static/llms-full.txt
 
 # Misc
 .DS_Store

package.json

@@ -3,11 +3,13 @@
   "version": "0.0.0",
   "private": true,
   "scripts": {
-    "build": "docusaurus build",
+    "build": "npm run build:llms && npm run build:docusaurus",
+    "build:docusaurus": "docusaurus build",
+    "build:llms": "node scripts/create-llms-files.js",
     "deploy": "docusaurus deploy",
     "prettier": "prettier --write \"./**/*.md\"",
     "prettier:check": "prettier --check \"./**/*.md\"",
-    "start": "docusaurus start"
+    "start": "npm run build:llms && docusaurus start"
   },
   "dependencies": {
     "@docusaurus/core": "3.8.1",

scripts/create-llms-files.js

@@ -0,0 +1,83 @@
+const fs = require('fs');
+const path = require('path');
+const sidebars = require('../sidebars.js').default;
+
+const docsDir = path.join(__dirname, '..', 'docs');
+const staticDir = path.join(__dirname, '..', 'static');
+const llmsTxtPath = path.join(staticDir, 'llms.txt');
+const llmsFullTxtPath = path.join(staticDir, 'llms-full.txt');
+
+function getMarkdownFiles(items) {
+  let files = [];
+  for (const item of items) {
+    if (typeof item === 'string') {
+      files.push(item);
+    } else if (item.type === 'category') {
+      files = files.concat(getMarkdownFiles(item.items));
+    }
+  }
+  return files;
+}
+
+function generateLlmsTxt() {
+  let content = '# Hinter Net\n\n';
+  content += `> Hinter Net is a peer-to-peer network for collaborative intelligence. It combines a secure, decentralized communication protocol (hinter-core) with a powerful, AI-assisted command center (hinter-cline) to help users build high-trust, private networks for sharing and contextualizing sensitive information.\n\n`;
+
+  const order = ['hinterNet', 'hinterCore', 'hinterCline'];
+
+  for (const key of order) {
+    const sidebar = sidebars[key];
+    if (sidebar) {
+      const category = sidebar.find(item => item.type === 'category');
+      if (category) {
+        content += `## ${category.label}\n\n`;
+        const files = getMarkdownFiles(category.items);
+        for (const file of files) {
+          const filePath = path.join(docsDir, `${file}.md`);
+          if (fs.existsSync(filePath)) {
+            const fileContent = fs.readFileSync(filePath, 'utf-8');
+            const match = fileContent.match(/---\s*sidebar_label:\s*"(.*?)"\s*---/);
+            const title = match ? match[1] : path.basename(file, '.md');
+            content += `- [${title}](/docs/${file}.md)\n`;
+          }
+        }
+        content += '\n';
+      }
+    }
+  }
+
+  fs.writeFileSync(llmsTxtPath, content);
+  console.log(`Successfully created ${llmsTxtPath}`);
+}
+
+function generateLlmsFullTxt() {
+  const llmsTxtContent = fs.readFileSync(llmsTxtPath, 'utf-8');
+  const links = llmsTxtContent.match(/- \[(.*?)\]\((.*?)\)/g);
+  let fullContent = '';
+
+  if (links) {
+    for (const link of links) {
+      const match = link.match(/- \[(.*?)\]\((.*?)\)/);
+      if (match) {
+        const url = match[2];
+        const filePath = path.join(__dirname, '..', url.substring(1));
+        if (fs.existsSync(filePath)) {
+          fullContent += fs.readFileSync(filePath, 'utf-8') + '\n\n';
+        }
+      }
+    }
+  }
+
+  fs.writeFileSync(llmsFullTxtPath, fullContent);
+  console.log(`Successfully created ${llmsFullTxtPath}`);
+}
+
+try {
+  if (!fs.existsSync(staticDir)) {
+    fs.mkdirSync(staticDir, { recursive: true });
+  }
+  generateLlmsTxt();
+  generateLlmsFullTxt();
+} catch (err) {
+  console.error('Error creating llms files:', err);
+}