From 05a6da8ed514334402386a28048571b971c39fa9 Mon Sep 17 00:00:00 2001 From: bbenligiray Date: Mon, 25 Aug 2025 15:05:40 +0300 Subject: [PATCH 1/6] Add script for generating llms.txt and llms-full.txt --- .gitignore | 3 + .prettierignore | 3 + docs/dapps/index.md | 6 ++ docs/oev-searchers/index.md | 6 ++ package.json | 5 +- scripts/generate-llms-files.js | 108 +++++++++++++++++++++++++++++++++ 6 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 scripts/generate-llms-files.js diff --git a/.gitignore b/.gitignore index 14230b1f..dd9c3394 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,9 @@ docs/.vitepress/dist/ docs/.vitepress/.temp .firebase/ +docs/public/llms.txt +docs/public/llms-full.txt + /indexes/content-files/ yarn-error.log diff --git a/.prettierignore b/.prettierignore index 8213aae5..2a17a3ed 100644 --- a/.prettierignore +++ b/.prettierignore @@ -14,6 +14,9 @@ pnpm-lock.yaml /docs/public/img /docs/public/api3-whitepaper-v1.0.3.pdf +docs/public/llms.txt +docs/public/llms-full.txt + # assets /docs/reference/ois/latest/assets /docs/reference/airnode/latest/assets diff --git a/docs/dapps/index.md b/docs/dapps/index.md index 1b615c3b..45534f7b 100644 --- a/docs/dapps/index.md +++ b/docs/dapps/index.md @@ -13,6 +13,12 @@ Api3 provides data feeds and pays dApps for using them. 2. Api3 enables the Oracle Extractable Value (OEV) resulting from the usage of these data feeds to be captured, and pays it to the respective dApps in the form of [OEV Rewards.](#oev-rewards) +::: info 💡 Tip + +For quick reference, you can copy-paste [`llms-full.txt`](https://docs.api3.org/llms-full.txt) to your choice of AI assistant. + +::: + ## Api3 Market Liquidity is increasingly shifting to newly launched L2 networks, and dApps that are able to branch out to these more quickly are at a significant competitive advantage. diff --git a/docs/oev-searchers/index.md b/docs/oev-searchers/index.md index 93a2baae..4d380b89 100644 --- a/docs/oev-searchers/index.md +++ b/docs/oev-searchers/index.md @@ -28,6 +28,12 @@ bound by rules enforced on-chain. The auction winner must pay the announced amount, which in return allows them to perform the oracle update and capture profitable opportunities. +::: info 💡 Tip + +For quick reference, you can copy-paste [`llms-full.txt`](https://docs.api3.org/llms-full.txt) to your choice of AI assistant. + +::: + ## Practical example Imagine an overcollateralized lending platform that uses Api3 price feeds. diff --git a/package.json b/package.json index 085f3c6c..3eaf40b8 100644 --- a/package.json +++ b/package.json @@ -6,11 +6,12 @@ "packageManager": "pnpm@9.15.9", "resolutions": {}, "scripts": { - "docs:dev": "pnpm vitepress dev docs", - "docs:build": "pnpm vitepress build docs;", + "docs:dev": "pnpm generate-llms-files && pnpm vitepress dev docs", + "docs:build": "pnpm generate-llms-files && pnpm vitepress build docs;", "docs:serve": "vitepress serve docs --port 8082", "format": "prettier --write --cache \"./**/*.{js,vue,md,json,yaml}\" --log-level silent", "format:check": "prettier --check --cache \"./**/*.{js,vue,md,json,yaml}\"", + "generate-llms-files": "node scripts/generate-llms-files.js", "prepare": "husky", "firebase:emulator": "pnpm docs:build; firebase emulators:start" }, diff --git a/scripts/generate-llms-files.js b/scripts/generate-llms-files.js new file mode 100644 index 00000000..b92ce9cc --- /dev/null +++ b/scripts/generate-llms-files.js @@ -0,0 +1,108 @@ +const fs = require('fs'); +const path = require('path'); +const dappsSidebar = require('../docs/dapps/sidebar.js'); +const oevSearchersSidebar = require('../docs/oev-searchers/sidebar.js'); + +const docsDir = path.join(__dirname, '..', 'docs'); +const staticDir = path.join(__dirname, '..', 'docs', 'public'); +const llmsTxtPath = path.join(staticDir, 'llms.txt'); +const llmsFullTxtPath = path.join(staticDir, 'llms-full.txt'); + +function getMarkdownFiles(items) { + let files = []; + for (const item of items) { + if (item.link) { + files.push(item.link); + } + if (item.items) { + files = files.concat(getMarkdownFiles(item.items)); + } + } + return files; +} + +function generateLlmsTxt() { + let content = '# Api3 Docs\n\n'; + content += `> Api3 Docs helps developers build dApps using Api3 data feeds and searchers recapture oracle extractable value (OEV).\n\n`; + + const sidebars = { + dapps: dappsSidebar, + 'oev-searchers': oevSearchersSidebar, + }; + + for (const key in sidebars) { + const sidebar = sidebars[key]; + if (sidebar) { + content += `## ${key}\n\n`; + const files = getMarkdownFiles(sidebar); + for (const file of files) { + const filePath = path.join(docsDir, `${file}.md`); + if (fs.existsSync(filePath)) { + const fileContent = fs.readFileSync(filePath, 'utf-8'); + const match = fileContent.match(/---\s*title:\s*(.*?)\s*---/); + const title = match ? match[1] : path.basename(file, '.md'); + const url = `https://docs.api3.org${file}`; + content += `- [${title}](${url.replace(/\/$/, '/index')}.md)\n`; + } else { + const indexPath = path.join(docsDir, file, 'index.md'); + if (fs.existsSync(indexPath)) { + const fileContent = fs.readFileSync(indexPath, 'utf-8'); + const match = fileContent.match(/---\s*title:\s*(.*?)\s*---/); + const title = match ? match[1] : path.basename(file); + const url = `https://docs.api3.org${file}`; + content += `- [${title}](${url.replace(/\/$/, '/index.md')})\n`; + } + } + } + content += '\n'; + } + } + + fs.writeFileSync(llmsTxtPath, content); + console.log(`Successfully created ${llmsTxtPath}`); +} + +function generateLlmsFullTxt() { + const llmsTxtContent = fs.readFileSync(llmsTxtPath, 'utf-8'); + const links = llmsTxtContent.match(/- \[(.*?)\]\((.*?)\)/g); + let fullContent = ''; + + if (links) { + for (const link of links) { + const match = link.match(/- \[(.*?)\]\((.*?)\)/); + if (match) { + const url = match[2].replace('https://docs.api3.org', ''); + const filePath = path.join(docsDir, url); + if (fs.existsSync(filePath)) { + let fileContent = fs.readFileSync(filePath, 'utf-8'); + const pageHeaderIndex = fileContent.indexOf('\n\n'); + if (pageHeaderIndex === -1) { + throw new Error(`Could not find PageHeader in ${filePath}`); + } + fileContent = fileContent.substring( + pageHeaderIndex + '\n\n'.length + ); + fullContent += fileContent + '\n\n'; + } + } + } + } + + fs.writeFileSync(llmsFullTxtPath, fullContent); + console.log(`Successfully created ${llmsFullTxtPath}`); +} + +function main() { + try { + if (!fs.existsSync(staticDir)) { + fs.mkdirSync(staticDir, { recursive: true }); + } + generateLlmsTxt(); + generateLlmsFullTxt(); + } catch (err) { + console.error('Error creating llms files:', err); + process.exit(1); + } +} + +main(); From a54c808a7d13d8cd877a7038c35f422ae8d0c20d Mon Sep 17 00:00:00 2001 From: bbenligiray Date: Mon, 25 Aug 2025 15:22:00 +0300 Subject: [PATCH 2/6] Use the title instead of the filename to identify links --- scripts/generate-llms-files.js | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/scripts/generate-llms-files.js b/scripts/generate-llms-files.js index b92ce9cc..1e1e4956 100644 --- a/scripts/generate-llms-files.js +++ b/scripts/generate-llms-files.js @@ -39,16 +39,28 @@ function generateLlmsTxt() { const filePath = path.join(docsDir, `${file}.md`); if (fs.existsSync(filePath)) { const fileContent = fs.readFileSync(filePath, 'utf-8'); - const match = fileContent.match(/---\s*title:\s*(.*?)\s*---/); - const title = match ? match[1] : path.basename(file, '.md'); + const lines = fileContent.split('\n'); + let title = path.basename(file, '.md'); + for (const line of lines) { + if (line.startsWith('title: ')) { + title = line.substring('title: '.length); + break; + } + } const url = `https://docs.api3.org${file}`; content += `- [${title}](${url.replace(/\/$/, '/index')}.md)\n`; } else { const indexPath = path.join(docsDir, file, 'index.md'); if (fs.existsSync(indexPath)) { const fileContent = fs.readFileSync(indexPath, 'utf-8'); - const match = fileContent.match(/---\s*title:\s*(.*?)\s*---/); - const title = match ? match[1] : path.basename(file); + const lines = fileContent.split('\n'); + let title = path.basename(file); + for (const line of lines) { + if (line.startsWith('title: ')) { + title = line.substring('title: '.length); + break; + } + } const url = `https://docs.api3.org${file}`; content += `- [${title}](${url.replace(/\/$/, '/index.md')})\n`; } From 84b65cec3c41de795860edede33cc09d6b007e13 Mon Sep 17 00:00:00 2001 From: bbenligiray Date: Mon, 25 Aug 2025 15:23:39 +0300 Subject: [PATCH 3/6] Don't validate llms links --- libs/link-validator-ignore.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libs/link-validator-ignore.json b/libs/link-validator-ignore.json index 938e318c..141c1950 100644 --- a/libs/link-validator-ignore.json +++ b/libs/link-validator-ignore.json @@ -7,5 +7,7 @@ "https://orbitlending.io/", "https://www.coinbase.com/blog/introducing-the-coinbase-price-oracle", "https://x.com", - "https://blastscan.io/address/0x5b0cf2b36a65a6BB085D501B971e4c102B9Cd473#readProxyContract#F17" + "https://blastscan.io/address/0x5b0cf2b36a65a6BB085D501B971e4c102B9Cd473#readProxyContract#F17", + "https://docs.api3.org/llms.txt", + "https://docs.api3.org/llms-full.txt" ] From 5f45d567b3c3192385c2117fa84fd0c3bd4863a0 Mon Sep 17 00:00:00 2001 From: bbenligiray Date: Tue, 26 Aug 2025 16:44:05 +0300 Subject: [PATCH 4/6] Force .html extension on llms.txt links --- scripts/generate-llms-files.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/generate-llms-files.js b/scripts/generate-llms-files.js index 1e1e4956..d09238e6 100644 --- a/scripts/generate-llms-files.js +++ b/scripts/generate-llms-files.js @@ -48,7 +48,7 @@ function generateLlmsTxt() { } } const url = `https://docs.api3.org${file}`; - content += `- [${title}](${url.replace(/\/$/, '/index')}.md)\n`; + content += `- [${title}](${url.replace(/\/$/, '/index')}.html)\n`; } else { const indexPath = path.join(docsDir, file, 'index.md'); if (fs.existsSync(indexPath)) { @@ -62,7 +62,7 @@ function generateLlmsTxt() { } } const url = `https://docs.api3.org${file}`; - content += `- [${title}](${url.replace(/\/$/, '/index.md')})\n`; + content += `- [${title}](${url.replace(/\/$/, '/index.html')})\n`; } } } @@ -83,7 +83,9 @@ function generateLlmsFullTxt() { for (const link of links) { const match = link.match(/- \[(.*?)\]\((.*?)\)/); if (match) { - const url = match[2].replace('https://docs.api3.org', ''); + const url = match[2] + .replace('https://docs.api3.org', '') + .replace('.html', '.md'); const filePath = path.join(docsDir, url); if (fs.existsSync(filePath)) { let fileContent = fs.readFileSync(filePath, 'utf-8'); From 7a91fe2485688f2d3af6ff0b9f6f5fa51cb18fdf Mon Sep 17 00:00:00 2001 From: bbenligiray Date: Tue, 26 Aug 2025 16:47:40 +0300 Subject: [PATCH 5/6] Declare the pageheader string pattern as a variable --- scripts/generate-llms-files.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/generate-llms-files.js b/scripts/generate-llms-files.js index d09238e6..546651b0 100644 --- a/scripts/generate-llms-files.js +++ b/scripts/generate-llms-files.js @@ -78,6 +78,7 @@ function generateLlmsFullTxt() { const llmsTxtContent = fs.readFileSync(llmsTxtPath, 'utf-8'); const links = llmsTxtContent.match(/- \[(.*?)\]\((.*?)\)/g); let fullContent = ''; + const pageHeader = '\n\n'; if (links) { for (const link of links) { @@ -89,12 +90,12 @@ function generateLlmsFullTxt() { const filePath = path.join(docsDir, url); if (fs.existsSync(filePath)) { let fileContent = fs.readFileSync(filePath, 'utf-8'); - const pageHeaderIndex = fileContent.indexOf('\n\n'); + const pageHeaderIndex = fileContent.indexOf(pageHeader); if (pageHeaderIndex === -1) { throw new Error(`Could not find PageHeader in ${filePath}`); } fileContent = fileContent.substring( - pageHeaderIndex + '\n\n'.length + pageHeaderIndex + pageHeader.length ); fullContent += fileContent + '\n\n'; } From 4fc15bd6d790491a8261b679967c5ab9df3f35fd Mon Sep 17 00:00:00 2001 From: bbenligiray Date: Tue, 26 Aug 2025 16:51:16 +0300 Subject: [PATCH 6/6] Add page headers to header text as a clue in llms-full.txt --- scripts/generate-llms-files.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/scripts/generate-llms-files.js b/scripts/generate-llms-files.js index 546651b0..597827d0 100644 --- a/scripts/generate-llms-files.js +++ b/scripts/generate-llms-files.js @@ -90,6 +90,15 @@ function generateLlmsFullTxt() { const filePath = path.join(docsDir, url); if (fs.existsSync(filePath)) { let fileContent = fs.readFileSync(filePath, 'utf-8'); + const lines = fileContent.split('\n'); + let pageHeaderValue = ''; + for (const line of lines) { + if (line.startsWith('pageHeader: ')) { + pageHeaderValue = line.substring('pageHeader: '.length); + break; + } + } + const pageHeaderIndex = fileContent.indexOf(pageHeader); if (pageHeaderIndex === -1) { throw new Error(`Could not find PageHeader in ${filePath}`); @@ -97,6 +106,13 @@ function generateLlmsFullTxt() { fileContent = fileContent.substring( pageHeaderIndex + pageHeader.length ); + const titleMatch = fileContent.match(/# (.*)/); + if (titleMatch && pageHeaderValue) { + fileContent = fileContent.replace( + /# (.*)/, + `# ${titleMatch[1]} (${pageHeaderValue})` + ); + } fullContent += fileContent + '\n\n'; } }