diff --git a/app/llms.mdx/[[...slug]]/route.ts b/app/llms.mdx/[[...slug]]/route.ts index 3ea37257..c7d923bf 100644 --- a/app/llms.mdx/[[...slug]]/route.ts +++ b/app/llms.mdx/[[...slug]]/route.ts @@ -1,6 +1,6 @@ import { notFound } from 'next/navigation'; import { type NextRequest, NextResponse } from 'next/server'; -import { getLLMText } from '@/lib/get-llm-text'; +import { getLLMText, shouldIncludeLLMPage } from '@/lib/get-llm-text'; import { appendMarkdownVaryHeader } from '@/lib/markdown-negotiation'; import { source } from '@/lib/source'; @@ -20,6 +20,9 @@ export async function GET(_req: NextRequest, { params }: { params: Promise<{ slu const page = getPage(slug); if (!page) notFound(); + // Pages opted out of LLM surfaces (llm: false) are not served as markdown. + if (!shouldIncludeLLMPage(page)) notFound(); + // This markdown duplicates the canonical HTML page, so keep it out of search // results. Crawlers may still fetch it: noindex only suppresses indexing. const headers = new Headers({ diff --git a/hooks/use-llms-txt.ts b/hooks/use-llms-txt.ts index b0c6cd7d..b7fedd0b 100644 --- a/hooks/use-llms-txt.ts +++ b/hooks/use-llms-txt.ts @@ -47,24 +47,27 @@ export function useLLMsTxt() { }); } -// Hook for getting the current page's markdown URL -export function useCurrentPageMarkdown() { - const pathname = usePathname(); +// Builds the public markdown URL for a page: the page path with a .md suffix +// (e.g. /overview/intro.md). The docs root has no markdown page, so it points +// at the /llms.txt index instead. Pure so the convention is unit-testable. +export function getPublicMarkdownUrl(pathname: string | null, origin: string) { + if (!pathname) return ""; - const getMarkdownUrl = () => { - if (!pathname) return ""; + // Remove /docs prefix if present + let basePath = pathname; + if (basePath.startsWith("/docs")) { + basePath = basePath.substring(5); + } - // Remove /docs prefix if present - let basePath = pathname; - if (basePath.startsWith("/docs")) { - basePath = basePath.substring(5); - } + const mdPath = basePath.startsWith("/") ? basePath : "/" + basePath; - const mdPath = basePath.startsWith("/") ? basePath : "/" + basePath; + return mdPath === "/" ? `${origin}/llms.txt` : `${origin}${mdPath}.md`; +} - const baseUrl = typeof window !== "undefined" ? window.location.origin : ""; - return `${baseUrl}/llms.mdx${mdPath === "/" ? "" : mdPath}`; - }; +// Hook for getting the current page's markdown URL +export function useCurrentPageMarkdown() { + const pathname = usePathname(); - return getMarkdownUrl(); + const origin = typeof window !== "undefined" ? window.location.origin : ""; + return getPublicMarkdownUrl(pathname, origin); } diff --git a/tests/e2e/llm-endpoints.test.ts b/tests/e2e/llm-endpoints.test.ts index ef388db9..235c2453 100644 --- a/tests/e2e/llm-endpoints.test.ts +++ b/tests/e2e/llm-endpoints.test.ts @@ -446,6 +446,13 @@ describe('.md suffix end-to-end', () => { expect(response.status).toBe(404); }); + test('returns 404 for an llm: false page at its .md URL', async () => { + const response = await fetch(`${BASE_URL}/cookbook/authors/hussufo.md`, { + headers: BROWSER_HEADERS, + }); + expect(response.status).toBe(404); + }); + test('still serves HTML at the canonical URL for browsers', async () => { const response = await fetch(`${BASE_URL}/overview/sessions-api/quickstart`, { headers: BROWSER_HEADERS, diff --git a/tests/get-llm-text.test.ts b/tests/get-llm-text.test.ts index 07ca4a06..87aefc24 100644 --- a/tests/get-llm-text.test.ts +++ b/tests/get-llm-text.test.ts @@ -1,7 +1,7 @@ // ABOUTME: Tests for getLLMText, which renders a page as LLM-facing markdown // ABOUTME: with an absolute URL and an optional llms.txt index pointer. import { describe, expect, test } from 'bun:test'; -import { getLLMText, LLMS_INDEX_POINTER } from '../lib/get-llm-text'; +import { getLLMText, LLMS_INDEX_POINTER, shouldIncludeLLMPage } from '../lib/get-llm-text'; // Minimal stand-in for a fumadocs page; typed any since building a real // InferPageType requires the full fumadocs loader. @@ -16,6 +16,26 @@ function fakePage(overrides: Record = {}): any { }; } +describe('shouldIncludeLLMPage', () => { + test('includes pages by default', () => { + expect(shouldIncludeLLMPage(fakePage())).toBe(true); + }); + + test('excludes pages with llm: false in page data', () => { + expect(shouldIncludeLLMPage(fakePage({ llm: false }))).toBe(false); + }); + + test('excludes pages with llm: false in raw frontmatter', () => { + const content = ['---', 'title: Hidden', 'llm: false', '---', '', 'Body.'].join('\n'); + expect(shouldIncludeLLMPage(fakePage({ content }))).toBe(false); + }); + + test('keeps pages whose frontmatter sets llm: true', () => { + const content = ['---', 'title: Visible', 'llm: true', '---', '', 'Body.'].join('\n'); + expect(shouldIncludeLLMPage(fakePage({ content }))).toBe(true); + }); +}); + describe('getLLMText', () => { test('renders the title and an absolute URL', async () => { const text = await getLLMText(fakePage()); diff --git a/tests/use-llms-txt.test.ts b/tests/use-llms-txt.test.ts new file mode 100644 index 00000000..4815d9a7 --- /dev/null +++ b/tests/use-llms-txt.test.ts @@ -0,0 +1,32 @@ +// ABOUTME: Tests the public markdown share URL built for the LLM share UI: +// ABOUTME: page path + .md suffix, with the docs root pointing at /llms.txt. +import { describe, expect, test } from 'bun:test'; +import { getPublicMarkdownUrl } from '../hooks/use-llms-txt'; + +const ORIGIN = 'https://docs.steel.dev'; + +describe('getPublicMarkdownUrl', () => { + test('appends .md to the page path', () => { + expect(getPublicMarkdownUrl('/overview/sessions-api/quickstart', ORIGIN)).toBe( + 'https://docs.steel.dev/overview/sessions-api/quickstart.md', + ); + }); + + test('never exposes the internal /llms.mdx rewrite target', () => { + expect(getPublicMarkdownUrl('/overview/intro', ORIGIN)).not.toContain('/llms.mdx'); + }); + + test('strips a legacy /docs prefix before appending .md', () => { + expect(getPublicMarkdownUrl('/docs/overview/intro', ORIGIN)).toBe( + 'https://docs.steel.dev/overview/intro.md', + ); + }); + + test('points the docs root at the llms.txt index', () => { + expect(getPublicMarkdownUrl('/', ORIGIN)).toBe('https://docs.steel.dev/llms.txt'); + }); + + test('returns an empty string without a pathname', () => { + expect(getPublicMarkdownUrl(null, ORIGIN)).toBe(''); + }); +});