fix(llm): honor llm:false on markdown routes and share the public .md url (#107)

The llms.mdx route served pages that opt out of LLM surfaces with
llm: false frontmatter (changelog entries, cookbook author pages) at
.md and content-negotiated URLs. It now 404s them via
shouldIncludeLLMPage, matching llms-full.txt and the llms.txt
generator.

The LLM share hook returned the internal /llms.mdx rewrite target.
It now returns the public convention: the page path with a .md
suffix, with the docs root pointing at /llms.txt since the landing
page has no markdown source.

Author: nibzard

app/llms.mdx/[[...slug]]/route.ts

@@ -1,6 +1,6 @@
 import { notFound } from 'next/navigation';
 import { type NextRequest, NextResponse } from 'next/server';
-import { getLLMText } from '@/lib/get-llm-text';
+import { getLLMText, shouldIncludeLLMPage } from '@/lib/get-llm-text';
 import { appendMarkdownVaryHeader } from '@/lib/markdown-negotiation';
 import { source } from '@/lib/source';
 
@@ -20,6 +20,9 @@ export async function GET(_req: NextRequest, { params }: { params: Promise<{ slu
   const page = getPage(slug);
   if (!page) notFound();
 
+  // Pages opted out of LLM surfaces (llm: false) are not served as markdown.
+  if (!shouldIncludeLLMPage(page)) notFound();
+
   // This markdown duplicates the canonical HTML page, so keep it out of search
   // results. Crawlers may still fetch it: noindex only suppresses indexing.
   const headers = new Headers({

hooks/use-llms-txt.ts

@@ -47,24 +47,27 @@ export function useLLMsTxt() {
   });
 }
 
-// Hook for getting the current page's markdown URL
-export function useCurrentPageMarkdown() {
-  const pathname = usePathname();
+// Builds the public markdown URL for a page: the page path with a .md suffix
+// (e.g. /overview/intro.md). The docs root has no markdown page, so it points
+// at the /llms.txt index instead. Pure so the convention is unit-testable.
+export function getPublicMarkdownUrl(pathname: string | null, origin: string) {
+  if (!pathname) return "";
 
-  const getMarkdownUrl = () => {
-    if (!pathname) return "";
+  // Remove /docs prefix if present
+  let basePath = pathname;
+  if (basePath.startsWith("/docs")) {
+    basePath = basePath.substring(5);
+  }
 
-    // Remove /docs prefix if present
-    let basePath = pathname;
-    if (basePath.startsWith("/docs")) {
-      basePath = basePath.substring(5);
-    }
+  const mdPath = basePath.startsWith("/") ? basePath : "/" + basePath;
 
-    const mdPath = basePath.startsWith("/") ? basePath : "/" + basePath;
+  return mdPath === "/" ? `${origin}/llms.txt` : `${origin}${mdPath}.md`;
+}
 
-    const baseUrl = typeof window !== "undefined" ? window.location.origin : "";
-    return `${baseUrl}/llms.mdx${mdPath === "/" ? "" : mdPath}`;
-  };
+// Hook for getting the current page's markdown URL
+export function useCurrentPageMarkdown() {
+  const pathname = usePathname();
 
-  return getMarkdownUrl();
+  const origin = typeof window !== "undefined" ? window.location.origin : "";
+  return getPublicMarkdownUrl(pathname, origin);
 }

tests/e2e/llm-endpoints.test.ts

@@ -472,6 +472,13 @@ describe('.md suffix end-to-end', () => {
     expect(response.status).toBe(404);
   });
 
+  test('returns 404 for an llm: false page at its .md URL', async () => {
+    const response = await fetch(`${BASE_URL}/cookbook/authors/hussufo.md`, {
+      headers: BROWSER_HEADERS,
+    });
+    expect(response.status).toBe(404);
+  });
+
   test('still serves HTML at the canonical URL for browsers', async () => {
     const response = await fetch(`${BASE_URL}/overview/sessions-api/quickstart`, {
       headers: BROWSER_HEADERS,

tests/get-llm-text.test.ts

@@ -1,7 +1,7 @@
 // ABOUTME: Tests for getLLMText, which renders a page as LLM-facing markdown
 // ABOUTME: with an absolute URL and an optional llms.txt index pointer.
 import { describe, expect, test } from 'bun:test';
-import { getLLMText, LLMS_INDEX_POINTER } from '../lib/get-llm-text';
+import { getLLMText, LLMS_INDEX_POINTER, shouldIncludeLLMPage } from '../lib/get-llm-text';
 
 // Minimal stand-in for a fumadocs page; typed any since building a real
 // InferPageType requires the full fumadocs loader.
@@ -16,6 +16,26 @@ function fakePage(overrides: Record<string, unknown> = {}): any {
   };
 }
 
+describe('shouldIncludeLLMPage', () => {
+  test('includes pages by default', () => {
+    expect(shouldIncludeLLMPage(fakePage())).toBe(true);
+  });
+
+  test('excludes pages with llm: false in page data', () => {
+    expect(shouldIncludeLLMPage(fakePage({ llm: false }))).toBe(false);
+  });
+
+  test('excludes pages with llm: false in raw frontmatter', () => {
+    const content = ['---', 'title: Hidden', 'llm: false', '---', '', 'Body.'].join('\n');
+    expect(shouldIncludeLLMPage(fakePage({ content }))).toBe(false);
+  });
+
+  test('keeps pages whose frontmatter sets llm: true', () => {
+    const content = ['---', 'title: Visible', 'llm: true', '---', '', 'Body.'].join('\n');
+    expect(shouldIncludeLLMPage(fakePage({ content }))).toBe(true);
+  });
+});
+
 describe('getLLMText', () => {
   test('renders the title and an absolute URL', async () => {
     const text = await getLLMText(fakePage());

tests/use-llms-txt.test.ts

@@ -0,0 +1,32 @@
+// ABOUTME: Tests the public markdown share URL built for the LLM share UI:
+// ABOUTME: page path + .md suffix, with the docs root pointing at /llms.txt.
+import { describe, expect, test } from 'bun:test';
+import { getPublicMarkdownUrl } from '../hooks/use-llms-txt';
+
+const ORIGIN = 'https://docs.steel.dev';
+
+describe('getPublicMarkdownUrl', () => {
+  test('appends .md to the page path', () => {
+    expect(getPublicMarkdownUrl('/overview/sessions-api/quickstart', ORIGIN)).toBe(
+      'https://docs.steel.dev/overview/sessions-api/quickstart.md',
+    );
+  });
+
+  test('never exposes the internal /llms.mdx rewrite target', () => {
+    expect(getPublicMarkdownUrl('/overview/intro', ORIGIN)).not.toContain('/llms.mdx');
+  });
+
+  test('strips a legacy /docs prefix before appending .md', () => {
+    expect(getPublicMarkdownUrl('/docs/overview/intro', ORIGIN)).toBe(
+      'https://docs.steel.dev/overview/intro.md',
+    );
+  });
+
+  test('points the docs root at the llms.txt index', () => {
+    expect(getPublicMarkdownUrl('/', ORIGIN)).toBe('https://docs.steel.dev/llms.txt');
+  });
+
+  test('returns an empty string without a pathname', () => {
+    expect(getPublicMarkdownUrl(null, ORIGIN)).toBe('');
+  });
+});