Skip to content

Commit 4b6631a

Browse files
nibzardnibzard
authored
feat(llms): serve markdown at .md-suffixed docs URLs (#81)
* feat(llms): serve markdown at .md-suffixed docs URLs Coding agents guess <docs-url>.md per the llms.txt convention; those requests 404'd because per-page markdown was only reachable at the internal /llms.mdx/<slug> route. The middleware now rewrites any .md-suffixed docs path to that route unconditionally. Moves the path-exclusion logic from middleware.ts into lib/markdown-negotiation.ts so the new resolveMarkdownPath helper can reuse it and both are unit-testable. * fix(tests): make e2e server probe independent of generated files The readiness check polled /llms.txt, which is gitignored and only exists after bun run generate-llms; CI runs bun test before that step, so the probe 404'd until the beforeAll timeout. Treat any HTTP response as server-ready instead. * fix(tests): wait for dev server exit before later test files run bun test runs every file in one process; the e2e teardown sent SIGTERM without awaiting exit, so the dev server could still hold CPU while the Chromium imagegen tests rendered, timing one out on slow CI runners.
1 parent a4d2505 commit 4b6631a

5 files changed

Lines changed: 234 additions & 51 deletions

File tree

‎lib/markdown-negotiation.ts‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,65 @@ export const MARKDOWN_USER_AGENT_SUBSTRINGS = [
3030

3131
const MARKDOWN_VARY_HEADERS = ['Accept', 'User-Agent'];
3232

33+
const EXCLUDED_EXACT_PATHS = new Set([
34+
'/.well-known/llms.txt',
35+
'/favicon.ico',
36+
'/llms-full.txt',
37+
'/llms.txt',
38+
'/overview/llms-full.txt',
39+
'/robots.txt',
40+
'/sitemap.xml',
41+
]);
42+
43+
const EXCLUDED_PATH_PREFIXES = ['/_next', '/api', '/llms.mdx', '/og'];
44+
const EXCLUDED_ASSET_EXTENSIONS = new Set([
45+
'.avif',
46+
'.css',
47+
'.gif',
48+
'.ico',
49+
'.jpeg',
50+
'.jpg',
51+
'.js',
52+
'.json',
53+
'.map',
54+
'.otf',
55+
'.pdf',
56+
'.png',
57+
'.svg',
58+
'.ttf',
59+
'.txt',
60+
'.webp',
61+
'.woff',
62+
'.woff2',
63+
'.xml',
64+
'.zip',
65+
]);
66+
67+
function matchesPathPrefix(pathname: string, prefix: string): boolean {
68+
return pathname === prefix || pathname.startsWith(`${prefix}/`);
69+
}
70+
71+
function hasExcludedAssetExtension(pathname: string): boolean {
72+
const extension = pathname.match(/\.[^./]+$/)?.[0]?.toLowerCase();
73+
return extension ? EXCLUDED_ASSET_EXTENSIONS.has(extension) : false;
74+
}
75+
76+
export function isNegotiableDocsPath(pathname: string): boolean {
77+
if (EXCLUDED_EXACT_PATHS.has(pathname)) return false;
78+
if (EXCLUDED_PATH_PREFIXES.some((prefix) => matchesPathPrefix(pathname, prefix))) return false;
79+
80+
return !hasExcludedAssetExtension(pathname);
81+
}
82+
83+
export function resolveMarkdownPath(pathname: string): string | null {
84+
if (!pathname.endsWith('.md')) return null;
85+
86+
const stripped = pathname.slice(0, -'.md'.length);
87+
if (!stripped || stripped === '/') return null;
88+
89+
return isNegotiableDocsPath(stripped) ? stripped : null;
90+
}
91+
3392
function acceptsMarkdownType(mediaType: string): boolean {
3493
return MARKDOWN_ACCEPT_TYPES.has(mediaType) || mediaType.endsWith('+markdown');
3594
}

‎middleware.ts‎

Lines changed: 14 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,62 +1,17 @@
11
import type { NextRequest } from 'next/server';
22
import { NextResponse } from 'next/server';
3-
import { appendMarkdownVaryHeader, shouldServeMarkdown } from '@/lib/markdown-negotiation';
4-
5-
const EXCLUDED_EXACT_PATHS = new Set([
6-
'/.well-known/llms.txt',
7-
'/favicon.ico',
8-
'/llms-full.txt',
9-
'/llms.txt',
10-
'/overview/llms-full.txt',
11-
'/robots.txt',
12-
'/sitemap.xml',
13-
]);
14-
15-
const EXCLUDED_PATH_PREFIXES = ['/_next', '/api', '/llms.mdx', '/og'];
16-
const EXCLUDED_ASSET_EXTENSIONS = new Set([
17-
'.avif',
18-
'.css',
19-
'.gif',
20-
'.ico',
21-
'.jpeg',
22-
'.jpg',
23-
'.js',
24-
'.json',
25-
'.map',
26-
'.otf',
27-
'.pdf',
28-
'.png',
29-
'.svg',
30-
'.ttf',
31-
'.txt',
32-
'.webp',
33-
'.woff',
34-
'.woff2',
35-
'.xml',
36-
'.zip',
37-
]);
3+
import {
4+
appendMarkdownVaryHeader,
5+
isNegotiableDocsPath,
6+
resolveMarkdownPath,
7+
shouldServeMarkdown,
8+
} from '@/lib/markdown-negotiation';
389

3910
function isProgrammaticClient(request: NextRequest): boolean {
4011
// Browsers always send Sec-Fetch-Dest; curl/WebFetch/python-requests do not
4112
return !request.headers.has('sec-fetch-dest');
4213
}
4314

44-
function matchesPathPrefix(pathname: string, prefix: string): boolean {
45-
return pathname === prefix || pathname.startsWith(`${prefix}/`);
46-
}
47-
48-
function hasExcludedAssetExtension(pathname: string): boolean {
49-
const extension = pathname.match(/\.[^./]+$/)?.[0]?.toLowerCase();
50-
return extension ? EXCLUDED_ASSET_EXTENSIONS.has(extension) : false;
51-
}
52-
53-
function isNegotiableDocsPath(pathname: string): boolean {
54-
if (EXCLUDED_EXACT_PATHS.has(pathname)) return false;
55-
if (EXCLUDED_PATH_PREFIXES.some((prefix) => matchesPathPrefix(pathname, prefix))) return false;
56-
57-
return !hasExcludedAssetExtension(pathname);
58-
}
59-
6015
function isNegotiableMethod(method: string): boolean {
6116
return method === 'GET' || method === 'HEAD';
6217
}
@@ -73,6 +28,14 @@ export default function middleware(request: NextRequest) {
7328
return NextResponse.next();
7429
}
7530

31+
// An explicit .md request gets markdown unconditionally, no header sniffing
32+
const markdownPath = resolveMarkdownPath(pathname);
33+
if (markdownPath) {
34+
const rewriteUrl = request.nextUrl.clone();
35+
rewriteUrl.pathname = `/llms.mdx${markdownPath}`;
36+
return NextResponse.rewrite(rewriteUrl);
37+
}
38+
7639
const wantsMarkdown = shouldServeMarkdown(request.headers);
7740

7841
if (pathname === '/' && (wantsMarkdown || isProgrammaticClient(request))) {

‎tests/e2e/md-suffix.test.ts‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
// ABOUTME: End-to-end tests that boot the Next.js dev server and verify
2+
// ABOUTME: .md-suffixed docs URLs serve markdown while canonical URLs stay HTML.
3+
import { afterAll, beforeAll, describe, expect, setDefaultTimeout, test } from 'bun:test';
4+
import { fileURLToPath } from 'node:url';
5+
6+
// The dev server compiles the middleware and markdown route on first request.
7+
setDefaultTimeout(120000);
8+
9+
const PROJECT_ROOT = fileURLToPath(new URL('../..', import.meta.url));
10+
const PORT = 3300 + Math.floor(Math.random() * 300);
11+
const BASE_URL = `http://localhost:${PORT}`;
12+
13+
const BROWSER_HEADERS = {
14+
accept: 'text/html,application/xhtml+xml',
15+
'sec-fetch-dest': 'document',
16+
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Safari/605.1.15',
17+
};
18+
19+
let server: ReturnType<typeof Bun.spawn>;
20+
21+
async function waitForServer(): Promise<void> {
22+
const deadline = Date.now() + 90000;
23+
while (Date.now() < deadline) {
24+
try {
25+
// Any HTTP response means the server is up; fetch throws until the
26+
// port accepts connections. Don't probe generated files like
27+
// /llms.txt, which don't exist in CI when tests run.
28+
await fetch(BASE_URL, { headers: BROWSER_HEADERS });
29+
return;
30+
} catch {
31+
// Server not accepting connections yet
32+
}
33+
await Bun.sleep(1000);
34+
}
35+
throw new Error(`Dev server did not become ready on port ${PORT}`);
36+
}
37+
38+
describe('.md suffix end-to-end', () => {
39+
beforeAll(async () => {
40+
server = Bun.spawn(['bunx', 'next', 'dev', '--turbopack', '-p', String(PORT)], {
41+
cwd: PROJECT_ROOT,
42+
stdout: 'ignore',
43+
stderr: 'ignore',
44+
});
45+
await waitForServer();
46+
});
47+
48+
afterAll(async () => {
49+
// Wait for the process to fully exit so the dev server's CPU and port
50+
// are released before later test files (Chromium renders) start.
51+
server?.kill();
52+
await server?.exited;
53+
});
54+
55+
test('serves markdown at a .md-suffixed docs URL', async () => {
56+
const response = await fetch(`${BASE_URL}/overview/sessions-api/quickstart.md`, {
57+
headers: BROWSER_HEADERS,
58+
});
59+
expect(response.status).toBe(200);
60+
expect(response.headers.get('content-type')).toStartWith('text/markdown');
61+
expect(await response.text()).toStartWith('# Quickstart');
62+
});
63+
64+
test('returns 404 for a .md URL with no matching page', async () => {
65+
const response = await fetch(`${BASE_URL}/nonexistent-page.md`, {
66+
headers: BROWSER_HEADERS,
67+
});
68+
expect(response.status).toBe(404);
69+
});
70+
71+
test('still serves HTML at the canonical URL for browsers', async () => {
72+
const response = await fetch(`${BASE_URL}/overview/sessions-api/quickstart`, {
73+
headers: BROWSER_HEADERS,
74+
});
75+
expect(response.status).toBe(200);
76+
expect(response.headers.get('content-type')).toStartWith('text/html');
77+
});
78+
});

‎tests/markdown-negotiation.test.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
// ABOUTME: Tests for resolveMarkdownPath, which maps .md-suffixed docs URLs
2+
// ABOUTME: to their canonical path so middleware can serve the markdown version.
3+
import { describe, expect, test } from 'bun:test';
4+
import { resolveMarkdownPath } from '../lib/markdown-negotiation';
5+
6+
describe('resolveMarkdownPath', () => {
7+
test('strips .md from a docs page path', () => {
8+
expect(resolveMarkdownPath('/overview/sessions-api/quickstart.md')).toBe(
9+
'/overview/sessions-api/quickstart',
10+
);
11+
});
12+
13+
test('strips .md from a top-level section path', () => {
14+
expect(resolveMarkdownPath('/cookbook.md')).toBe('/cookbook');
15+
});
16+
17+
test('returns null for paths without the .md suffix', () => {
18+
expect(resolveMarkdownPath('/overview/sessions-api/quickstart')).toBeNull();
19+
expect(resolveMarkdownPath('/')).toBeNull();
20+
});
21+
22+
test('returns null for a bare /.md', () => {
23+
expect(resolveMarkdownPath('/.md')).toBeNull();
24+
});
25+
26+
test('returns null when the stripped path is an excluded exact path', () => {
27+
expect(resolveMarkdownPath('/llms-full.txt.md')).toBeNull();
28+
expect(resolveMarkdownPath('/llms.txt.md')).toBeNull();
29+
});
30+
31+
test('returns null when the stripped path is under an excluded prefix', () => {
32+
expect(resolveMarkdownPath('/llms.mdx/overview.md')).toBeNull();
33+
expect(resolveMarkdownPath('/api/search.md')).toBeNull();
34+
});
35+
36+
test('returns null when the stripped path is a static asset', () => {
37+
expect(resolveMarkdownPath('/images/logo.png.md')).toBeNull();
38+
});
39+
});

‎tests/middleware.test.ts‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
// ABOUTME: Integration tests for the docs middleware, verifying .md-suffixed
2+
// ABOUTME: URLs rewrite to the /llms.mdx markdown route and others pass through.
3+
import { describe, expect, test } from 'bun:test';
4+
import { NextRequest } from 'next/server';
5+
import middleware from '../middleware';
6+
7+
function browserRequest(url: string): NextRequest {
8+
return new NextRequest(url, {
9+
headers: {
10+
accept: 'text/html,application/xhtml+xml',
11+
'sec-fetch-dest': 'document',
12+
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Safari/605.1.15',
13+
},
14+
});
15+
}
16+
17+
describe('middleware .md suffix handling', () => {
18+
test('rewrites a .md-suffixed docs URL to the llms.mdx route', () => {
19+
const response = middleware(browserRequest('http://localhost/overview/steel-cli.md'));
20+
const rewrite = response.headers.get('x-middleware-rewrite');
21+
expect(rewrite).not.toBeNull();
22+
expect(new URL(rewrite as string).pathname).toBe('/llms.mdx/overview/steel-cli');
23+
});
24+
25+
test('rewrites .md URLs even for markdown user agents', () => {
26+
const request = new NextRequest('http://localhost/cookbook/playwright.md', {
27+
headers: { 'user-agent': 'claude-code/1.0' },
28+
});
29+
const response = middleware(request);
30+
const rewrite = response.headers.get('x-middleware-rewrite');
31+
expect(rewrite).not.toBeNull();
32+
expect(new URL(rewrite as string).pathname).toBe('/llms.mdx/cookbook/playwright');
33+
});
34+
35+
test('leaves canonical docs URLs from browsers untouched', () => {
36+
const response = middleware(browserRequest('http://localhost/overview/steel-cli'));
37+
expect(response.headers.get('x-middleware-rewrite')).toBeNull();
38+
});
39+
40+
test('does not rewrite excluded .md paths', () => {
41+
const response = middleware(browserRequest('http://localhost/llms-full.txt.md'));
42+
expect(response.headers.get('x-middleware-rewrite')).toBeNull();
43+
});
44+
});

0 commit comments

Comments
 (0)