feat(http): add markdown content negotiation (#50)

Author: fukouda

app/llms.mdx/[[...slug]]/route.ts

@@ -1,17 +1,30 @@
 import { notFound } from 'next/navigation';
 import { type NextRequest, NextResponse } from 'next/server';
 import { getLLMText } from '@/lib/get-llm-text';
+import { appendMarkdownVaryHeader } from '@/lib/markdown-negotiation';
 import { source } from '@/lib/source';
 
 export const revalidate = false;
 
+function getPage(slug?: string[]) {
+  let page = source.getPage(slug);
+  if (!page && slug?.[0] !== 'en') {
+    page = source.getPage(['en', ...(slug ?? [])]);
+  }
+
+  return page;
+}
+
 export async function GET(_req: NextRequest, { params }: { params: Promise<{ slug?: string[] }> }) {
   const { slug } = await params;
-  const page = source.getPage(slug);
+  const page = getPage(slug);
   if (!page) notFound();
 
+  const headers = new Headers({ 'Content-Type': 'text/markdown; charset=utf-8' });
+  appendMarkdownVaryHeader(headers);
+
   return new NextResponse(await getLLMText(page), {
-    headers: { 'Content-Type': 'text/markdown; charset=utf-8' },
+    headers,
   });
 }
 

lib/markdown-negotiation.ts

@@ -0,0 +1,88 @@
+const MARKDOWN_ACCEPT_TYPES = new Set([
+  'application/markdown',
+  'text/markdown',
+  'text/x-markdown',
+  'text/vnd.daringfireball.markdown',
+]);
+
+export const EXACT_MARKDOWN_USER_AGENTS = [
+  'anthropic-ai',
+  'chatgpt-user',
+  'claudebot',
+  'gptbot',
+  'oai-searchbot',
+  'perplexitybot',
+];
+
+export const MARKDOWN_USER_AGENT_SUBSTRINGS = [
+  'anthropic',
+  'chatgpt',
+  'claude',
+  'copilot',
+  'cursor',
+  'gemini',
+  'gptbot',
+  'mistral',
+  'oai-searchbot',
+  'openai',
+  'perplexity',
+];
+
+const MARKDOWN_VARY_HEADERS = ['Accept', 'User-Agent'];
+
+function acceptsMarkdownType(mediaType: string): boolean {
+  return MARKDOWN_ACCEPT_TYPES.has(mediaType) || mediaType.endsWith('+markdown');
+}
+
+function hasNonZeroQuality(params: string[]): boolean {
+  const qualityParam = params.find((param) => param.toLowerCase().startsWith('q='));
+  if (!qualityParam) return true;
+
+  const quality = Number.parseFloat(qualityParam.slice(2));
+  return Number.isNaN(quality) || quality > 0;
+}
+
+export function acceptsMarkdown(acceptHeader: string | null): boolean {
+  if (!acceptHeader) return false;
+
+  return acceptHeader.split(',').some((entry) => {
+    const [rawMediaType, ...rawParams] = entry.split(';').map((part) => part.trim());
+    const mediaType = rawMediaType.toLowerCase();
+
+    return acceptsMarkdownType(mediaType) && hasNonZeroQuality(rawParams);
+  });
+}
+
+function normalizeUserAgent(userAgent: string): string {
+  return userAgent.toLowerCase().replace(/\s+/g, ' ').trim();
+}
+
+export function isMarkdownUserAgent(userAgentHeader: string | null): boolean {
+  if (!userAgentHeader) return false;
+
+  const userAgent = normalizeUserAgent(userAgentHeader);
+
+  return (
+    EXACT_MARKDOWN_USER_AGENTS.includes(userAgent) ||
+    MARKDOWN_USER_AGENT_SUBSTRINGS.some((match) => userAgent.includes(match))
+  );
+}
+
+export function shouldServeMarkdown(headers: Headers): boolean {
+  return acceptsMarkdown(headers.get('accept')) || isMarkdownUserAgent(headers.get('user-agent'));
+}
+
+export function appendMarkdownVaryHeader(headers: Headers) {
+  const existingValues = new Set(
+    (headers.get('Vary') ?? '')
+      .split(',')
+      .map((value) => value.trim())
+      .filter(Boolean),
+  );
+
+  for (const value of MARKDOWN_VARY_HEADERS) {
+    existingValues.add(value);
+  }
+
+  headers.set('Vary', [...existingValues].join(', '));
+}

middleware.ts

@@ -1,21 +1,97 @@
 import type { NextRequest } from 'next/server';
 import { NextResponse } from 'next/server';
+import { appendMarkdownVaryHeader, shouldServeMarkdown } from '@/lib/markdown-negotiation';
+
+const EXCLUDED_EXACT_PATHS = new Set([
+  '/.well-known/llms.txt',
+  '/favicon.ico',
+  '/llms-full.txt',
+  '/llms.txt',
+  '/overview/llms-full.txt',
+  '/robots.txt',
+  '/sitemap.xml',
+]);
+
+const EXCLUDED_PATH_PREFIXES = ['/_next', '/api', '/llms.mdx', '/og'];
+const EXCLUDED_ASSET_EXTENSIONS = new Set([
+  '.avif',
+  '.css',
+  '.gif',
+  '.ico',
+  '.jpeg',
+  '.jpg',
+  '.js',
+  '.json',
+  '.map',
+  '.otf',
+  '.pdf',
+  '.png',
+  '.svg',
+  '.ttf',
+  '.txt',
+  '.webp',
+  '.woff',
+  '.woff2',
+  '.xml',
+  '.zip',
+]);
 
 function isProgrammaticClient(request: NextRequest): boolean {
   // Browsers always send Sec-Fetch-Dest; curl/WebFetch/python-requests do not
   return !request.headers.has('sec-fetch-dest');
 }
 
+function matchesPathPrefix(pathname: string, prefix: string): boolean {
+  return pathname === prefix || pathname.startsWith(`${prefix}/`);
+}
+
+function hasExcludedAssetExtension(pathname: string): boolean {
+  const extension = pathname.match(/\.[^./]+$/)?.[0]?.toLowerCase();
+  return extension ? EXCLUDED_ASSET_EXTENSIONS.has(extension) : false;
+}
+
+function isNegotiableDocsPath(pathname: string): boolean {
+  if (EXCLUDED_EXACT_PATHS.has(pathname)) return false;
+  if (EXCLUDED_PATH_PREFIXES.some((prefix) => matchesPathPrefix(pathname, prefix))) return false;
+
+  return !hasExcludedAssetExtension(pathname);
+}
+
+function isNegotiableMethod(method: string): boolean {
+  return method === 'GET' || method === 'HEAD';
+}
+
+function withMarkdownVary(response: NextResponse): NextResponse {
+  appendMarkdownVaryHeader(response.headers);
+  return response;
+}
+
 export default function middleware(request: NextRequest) {
   const { pathname } = request.nextUrl;
 
-  if (pathname === '/' && isProgrammaticClient(request)) {
+  if (!isNegotiableMethod(request.method)) {
+    return NextResponse.next();
+  }
+
+  const wantsMarkdown = shouldServeMarkdown(request.headers);
+
+  if (pathname === '/' && (wantsMarkdown || isProgrammaticClient(request))) {
     const redirectUrl = request.nextUrl.clone();
     redirectUrl.pathname = '/llms.txt';
-    return NextResponse.redirect(redirectUrl);
+    return withMarkdownVary(NextResponse.redirect(redirectUrl));
+  }
+
+  if (!isNegotiableDocsPath(pathname)) {
+    return NextResponse.next();
+  }
+
+  if (wantsMarkdown) {
+    const rewriteUrl = request.nextUrl.clone();
+    rewriteUrl.pathname = `/llms.mdx${pathname}`;
+    return withMarkdownVary(NextResponse.rewrite(rewriteUrl));
   }
 
-  return NextResponse.next();
+  return withMarkdownVary(NextResponse.next());
 }
 
 export const config = {