test(middleware,robots): guard the vary header and crawl allowlist (#105)

The middleware suite only asserted that the homepage does not vary, so
removing withMarkdownVary kept every test green while shipping a
cache-poisoning regression. Add a positive assertion that negotiated
docs URLs carry Vary: Accept, User-Agent on both the markdown rewrite
and the HTML pass-through.

The robots suite checked for Allow: / but never for blocking rules, so
a blanket Disallow: / would pass. Assert that no user-agent group
contains any Disallow directive, matching the current robots.txt.

Author: nibzard

tests/middleware.test.ts

@@ -83,6 +83,22 @@ describe('middleware .md suffix handling', () => {
     expect(varyTokens(response)).not.toEqual(expect.arrayContaining(['accept', 'user-agent']));
   });
 
+  test('varies negotiated docs URLs on accept and user-agent', () => {
+    // The markdown rewrite and the HTML pass-through share one cache key,
+    // so both responses must carry the Vary tokens from appendMarkdownVaryHeader.
+    const rewritten = middleware(
+      new NextRequest('http://localhost/overview/steel-cli', {
+        headers: { accept: 'text/html', 'user-agent': 'claude-code/1.0' },
+      }),
+    );
+    expect(rewritten.headers.get('x-middleware-rewrite')).not.toBeNull();
+    expect(varyTokens(rewritten)).toEqual(expect.arrayContaining(['accept', 'user-agent']));
+
+    const passedThrough = middleware(browserRequest('http://localhost/overview/steel-cli'));
+    expect(passedThrough.headers.get('x-middleware-rewrite')).toBeNull();
+    expect(varyTokens(passedThrough)).toEqual(expect.arrayContaining(['accept', 'user-agent']));
+  });
+
   test('passes browser navigation at the homepage through', () => {
     const response = middleware(browserRequest('http://localhost/'));
     expect(response.headers.get('x-middleware-rewrite')).toBeNull();

tests/robots-txt.test.ts

@@ -95,4 +95,14 @@ describe('robots.txt content signals', () => {
     expect(groups.get('ClaudeBot')).toContain('Allow: /');
     expect(ROBOTS).toContain('Sitemap: https://docs.steel.dev/sitemap.xml');
   });
+
+  test('contains no Disallow directives in any group', () => {
+    // The docs are fully public; a blanket Disallow would silently deindex the site.
+    for (const [agent, directives] of groups) {
+      const disallows = directives.filter((directive) =>
+        directive.toLowerCase().startsWith('disallow:'),
+      );
+      expect(disallows, `${agent} blocks crawling with ${disallows.join(', ')}`).toEqual([]);
+    }
+  });
 });