feat(robots): declare Content Signals for AI crawlers (#88)

robots.txt says which crawlers may fetch the docs but says nothing about
what they may do with the content once fetched. Content Signals
(contentsignals.org, draft-romm-aipref-contentsignals) covers that second
half, and agent-readiness scanners check for it.

Declare search=yes, ai-input=yes, ai-train=yes. That restates in
machine-readable form what robots.txt already grants: every AI crawler
here, including the training crawlers GPTBot and Google-Extended, is
allowed. Flip a value if that intent ever changes.

The declaration is repeated in every User-agent group because a crawler
matching a named group ignores the wildcard group entirely, so a single
declaration under `*` would not reach GPTBot or ClaudeBot.

Author: nibzard

public/robots.txt

@@ -1,31 +1,52 @@
+# Content Signals (https://contentsignals.org) state how this site's content
+# may be used once it has been crawled. Steel's docs are public and written to
+# be read by agents, so all three signals are open:
+#
+#   search:   appearing in search results
+#   ai-input: use as input for an AI answer, such as RAG or grounding
+#   ai-train: use as training data for a model
+#
+# A crawler that matches a named group below ignores the wildcard group, so the
+# declaration is repeated in every group rather than stated once.
+
 User-agent: *
+Content-Signal: search=yes, ai-input=yes, ai-train=yes
 Allow: /
 
 User-agent: GPTBot
+Content-Signal: search=yes, ai-input=yes, ai-train=yes
 Allow: /
 
 User-agent: ChatGPT-User
+Content-Signal: search=yes, ai-input=yes, ai-train=yes
 Allow: /
 
 User-agent: OAI-SearchBot
+Content-Signal: search=yes, ai-input=yes, ai-train=yes
 Allow: /
 
 User-agent: PerplexityBot
+Content-Signal: search=yes, ai-input=yes, ai-train=yes
 Allow: /
 
 User-agent: Perplexity-User
+Content-Signal: search=yes, ai-input=yes, ai-train=yes
 Allow: /
 
 User-agent: ClaudeBot
+Content-Signal: search=yes, ai-input=yes, ai-train=yes
 Allow: /
 
 User-agent: Claude-User
+Content-Signal: search=yes, ai-input=yes, ai-train=yes
 Allow: /
 
 User-agent: Claude-SearchBot
+Content-Signal: search=yes, ai-input=yes, ai-train=yes
 Allow: /
 
 User-agent: Google-Extended
+Content-Signal: search=yes, ai-input=yes, ai-train=yes
 Allow: /
 
 Sitemap: https://docs.steel.dev/sitemap.xml

tests/robots-txt.test.ts

@@ -0,0 +1,79 @@
+// ABOUTME: Tests for public/robots.txt, covering the Content Signals declaration
+// ABOUTME: that states how AI crawlers may use the docs, plus the crawl directives.
+import { describe, expect, test } from 'bun:test';
+import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+
+const ROBOTS = readFileSync(
+  fileURLToPath(new URL('../public/robots.txt', import.meta.url)),
+  'utf8',
+);
+
+/** Splits robots.txt into its User-agent groups, keyed by agent name. */
+function parseGroups(source: string): Map<string, string[]> {
+  const groups = new Map<string, string[]>();
+  let current: string[] | null = null;
+
+  for (const line of source.split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) continue;
+
+    const [rawKey, ...rawValue] = trimmed.split(':');
+    const key = rawKey.trim().toLowerCase();
+    const value = rawValue.join(':').trim();
+
+    if (key === 'user-agent') {
+      current = [];
+      groups.set(value, current);
+    } else if (current) {
+      current.push(trimmed);
+    }
+  }
+
+  return groups;
+}
+
+const groups = parseGroups(ROBOTS);
+
+describe('robots.txt content signals', () => {
+  test('declares a Content-Signal in every User-agent group', () => {
+    // A crawler matching a specific group ignores the wildcard group entirely,
+    // so the declaration has to be repeated rather than stated once.
+    expect(groups.size).toBeGreaterThan(0);
+
+    for (const [agent, directives] of groups) {
+      const signal = directives.find((directive) =>
+        directive.toLowerCase().startsWith('content-signal:'),
+      );
+      expect(signal, `${agent} is missing a Content-Signal directive`).toBeDefined();
+    }
+  });
+
+  test('declares all three signals with valid values', () => {
+    for (const [agent, directives] of groups) {
+      const signal = directives.find((directive) =>
+        directive.toLowerCase().startsWith('content-signal:'),
+      );
+      const declared = new Map(
+        (signal ?? '')
+          .slice('content-signal:'.length)
+          .split(',')
+          .map((entry) => entry.trim().split('=') as [string, string]),
+      );
+
+      for (const name of ['search', 'ai-input', 'ai-train']) {
+        expect([...declared.keys()], `${agent} is missing the ${name} signal`).toContain(name);
+        expect(['yes', 'no'], `${agent} declares an invalid ${name} value`).toContain(
+          declared.get(name) ?? '',
+        );
+      }
+    }
+  });
+
+  test('keeps the crawl directives, sitemap and host', () => {
+    expect(groups.get('*')).toContain('Allow: /');
+    expect(groups.get('ClaudeBot')).toContain('Allow: /');
+    expect(ROBOTS).toContain('Sitemap: https://docs.steel.dev/sitemap.xml');
+    expect(ROBOTS).toContain('Host: docs.steel.dev');
+  });
+});