fix(core): handle .md webfetch URLs

packages/core/src/tool/plugin/webfetch.ts

@@ -50,11 +50,16 @@ const acceptHeader = (format: Format) => {
   }
 }
 
-const headers = (format: Format, userAgent: string) => ({
-  "User-Agent": userAgent,
-  Accept: acceptHeader(format),
-  "Accept-Language": "en-US,en;q=0.9",
-})
+const headers = (url: string, format: Format, userAgent: string) => {
+  const pathname = new URL(url).pathname.toLowerCase()
+  const isMarkdownPath = pathname.endsWith(".md") || pathname.endsWith(".markdown")
+
+  return {
+    "User-Agent": userAgent,
+    Accept: isMarkdownPath ? "*/*" : acceptHeader(format),
+    "Accept-Language": "en-US,en;q=0.9",
+  }
+}
 
 const openCodeUserAgent =
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OpenCode-User/1.0; +https://opencode.ai"
@@ -66,7 +71,7 @@ const isCloudflareChallenge = (error: unknown) => {
 }
 
 const request = (url: string, format: Format, userAgent = openCodeUserAgent) =>
-  HttpClientRequest.get(url).pipe(HttpClientRequest.setHeaders(headers(format, userAgent)))
+  HttpClientRequest.get(url).pipe(HttpClientRequest.setHeaders(headers(url, format, userAgent)))
 
 const assertHttpUrl = (url: URL) => {
   if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error("URL must use http:// or https://")

packages/core/test/tool-webfetch.test.ts

@@ -455,6 +455,27 @@ describe("WebFetchTool registration", () => {
     }),
   )
 
+  it.effect("does not negotiate markdown for .md and .markdown URLs", () =>
+    Effect.gen(function* () {
+      reset()
+      const registry = yield* Tool.Service
+      const urls = [
+        "https://docs.example.com/page.md?source=llms.txt",
+        "https://docs.example.com/page.markdown?source=llms.txt",
+      ]
+
+      yield* Effect.forEach(urls, (url) =>
+        Effect.forEach(["markdown", "text", "html"] as const, (format) =>
+          executeTool(registry, call({ url, format }, `call-webfetch-${format}`)),
+        ),
+      )
+
+      expect(requests.map((request) => request.headers.accept)).toEqual(
+        Array.from({ length: urls.length * 3 }, () => "*/*"),
+      )
+    }),
+  )
+
   it.effect("accepts localhost URLs with the same requested-URL permission check", () =>
     Effect.gen(function* () {
       reset()