fix(core): handle .md webfetch URLs

packages/core/src/tool/plugin/webfetch.ts

@@ -50,9 +50,9 @@ const acceptHeader = (format: Format) => {
   }
 }
 
-const headers = (format: Format, userAgent: string) => ({
+const headers = (url: string, format: Format, userAgent: string) => ({
   "User-Agent": userAgent,
-  Accept: acceptHeader(format),
+  Accept: new URL(url).pathname.toLowerCase().endsWith(".md") ? "*/*" : acceptHeader(format),
   "Accept-Language": "en-US,en;q=0.9",
 })
 
@@ -66,7 +66,7 @@ const isCloudflareChallenge = (error: unknown) => {
 }
 
 const request = (url: string, format: Format, userAgent = openCodeUserAgent) =>
-  HttpClientRequest.get(url).pipe(HttpClientRequest.setHeaders(headers(format, userAgent)))
+  HttpClientRequest.get(url).pipe(HttpClientRequest.setHeaders(headers(url, format, userAgent)))
 
 const assertHttpUrl = (url: URL) => {
   if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error("URL must use http:// or https://")

packages/core/test/tool-webfetch.test.ts

@@ -455,6 +455,20 @@ describe("WebFetchTool registration", () => {
     }),
   )
 
+  it.effect("does not negotiate markdown for .md URLs", () =>
+    Effect.gen(function* () {
+      reset()
+      const registry = yield* Tool.Service
+      const url = "https://docs.example.com/page.md?source=llms.txt"
+
+      yield* Effect.forEach(["markdown", "text", "html"] as const, (format) =>
+        executeTool(registry, call({ url, format }, `call-webfetch-${format}`)),
+      )
+
+      expect(requests.map((request) => request.headers.accept)).toEqual(["*/*", "*/*", "*/*"])
+    }),
+  )
+
   it.effect("accepts localhost URLs with the same requested-URL permission check", () =>
     Effect.gen(function* () {
       reset()