diff --git a/packages/core/src/tool/plugin/webfetch.ts b/packages/core/src/tool/plugin/webfetch.ts index 3c9cf10926c4..5f59790721d0 100644 --- a/packages/core/src/tool/plugin/webfetch.ts +++ b/packages/core/src/tool/plugin/webfetch.ts @@ -50,11 +50,16 @@ const acceptHeader = (format: Format) => { } } -const headers = (format: Format, userAgent: string) => ({ - "User-Agent": userAgent, - Accept: acceptHeader(format), - "Accept-Language": "en-US,en;q=0.9", -}) +const headers = (url: string, format: Format, userAgent: string) => { + const pathname = new URL(url).pathname.toLowerCase() + const isMarkdownPath = pathname.endsWith(".md") || pathname.endsWith(".markdown") + + return { + "User-Agent": userAgent, + Accept: isMarkdownPath ? "*/*" : acceptHeader(format), + "Accept-Language": "en-US,en;q=0.9", + } +} const openCodeUserAgent = "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OpenCode-User/1.0; +https://opencode.ai" @@ -66,7 +71,7 @@ const isCloudflareChallenge = (error: unknown) => { } const request = (url: string, format: Format, userAgent = openCodeUserAgent) => - HttpClientRequest.get(url).pipe(HttpClientRequest.setHeaders(headers(format, userAgent))) + HttpClientRequest.get(url).pipe(HttpClientRequest.setHeaders(headers(url, format, userAgent))) const assertHttpUrl = (url: URL) => { if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error("URL must use http:// or https://") diff --git a/packages/core/test/tool-webfetch.test.ts b/packages/core/test/tool-webfetch.test.ts index 1b6cbe85f317..b1cab6c80cbd 100644 --- a/packages/core/test/tool-webfetch.test.ts +++ b/packages/core/test/tool-webfetch.test.ts @@ -455,6 +455,27 @@ describe("WebFetchTool registration", () => { }), ) + it.effect("does not negotiate markdown for .md and .markdown URLs", () => + Effect.gen(function* () { + reset() + const registry = yield* Tool.Service + const urls = [ + "https://docs.example.com/page.md?source=llms.txt", + "https://docs.example.com/page.markdown?source=llms.txt", + ] + + yield* Effect.forEach(urls, (url) => + Effect.forEach(["markdown", "text", "html"] as const, (format) => + executeTool(registry, call({ url, format }, `call-webfetch-${format}`)), + ), + ) + + expect(requests.map((request) => request.headers.accept)).toEqual( + Array.from({ length: urls.length * 3 }, () => "*/*"), + ) + }), + ) + it.effect("accepts localhost URLs with the same requested-URL permission check", () => Effect.gen(function* () { reset()