diff --git a/packages/app/test/mimir-sticky-toolsets.test.ts b/packages/app/test/mimir-sticky-toolsets.test.ts index 02de5a687..8b3d9fab9 100644 --- a/packages/app/test/mimir-sticky-toolsets.test.ts +++ b/packages/app/test/mimir-sticky-toolsets.test.ts @@ -1,7 +1,8 @@ // @vitest-environment node // SPDX-License-Identifier: Apache-2.0 // Copyright (C) 2026 The SlicerX contributors -// On a capped mimir cloud chat the tools a request carries follow the chat, not only the message. The worst case is the +// On a capped mimir cloud chat (a proxy deployed before tool search, which refuses the catalog the chat's first request +// carries, so the chat goes on the per-turn pick) the tools a request carries follow the chat, not only the message. The worst case is the // desktop app with every connector and twelve printing printers (well over 128 tools offered, 128 sent). A "yes please" after // mimir offers to print gets the printer tools, the list is the same request after request while the chat stays on its // topic (so the proxy's prompt cache keeps hitting), and every request stays at 128 tools or under. Each request is read @@ -55,14 +56,30 @@ function reply(step: { text?: string; call?: { name: string; input?: unknown } } ) } -/** A transport that keeps each request and plays the replies in order. */ -function transport(steps: { text?: string; call?: { name: string; input?: unknown } }[]): LlmTransport & { sent: LlmHttpRequest[] } { +/** + * What a mimir cloud proxy deployed before tool search answered a request that sends the tool catalog (its shape.ts + * then): a 400 for more than 128 tools, or for the tool search tool, as the desktop host raises it. + */ +function oldRefusal(req: LlmHttpRequest): Error | undefined { + const tools = (JSON.parse(req.body) as { tools?: { type?: string }[] }).tools ?? [] + const search = tools.find((t) => t.type !== undefined && t.type !== 'custom' && !t.type.startsWith('web_search_')) + const words = tools.length > 128 ? 'tools must be a list of at most 128' : search ? `tool ${String(search.type)} is not allowed` : null + return words ? Object.assign(new Error(`400: ${words}`), { status: 400, code: 'invalid_request' }) : undefined +} + +/** + * A transport that keeps each request and plays the replies in order. `old`: a mimir cloud proxy deployed before tool + * search, which refuses a catalog (oldRefusal); a refused request is not kept and plays no reply. + */ +function transport(steps: { text?: string; call?: { name: string; input?: unknown } }[], old = false): LlmTransport & { sent: LlmHttpRequest[] } { const sent: LlmHttpRequest[] = [] let i = 0 return { sent, available: async () => true, async *stream(req) { + const no = old ? oldRefusal(req) : undefined + if (no) throw no sent.push(req) yield reply(steps[i++] ?? { text: 'Done.' }) }, @@ -91,11 +108,11 @@ async function registerApp(platform: Platform): Promise<() => void> { return off } -/** A fresh mimir cloud chat in the desktop app, with the replies it plays. */ +/** A fresh mimir cloud chat in the desktop app on a proxy deployed before tool search, with the replies it plays. */ async function chat(steps: Parameters[0]) { resetTabMimir() set({ pilot: { mode: 'on', provider: 'mimir-cloud' } as never }) - const llm = transport(steps) + const llm = transport(steps, true) const pilot = await tabMimir(hostFor('desktop', llm), withCloud(NEUTRAL), 'cloud') const say = async (message: string): Promise => { const at = llm.sent.length diff --git a/packages/app/test/mimir-tool-cap.test.ts b/packages/app/test/mimir-tool-cap.test.ts index aee58c5b1..b9c0851fe 100644 --- a/packages/app/test/mimir-tool-cap.test.ts +++ b/packages/app/test/mimir-tool-cap.test.ts @@ -2,13 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 // Copyright (C) 2026 The SlicerX contributors // No mimir tab request loads more than 128 tools. OpenAI-style APIs and local servers refuse a longer list, and so does -// mimir cloud's proxy for the tools that load up front (supabase/functions/mimir/shape.ts). Once the proxy has said it -// takes tool search (the event its streams open with), a new chat on mimir cloud carries the tab's whole catalog, most -// of it deferred behind tool search; a proxy that never said so gets the per-turn pick it always got. The tab is made +// mimir cloud's proxy for the tools that load up front (supabase/functions/mimir/shape.ts). A chat on mimir cloud carries +// the tab's whole catalog from its first request, most of it deferred behind tool search; a proxy deployed before tool +// search refuses that, and gets the request again as the per-turn pick it always got. The tab is made // the way the app makes it, with the commands the app registers at startup, its features' commands, a command per // connected printer and every connector's tools, and each request is read back from the HTTP body the provider adapter // wrote. Every kind of turn is checked in the desktop app (which slices and prints) and the browser build (which only -// models), on mimir cloud before and after its proxy takes tool search, a key and a local model, in the neutral edition +// models), on mimir cloud on a proxy deployed before tool search and on one with it, a key and a local model, in the neutral edition // and in one with the printer and web tools off. Last, a recorded chat's requests go through the proxy's own shape(): // the same tools on every request. import type { EditionConfig } from '@slicerx/edition-config' @@ -56,18 +56,32 @@ const PROXY_SAYS = (() => { return JSON.parse(`"${lit.slice(1, -1).replaceAll('"', '\\"')}"`) as string })() +/** + * What a mimir cloud proxy deployed before tool search answered a request that sends the tool catalog (its shape.ts + * then): a 400 for more than 128 tools, or for the tool search tool, as the desktop host raises it. + */ +function oldRefusal(req: LlmHttpRequest): Error | undefined { + const tools = (JSON.parse(req.body) as { tools?: { type?: string }[] }).tools ?? [] + const search = tools.find((t) => t.type !== undefined && t.type !== 'custom' && !t.type.startsWith('web_search_')) + const words = tools.length > 128 ? 'tools must be a list of at most 128' : search ? `tool ${String(search.type)} is not allowed` : null + return words ? Object.assign(new Error(`400: ${words}`), { status: 400, code: 'invalid_request' }) : undefined +} + /** * A transport that keeps each request and answers none of them, so every turn ends after its first request. `proxy`: - * a mimir cloud proxy that takes tool search, so each answer opens with its event before it fails. + * 'search', a mimir cloud proxy that takes tool search, so each answer opens with its event before it fails; 'old', one + * deployed before tool search, which refuses a catalog (oldRefusal) and keeps no refused request. */ -function transport(proxy = false): LlmTransport & { sent: LlmHttpRequest[] } { +function transport(proxy?: 'search' | 'old'): LlmTransport & { sent: LlmHttpRequest[] } { const sent: LlmHttpRequest[] = [] return { sent, available: async () => true, async *stream(req) { + const no = proxy === 'old' ? oldRefusal(req) : undefined + if (no) throw no sent.push(req) - if (proxy) yield new TextEncoder().encode(PROXY_SAYS) + if (proxy === 'search') yield new TextEncoder().encode(PROXY_SAYS) throw new Error('not answered in this test') }, } @@ -87,8 +101,8 @@ const EDITIONS: [string, EditionConfig][] = [ ] /** - * How mimir runs: mimir cloud on a proxy that never said it takes tool search (deployed before it), mimir cloud once its - * proxy has said so, a key on an OpenAI-style API, and a local model. + * How mimir runs: mimir cloud on a proxy deployed before tool search (which refuses the catalog), mimir cloud on one that + * takes it, a key on an OpenAI-style API, and a local model. */ const MODES = { 'mimir cloud': { mode: 'on', provider: 'mimir-cloud' }, @@ -127,15 +141,15 @@ function loadedNames(req: LlmHttpRequest): string[] { const SEARCH = 'tool_search_tool_bm25' /** - * The first request of a turn in a fresh chat, after `before` when given, and how many tools the tab offers. With tool - * search, a first chat has heard the proxy say it takes it. + * The first request the proxy took of a turn in a fresh chat, after `before` when given, and how many tools the tab + * offers. With tool search, a first chat has run before it. */ async function turn(platform: Platform, edition: EditionConfig, mode: Mode, t: (typeof TURNS)[number]): Promise<{ req: LlmHttpRequest; offered: number }> { resetTabMimir() set({ pilot: MODES[mode] as never }) const searching = mode === 'mimir cloud, tool search' - const llm = transport(searching) const cloud = MODES[mode].provider === 'mimir-cloud' + const llm = transport(searching ? 'search' : cloud ? 'old' : undefined) const pilot = await tabMimir(hostFor(platform, llm), cloud ? withCloud(edition) : edition, cloud ? 'cloud' : null) if (searching) for await (const _ of pilot.run('warm', 'hello')) void _ if (t.before) for await (const _ of pilot.run('s', t.before)) void _ @@ -208,7 +222,7 @@ describe('no mimir tab request loads more than 128 tools', () => { }, 120_000) } - it('sends the per-turn pick on mimir cloud until its proxy says it takes tool search: no deferral, search tool or tool change', async () => { + it('sends the per-turn pick on mimir cloud once a proxy deployed before tool search refused the catalog: no deferral, search tool or tool change', async () => { const off = await registerApp('desktop') try { for (const t of TURNS) { diff --git a/packages/pilot/src/provider/client.ts b/packages/pilot/src/provider/client.ts index 12324b0ac..924e60214 100644 --- a/packages/pilot/src/provider/client.ts +++ b/packages/pilot/src/provider/client.ts @@ -22,9 +22,9 @@ export const MIMIR_CLOUD = 'mimir-cloud' /** * The most tools that load up front in one mimir cloud request (supabase/functions/mimir/shape.ts, MAX_LOADED_TOOLS); * it refuses more. The desktop app's tab offers more than that (the design tools, slicing, printers, the app's - * commands, a command per printer and the connectors' tools). Once the proxy has said it takes tool search, a new chat - * on a model with it sends them all, most deferred (tool-search.ts); until then, and on any other model, a turn sends - * the tools it needs first (toolsets.ts). + * commands, a command per printer and the connectors' tools). A chat on a model with tool search sends them all, most + * deferred (tool-search.ts), from its first request; on a proxy that refuses that (refusesToolSearch), and on any other + * model, a turn sends the tools it needs first (toolsets.ts). */ export const MIMIR_CLOUD_MAX_TOOLS = 128 @@ -76,31 +76,57 @@ export function adapterFor(provider: string): LlmAdapter { return make() } -/** Joins an adapter with the host transport. The transport adds the key; this code never sees it. */ /** - * The transports whose mimir cloud proxy said it takes a request that sends the whole tool catalog through tool search - * (the `mimir` event its streams open with, supabase/functions/mimir/handler.ts). Kept per transport, so a chat runtime + * What a mimir cloud proxy deployed before tool search answers a request that sends the whole catalog: its shape.ts + * then took at most 128 tools and no tool search tool, and refused anything else with a 400 and these words. + */ +const OLD_SHAPE = /^(?:tools must be a list of at most 128|tool tool_search_tool_(?:bm25|regex)_\d+ is not allowed)$/ + +/** + * Whether a failed mimir cloud request was refused by a proxy deployed before tool search, for the request's shape: + * a 400 with its invalid request code (or none) and one of its old words (OLD_SHAPE). The desktop host puts the status + * before the words (`400: ...`); the browser's keeps it apart. + */ +export function refusesToolSearch(e: unknown): boolean { + const status = e && typeof e === 'object' && 'status' in e ? (e as { status: unknown }).status : undefined + const msg = errorText(e) + const said = /^400: /.test(msg) + if (typeof status === 'number' ? status !== 400 : !said) return false + const code = errorCode(e) + if (code !== undefined && code !== 'invalid_request') return false + return OLD_SHAPE.test(said ? msg.slice(5) : msg) +} + +/** + * The transports whose mimir cloud proxy refused a request that sends the whole tool catalog through tool search + * (refusesToolSearch). Their chats send the per-turn pick from then on, until the proxy opens a reply with its `mimir` + * event (supabase/functions/mimir/handler.ts), which says it takes tool search. Kept per transport, so a chat runtime * made again on the same host knows it at once. */ -const searchingProxies = new WeakSet() +const plainProxies = new WeakSet() +/** Joins an adapter with the host transport. The transport adds the key; this code never sees it. */ export function createTransportClient(transport: LlmTransport, provider: string, opts: { baseUrl?: string; share?: () => CloudShare } = {}): LlmClient { const adapter = adapterFor(provider) const cloud = provider === MIMIR_CLOUD // mimir cloud's refusals (daily caps, a pause) are shown as the proxy wrote them, never retried elsewhere - const failure = (e: unknown): LlmEvent => { + // a tool search request an old proxy refused marks the transport, and the runtime sends it once more without + const failure = (e: unknown, req: LlmRequest): LlmEvent => { if (!cloud) return { type: 'error', message: errorText(e), retryable: true } const code = errorCode(e) - return { type: 'error', message: errorText(e).replace(/^\d{3}: /, ''), retryable: false, ...(code ? { code } : {}) } + const refused = req.toolSearch === true && refusesToolSearch(e) + if (refused) plainProxies.add(transport) + return { type: 'error', message: errorText(e).replace(/^\d{3}: /, ''), retryable: false, ...(code ? { code } : {}), ...(refused ? { searchRefused: true as const } : {}) } } return { provider, // the proxy's own limit, though the Messages API it speaks has none; a tool search request carries more, deferred ...(cloud ? { maxTools: MIMIR_CLOUD_MAX_TOOLS } : adapter.maxTools === undefined ? {} : { maxTools: adapter.maxTools }), - // the Messages API searches deferred tools; through mimir cloud only once its proxy has said it passes them, so an - // app that reaches a proxy deployed before tool search keeps the per-turn pick + // the Messages API searches deferred tools, and so does mimir cloud from the first request, so the first chat sends + // the same tools as every later one; an app that reaches a proxy deployed before tool search keeps the per-turn pick + // once that proxy has refused the catalog get toolSearch(): boolean { - return adapter.id === 'anthropic' && (!cloud || searchingProxies.has(transport)) + return adapter.id === 'anthropic' && (!cloud || !plainProxies.has(transport)) }, async *stream(req: LlmRequest, signal?: AbortSignal): AsyncIterable { let http = adapter.build(req, opts.baseUrl === undefined ? {} : { baseUrl: opts.baseUrl }) @@ -110,7 +136,7 @@ export function createTransportClient(transport: LlmTransport, provider: string, try { body = cloud ? transport.stream(http, signal, (h) => (headers = h)) : transport.stream(http, signal) } catch (e) { - yield failure(e) + yield failure(e, req) yield { type: 'done', stop: 'error' } return } @@ -126,12 +152,13 @@ export function createTransportClient(transport: LlmTransport, provider: string, const model = cloudModel(headers) if (model) yield { type: 'model', model } for await (const ev of adapter.parse(replay(first, it))) { - if (ev.type === 'tool_search') searchingProxies.add(transport) + // the proxy says it takes tool search: a refusal before (a proxy deployed since) no longer holds + if (ev.type === 'tool_search') plainProxies.delete(transport) else yield ev } } catch (e) { if (signal?.aborted) throw e - yield failure(e) + yield failure(e, req) yield { type: 'done', stop: 'error' } } }, diff --git a/packages/pilot/src/provider/types.ts b/packages/pilot/src/provider/types.ts index e72af4b08..7462f4699 100644 --- a/packages/pilot/src/provider/types.ts +++ b/packages/pilot/src/provider/types.ts @@ -92,13 +92,18 @@ export type LlmEvent = /** `webSearches`: the hosted web searches the reply ran, when the provider reports them. */ | { type: 'usage'; inputTokens: number; outputTokens: number; cacheReadTokens?: number; cacheWriteTokens?: number; webSearches?: number } | { type: 'done'; stop: LlmStop; raw?: unknown[] } - /** `code` is mimir cloud's refusal code (`day_used`, `paused`, ...). */ - | { type: 'error'; message: string; retryable: boolean; code?: string } + /** + * `code` is mimir cloud's refusal code (`day_used`, `paused`, ...). `searchRefused`: a mimir cloud proxy deployed before + * tool search refused this tool search request for its shape; the client no longer takes tool search on it + * (LlmClient.toolSearch), and the runtime sends the request once more with the per-turn pick. + */ + | { type: 'error'; message: string; retryable: boolean; code?: string; searchRefused?: true } /** The model that answered, when the reply names it (mimir cloud's `x-sx-model`). Comes before the reply's text. */ | { type: 'model'; model: string } /** * mimir cloud's proxy takes a request that sends the whole tool catalog through tool search (its `mimir` stream event, - * supabase/functions/mimir/handler.ts). The transport client keeps it (LlmClient.toolSearch) and passes it on no further. + * supabase/functions/mimir/handler.ts). The transport client takes back an earlier refusal on it (LlmClient.toolSearch) + * and passes it on no further. */ | { type: 'tool_search' } @@ -118,7 +123,8 @@ export interface LlmClient { maxTools?: number /** * The provider runs a tool search over deferred tools and takes tool changes mid-chat (LlmRequest.toolSearch). It can - * turn true while the client runs: mimir cloud's proxy says so in its replies. + * change while the client runs: a mimir cloud proxy deployed before tool search turns it false by refusing a tool + * search request, and true again by saying in a reply that it takes tool search. */ readonly toolSearch?: boolean stream(req: LlmRequest, signal?: AbortSignal): AsyncIterable diff --git a/packages/pilot/src/runtime.ts b/packages/pilot/src/runtime.ts index c10b5d7c6..83ca71a2b 100644 --- a/packages/pilot/src/runtime.ts +++ b/packages/pilot/src/runtime.ts @@ -885,9 +885,9 @@ export function createPilot(opts: CreatePilotOptions): Pilot & { /** Each chat's tool catalog on a model with tool search (tool-search.ts): declared once, only ever appended to. */ const catalogs = new Map() /** - * Whether each chat sends its tool catalog (tool-search.ts) or the per-turn pick, fixed at its first request. mimir - * cloud's proxy says it takes tool search only in its replies, so the chat that first hears it keeps the pick it - * started with, and no chat's tool list changes shape partway through; the next chat sends the catalog. + * Whether each chat sends its tool catalog (tool-search.ts) or the per-turn pick, fixed at its first request, so no + * chat's tool list changes shape partway through. The one change: a mimir cloud proxy deployed before tool search + * refuses the catalog, and the chat goes on the pick from that request, sent once more. */ const searchChats = new Map() /** @@ -1647,6 +1647,8 @@ export function createPilot(opts: CreatePilotOptions): Pilot & { let slowed = 0 // a request lost to the network is sent once more before the run says so (NETWORK_RETRY_MS, NETWORK_LINE) let netRetried = false + // a tool search request that a proxy deployed before tool search refused goes once more with the per-turn pick, never twice + let searchResent = false // the run's limits; an approved plan raises them to fit its features (planLimits) let maxSteps = opts.config.maxSteps let maxToolCalls = opts.config.maxToolCalls @@ -1752,6 +1754,7 @@ export function createPilot(opts: CreatePilotOptions): Pilot & { let failed: string | null = null let netDown = false let failedCode: string | undefined + let searchRefused = false let cut = false const model = modelFor(opts.config, tier) if (capped.report) { @@ -1826,6 +1829,7 @@ export function createPilot(opts: CreatePilotOptions): Pilot & { case 'error': failed = ev.message failedCode = ev.code + searchRefused = ev.searchRefused === true netDown = networkLost({ message: ev.message }) break case 'done': @@ -1884,6 +1888,16 @@ export function createPilot(opts: CreatePilotOptions): Pilot & { stopReason = 'error' break } + if (failed !== null && searchRefused && searching && !searchResent && !text && toolCalls.length === 0) { + // a proxy deployed before tool search refused the catalog: the chat drops it and sends the same step once more, + // unseen, with the per-turn pick (the client takes no tool search on this transport now, so it cannot come again) + searchResent = true + dropUnanswered(sessionId, messages) + searchChats.delete(sessionId) + catalogs.delete(sessionId) + step-- + continue + } if (failed !== null) { emit({ type: 'error', message: failed, retryable: true, ...(failedCode ? { code: failedCode } : {}) }) stopReason = 'error' diff --git a/packages/pilot/src/tool-search.ts b/packages/pilot/src/tool-search.ts index 23361f566..a04eaca2b 100644 --- a/packages/pilot/src/tool-search.ts +++ b/packages/pilot/src/tool-search.ts @@ -6,7 +6,7 @@ // never edited: a tool the app offers later is appended deferred, and offering or withdrawing a tool mid-chat (a run // limited to read-only tools, a printer or connector that comes or goes) is a tool change appended to the chat // (LlmMessage `tools`). Local models and providers without tool search keep their per-turn lists (toolsets.ts), and so -// does a chat on mimir cloud until its proxy has said it takes tool search (LlmClient.toolSearch, runtime.ts). +// does a chat on mimir cloud whose proxy, deployed before tool search, refused the catalog (LlmClient.toolSearch, runtime.ts). import type { LlmToolDef } from './provider/types' /** diff --git a/packages/pilot/test/sticky-toolsets.test.ts b/packages/pilot/test/sticky-toolsets.test.ts index 8c53ce398..93184cde7 100644 --- a/packages/pilot/test/sticky-toolsets.test.ts +++ b/packages/pilot/test/sticky-toolsets.test.ts @@ -736,7 +736,11 @@ describe('a paint ask in a capped design chat', () => { }) describe('words that ask for a held-back tool on a capped design chat', () => { - /** A design chat over the limit with the more design tools and the trending models command, whose model builds once and stops. */ + /** + * A design chat over the limit with the more design tools and the trending models command, whose model builds once and + * stops, on a proxy deployed before tool search: it refuses the chat's first request, the whole catalog, so the chat + * goes on the per-turn pick (left out of `sent`). + */ function chat() { const named = (name: string): PilotTool => defineTool({ name, version: '1.0.0', description: `${name} does one thing`, source: 'command', permission: 'read', input: z.object({}).passthrough(), run: async () => ({ summary: `${name} ran` }) }) as PilotTool @@ -746,8 +750,9 @@ describe('words that ask for a held-back tool on a capped design chat', () => { const llm = { available: async () => true, async *stream(req: LlmHttpRequest) { + const body = JSON.parse(req.body) as { messages: { role: string; content: unknown }[]; tools?: unknown[] } + if ((body.tools ?? []).length > 128) throw Object.assign(new Error('400: tools must be a list of at most 128'), { status: 400, code: 'invalid_request' }) sent.push(req) - const body = JSON.parse(req.body) as { messages: { role: string; content: unknown }[] } yield wire('mimir-cloud', JSON.stringify(body.messages.at(-1)?.content ?? '').includes('tool_result') ? { text: 'Done.' } : { call: 'geom.t0' }) }, } as unknown as LlmTransport diff --git a/packages/pilot/test/tool-search.test.ts b/packages/pilot/test/tool-search.test.ts index e855ddd5c..80acdfc3d 100644 --- a/packages/pilot/test/tool-search.test.ts +++ b/packages/pilot/test/tool-search.test.ts @@ -3,13 +3,15 @@ // A Claude model with tool search gets the chat's whole tool catalog on every request: the same bytes in the same order // from the first request to the last, the core loaded and every other tool deferred behind the search tool. A tool the // app offers or withdraws mid-chat is a tool change appended to the chat, never an edit of the list. The model's own -// searches come back whole and go back unchanged. A local model and an OpenAI-style key keep their per-turn lists, and -// so does mimir cloud until its proxy has said it takes tool search: a proxy deployed before it gets what it always got. +// searches come back whole and go back unchanged. A local model and an OpenAI-style key keep their per-turn lists. mimir +// cloud sends the catalog from its first request; a proxy deployed before tool search refuses it, and gets the request +// once more as the per-turn pick, and that pick from then on. import { readFileSync } from 'node:fs' import { DEFAULT_POLICY, type ApprovalHost, type LlmHttpRequest, type LlmTransport, type PilotMachine, type PrinterHost } from '@slicerx/contracts' import { describe, expect, it } from 'vitest' import { z } from 'zod' import { createPilot, createTransportClient, DEFAULT_CONFIG, defineTool, type PilotTool, type Toolset } from '../src/index' +import { refusesToolSearch } from '../src/provider/client' import { createAnthropicAdapter, TOOL_CHANGES_BETA, TOOL_SEARCH_TOOL, toolSearchModel } from '../src/provider/anthropic' import type { LlmEvent, LlmRequest } from '../src/provider/types' import { catalogStep, SEARCH_CORE, type OfferedTool } from '../src/tool-search' @@ -85,8 +87,28 @@ const PROXY_SAYS = (() => { return JSON.parse(`"${lit.slice(1, -1).replaceAll('"', '\\"')}"`) as string })() -/** A provider that plays `steps` in order; `proxy` is a mimir cloud proxy that takes tool search, so each reply opens with its event. */ -function provider(steps: Step[], proxy = false): LlmTransport & { sent: LlmHttpRequest[] } { +/** A refusal as the desktop host raises it: the status before the proxy's words, and its code apart. */ +const refusal = (status: number, words: string, code = 'invalid_request'): Error => Object.assign(new Error(`${status}: ${words}`), { status, code }) + +/** The words a proxy deployed before tool search refused a catalog with (its shape.ts before tool search). */ +const OLD_TOO_MANY = 'tools must be a list of at most 128' + +/** + * What a proxy deployed before tool search did with a request: refused more than 128 tools, or a tool search tool, with + * a 400 and its words; anything else it took. + */ +function oldShape(req: LlmHttpRequest): Error | undefined { + const tools = toolsOf(req) + if (tools.length > 128) return refusal(400, OLD_TOO_MANY) + const search = tools.find((t) => typeof t['type'] === 'string' && t['type'] !== 'custom' && !String(t['type']).startsWith('web_search_')) + return search ? refusal(400, `tool ${String(search['type'])} is not allowed`) : undefined +} + +/** + * A provider that plays `steps` in order; `proxy` is a mimir cloud proxy that takes tool search, so each reply opens with + * its event; `refuse` answers a request with an error instead of a step. + */ +function provider(steps: Step[], proxy = false, refuse?: (req: LlmHttpRequest) => Error | undefined): LlmTransport & { sent: LlmHttpRequest[] } { const sent: LlmHttpRequest[] = [] let i = 0 return { @@ -94,6 +116,8 @@ function provider(steps: Step[], proxy = false): LlmTransport & { sent: LlmHttpR available: async () => true, async *stream(req) { sent.push(req) + const no = refuse?.(req) + if (no) throw no if (proxy) yield enc.encode(PROXY_SAYS) yield wire(steps[i++] ?? { text: 'Done.' }) }, @@ -108,14 +132,15 @@ const systemMessages = (req: LlmHttpRequest): Obj[] => messagesOf(req).filter((m const name = (t: Obj): string => String(t['name']).replaceAll('__', '.') /** - * A chat runtime on `kind`. On mimir cloud the proxy takes tool search unless `proxy` is 'old', and a first chat has - * heard it say so (`warm`, its requests left out of `sent`); `warm: false` leaves that out. + * A chat runtime on `kind`. On mimir cloud the proxy takes tool search unless `proxy` is 'old' (deployed before it, so it + * refuses a catalog, oldShape), and a first chat has run (`warm`, its requests left out of `sent`); `warm: false` leaves + * that out. */ async function setup(kind: Kind, steps: Step[], model = 'claude-haiku-5-5', extra: Partial[0]> = {}, opts: { proxy?: 'old' | 'new'; warm?: boolean } = {}) { const cloud = kind === 'mimir-cloud' const proxy = cloud && opts.proxy !== 'old' const warm = proxy && opts.warm !== false - const llm = provider(warm ? [{ text: 'Hi.' }, ...steps] : steps, proxy) + const llm = provider(warm ? [{ text: 'Hi.' }, ...steps] : steps, proxy, cloud && !proxy ? oldShape : undefined) const host = { printers: { list: async () => [], plugins: async () => [] } as unknown as PrinterHost, approvals: {} as ApprovalHost, llm } const { models: _tiers, ...base } = DEFAULT_CONFIG const pilot = createPilot({ host, config: { ...base, provider: kind, model, maxSteps: 6 }, policy: DEFAULT_POLICY, builtins: false, tools: catalog(), ...extra }) @@ -332,56 +357,163 @@ describe('a chat on a model with tool search', () => { }) }) -describe('mimir cloud before and after its proxy takes tool search', () => { +describe('mimir cloud on a proxy with tool search and on one deployed before it', () => { const capped = (r: LlmHttpRequest): boolean => toolsOf(r).length <= 128 && !toolsOf(r).some((t) => 'defer_loading' in t || t['type'] === TOOL_SEARCH_TOOL.type) && systemMessages(r).length === 0 && r.headers['anthropic-beta'] === undefined + const searchTools = (r: LlmHttpRequest): string[] => toolsOf(r).filter((t) => t['defer_loading'] !== true).map(name) + + it('sends the catalog from the first request on a fresh transport, the same tool list as a later chat', async () => { + const { llm, say } = await setup('mimir-cloud', [{ text: 'Hi.' }, { text: 'Still here.' }, { text: 'New chat.' }], 'claude-haiku-5-5', {}, { warm: false }) + const first = [...(await say('hello')), ...(await say('and now?'))] + expect(first).toHaveLength(2) + const tools = toolsOf(first[0]!) + expect(tools).toHaveLength(SEARCH_CORE.length + 150 + 1) + expect(searchTools(first[0]!)).toEqual([...SEARCH_CORE, 'tool_search_tool_bm25']) + const next = await say('a new chat', 's2') + // byte for byte, so the next chat reads the first chat's prefix from the cache + expect([toolsJson(first[1]!), toolsJson(next[0]!)]).toEqual([toolsJson(first[0]!), toolsJson(first[0]!)]) + expect(llm.sent).toHaveLength(3) + }) - it('on a proxy that never said so sends the per-turn pick: no deferral, no search tool and no tool changes', async () => { + it('on a proxy deployed before tool search sends the refused catalog once more as the per-turn pick, and the pick from then on', async () => { const planner = { plan: (from: PilotMachine, to: PilotMachine) => ({ from, to, changes: [], warnings: [] }) } - const { llm, pilot, say } = await setup('mimir-cloud', [{ text: 'Hi.' }, { call: 'settings.t1' }, { text: 'Explained.' }, { text: 'Here.' }], 'claude-haiku-5-5', { planner }, { proxy: 'old' }) - await say('hello') + const { llm, pilot, say } = await setup('mimir-cloud', [{ text: 'Hi.' }, { call: 'settings.t1' }, { text: 'Explained.' }, { text: 'Here.' }, { text: 'Again.' }], 'claude-haiku-5-5', { planner }, { proxy: 'old' }) + const errors: string[] = [] + for await (const e of pilot.run('s1', 'hello')) if (e.type === 'error') errors.push(e.message) + // the refusal never reaches the chat: the same step goes again at once, plain + expect(errors).toEqual([]) + expect(llm.sent).toHaveLength(2) + expect(toolsOf(llm.sent[0]!).length).toBeGreaterThan(128) + expect(searchTools(llm.sent[0]!)).toContain('tool_search_tool_bm25') + expect(capped(llm.sent[1]!)).toBe(true) // a read-only run, which on the catalog would withdraw tools by a tool change const a1 = { printer: 'A1', material: 'PLA', nozzle: 0.4 } for await (const _ of pilot.switchMachine('s1', a1, { ...a1, material: 'PETG' }, { narrate: true })) void _ await say('and now?') await say('a new chat', 's2') expect(pilot.tools().length).toBeGreaterThan(128) - expect(llm.sent.length).toBeGreaterThanOrEqual(4) - expect(llm.sent.filter((r) => !capped(r)).map((r) => toolsOf(r).length)).toEqual([]) + expect(llm.sent.length).toBeGreaterThanOrEqual(5) + expect(llm.sent.slice(1).filter((r) => !capped(r)).map((r) => toolsOf(r).length)).toEqual([]) + // a chat runtime made again on the same transport starts plain too + const host = { printers: { list: async () => [], plugins: async () => [] } as unknown as PrinterHost, approvals: {} as ApprovalHost, llm } + const { models: _tiers, ...base } = DEFAULT_CONFIG + const again = createPilot({ host, config: { ...base, provider: 'mimir-cloud', model: 'claude-haiku-5-5', maxSteps: 6 }, policy: DEFAULT_POLICY, builtins: false, tools: catalog() }) + const at = llm.sent.length + for await (const _ of again.run('s3', 'hello')) void _ + expect(llm.sent.slice(at).map(capped)).toEqual([true]) }) - it('keeps the pick for the whole chat that first hears it, and sends the catalog from the next chat on', async () => { - const { llm, say } = await setup('mimir-cloud', [{ text: 'Hi.' }, { text: 'Still here.' }, { text: 'New chat.' }], 'claude-haiku-5-5', {}, { warm: false }) - const first = [...(await say('hello')), ...(await say('and now?'))] - expect(first).toHaveLength(2) - expect(first.every(capped)).toBe(true) - const next = await say('a new chat', 's2') - const tools = toolsOf(next[0]!) - expect(tools).toHaveLength(SEARCH_CORE.length + 150 + 1) - expect(tools.filter((t) => t['defer_loading'] !== true).map(name)).toEqual([...SEARCH_CORE, 'tool_search_tool_bm25']) - expect(llm.sent).toHaveLength(3) + it('sends a catalog with no more than 128 tools once more too, when the old proxy refuses its search tool', async () => { + const few = catalog().slice(-40) + const llm = provider([{ text: 'Hi.' }], false, oldShape) + const host = { printers: { list: async () => [], plugins: async () => [] } as unknown as PrinterHost, approvals: {} as ApprovalHost, llm } + const { models: _tiers, ...base } = DEFAULT_CONFIG + const pilot = createPilot({ host, config: { ...base, provider: 'mimir-cloud', model: 'claude-haiku-5-5', maxSteps: 6 }, policy: DEFAULT_POLICY, builtins: false, tools: few }) + for await (const _ of pilot.run('s1', 'hello')) void _ + expect(llm.sent).toHaveLength(2) + expect(toolsOf(llm.sent[0]!).length).toBeLessThanOrEqual(128) + expect(searchTools(llm.sent[0]!)).toContain('tool_search_tool_bm25') + expect(capped(llm.sent[1]!)).toBe(true) + }) + + it('sends no request again for any other refusal, and keeps tool search on', async () => { + const others = [ + refusal(400, 'messages must be a list of 1 to 400'), + // the words of today's proxy for too many tools up front, a request the app got wrong + refusal(400, 'at most 128 tools load up front; defer the rest behind tool search'), + // the old words, but not as a 400 for the request's shape + refusal(502, OLD_TOO_MANY, 'provider_error'), + refusal(400, OLD_TOO_MANY, 'provider_error'), + ] + for (const no of others) { + const llm = provider([{ text: 'Hi.' }], false, () => no) + const host = { printers: { list: async () => [], plugins: async () => [] } as unknown as PrinterHost, approvals: {} as ApprovalHost, llm } + const { models: _tiers, ...base } = DEFAULT_CONFIG + const pilot = createPilot({ host, config: { ...base, provider: 'mimir-cloud', model: 'claude-haiku-5-5', maxSteps: 6 }, policy: DEFAULT_POLICY, builtins: false, tools: catalog() }) + const errors: string[] = [] + for await (const e of pilot.run('s1', 'hello')) if (e.type === 'error') errors.push(e.message) + expect(llm.sent, no.message).toHaveLength(1) + expect(errors, no.message).toEqual([no.message.replace(/^\d{3}: /, '')]) + expect(createTransportClient(llm, 'mimir-cloud').toolSearch, no.message).toBe(true) + } }) - it('hears the event from mimir cloud only, keeps it per transport, and passes it on no further', async () => { + it('sends the request once more at most, even when the old words come back for the pick', async () => { + const llm = provider([{ text: 'Hi.' }], false, () => refusal(400, OLD_TOO_MANY)) + const host = { printers: { list: async () => [], plugins: async () => [] } as unknown as PrinterHost, approvals: {} as ApprovalHost, llm } + const { models: _tiers, ...base } = DEFAULT_CONFIG + const pilot = createPilot({ host, config: { ...base, provider: 'mimir-cloud', model: 'claude-haiku-5-5', maxSteps: 6 }, policy: DEFAULT_POLICY, builtins: false, tools: catalog() }) + const errors: string[] = [] + for await (const e of pilot.run('s1', 'hello')) if (e.type === 'error') errors.push(e.message) + expect(llm.sent).toHaveLength(2) + expect(capped(llm.sent[1]!)).toBe(true) + expect(errors).toEqual([OLD_TOO_MANY]) + }) + + it("leaves every other provider's first request as it was", async () => { + // a key on the Messages API sends the catalog first; an OpenAI-style key and a model without tool search send the pick + for (const [kind, model, search] of [['anthropic', 'claude-haiku-5-5', true], ['openai', 'gpt-5.6-luna', false], ['mimir-cloud', 'claude-haiku-4-5', false]] as const) { + const { llm, say } = await setup(kind, [{ text: 'Hi.' }], model, {}, { warm: false }) + await say('hello') + if (search) expect(searchTools(llm.sent[0]!), kind).toEqual([...SEARCH_CORE, 'tool_search_tool_bm25']) + else { + const tools = toolsOf(llm.sent[0]!) + expect(tools.length, kind).toBeLessThanOrEqual(128) + expect(tools.some((t) => 'defer_loading' in t || t['type'] === TOOL_SEARCH_TOOL.type), kind).toBe(false) + } + } + }) + + it('reads the old proxy refusal from either host, and nothing else', () => { + // the desktop host puts the status before the words; the browser host keeps it apart + expect(refusesToolSearch(refusal(400, OLD_TOO_MANY))).toBe(true) + expect(refusesToolSearch(Object.assign(new Error(OLD_TOO_MANY), { status: 400, code: 'invalid_request' }))).toBe(true) + expect(refusesToolSearch(refusal(400, 'tool tool_search_tool_bm25_20251119 is not allowed'))).toBe(true) + expect(refusesToolSearch(new Error(`400: ${OLD_TOO_MANY}`))).toBe(true) + expect(refusesToolSearch(new Error(OLD_TOO_MANY))).toBe(false) + expect(refusesToolSearch(refusal(400, 'tool web_fetch_20250910 is not allowed'))).toBe(false) + expect(refusesToolSearch(refusal(400, 'tools must be a list of at most 512'))).toBe(false) + expect(refusesToolSearch(refusal(429, OLD_TOO_MANY, 'day_used'))).toBe(false) + }) + + it('hears the event from mimir cloud only, takes back a refusal by it, keeps both per transport, and passes it on no further', async () => { const said = (): LlmTransport & { sent: LlmHttpRequest[] } => provider([{ text: 'Hi.' }, { text: 'Hi.' }], true) const ask: LlmRequest = { model: 'claude-haiku-5-5', messages: [{ role: 'user', content: 'hello' }], tools: [] } - const events = async (c: ReturnType): Promise => { - const out: string[] = [] - for await (const e of c.stream(ask)) out.push(e.type) + const events = async (c: ReturnType, req = ask): Promise => { + const out: LlmEvent[] = [] + for await (const e of c.stream(req)) out.push(e) return out } - const llm = said() - // a key on the Messages API takes tool search without the event, and an event there changes nothing + // a transport whose proxy refuses the first request for its shape, and then takes tool search + let refuse = true + const inner = said() + const llm: LlmTransport = { + available: async () => true, + async *stream(req, signal) { + if (refuse) { + refuse = false + throw refusal(400, OLD_TOO_MANY) + } + yield* inner.stream(req, signal) + }, + } const direct = createTransportClient(llm, 'anthropic') const cloud = createTransportClient(llm, 'mimir-cloud') - expect([direct.toolSearch, cloud.toolSearch]).toEqual([true, false]) - expect(await events(direct)).not.toContain('tool_search') - expect(cloud.toolSearch).toBe(false) - expect(await events(cloud)).toEqual(['text', 'usage', 'done']) + expect([direct.toolSearch, cloud.toolSearch]).toEqual([true, true]) + // a plain request refused with the old words leaves tool search alone: only a tool search request is the shape + expect((await events(cloud)).map((e) => e.type)).toEqual(['error', 'done']) expect(cloud.toolSearch).toBe(true) - // a runtime made again on the same transport knows at once; another transport, or an OpenAI-style key, does not - expect(createTransportClient(llm, 'mimir-cloud').toolSearch).toBe(true) - expect(createTransportClient(said(), 'mimir-cloud').toolSearch).toBe(false) + refuse = true + const refused = await events(cloud, { ...ask, toolSearch: true }) + expect(refused[0]).toEqual({ type: 'error', message: OLD_TOO_MANY, retryable: false, code: 'invalid_request', searchRefused: true }) + expect(cloud.toolSearch).toBe(false) + // a runtime made again on the same transport knows at once; another transport, or a key on the Messages API, does not + expect(createTransportClient(llm, 'mimir-cloud').toolSearch).toBe(false) + expect(createTransportClient(said(), 'mimir-cloud').toolSearch).toBe(true) + expect(direct.toolSearch).toBe(true) expect(createTransportClient(llm, 'openai').toolSearch).toBe(false) + // the proxy's event takes the refusal back, and goes no further + expect((await events(cloud)).map((e) => e.type)).toEqual(['text', 'usage', 'done']) + expect(cloud.toolSearch).toBe(true) // the adapter reads it, and only the proxy's own event with its flag set const parsed: LlmEvent[] = [] async function* chunks(): AsyncIterable {