From e4bbcb53c636deabdf01dac8c940fbcbf6751c5e Mon Sep 17 00:00:00 2001 From: "subydoo-ci[bot]" <343861189+subydoo-ci[bot]@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:06:26 -0800 Subject: [PATCH] fix(pilot): mimir's first chat uses tool search too, so the next chat reuses its cache On mimir cloud the app turned tool search on only after a reply opened with the proxy's mimir event, so the first chat after launch went out with the 128-tool per-turn pick and every later chat with the deferred catalog. The two tool lists share no prefix, so the next chat wrote the whole prompt cache again. Tool search is now on from the first request. A proxy deployed before tool search refuses that request with a 400 ("tools must be a list of at most 128", or "tool tool_search_tool_bm25_... is not allowed" for a short catalog). On that refusal the client marks the transport as plain and the runtime sends the same step once more with the per-turn pick, without showing the refusal. It never resends twice, and any other error behaves as before. The proxy's mimir event still clears the mark. Other providers are unchanged. Signed-off-by: subydoo-ci[bot] <343861189+subydoo-ci[bot]@users.noreply.github.com> --- .../app/test/mimir-sticky-toolsets.test.ts | 27 ++- packages/app/test/mimir-tool-cap.test.ts | 40 ++-- packages/pilot/src/provider/client.ts | 57 +++-- packages/pilot/src/provider/types.ts | 14 +- packages/pilot/src/runtime.ts | 20 +- packages/pilot/src/tool-search.ts | 2 +- packages/pilot/test/sticky-toolsets.test.ts | 9 +- packages/pilot/test/tool-search.test.ts | 204 ++++++++++++++---- 8 files changed, 294 insertions(+), 79 deletions(-) diff --git a/packages/app/test/mimir-sticky-toolsets.test.ts b/packages/app/test/mimir-sticky-toolsets.test.ts index 02de5a687..8b3d9fab9 100644 --- a/packages/app/test/mimir-sticky-toolsets.test.ts +++ b/packages/app/test/mimir-sticky-toolsets.test.ts @@ -1,7 +1,8 @@ // @vitest-environment node // SPDX-License-Identifier: Apache-2.0 // Copyright (C) 2026 The SlicerX contributors -// On a capped mimir cloud chat the tools a request carries follow the chat, not only the message. The worst case is the +// On a capped mimir cloud chat (a proxy deployed before tool search, which refuses the catalog the chat's first request +// carries, so the chat goes on the per-turn pick) the tools a request carries follow the chat, not only the message. The worst case is the // desktop app with every connector and twelve printing printers (well over 128 tools offered, 128 sent). A "yes please" after // mimir offers to print gets the printer tools, the list is the same request after request while the chat stays on its // topic (so the proxy's prompt cache keeps hitting), and every request stays at 128 tools or under. Each request is read @@ -55,14 +56,30 @@ function reply(step: { text?: string; call?: { name: string; input?: unknown } } ) } -/** A transport that keeps each request and plays the replies in order. */ -function transport(steps: { text?: string; call?: { name: string; input?: unknown } }[]): LlmTransport & { sent: LlmHttpRequest[] } { +/** + * What a mimir cloud proxy deployed before tool search answered a request that sends the tool catalog (its shape.ts + * then): a 400 for more than 128 tools, or for the tool search tool, as the desktop host raises it. + */ +function oldRefusal(req: LlmHttpRequest): Error | undefined { + const tools = (JSON.parse(req.body) as { tools?: { type?: string }[] }).tools ?? [] + const search = tools.find((t) => t.type !== undefined && t.type !== 'custom' && !t.type.startsWith('web_search_')) + const words = tools.length > 128 ? 'tools must be a list of at most 128' : search ? `tool ${String(search.type)} is not allowed` : null + return words ? Object.assign(new Error(`400: ${words}`), { status: 400, code: 'invalid_request' }) : undefined +} + +/** + * A transport that keeps each request and plays the replies in order. `old`: a mimir cloud proxy deployed before tool + * search, which refuses a catalog (oldRefusal); a refused request is not kept and plays no reply. + */ +function transport(steps: { text?: string; call?: { name: string; input?: unknown } }[], old = false): LlmTransport & { sent: LlmHttpRequest[] } { const sent: LlmHttpRequest[] = [] let i = 0 return { sent, available: async () => true, async *stream(req) { + const no = old ? oldRefusal(req) : undefined + if (no) throw no sent.push(req) yield reply(steps[i++] ?? { text: 'Done.' }) }, @@ -91,11 +108,11 @@ async function registerApp(platform: Platform): Promise<() => void> { return off } -/** A fresh mimir cloud chat in the desktop app, with the replies it plays. */ +/** A fresh mimir cloud chat in the desktop app on a proxy deployed before tool search, with the replies it plays. */ async function chat(steps: Parameters[0]) { resetTabMimir() set({ pilot: { mode: 'on', provider: 'mimir-cloud' } as never }) - const llm = transport(steps) + const llm = transport(steps, true) const pilot = await tabMimir(hostFor('desktop', llm), withCloud(NEUTRAL), 'cloud') const say = async (message: string): Promise => { const at = llm.sent.length diff --git a/packages/app/test/mimir-tool-cap.test.ts b/packages/app/test/mimir-tool-cap.test.ts index aee58c5b1..b9c0851fe 100644 --- a/packages/app/test/mimir-tool-cap.test.ts +++ b/packages/app/test/mimir-tool-cap.test.ts @@ -2,13 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 // Copyright (C) 2026 The SlicerX contributors // No mimir tab request loads more than 128 tools. OpenAI-style APIs and local servers refuse a longer list, and so does -// mimir cloud's proxy for the tools that load up front (supabase/functions/mimir/shape.ts). Once the proxy has said it -// takes tool search (the event its streams open with), a new chat on mimir cloud carries the tab's whole catalog, most -// of it deferred behind tool search; a proxy that never said so gets the per-turn pick it always got. The tab is made +// mimir cloud's proxy for the tools that load up front (supabase/functions/mimir/shape.ts). A chat on mimir cloud carries +// the tab's whole catalog from its first request, most of it deferred behind tool search; a proxy deployed before tool +// search refuses that, and gets the request again as the per-turn pick it always got. The tab is made // the way the app makes it, with the commands the app registers at startup, its features' commands, a command per // connected printer and every connector's tools, and each request is read back from the HTTP body the provider adapter // wrote. Every kind of turn is checked in the desktop app (which slices and prints) and the browser build (which only -// models), on mimir cloud before and after its proxy takes tool search, a key and a local model, in the neutral edition +// models), on mimir cloud on a proxy deployed before tool search and on one with it, a key and a local model, in the neutral edition // and in one with the printer and web tools off. Last, a recorded chat's requests go through the proxy's own shape(): // the same tools on every request. import type { EditionConfig } from '@slicerx/edition-config' @@ -56,18 +56,32 @@ const PROXY_SAYS = (() => { return JSON.parse(`"${lit.slice(1, -1).replaceAll('"', '\\"')}"`) as string })() +/** + * What a mimir cloud proxy deployed before tool search answered a request that sends the tool catalog (its shape.ts + * then): a 400 for more than 128 tools, or for the tool search tool, as the desktop host raises it. + */ +function oldRefusal(req: LlmHttpRequest): Error | undefined { + const tools = (JSON.parse(req.body) as { tools?: { type?: string }[] }).tools ?? [] + const search = tools.find((t) => t.type !== undefined && t.type !== 'custom' && !t.type.startsWith('web_search_')) + const words = tools.length > 128 ? 'tools must be a list of at most 128' : search ? `tool ${String(search.type)} is not allowed` : null + return words ? Object.assign(new Error(`400: ${words}`), { status: 400, code: 'invalid_request' }) : undefined +} + /** * A transport that keeps each request and answers none of them, so every turn ends after its first request. `proxy`: - * a mimir cloud proxy that takes tool search, so each answer opens with its event before it fails. + * 'search', a mimir cloud proxy that takes tool search, so each answer opens with its event before it fails; 'old', one + * deployed before tool search, which refuses a catalog (oldRefusal) and keeps no refused request. */ -function transport(proxy = false): LlmTransport & { sent: LlmHttpRequest[] } { +function transport(proxy?: 'search' | 'old'): LlmTransport & { sent: LlmHttpRequest[] } { const sent: LlmHttpRequest[] = [] return { sent, available: async () => true, async *stream(req) { + const no = proxy === 'old' ? oldRefusal(req) : undefined + if (no) throw no sent.push(req) - if (proxy) yield new TextEncoder().encode(PROXY_SAYS) + if (proxy === 'search') yield new TextEncoder().encode(PROXY_SAYS) throw new Error('not answered in this test') }, } @@ -87,8 +101,8 @@ const EDITIONS: [string, EditionConfig][] = [ ] /** - * How mimir runs: mimir cloud on a proxy that never said it takes tool search (deployed before it), mimir cloud once its - * proxy has said so, a key on an OpenAI-style API, and a local model. + * How mimir runs: mimir cloud on a proxy deployed before tool search (which refuses the catalog), mimir cloud on one that + * takes it, a key on an OpenAI-style API, and a local model. */ const MODES = { 'mimir cloud': { mode: 'on', provider: 'mimir-cloud' }, @@ -127,15 +141,15 @@ function loadedNames(req: LlmHttpRequest): string[] { const SEARCH = 'tool_search_tool_bm25' /** - * The first request of a turn in a fresh chat, after `before` when given, and how many tools the tab offers. With tool - * search, a first chat has heard the proxy say it takes it. + * The first request the proxy took of a turn in a fresh chat, after `before` when given, and how many tools the tab + * offers. With tool search, a first chat has run before it. */ async function turn(platform: Platform, edition: EditionConfig, mode: Mode, t: (typeof TURNS)[number]): Promise<{ req: LlmHttpRequest; offered: number }> { resetTabMimir() set({ pilot: MODES[mode] as never }) const searching = mode === 'mimir cloud, tool search' - const llm = transport(searching) const cloud = MODES[mode].provider === 'mimir-cloud' + const llm = transport(searching ? 'search' : cloud ? 'old' : undefined) const pilot = await tabMimir(hostFor(platform, llm), cloud ? withCloud(edition) : edition, cloud ? 'cloud' : null) if (searching) for await (const _ of pilot.run('warm', 'hello')) void _ if (t.before) for await (const _ of pilot.run('s', t.before)) void _ @@ -208,7 +222,7 @@ describe('no mimir tab request loads more than 128 tools', () => { }, 120_000) } - it('sends the per-turn pick on mimir cloud until its proxy says it takes tool search: no deferral, search tool or tool change', async () => { + it('sends the per-turn pick on mimir cloud once a proxy deployed before tool search refused the catalog: no deferral, search tool or tool change', async () => { const off = await registerApp('desktop') try { for (const t of TURNS) { diff --git a/packages/pilot/src/provider/client.ts b/packages/pilot/src/provider/client.ts index 12324b0ac..924e60214 100644 --- a/packages/pilot/src/provider/client.ts +++ b/packages/pilot/src/provider/client.ts @@ -22,9 +22,9 @@ export const MIMIR_CLOUD = 'mimir-cloud' /** * The most tools that load up front in one mimir cloud request (supabase/functions/mimir/shape.ts, MAX_LOADED_TOOLS); * it refuses more. The desktop app's tab offers more than that (the design tools, slicing, printers, the app's - * commands, a command per printer and the connectors' tools). Once the proxy has said it takes tool search, a new chat - * on a model with it sends them all, most deferred (tool-search.ts); until then, and on any other model, a turn sends - * the tools it needs first (toolsets.ts). + * commands, a command per printer and the connectors' tools). A chat on a model with tool search sends them all, most + * deferred (tool-search.ts), from its first request; on a proxy that refuses that (refusesToolSearch), and on any other + * model, a turn sends the tools it needs first (toolsets.ts). */ export const MIMIR_CLOUD_MAX_TOOLS = 128 @@ -76,31 +76,57 @@ export function adapterFor(provider: string): LlmAdapter { return make() } -/** Joins an adapter with the host transport. The transport adds the key; this code never sees it. */ /** - * The transports whose mimir cloud proxy said it takes a request that sends the whole tool catalog through tool search - * (the `mimir` event its streams open with, supabase/functions/mimir/handler.ts). Kept per transport, so a chat runtime + * What a mimir cloud proxy deployed before tool search answers a request that sends the whole catalog: its shape.ts + * then took at most 128 tools and no tool search tool, and refused anything else with a 400 and these words. + */ +const OLD_SHAPE = /^(?:tools must be a list of at most 128|tool tool_search_tool_(?:bm25|regex)_\d+ is not allowed)$/ + +/** + * Whether a failed mimir cloud request was refused by a proxy deployed before tool search, for the request's shape: + * a 400 with its invalid request code (or none) and one of its old words (OLD_SHAPE). The desktop host puts the status + * before the words (`400: ...`); the browser's keeps it apart. + */ +export function refusesToolSearch(e: unknown): boolean { + const status = e && typeof e === 'object' && 'status' in e ? (e as { status: unknown }).status : undefined + const msg = errorText(e) + const said = /^400: /.test(msg) + if (typeof status === 'number' ? status !== 400 : !said) return false + const code = errorCode(e) + if (code !== undefined && code !== 'invalid_request') return false + return OLD_SHAPE.test(said ? msg.slice(5) : msg) +} + +/** + * The transports whose mimir cloud proxy refused a request that sends the whole tool catalog through tool search + * (refusesToolSearch). Their chats send the per-turn pick from then on, until the proxy opens a reply with its `mimir` + * event (supabase/functions/mimir/handler.ts), which says it takes tool search. Kept per transport, so a chat runtime * made again on the same host knows it at once. */ -const searchingProxies = new WeakSet() +const plainProxies = new WeakSet() +/** Joins an adapter with the host transport. The transport adds the key; this code never sees it. */ export function createTransportClient(transport: LlmTransport, provider: string, opts: { baseUrl?: string; share?: () => CloudShare } = {}): LlmClient { const adapter = adapterFor(provider) const cloud = provider === MIMIR_CLOUD // mimir cloud's refusals (daily caps, a pause) are shown as the proxy wrote them, never retried elsewhere - const failure = (e: unknown): LlmEvent => { + // a tool search request an old proxy refused marks the transport, and the runtime sends it once more without + const failure = (e: unknown, req: LlmRequest): LlmEvent => { if (!cloud) return { type: 'error', message: errorText(e), retryable: true } const code = errorCode(e) - return { type: 'error', message: errorText(e).replace(/^\d{3}: /, ''), retryable: false, ...(code ? { code } : {}) } + const refused = req.toolSearch === true && refusesToolSearch(e) + if (refused) plainProxies.add(transport) + return { type: 'error', message: errorText(e).replace(/^\d{3}: /, ''), retryable: false, ...(code ? { code } : {}), ...(refused ? { searchRefused: true as const } : {}) } } return { provider, // the proxy's own limit, though the Messages API it speaks has none; a tool search request carries more, deferred ...(cloud ? { maxTools: MIMIR_CLOUD_MAX_TOOLS } : adapter.maxTools === undefined ? {} : { maxTools: adapter.maxTools }), - // the Messages API searches deferred tools; through mimir cloud only once its proxy has said it passes them, so an - // app that reaches a proxy deployed before tool search keeps the per-turn pick + // the Messages API searches deferred tools, and so does mimir cloud from the first request, so the first chat sends + // the same tools as every later one; an app that reaches a proxy deployed before tool search keeps the per-turn pick + // once that proxy has refused the catalog get toolSearch(): boolean { - return adapter.id === 'anthropic' && (!cloud || searchingProxies.has(transport)) + return adapter.id === 'anthropic' && (!cloud || !plainProxies.has(transport)) }, async *stream(req: LlmRequest, signal?: AbortSignal): AsyncIterable { let http = adapter.build(req, opts.baseUrl === undefined ? {} : { baseUrl: opts.baseUrl }) @@ -110,7 +136,7 @@ export function createTransportClient(transport: LlmTransport, provider: string, try { body = cloud ? transport.stream(http, signal, (h) => (headers = h)) : transport.stream(http, signal) } catch (e) { - yield failure(e) + yield failure(e, req) yield { type: 'done', stop: 'error' } return } @@ -126,12 +152,13 @@ export function createTransportClient(transport: LlmTransport, provider: string, const model = cloudModel(headers) if (model) yield { type: 'model', model } for await (const ev of adapter.parse(replay(first, it))) { - if (ev.type === 'tool_search') searchingProxies.add(transport) + // the proxy says it takes tool search: a refusal before (a proxy deployed since) no longer holds + if (ev.type === 'tool_search') plainProxies.delete(transport) else yield ev } } catch (e) { if (signal?.aborted) throw e - yield failure(e) + yield failure(e, req) yield { type: 'done', stop: 'error' } } }, diff --git a/packages/pilot/src/provider/types.ts b/packages/pilot/src/provider/types.ts index e72af4b08..7462f4699 100644 --- a/packages/pilot/src/provider/types.ts +++ b/packages/pilot/src/provider/types.ts @@ -92,13 +92,18 @@ export type LlmEvent = /** `webSearches`: the hosted web searches the reply ran, when the provider reports them. */ | { type: 'usage'; inputTokens: number; outputTokens: number; cacheReadTokens?: number; cacheWriteTokens?: number; webSearches?: number } | { type: 'done'; stop: LlmStop; raw?: unknown[] } - /** `code` is mimir cloud's refusal code (`day_used`, `paused`, ...). */ - | { type: 'error'; message: string; retryable: boolean; code?: string } + /** + * `code` is mimir cloud's refusal code (`day_used`, `paused`, ...). `searchRefused`: a mimir cloud proxy deployed before + * tool search refused this tool search request for its shape; the client no longer takes tool search on it + * (LlmClient.toolSearch), and the runtime sends the request once more with the per-turn pick. + */ + | { type: 'error'; message: string; retryable: boolean; code?: string; searchRefused?: true } /** The model that answered, when the reply names it (mimir cloud's `x-sx-model`). Comes before the reply's text. */ | { type: 'model'; model: string } /** * mimir cloud's proxy takes a request that sends the whole tool catalog through tool search (its `mimir` stream event, - * supabase/functions/mimir/handler.ts). The transport client keeps it (LlmClient.toolSearch) and passes it on no further. + * supabase/functions/mimir/handler.ts). The transport client takes back an earlier refusal on it (LlmClient.toolSearch) + * and passes it on no further. */ | { type: 'tool_search' } @@ -118,7 +123,8 @@ export interface LlmClient { maxTools?: number /** * The provider runs a tool search over deferred tools and takes tool changes mid-chat (LlmRequest.toolSearch). It can - * turn true while the client runs: mimir cloud's proxy says so in its replies. + * change while the client runs: a mimir cloud proxy deployed before tool search turns it false by refusing a tool + * search request, and true again by saying in a reply that it takes tool search. */ readonly toolSearch?: boolean stream(req: LlmRequest, signal?: AbortSignal): AsyncIterable diff --git a/packages/pilot/src/runtime.ts b/packages/pilot/src/runtime.ts index c10b5d7c6..83ca71a2b 100644 --- a/packages/pilot/src/runtime.ts +++ b/packages/pilot/src/runtime.ts @@ -885,9 +885,9 @@ export function createPilot(opts: CreatePilotOptions): Pilot & { /** Each chat's tool catalog on a model with tool search (tool-search.ts): declared once, only ever appended to. */ const catalogs = new Map() /** - * Whether each chat sends its tool catalog (tool-search.ts) or the per-turn pick, fixed at its first request. mimir - * cloud's proxy says it takes tool search only in its replies, so the chat that first hears it keeps the pick it - * started with, and no chat's tool list changes shape partway through; the next chat sends the catalog. + * Whether each chat sends its tool catalog (tool-search.ts) or the per-turn pick, fixed at its first request, so no + * chat's tool list changes shape partway through. The one change: a mimir cloud proxy deployed before tool search + * refuses the catalog, and the chat goes on the pick from that request, sent once more. */ const searchChats = new Map() /** @@ -1647,6 +1647,8 @@ export function createPilot(opts: CreatePilotOptions): Pilot & { let slowed = 0 // a request lost to the network is sent once more before the run says so (NETWORK_RETRY_MS, NETWORK_LINE) let netRetried = false + // a tool search request that a proxy deployed before tool search refused goes once more with the per-turn pick, never twice + let searchResent = false // the run's limits; an approved plan raises them to fit its features (planLimits) let maxSteps = opts.config.maxSteps let maxToolCalls = opts.config.maxToolCalls @@ -1752,6 +1754,7 @@ export function createPilot(opts: CreatePilotOptions): Pilot & { let failed: string | null = null let netDown = false let failedCode: string | undefined + let searchRefused = false let cut = false const model = modelFor(opts.config, tier) if (capped.report) { @@ -1826,6 +1829,7 @@ export function createPilot(opts: CreatePilotOptions): Pilot & { case 'error': failed = ev.message failedCode = ev.code + searchRefused = ev.searchRefused === true netDown = networkLost({ message: ev.message }) break case 'done': @@ -1884,6 +1888,16 @@ export function createPilot(opts: CreatePilotOptions): Pilot & { stopReason = 'error' break } + if (failed !== null && searchRefused && searching && !searchResent && !text && toolCalls.length === 0) { + // a proxy deployed before tool search refused the catalog: the chat drops it and sends the same step once more, + // unseen, with the per-turn pick (the client takes no tool search on this transport now, so it cannot come again) + searchResent = true + dropUnanswered(sessionId, messages) + searchChats.delete(sessionId) + catalogs.delete(sessionId) + step-- + continue + } if (failed !== null) { emit({ type: 'error', message: failed, retryable: true, ...(failedCode ? { code: failedCode } : {}) }) stopReason = 'error' diff --git a/packages/pilot/src/tool-search.ts b/packages/pilot/src/tool-search.ts index 23361f566..a04eaca2b 100644 --- a/packages/pilot/src/tool-search.ts +++ b/packages/pilot/src/tool-search.ts @@ -6,7 +6,7 @@ // never edited: a tool the app offers later is appended deferred, and offering or withdrawing a tool mid-chat (a run // limited to read-only tools, a printer or connector that comes or goes) is a tool change appended to the chat // (LlmMessage `tools`). Local models and providers without tool search keep their per-turn lists (toolsets.ts), and so -// does a chat on mimir cloud until its proxy has said it takes tool search (LlmClient.toolSearch, runtime.ts). +// does a chat on mimir cloud whose proxy, deployed before tool search, refused the catalog (LlmClient.toolSearch, runtime.ts). import type { LlmToolDef } from './provider/types' /** diff --git a/packages/pilot/test/sticky-toolsets.test.ts b/packages/pilot/test/sticky-toolsets.test.ts index 8c53ce398..93184cde7 100644 --- a/packages/pilot/test/sticky-toolsets.test.ts +++ b/packages/pilot/test/sticky-toolsets.test.ts @@ -736,7 +736,11 @@ describe('a paint ask in a capped design chat', () => { }) describe('words that ask for a held-back tool on a capped design chat', () => { - /** A design chat over the limit with the more design tools and the trending models command, whose model builds once and stops. */ + /** + * A design chat over the limit with the more design tools and the trending models command, whose model builds once and + * stops, on a proxy deployed before tool search: it refuses the chat's first request, the whole catalog, so the chat + * goes on the per-turn pick (left out of `sent`). + */ function chat() { const named = (name: string): PilotTool => defineTool({ name, version: '1.0.0', description: `${name} does one thing`, source: 'command', permission: 'read', input: z.object({}).passthrough(), run: async () => ({ summary: `${name} ran` }) }) as PilotTool @@ -746,8 +750,9 @@ describe('words that ask for a held-back tool on a capped design chat', () => { const llm = { available: async () => true, async *stream(req: LlmHttpRequest) { + const body = JSON.parse(req.body) as { messages: { role: string; content: unknown }[]; tools?: unknown[] } + if ((body.tools ?? []).length > 128) throw Object.assign(new Error('400: tools must be a list of at most 128'), { status: 400, code: 'invalid_request' }) sent.push(req) - const body = JSON.parse(req.body) as { messages: { role: string; content: unknown }[] } yield wire('mimir-cloud', JSON.stringify(body.messages.at(-1)?.content ?? '').includes('tool_result') ? { text: 'Done.' } : { call: 'geom.t0' }) }, } as unknown as LlmTransport diff --git a/packages/pilot/test/tool-search.test.ts b/packages/pilot/test/tool-search.test.ts index e855ddd5c..80acdfc3d 100644 --- a/packages/pilot/test/tool-search.test.ts +++ b/packages/pilot/test/tool-search.test.ts @@ -3,13 +3,15 @@ // A Claude model with tool search gets the chat's whole tool catalog on every request: the same bytes in the same order // from the first request to the last, the core loaded and every other tool deferred behind the search tool. A tool the // app offers or withdraws mid-chat is a tool change appended to the chat, never an edit of the list. The model's own -// searches come back whole and go back unchanged. A local model and an OpenAI-style key keep their per-turn lists, and -// so does mimir cloud until its proxy has said it takes tool search: a proxy deployed before it gets what it always got. +// searches come back whole and go back unchanged. A local model and an OpenAI-style key keep their per-turn lists. mimir +// cloud sends the catalog from its first request; a proxy deployed before tool search refuses it, and gets the request +// once more as the per-turn pick, and that pick from then on. import { readFileSync } from 'node:fs' import { DEFAULT_POLICY, type ApprovalHost, type LlmHttpRequest, type LlmTransport, type PilotMachine, type PrinterHost } from '@slicerx/contracts' import { describe, expect, it } from 'vitest' import { z } from 'zod' import { createPilot, createTransportClient, DEFAULT_CONFIG, defineTool, type PilotTool, type Toolset } from '../src/index' +import { refusesToolSearch } from '../src/provider/client' import { createAnthropicAdapter, TOOL_CHANGES_BETA, TOOL_SEARCH_TOOL, toolSearchModel } from '../src/provider/anthropic' import type { LlmEvent, LlmRequest } from '../src/provider/types' import { catalogStep, SEARCH_CORE, type OfferedTool } from '../src/tool-search' @@ -85,8 +87,28 @@ const PROXY_SAYS = (() => { return JSON.parse(`"${lit.slice(1, -1).replaceAll('"', '\\"')}"`) as string })() -/** A provider that plays `steps` in order; `proxy` is a mimir cloud proxy that takes tool search, so each reply opens with its event. */ -function provider(steps: Step[], proxy = false): LlmTransport & { sent: LlmHttpRequest[] } { +/** A refusal as the desktop host raises it: the status before the proxy's words, and its code apart. */ +const refusal = (status: number, words: string, code = 'invalid_request'): Error => Object.assign(new Error(`${status}: ${words}`), { status, code }) + +/** The words a proxy deployed before tool search refused a catalog with (its shape.ts before tool search). */ +const OLD_TOO_MANY = 'tools must be a list of at most 128' + +/** + * What a proxy deployed before tool search did with a request: refused more than 128 tools, or a tool search tool, with + * a 400 and its words; anything else it took. + */ +function oldShape(req: LlmHttpRequest): Error | undefined { + const tools = toolsOf(req) + if (tools.length > 128) return refusal(400, OLD_TOO_MANY) + const search = tools.find((t) => typeof t['type'] === 'string' && t['type'] !== 'custom' && !String(t['type']).startsWith('web_search_')) + return search ? refusal(400, `tool ${String(search['type'])} is not allowed`) : undefined +} + +/** + * A provider that plays `steps` in order; `proxy` is a mimir cloud proxy that takes tool search, so each reply opens with + * its event; `refuse` answers a request with an error instead of a step. + */ +function provider(steps: Step[], proxy = false, refuse?: (req: LlmHttpRequest) => Error | undefined): LlmTransport & { sent: LlmHttpRequest[] } { const sent: LlmHttpRequest[] = [] let i = 0 return { @@ -94,6 +116,8 @@ function provider(steps: Step[], proxy = false): LlmTransport & { sent: LlmHttpR available: async () => true, async *stream(req) { sent.push(req) + const no = refuse?.(req) + if (no) throw no if (proxy) yield enc.encode(PROXY_SAYS) yield wire(steps[i++] ?? { text: 'Done.' }) }, @@ -108,14 +132,15 @@ const systemMessages = (req: LlmHttpRequest): Obj[] => messagesOf(req).filter((m const name = (t: Obj): string => String(t['name']).replaceAll('__', '.') /** - * A chat runtime on `kind`. On mimir cloud the proxy takes tool search unless `proxy` is 'old', and a first chat has - * heard it say so (`warm`, its requests left out of `sent`); `warm: false` leaves that out. + * A chat runtime on `kind`. On mimir cloud the proxy takes tool search unless `proxy` is 'old' (deployed before it, so it + * refuses a catalog, oldShape), and a first chat has run (`warm`, its requests left out of `sent`); `warm: false` leaves + * that out. */ async function setup(kind: Kind, steps: Step[], model = 'claude-haiku-5-5', extra: Partial[0]> = {}, opts: { proxy?: 'old' | 'new'; warm?: boolean } = {}) { const cloud = kind === 'mimir-cloud' const proxy = cloud && opts.proxy !== 'old' const warm = proxy && opts.warm !== false - const llm = provider(warm ? [{ text: 'Hi.' }, ...steps] : steps, proxy) + const llm = provider(warm ? [{ text: 'Hi.' }, ...steps] : steps, proxy, cloud && !proxy ? oldShape : undefined) const host = { printers: { list: async () => [], plugins: async () => [] } as unknown as PrinterHost, approvals: {} as ApprovalHost, llm } const { models: _tiers, ...base } = DEFAULT_CONFIG const pilot = createPilot({ host, config: { ...base, provider: kind, model, maxSteps: 6 }, policy: DEFAULT_POLICY, builtins: false, tools: catalog(), ...extra }) @@ -332,56 +357,163 @@ describe('a chat on a model with tool search', () => { }) }) -describe('mimir cloud before and after its proxy takes tool search', () => { +describe('mimir cloud on a proxy with tool search and on one deployed before it', () => { const capped = (r: LlmHttpRequest): boolean => toolsOf(r).length <= 128 && !toolsOf(r).some((t) => 'defer_loading' in t || t['type'] === TOOL_SEARCH_TOOL.type) && systemMessages(r).length === 0 && r.headers['anthropic-beta'] === undefined + const searchTools = (r: LlmHttpRequest): string[] => toolsOf(r).filter((t) => t['defer_loading'] !== true).map(name) + + it('sends the catalog from the first request on a fresh transport, the same tool list as a later chat', async () => { + const { llm, say } = await setup('mimir-cloud', [{ text: 'Hi.' }, { text: 'Still here.' }, { text: 'New chat.' }], 'claude-haiku-5-5', {}, { warm: false }) + const first = [...(await say('hello')), ...(await say('and now?'))] + expect(first).toHaveLength(2) + const tools = toolsOf(first[0]!) + expect(tools).toHaveLength(SEARCH_CORE.length + 150 + 1) + expect(searchTools(first[0]!)).toEqual([...SEARCH_CORE, 'tool_search_tool_bm25']) + const next = await say('a new chat', 's2') + // byte for byte, so the next chat reads the first chat's prefix from the cache + expect([toolsJson(first[1]!), toolsJson(next[0]!)]).toEqual([toolsJson(first[0]!), toolsJson(first[0]!)]) + expect(llm.sent).toHaveLength(3) + }) - it('on a proxy that never said so sends the per-turn pick: no deferral, no search tool and no tool changes', async () => { + it('on a proxy deployed before tool search sends the refused catalog once more as the per-turn pick, and the pick from then on', async () => { const planner = { plan: (from: PilotMachine, to: PilotMachine) => ({ from, to, changes: [], warnings: [] }) } - const { llm, pilot, say } = await setup('mimir-cloud', [{ text: 'Hi.' }, { call: 'settings.t1' }, { text: 'Explained.' }, { text: 'Here.' }], 'claude-haiku-5-5', { planner }, { proxy: 'old' }) - await say('hello') + const { llm, pilot, say } = await setup('mimir-cloud', [{ text: 'Hi.' }, { call: 'settings.t1' }, { text: 'Explained.' }, { text: 'Here.' }, { text: 'Again.' }], 'claude-haiku-5-5', { planner }, { proxy: 'old' }) + const errors: string[] = [] + for await (const e of pilot.run('s1', 'hello')) if (e.type === 'error') errors.push(e.message) + // the refusal never reaches the chat: the same step goes again at once, plain + expect(errors).toEqual([]) + expect(llm.sent).toHaveLength(2) + expect(toolsOf(llm.sent[0]!).length).toBeGreaterThan(128) + expect(searchTools(llm.sent[0]!)).toContain('tool_search_tool_bm25') + expect(capped(llm.sent[1]!)).toBe(true) // a read-only run, which on the catalog would withdraw tools by a tool change const a1 = { printer: 'A1', material: 'PLA', nozzle: 0.4 } for await (const _ of pilot.switchMachine('s1', a1, { ...a1, material: 'PETG' }, { narrate: true })) void _ await say('and now?') await say('a new chat', 's2') expect(pilot.tools().length).toBeGreaterThan(128) - expect(llm.sent.length).toBeGreaterThanOrEqual(4) - expect(llm.sent.filter((r) => !capped(r)).map((r) => toolsOf(r).length)).toEqual([]) + expect(llm.sent.length).toBeGreaterThanOrEqual(5) + expect(llm.sent.slice(1).filter((r) => !capped(r)).map((r) => toolsOf(r).length)).toEqual([]) + // a chat runtime made again on the same transport starts plain too + const host = { printers: { list: async () => [], plugins: async () => [] } as unknown as PrinterHost, approvals: {} as ApprovalHost, llm } + const { models: _tiers, ...base } = DEFAULT_CONFIG + const again = createPilot({ host, config: { ...base, provider: 'mimir-cloud', model: 'claude-haiku-5-5', maxSteps: 6 }, policy: DEFAULT_POLICY, builtins: false, tools: catalog() }) + const at = llm.sent.length + for await (const _ of again.run('s3', 'hello')) void _ + expect(llm.sent.slice(at).map(capped)).toEqual([true]) }) - it('keeps the pick for the whole chat that first hears it, and sends the catalog from the next chat on', async () => { - const { llm, say } = await setup('mimir-cloud', [{ text: 'Hi.' }, { text: 'Still here.' }, { text: 'New chat.' }], 'claude-haiku-5-5', {}, { warm: false }) - const first = [...(await say('hello')), ...(await say('and now?'))] - expect(first).toHaveLength(2) - expect(first.every(capped)).toBe(true) - const next = await say('a new chat', 's2') - const tools = toolsOf(next[0]!) - expect(tools).toHaveLength(SEARCH_CORE.length + 150 + 1) - expect(tools.filter((t) => t['defer_loading'] !== true).map(name)).toEqual([...SEARCH_CORE, 'tool_search_tool_bm25']) - expect(llm.sent).toHaveLength(3) + it('sends a catalog with no more than 128 tools once more too, when the old proxy refuses its search tool', async () => { + const few = catalog().slice(-40) + const llm = provider([{ text: 'Hi.' }], false, oldShape) + const host = { printers: { list: async () => [], plugins: async () => [] } as unknown as PrinterHost, approvals: {} as ApprovalHost, llm } + const { models: _tiers, ...base } = DEFAULT_CONFIG + const pilot = createPilot({ host, config: { ...base, provider: 'mimir-cloud', model: 'claude-haiku-5-5', maxSteps: 6 }, policy: DEFAULT_POLICY, builtins: false, tools: few }) + for await (const _ of pilot.run('s1', 'hello')) void _ + expect(llm.sent).toHaveLength(2) + expect(toolsOf(llm.sent[0]!).length).toBeLessThanOrEqual(128) + expect(searchTools(llm.sent[0]!)).toContain('tool_search_tool_bm25') + expect(capped(llm.sent[1]!)).toBe(true) + }) + + it('sends no request again for any other refusal, and keeps tool search on', async () => { + const others = [ + refusal(400, 'messages must be a list of 1 to 400'), + // the words of today's proxy for too many tools up front, a request the app got wrong + refusal(400, 'at most 128 tools load up front; defer the rest behind tool search'), + // the old words, but not as a 400 for the request's shape + refusal(502, OLD_TOO_MANY, 'provider_error'), + refusal(400, OLD_TOO_MANY, 'provider_error'), + ] + for (const no of others) { + const llm = provider([{ text: 'Hi.' }], false, () => no) + const host = { printers: { list: async () => [], plugins: async () => [] } as unknown as PrinterHost, approvals: {} as ApprovalHost, llm } + const { models: _tiers, ...base } = DEFAULT_CONFIG + const pilot = createPilot({ host, config: { ...base, provider: 'mimir-cloud', model: 'claude-haiku-5-5', maxSteps: 6 }, policy: DEFAULT_POLICY, builtins: false, tools: catalog() }) + const errors: string[] = [] + for await (const e of pilot.run('s1', 'hello')) if (e.type === 'error') errors.push(e.message) + expect(llm.sent, no.message).toHaveLength(1) + expect(errors, no.message).toEqual([no.message.replace(/^\d{3}: /, '')]) + expect(createTransportClient(llm, 'mimir-cloud').toolSearch, no.message).toBe(true) + } }) - it('hears the event from mimir cloud only, keeps it per transport, and passes it on no further', async () => { + it('sends the request once more at most, even when the old words come back for the pick', async () => { + const llm = provider([{ text: 'Hi.' }], false, () => refusal(400, OLD_TOO_MANY)) + const host = { printers: { list: async () => [], plugins: async () => [] } as unknown as PrinterHost, approvals: {} as ApprovalHost, llm } + const { models: _tiers, ...base } = DEFAULT_CONFIG + const pilot = createPilot({ host, config: { ...base, provider: 'mimir-cloud', model: 'claude-haiku-5-5', maxSteps: 6 }, policy: DEFAULT_POLICY, builtins: false, tools: catalog() }) + const errors: string[] = [] + for await (const e of pilot.run('s1', 'hello')) if (e.type === 'error') errors.push(e.message) + expect(llm.sent).toHaveLength(2) + expect(capped(llm.sent[1]!)).toBe(true) + expect(errors).toEqual([OLD_TOO_MANY]) + }) + + it("leaves every other provider's first request as it was", async () => { + // a key on the Messages API sends the catalog first; an OpenAI-style key and a model without tool search send the pick + for (const [kind, model, search] of [['anthropic', 'claude-haiku-5-5', true], ['openai', 'gpt-5.6-luna', false], ['mimir-cloud', 'claude-haiku-4-5', false]] as const) { + const { llm, say } = await setup(kind, [{ text: 'Hi.' }], model, {}, { warm: false }) + await say('hello') + if (search) expect(searchTools(llm.sent[0]!), kind).toEqual([...SEARCH_CORE, 'tool_search_tool_bm25']) + else { + const tools = toolsOf(llm.sent[0]!) + expect(tools.length, kind).toBeLessThanOrEqual(128) + expect(tools.some((t) => 'defer_loading' in t || t['type'] === TOOL_SEARCH_TOOL.type), kind).toBe(false) + } + } + }) + + it('reads the old proxy refusal from either host, and nothing else', () => { + // the desktop host puts the status before the words; the browser host keeps it apart + expect(refusesToolSearch(refusal(400, OLD_TOO_MANY))).toBe(true) + expect(refusesToolSearch(Object.assign(new Error(OLD_TOO_MANY), { status: 400, code: 'invalid_request' }))).toBe(true) + expect(refusesToolSearch(refusal(400, 'tool tool_search_tool_bm25_20251119 is not allowed'))).toBe(true) + expect(refusesToolSearch(new Error(`400: ${OLD_TOO_MANY}`))).toBe(true) + expect(refusesToolSearch(new Error(OLD_TOO_MANY))).toBe(false) + expect(refusesToolSearch(refusal(400, 'tool web_fetch_20250910 is not allowed'))).toBe(false) + expect(refusesToolSearch(refusal(400, 'tools must be a list of at most 512'))).toBe(false) + expect(refusesToolSearch(refusal(429, OLD_TOO_MANY, 'day_used'))).toBe(false) + }) + + it('hears the event from mimir cloud only, takes back a refusal by it, keeps both per transport, and passes it on no further', async () => { const said = (): LlmTransport & { sent: LlmHttpRequest[] } => provider([{ text: 'Hi.' }, { text: 'Hi.' }], true) const ask: LlmRequest = { model: 'claude-haiku-5-5', messages: [{ role: 'user', content: 'hello' }], tools: [] } - const events = async (c: ReturnType): Promise => { - const out: string[] = [] - for await (const e of c.stream(ask)) out.push(e.type) + const events = async (c: ReturnType, req = ask): Promise => { + const out: LlmEvent[] = [] + for await (const e of c.stream(req)) out.push(e) return out } - const llm = said() - // a key on the Messages API takes tool search without the event, and an event there changes nothing + // a transport whose proxy refuses the first request for its shape, and then takes tool search + let refuse = true + const inner = said() + const llm: LlmTransport = { + available: async () => true, + async *stream(req, signal) { + if (refuse) { + refuse = false + throw refusal(400, OLD_TOO_MANY) + } + yield* inner.stream(req, signal) + }, + } const direct = createTransportClient(llm, 'anthropic') const cloud = createTransportClient(llm, 'mimir-cloud') - expect([direct.toolSearch, cloud.toolSearch]).toEqual([true, false]) - expect(await events(direct)).not.toContain('tool_search') - expect(cloud.toolSearch).toBe(false) - expect(await events(cloud)).toEqual(['text', 'usage', 'done']) + expect([direct.toolSearch, cloud.toolSearch]).toEqual([true, true]) + // a plain request refused with the old words leaves tool search alone: only a tool search request is the shape + expect((await events(cloud)).map((e) => e.type)).toEqual(['error', 'done']) expect(cloud.toolSearch).toBe(true) - // a runtime made again on the same transport knows at once; another transport, or an OpenAI-style key, does not - expect(createTransportClient(llm, 'mimir-cloud').toolSearch).toBe(true) - expect(createTransportClient(said(), 'mimir-cloud').toolSearch).toBe(false) + refuse = true + const refused = await events(cloud, { ...ask, toolSearch: true }) + expect(refused[0]).toEqual({ type: 'error', message: OLD_TOO_MANY, retryable: false, code: 'invalid_request', searchRefused: true }) + expect(cloud.toolSearch).toBe(false) + // a runtime made again on the same transport knows at once; another transport, or a key on the Messages API, does not + expect(createTransportClient(llm, 'mimir-cloud').toolSearch).toBe(false) + expect(createTransportClient(said(), 'mimir-cloud').toolSearch).toBe(true) + expect(direct.toolSearch).toBe(true) expect(createTransportClient(llm, 'openai').toolSearch).toBe(false) + // the proxy's event takes the refusal back, and goes no further + expect((await events(cloud)).map((e) => e.type)).toEqual(['text', 'usage', 'done']) + expect(cloud.toolSearch).toBe(true) // the adapter reads it, and only the proxy's own event with its flag set const parsed: LlmEvent[] = [] async function* chunks(): AsyncIterable {