Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 22 additions & 5 deletions packages/app/test/mimir-sticky-toolsets.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
// @vitest-environment node
// SPDX-License-Identifier: Apache-2.0
// Copyright (C) 2026 The SlicerX contributors
// On a capped mimir cloud chat the tools a request carries follow the chat, not only the message. The worst case is the
// On a capped mimir cloud chat (a proxy deployed before tool search, which refuses the catalog the chat's first request
// carries, so the chat goes on the per-turn pick) the tools a request carries follow the chat, not only the message. The worst case is the
// desktop app with every connector and twelve printing printers (well over 128 tools offered, 128 sent). A "yes please" after
// mimir offers to print gets the printer tools, the list is the same request after request while the chat stays on its
// topic (so the proxy's prompt cache keeps hitting), and every request stays at 128 tools or under. Each request is read
Expand Down Expand Up @@ -55,14 +56,30 @@ function reply(step: { text?: string; call?: { name: string; input?: unknown } }
)
}

/** A transport that keeps each request and plays the replies in order. */
function transport(steps: { text?: string; call?: { name: string; input?: unknown } }[]): LlmTransport & { sent: LlmHttpRequest[] } {
/**
* What a mimir cloud proxy deployed before tool search answered a request that sends the tool catalog (its shape.ts
* then): a 400 for more than 128 tools, or for the tool search tool, as the desktop host raises it.
*/
function oldRefusal(req: LlmHttpRequest): Error | undefined {
const tools = (JSON.parse(req.body) as { tools?: { type?: string }[] }).tools ?? []
const search = tools.find((t) => t.type !== undefined && t.type !== 'custom' && !t.type.startsWith('web_search_'))
const words = tools.length > 128 ? 'tools must be a list of at most 128' : search ? `tool ${String(search.type)} is not allowed` : null
return words ? Object.assign(new Error(`400: ${words}`), { status: 400, code: 'invalid_request' }) : undefined
}

/**
* A transport that keeps each request and plays the replies in order. `old`: a mimir cloud proxy deployed before tool
* search, which refuses a catalog (oldRefusal); a refused request is not kept and plays no reply.
*/
function transport(steps: { text?: string; call?: { name: string; input?: unknown } }[], old = false): LlmTransport & { sent: LlmHttpRequest[] } {
const sent: LlmHttpRequest[] = []
let i = 0
return {
sent,
available: async () => true,
async *stream(req) {
const no = old ? oldRefusal(req) : undefined
if (no) throw no
sent.push(req)
yield reply(steps[i++] ?? { text: 'Done.' })
},
Expand Down Expand Up @@ -91,11 +108,11 @@ async function registerApp(platform: Platform): Promise<() => void> {
return off
}

/** A fresh mimir cloud chat in the desktop app, with the replies it plays. */
/** A fresh mimir cloud chat in the desktop app on a proxy deployed before tool search, with the replies it plays. */
async function chat(steps: Parameters<typeof transport>[0]) {
resetTabMimir()
set({ pilot: { mode: 'on', provider: 'mimir-cloud' } as never })
const llm = transport(steps)
const llm = transport(steps, true)
const pilot = await tabMimir(hostFor('desktop', llm), withCloud(NEUTRAL), 'cloud')
const say = async (message: string): Promise<LlmHttpRequest[]> => {
const at = llm.sent.length
Expand Down
40 changes: 27 additions & 13 deletions packages/app/test/mimir-tool-cap.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@
// SPDX-License-Identifier: Apache-2.0
// Copyright (C) 2026 The SlicerX contributors
// No mimir tab request loads more than 128 tools. OpenAI-style APIs and local servers refuse a longer list, and so does
// mimir cloud's proxy for the tools that load up front (supabase/functions/mimir/shape.ts). Once the proxy has said it
// takes tool search (the event its streams open with), a new chat on mimir cloud carries the tab's whole catalog, most
// of it deferred behind tool search; a proxy that never said so gets the per-turn pick it always got. The tab is made
// mimir cloud's proxy for the tools that load up front (supabase/functions/mimir/shape.ts). A chat on mimir cloud carries
// the tab's whole catalog from its first request, most of it deferred behind tool search; a proxy deployed before tool
// search refuses that, and gets the request again as the per-turn pick it always got. The tab is made
// the way the app makes it, with the commands the app registers at startup, its features' commands, a command per
// connected printer and every connector's tools, and each request is read back from the HTTP body the provider adapter
// wrote. Every kind of turn is checked in the desktop app (which slices and prints) and the browser build (which only
// models), on mimir cloud before and after its proxy takes tool search, a key and a local model, in the neutral edition
// models), on mimir cloud on a proxy deployed before tool search and on one with it, a key and a local model, in the neutral edition
// and in one with the printer and web tools off. Last, a recorded chat's requests go through the proxy's own shape():
// the same tools on every request.
import type { EditionConfig } from '@slicerx/edition-config'
Expand Down Expand Up @@ -56,18 +56,32 @@ const PROXY_SAYS = (() => {
return JSON.parse(`"${lit.slice(1, -1).replaceAll('"', '\\"')}"`) as string
})()

/**
* What a mimir cloud proxy deployed before tool search answered a request that sends the tool catalog (its shape.ts
* then): a 400 for more than 128 tools, or for the tool search tool, as the desktop host raises it.
*/
function oldRefusal(req: LlmHttpRequest): Error | undefined {
const tools = (JSON.parse(req.body) as { tools?: { type?: string }[] }).tools ?? []
const search = tools.find((t) => t.type !== undefined && t.type !== 'custom' && !t.type.startsWith('web_search_'))
const words = tools.length > 128 ? 'tools must be a list of at most 128' : search ? `tool ${String(search.type)} is not allowed` : null
return words ? Object.assign(new Error(`400: ${words}`), { status: 400, code: 'invalid_request' }) : undefined
}

/**
* A transport that keeps each request and answers none of them, so every turn ends after its first request. `proxy`:
* a mimir cloud proxy that takes tool search, so each answer opens with its event before it fails.
* 'search', a mimir cloud proxy that takes tool search, so each answer opens with its event before it fails; 'old', one
* deployed before tool search, which refuses a catalog (oldRefusal) and keeps no refused request.
*/
function transport(proxy = false): LlmTransport & { sent: LlmHttpRequest[] } {
function transport(proxy?: 'search' | 'old'): LlmTransport & { sent: LlmHttpRequest[] } {
const sent: LlmHttpRequest[] = []
return {
sent,
available: async () => true,
async *stream(req) {
const no = proxy === 'old' ? oldRefusal(req) : undefined
if (no) throw no
sent.push(req)
if (proxy) yield new TextEncoder().encode(PROXY_SAYS)
if (proxy === 'search') yield new TextEncoder().encode(PROXY_SAYS)
throw new Error('not answered in this test')
},
}
Expand All @@ -87,8 +101,8 @@ const EDITIONS: [string, EditionConfig][] = [
]

/**
* How mimir runs: mimir cloud on a proxy that never said it takes tool search (deployed before it), mimir cloud once its
* proxy has said so, a key on an OpenAI-style API, and a local model.
* How mimir runs: mimir cloud on a proxy deployed before tool search (which refuses the catalog), mimir cloud on one that
* takes it, a key on an OpenAI-style API, and a local model.
*/
const MODES = {
'mimir cloud': { mode: 'on', provider: 'mimir-cloud' },
Expand Down Expand Up @@ -127,15 +141,15 @@ function loadedNames(req: LlmHttpRequest): string[] {
const SEARCH = 'tool_search_tool_bm25'

/**
* The first request of a turn in a fresh chat, after `before` when given, and how many tools the tab offers. With tool
* search, a first chat has heard the proxy say it takes it.
* The first request the proxy took of a turn in a fresh chat, after `before` when given, and how many tools the tab
* offers. With tool search, a first chat has run before it.
*/
async function turn(platform: Platform, edition: EditionConfig, mode: Mode, t: (typeof TURNS)[number]): Promise<{ req: LlmHttpRequest; offered: number }> {
resetTabMimir()
set({ pilot: MODES[mode] as never })
const searching = mode === 'mimir cloud, tool search'
const llm = transport(searching)
const cloud = MODES[mode].provider === 'mimir-cloud'
const llm = transport(searching ? 'search' : cloud ? 'old' : undefined)
const pilot = await tabMimir(hostFor(platform, llm), cloud ? withCloud(edition) : edition, cloud ? 'cloud' : null)
if (searching) for await (const _ of pilot.run('warm', 'hello')) void _
if (t.before) for await (const _ of pilot.run('s', t.before)) void _
Expand Down Expand Up @@ -208,7 +222,7 @@ describe('no mimir tab request loads more than 128 tools', () => {
}, 120_000)
}

it('sends the per-turn pick on mimir cloud until its proxy says it takes tool search: no deferral, search tool or tool change', async () => {
it('sends the per-turn pick on mimir cloud once a proxy deployed before tool search refused the catalog: no deferral, search tool or tool change', async () => {
const off = await registerApp('desktop')
try {
for (const t of TURNS) {
Expand Down
57 changes: 42 additions & 15 deletions packages/pilot/src/provider/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@ export const MIMIR_CLOUD = 'mimir-cloud'
/**
* The most tools that load up front in one mimir cloud request (supabase/functions/mimir/shape.ts, MAX_LOADED_TOOLS);
* it refuses more. The desktop app's tab offers more than that (the design tools, slicing, printers, the app's
* commands, a command per printer and the connectors' tools). Once the proxy has said it takes tool search, a new chat
* on a model with it sends them all, most deferred (tool-search.ts); until then, and on any other model, a turn sends
* the tools it needs first (toolsets.ts).
* commands, a command per printer and the connectors' tools). A chat on a model with tool search sends them all, most
* deferred (tool-search.ts), from its first request; on a proxy that refuses that (refusesToolSearch), and on any other
* model, a turn sends the tools it needs first (toolsets.ts).
*/
export const MIMIR_CLOUD_MAX_TOOLS = 128

Expand Down Expand Up @@ -76,31 +76,57 @@ export function adapterFor(provider: string): LlmAdapter {
return make()
}

/** Joins an adapter with the host transport. The transport adds the key; this code never sees it. */
/**
* The transports whose mimir cloud proxy said it takes a request that sends the whole tool catalog through tool search
* (the `mimir` event its streams open with, supabase/functions/mimir/handler.ts). Kept per transport, so a chat runtime
* What a mimir cloud proxy deployed before tool search answers a request that sends the whole catalog: its shape.ts
* then took at most 128 tools and no tool search tool, and refused anything else with a 400 and these words.
*/
const OLD_SHAPE = /^(?:tools must be a list of at most 128|tool tool_search_tool_(?:bm25|regex)_\d+ is not allowed)$/

/**
* Whether a failed mimir cloud request was refused by a proxy deployed before tool search, for the request's shape:
* a 400 with its invalid request code (or none) and one of its old words (OLD_SHAPE). The desktop host puts the status
* before the words (`400: ...`); the browser's keeps it apart.
*/
export function refusesToolSearch(e: unknown): boolean {
const status = e && typeof e === 'object' && 'status' in e ? (e as { status: unknown }).status : undefined
const msg = errorText(e)
const said = /^400: /.test(msg)
if (typeof status === 'number' ? status !== 400 : !said) return false
const code = errorCode(e)
if (code !== undefined && code !== 'invalid_request') return false
return OLD_SHAPE.test(said ? msg.slice(5) : msg)
}

/**
* The transports whose mimir cloud proxy refused a request that sends the whole tool catalog through tool search
* (refusesToolSearch). Their chats send the per-turn pick from then on, until the proxy opens a reply with its `mimir`
* event (supabase/functions/mimir/handler.ts), which says it takes tool search. Kept per transport, so a chat runtime
* made again on the same host knows it at once.
*/
const searchingProxies = new WeakSet<LlmTransport>()
const plainProxies = new WeakSet<LlmTransport>()

/** Joins an adapter with the host transport. The transport adds the key; this code never sees it. */
export function createTransportClient(transport: LlmTransport, provider: string, opts: { baseUrl?: string; share?: () => CloudShare } = {}): LlmClient {
const adapter = adapterFor(provider)
const cloud = provider === MIMIR_CLOUD
// mimir cloud's refusals (daily caps, a pause) are shown as the proxy wrote them, never retried elsewhere
const failure = (e: unknown): LlmEvent => {
// a tool search request an old proxy refused marks the transport, and the runtime sends it once more without
const failure = (e: unknown, req: LlmRequest): LlmEvent => {
if (!cloud) return { type: 'error', message: errorText(e), retryable: true }
const code = errorCode(e)
return { type: 'error', message: errorText(e).replace(/^\d{3}: /, ''), retryable: false, ...(code ? { code } : {}) }
const refused = req.toolSearch === true && refusesToolSearch(e)
if (refused) plainProxies.add(transport)
return { type: 'error', message: errorText(e).replace(/^\d{3}: /, ''), retryable: false, ...(code ? { code } : {}), ...(refused ? { searchRefused: true as const } : {}) }
}
return {
provider,
// the proxy's own limit, though the Messages API it speaks has none; a tool search request carries more, deferred
...(cloud ? { maxTools: MIMIR_CLOUD_MAX_TOOLS } : adapter.maxTools === undefined ? {} : { maxTools: adapter.maxTools }),
// the Messages API searches deferred tools; through mimir cloud only once its proxy has said it passes them, so an
// app that reaches a proxy deployed before tool search keeps the per-turn pick
// the Messages API searches deferred tools, and so does mimir cloud from the first request, so the first chat sends
// the same tools as every later one; an app that reaches a proxy deployed before tool search keeps the per-turn pick
// once that proxy has refused the catalog
get toolSearch(): boolean {
return adapter.id === 'anthropic' && (!cloud || searchingProxies.has(transport))
return adapter.id === 'anthropic' && (!cloud || !plainProxies.has(transport))
},
async *stream(req: LlmRequest, signal?: AbortSignal): AsyncIterable<LlmEvent> {
let http = adapter.build(req, opts.baseUrl === undefined ? {} : { baseUrl: opts.baseUrl })
Expand All @@ -110,7 +136,7 @@ export function createTransportClient(transport: LlmTransport, provider: string,
try {
body = cloud ? transport.stream(http, signal, (h) => (headers = h)) : transport.stream(http, signal)
} catch (e) {
yield failure(e)
yield failure(e, req)
yield { type: 'done', stop: 'error' }
return
}
Expand All @@ -126,12 +152,13 @@ export function createTransportClient(transport: LlmTransport, provider: string,
const model = cloudModel(headers)
if (model) yield { type: 'model', model }
for await (const ev of adapter.parse(replay(first, it))) {
if (ev.type === 'tool_search') searchingProxies.add(transport)
// the proxy says it takes tool search: a refusal before (a proxy deployed since) no longer holds
if (ev.type === 'tool_search') plainProxies.delete(transport)
else yield ev
}
} catch (e) {
if (signal?.aborted) throw e
yield failure(e)
yield failure(e, req)
yield { type: 'done', stop: 'error' }
}
},
Expand Down
14 changes: 10 additions & 4 deletions packages/pilot/src/provider/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -92,13 +92,18 @@ export type LlmEvent =
/** `webSearches`: the hosted web searches the reply ran, when the provider reports them. */
| { type: 'usage'; inputTokens: number; outputTokens: number; cacheReadTokens?: number; cacheWriteTokens?: number; webSearches?: number }
| { type: 'done'; stop: LlmStop; raw?: unknown[] }
/** `code` is mimir cloud's refusal code (`day_used`, `paused`, ...). */
| { type: 'error'; message: string; retryable: boolean; code?: string }
/**
* `code` is mimir cloud's refusal code (`day_used`, `paused`, ...). `searchRefused`: a mimir cloud proxy deployed before
* tool search refused this tool search request for its shape; the client no longer takes tool search on it
* (LlmClient.toolSearch), and the runtime sends the request once more with the per-turn pick.
*/
| { type: 'error'; message: string; retryable: boolean; code?: string; searchRefused?: true }
/** The model that answered, when the reply names it (mimir cloud's `x-sx-model`). Comes before the reply's text. */
| { type: 'model'; model: string }
/**
* mimir cloud's proxy takes a request that sends the whole tool catalog through tool search (its `mimir` stream event,
* supabase/functions/mimir/handler.ts). The transport client keeps it (LlmClient.toolSearch) and passes it on no further.
* supabase/functions/mimir/handler.ts). The transport client takes back an earlier refusal on it (LlmClient.toolSearch)
* and passes it on no further.
*/
| { type: 'tool_search' }

Expand All @@ -118,7 +123,8 @@ export interface LlmClient {
maxTools?: number
/**
* The provider runs a tool search over deferred tools and takes tool changes mid-chat (LlmRequest.toolSearch). It can
* turn true while the client runs: mimir cloud's proxy says so in its replies.
* change while the client runs: a mimir cloud proxy deployed before tool search turns it false by refusing a tool
* search request, and true again by saying in a reply that it takes tool search.
*/
readonly toolSearch?: boolean
stream(req: LlmRequest, signal?: AbortSignal): AsyncIterable<LlmEvent>
Expand Down
Loading
Loading