Skip to content

Commit ced73dc

Browse files
authored
feat(ai): expose compaction on the Promise client (#59987)
1 parent 3364bf5 commit ced73dc

4 files changed

Lines changed: 99 additions & 1 deletion

File tree

‎packages/ai/README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1244,6 +1244,8 @@ const compacted = Effect.gen(function* () {
12441244
})
12451245
```
12461246

1247+
From the Promise API, `await ai.llm.compact(request)` takes the same request and options (plus `signal`) and returns the same result.
1248+
12471249
`replacement` replaces the complete input window. Do not append it to the original transcript or extract only the encrypted item: the provider may retain additional messages in its output. Retained user and assistant messages remain ordinary messages with typed text, media, or reasoning parts, in their original order. Provider-specific message IDs, status, and phase use `providerMetadata`, not a raw output array hidden in an assistant message. Unsupported returned item types fail explicitly.
12481250

12491251
The selected model carries explicit-compaction capability through request construction and updates. Calls using unsupported routes fail type checking. When the model is selected dynamically, narrow the request with `LLMClient.canCompact(request)` before calling `LLMClient.compact`; a model or route switch does not inherit the old capability. Runtime validation still rejects unsupported calls from untyped consumers. Capability describes the route's API, not whether every model or custom deployment supports the operation.

‎packages/ai/src/index.ts‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,22 @@ import type { AwaitOptions, Event, Generation, Snapshot } from "./generation.js"
55
import { Image, type ImageModel, type ImageRequest, type ImageRequestInput } from "./image.js"
66
import { LLM } from "./effect.js"
77
import { Media } from "./media.js"
8+
import {
9+
LLMClient,
10+
type CheckpointRequest,
11+
type CompactionRequest,
12+
type EndpointCompactOptions,
13+
type TriggerCompactOptions,
14+
} from "./route/client.js"
815
import { RequestExecutor } from "./route/executor.js"
9-
import { AIError, InvalidRequestError, LanguageModel, type LLMRequest } from "./schema/index.js"
16+
import {
17+
AIError,
18+
InvalidRequestError,
19+
LanguageModel,
20+
type CompactionCheckpointResponse,
21+
type CompactionResponse,
22+
type LLMRequest,
23+
} from "./schema/index.js"
1024
import type {
1125
GenerateObjectDynamicOptions,
1226
GenerateObjectOptions,
@@ -113,6 +127,21 @@ export const make = <R = never>(options: Options<R> = {}) => {
113127
return run("schema" in input ? LLM.generateObject(input) : LLM.generateObject(input), options)
114128
}
115129

130+
function compact(
131+
request: CheckpointRequest,
132+
options: TriggerCompactOptions & RunOptions,
133+
): Promise<CompactionCheckpointResponse>
134+
function compact(request: CompactionRequest, options?: EndpointCompactOptions & RunOptions): Promise<CompactionResponse>
135+
// The overloads pair each request type with its mechanism; `LLMClient.compact` checks the route's capability at runtime.
136+
function compact(
137+
request: CheckpointRequest | CompactionRequest,
138+
options?: (TriggerCompactOptions | EndpointCompactOptions) & RunOptions,
139+
) {
140+
return options?.mechanism === "trigger"
141+
? run(LLMClient.compact(request as CheckpointRequest, options), options)
142+
: run(LLMClient.compact(request as CompactionRequest, options), options)
143+
}
144+
116145
const handle = <Response>(generation: Generation<Response>): GenerationHandle<Response> => ({
117146
...generation.snapshot,
118147
token: generation.token,
@@ -172,6 +201,8 @@ export const make = <R = never>(options: Options<R> = {}) => {
172201
iterate(LLM.loop(input), options),
173202
/** Decode the model's output against `schema` (or `jsonSchema`, unvalidated) like `LLM.generateObject`. */
174203
generateObject,
204+
/** Compact a Responses request like `LLMClient.compact`: a replacement window, or a checkpoint with `mechanism: "trigger"`. */
205+
compact,
175206
},
176207
image: {
177208
request: Image.request,

‎packages/ai/test/promise.test.ts‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -458,6 +458,54 @@ describe("AI promise client", () => {
458458
await ai.dispose()
459459
})
460460

461+
test("compacts through the endpoint and the trigger, and rejects unsupported routes with AIError", async () => {
462+
const checkpoint = { type: "compaction", id: "cmp_1", encrypted_content: "opaque" }
463+
const paths: Array<string> = []
464+
const ai = AI.make({
465+
layer: RequestExecutor.layer.pipe(
466+
Layer.provide(
467+
handlerLayer((input) =>
468+
Effect.gen(function* () {
469+
const path = new URL((yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)).url).pathname
470+
paths.push(path)
471+
if (path === "/v1/responses/compact")
472+
return json(input, {
473+
object: "response.compaction",
474+
output: [checkpoint],
475+
usage: { input_tokens: 100, output_tokens: 10, total_tokens: 110 },
476+
})
477+
expect(JSON.parse(input.text).input.at(-1)).toEqual({ type: "compaction_trigger" })
478+
return input.respond(
479+
sseEvents({ type: "response.completed", response: { id: "resp_1", output: [checkpoint] } }),
480+
{ headers: { "content-type": "text/event-stream" } },
481+
)
482+
}),
483+
),
484+
),
485+
),
486+
})
487+
const request = ai.llm.request({ model: openai.responses("gpt-5"), prompt: "Hello" })
488+
489+
const compacted = await ai.llm.compact(request)
490+
expect(compacted.replacement.flatMap((message) => message.content.map((part) => part.type))).toEqual([
491+
"compaction",
492+
])
493+
expect(compacted.usage?.totalTokens).toBe(110)
494+
495+
const triggered = await ai.llm.compact(request, { mechanism: "trigger" })
496+
expect(triggered.checkpoint).toMatchObject({ id: "cmp_1", encrypted: "opaque" })
497+
expect(triggered.responseID).toBe("resp_1")
498+
expect(paths).toEqual(["/v1/responses/compact", "/v1/responses"])
499+
500+
const unsupported = await ai.llm
501+
// @ts-expect-error Untyped callers on routes without a compact endpoint must reject with AIError.
502+
.compact(ai.llm.request({ model: openai.chat("gpt-4o-mini"), prompt: "Hello" }))
503+
.catch((error: unknown) => error)
504+
expect(unsupported instanceof AIError && unsupported.reason._tag).toBe("UnsupportedOperation")
505+
expect(paths).toHaveLength(2)
506+
await ai.dispose()
507+
})
508+
461509
test("the default client is created lazily and can be disposed", async () => {
462510
expect(typeof AI.ai.llm.generate).toBe("function")
463511
expect(typeof AI.ai.image.generate).toBe("function")

‎packages/ai/test/provider-options/checkpoint.types.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import {
1616
OpenAICompatibleResponses,
1717
XAI,
1818
} from "../../src/providers.js"
19+
import { ai } from "../../src/index.js"
1920
import type { WebSocketChannelExecutor } from "../../src/route.js"
2021
import type { RoutePatch } from "../../src/route/client.js"
2122
import type { OpenAIResponsesBody } from "../../src/protocols/openai-responses.js"
@@ -127,3 +128,19 @@ if (LLMClient.canCompact(dynamic)) {
127128
}
128129
// @ts-expect-error A dynamic model override cannot preserve trigger support.
129130
LLMClient.compact(LLMRequest.update(request, patch), { mechanism: "trigger" })
131+
132+
void ai.llm.compact(request).then((result) => result satisfies CompactionResponse)
133+
void ai.llm.compact(request, { mechanism: "trigger", webSocket, signal: AbortSignal.timeout(1) }).then((result) => {
134+
result satisfies CompactionCheckpointResponse
135+
// @ts-expect-error A trigger does not return replacement history.
136+
result.replacement
137+
})
138+
// @ts-expect-error Endpoint compaction does not accept a WebSocket executor.
139+
void ai.llm.compact(request, { webSocket })
140+
// @ts-expect-error Azure must not inherit OpenAI's trigger operation.
141+
void ai.llm.compact(LLM.request({ model: azure }), { mechanism: "trigger" })
142+
// @ts-expect-error OpenAI Chat does not expose explicit compaction.
143+
void ai.llm.compact(unsupported.openai)
144+
// @ts-expect-error Dynamic models must be narrowed.
145+
void ai.llm.compact(dynamic)
146+
if (LLMClient.canCompact(dynamic, { mechanism: "trigger" })) void ai.llm.compact(dynamic, { mechanism: "trigger" })

0 commit comments

Comments
 (0)