Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions packages/client/src/promise/generated/types.ts

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions packages/core/src/session/prompt.ts
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ const materializeAttachment = Effect.fn("SessionPrompt.materializeAttachment")(f
return FileAttachment.create({
data: Base64.make(Buffer.from(blob.data).toString("base64")),
mime: blob.mime,
source: { type: "inline" },
source: input.source ?? { type: "inline" },
name: input.name,
description: input.description,
mention: input.mention,
Expand All @@ -109,7 +109,7 @@ const materializeAttachment = Effect.fn("SessionPrompt.materializeAttachment")(f
const resolved = input.uri.startsWith("data:")
? {
bytes: yield* decodeDataURL(input.uri, label),
source: { type: "inline" as const },
source: input.source ?? { type: "inline" as const },
start: undefined,
end: undefined,
name: undefined,
Expand Down
41 changes: 41 additions & 0 deletions packages/core/test/session-prompt.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ import { ConfigExperimental } from "@opencode/schema/config/experimental"
import { SessionStored } from "@opencode/core/session/stored"
import { SessionBlob } from "@opencode/core/session/blob"
import { SessionTransfer } from "@opencode/core/session/transfer"
import { PromptInput } from "@opencode/schema/prompt-input"

const executionCalls: Session.ID[] = []
const interruptCalls: Session.ID[] = []
Expand Down Expand Up @@ -534,6 +535,46 @@ describe("Session.prompt", () => {
}),
)

it.live("admits captured image bytes with their source path after the source disappears", () =>
Effect.gen(function* () {
yield* setup
const session = yield* Session.Service
const directory = yield* tmpdirScoped("opencode-session-prompt-")
const source = path.join(directory.path, "screenshot.png")
yield* Effect.promise(() => Bun.write(source, Buffer.from(pngBase64, "base64")))
const bytes = yield* Effect.promise(() => Bun.file(source).arrayBuffer())
yield* Effect.promise(() => Bun.file(source).delete())
const file = {
uri: `data:image/png;base64,${Buffer.from(bytes).toString("base64")}`,
source: { type: "uri", uri: pathToFileURL(source).href },
name: "screenshot.png",
mention: { start: 0, end: 9, text: "[Image 1]" },
} satisfies PromptInput.FileAttachment

const message = yield* session.prompt({
sessionID,
text: "[Image 1] Inspect this image",
files: [Schema.decodeUnknownSync(PromptInput.FileAttachment)(file)],
resume: false,
})

expect(message.payload.files).toEqual([
{
data: pngBase64,
mime: "image/png",
source: file.source,
name: file.name,
mention: file.mention,
},
])
const stored = yield* admitted(message.id)
expect(stored?.type === "user" ? stored.payload.files : undefined).toEqual(message.payload.files)
expect(
Schema.encodeSync(PromptInput.FileAttachment)(Schema.decodeUnknownSync(PromptInput.FileAttachment)(file)),
).toEqual(file)
}),
)

it.effect("stores media refs at admission when the session's config enables media_blob", () =>
Effect.gen(function* () {
yield* setup
Expand Down
10 changes: 8 additions & 2 deletions packages/protocol/openapi.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 3 additions & 1 deletion packages/schema/src/prompt-input.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
export * as PromptInput from "./prompt-input.js"

import { Schema } from "effect"
import { AgentAttachment, PromptMention } from "./prompt.js"
import { AgentAttachment, FileSource, PromptMention } from "./prompt.js"
import { optional } from "./schema.js"
import { Skill } from "./skill.js"

export interface FileAttachment extends Schema.Schema.Type<typeof FileAttachment> {}
export const FileAttachment = Schema.Struct({
uri: Schema.String,
// Source metadata does not require reopening the original file.
source: FileSource.pipe(optional),
name: Schema.String.pipe(optional),
description: Schema.String.pipe(optional),
mention: PromptMention.pipe(optional),
Expand Down
6 changes: 2 additions & 4 deletions packages/tui/src/component/prompt/local-attachment.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import { existsSync } from "node:fs"
import path from "node:path"
import { pathToFileURL } from "node:url"

Expand Down Expand Up @@ -61,14 +60,13 @@ export async function resolvePastedAttachments(text: string, platform: string) {
})
}

// A local server reads the file itself so the model sees its path. The pasted bytes are the
// fallback for remote servers and for files deleted before submit, like screenshot drags.
// Keep the captured bytes: a local server may not have the terminal's access to the source file.
export function promptFilesForServer<T extends { uri: string; path?: string }>(
files: readonly T[] | undefined,
localServer: boolean,
) {
return files?.map(({ path: file, ...rest }) =>
file && localServer && existsSync(file) ? { ...rest, uri: pathToFileURL(file).href } : rest,
file && localServer ? { ...rest, source: { type: "uri" as const, uri: pathToFileURL(file).href } } : rest,
)
}

Expand Down
25 changes: 21 additions & 4 deletions packages/tui/test/prompt/local-attachment.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ describe("prompt local attachments", () => {
expect(await resolvePastedAttachments(Array(33).fill(file).join(" "), process.platform)).toBeUndefined()
})

test("sends pasted files by path to a local server and by bytes otherwise", async () => {
test("keeps pasted bytes with source paths only for a local server", async () => {
await using tmp = await tmpdir()
const image = path.join(tmp.path, "scrëen shot.png")
const photo = path.join(tmp.path, "photo.jpg")
Expand All @@ -204,10 +204,26 @@ describe("prompt local attachments", () => {

const mentions = files.map((file) => file.mention)

await Bun.write(image, new Uint8Array([10, 11, 12]))
expect(promptFilesForServer(files, true)).toEqual([
{ uri: pathToFileURL(image).href, name: "scrëen shot.png", mention: mentions[0] },
{ uri: pathToFileURL(photo).href, name: "photo.jpg", mention: mentions[1] },
{ uri: pathToFileURL(pdf).href, name: "report.pdf", mention: mentions[2] },
{
uri: "data:image/png;base64,AQID",
source: { type: "uri", uri: pathToFileURL(image).href },
name: "scrëen shot.png",
mention: mentions[0],
},
{
uri: "data:image/jpeg;base64,BwgJ",
source: { type: "uri", uri: pathToFileURL(photo).href },
name: "photo.jpg",
mention: mentions[1],
},
{
uri: "data:application/pdf;base64,BAUG",
source: { type: "uri", uri: pathToFileURL(pdf).href },
name: "report.pdf",
mention: mentions[2],
},
{ uri: "data:image/png;base64,Cg==", name: "clipboard" },
])
expect(promptFilesForServer(files, false)).toEqual([
Expand All @@ -220,6 +236,7 @@ describe("prompt local attachments", () => {
await Bun.file(image).delete()
expect(promptFilesForServer(files, true)?.[0]).toEqual({
uri: "data:image/png;base64,AQID",
source: { type: "uri", uri: pathToFileURL(image).href },
name: "scrëen shot.png",
mention: mentions[0],
})
Expand Down
Loading