diff --git a/packages/happy-cli/src/api/types.ts b/packages/happy-cli/src/api/types.ts index cfcee3e42..88b5c5f23 100644 --- a/packages/happy-cli/src/api/types.ts +++ b/packages/happy-cli/src/api/types.ts @@ -222,7 +222,11 @@ export const UserMessageSchema = z.object({ role: z.literal('user'), content: z.object({ type: z.literal('text'), - text: z.string() + text: z.string(), + images: z.array(z.object({ + base64: z.string(), + mediaType: z.string(), + })).optional(), }), localKey: z.string().optional(), // Mobile messages include this meta: MessageMetaSchema.optional() diff --git a/packages/happy-cli/src/claude/claudeRemote.ts b/packages/happy-cli/src/claude/claudeRemote.ts index d93215c8c..bf5959489 100644 --- a/packages/happy-cli/src/claude/claudeRemote.ts +++ b/packages/happy-cli/src/claude/claudeRemote.ts @@ -30,7 +30,7 @@ export async function claudeRemote(opts: { jsRuntime?: JsRuntime, // Dynamic parameters - nextMessage: () => Promise<{ message: string, mode: EnhancedMode } | null>, + nextMessage: () => Promise<{ message: string, mode: EnhancedMode, images?: Array<{ base64: string, mediaType: string }> } | null>, onReady: () => void, isAborted: (toolCallId: string) => boolean, @@ -81,6 +81,26 @@ export async function claudeRemote(opts: { }); } + // Build content for SDK message: string for text-only, content array for images+text + function buildContent(text: string, images?: Array<{ base64: string, mediaType: string }>): string | Array { + if (!images || images.length === 0) { + return text; + } + const contentParts: Array = []; + for (const img of images) { + contentParts.push({ + type: 'image', + source: { + type: 'base64', + media_type: img.mediaType || 'image/png', + data: img.base64, + }, + }); + } + contentParts.push({ type: 'text', text }); + return contentParts; + } + // Get initial message const initial = await opts.nextMessage(); if (!initial) { // No initial message - exit @@ -151,7 +171,7 @@ export async function claudeRemote(opts: { type: 'user', message: { role: 'user', - content: initial.message, + content: buildContent(initial.message, initial.images), }, }); @@ -213,7 +233,7 @@ export async function claudeRemote(opts: { return; } mode = next.mode; - messages.push({ type: 'user', message: { role: 'user', content: next.message } }); + messages.push({ type: 'user', message: { role: 'user', content: buildContent(next.message, next.images) } }); } // Handle tool result diff --git a/packages/happy-cli/src/claude/claudeRemoteLauncher.ts b/packages/happy-cli/src/claude/claudeRemoteLauncher.ts index 79f6e980f..b42ed49ca 100644 --- a/packages/happy-cli/src/claude/claudeRemoteLauncher.ts +++ b/packages/happy-cli/src/claude/claudeRemoteLauncher.ts @@ -315,6 +315,7 @@ export async function claudeRemoteLauncher(session: Session): Promise<'switch' | let pending: { message: string; mode: EnhancedMode; + images?: Array<{ base64: string, mediaType: string }>; } | null = null; // Track session ID to detect when it actually changes @@ -379,7 +380,8 @@ export async function claudeRemoteLauncher(session: Session): Promise<'switch' | permissionHandler.handleModeChange(mode.permissionMode); return { message: msg.message, - mode: msg.mode + mode: msg.mode, + images: msg.images, } } diff --git a/packages/happy-cli/src/claude/runClaude.ts b/packages/happy-cli/src/claude/runClaude.ts index b626c62b6..284674813 100644 --- a/packages/happy-cli/src/claude/runClaude.ts +++ b/packages/happy-cli/src/claude/runClaude.ts @@ -380,7 +380,7 @@ export async function runClaude(credentials: Credentials, options: StartOptions allowedTools: messageAllowedTools, disallowedTools: messageDisallowedTools }; - messageQueue.push(message.content.text, enhancedMode); + messageQueue.push(message.content.text, enhancedMode, (message.content as any).images); logger.debugLargeJson('User message pushed to queue:', message) }); diff --git a/packages/happy-cli/src/utils/MessageQueue2.ts b/packages/happy-cli/src/utils/MessageQueue2.ts index 45f254ed9..93535e04c 100644 --- a/packages/happy-cli/src/utils/MessageQueue2.ts +++ b/packages/happy-cli/src/utils/MessageQueue2.ts @@ -1,10 +1,16 @@ import { logger } from "@/ui/logger"; +export interface ImageAttachment { + base64: string; + mediaType: string; +} + interface QueueItem { message: string; mode: T; modeHash: string; isolate?: boolean; // If true, this message must be processed alone + images?: ImageAttachment[]; } /** @@ -37,7 +43,7 @@ export class MessageQueue2 { /** * Push a message to the queue with a mode. */ - push(message: string, mode: T): void { + push(message: string, mode: T, images?: ImageAttachment[]): void { if (this.closed) { throw new Error('Cannot push to closed queue'); } @@ -49,7 +55,8 @@ export class MessageQueue2 { message, mode, modeHash, - isolate: false + isolate: false, + images, }); // Trigger message handler if set @@ -221,7 +228,7 @@ export class MessageQueue2 { * Wait for messages and return all messages with the same mode as a single string * Returns { message: string, mode: T } or null if aborted/closed */ - async waitForMessagesAndGetAsString(abortSignal?: AbortSignal): Promise<{ message: string, mode: T, isolate: boolean, hash: string } | null> { + async waitForMessagesAndGetAsString(abortSignal?: AbortSignal): Promise<{ message: string, mode: T, isolate: boolean, hash: string, images?: ImageAttachment[] } | null> { // If we have messages, return them immediately if (this.queue.length > 0) { return this.collectBatch(); @@ -245,13 +252,14 @@ export class MessageQueue2 { /** * Collect a batch of messages with the same mode, respecting isolation requirements */ - private collectBatch(): { message: string, mode: T, hash: string, isolate: boolean } | null { + private collectBatch(): { message: string, mode: T, hash: string, isolate: boolean, images?: ImageAttachment[] } | null { if (this.queue.length === 0) { return null; } const firstItem = this.queue[0]; const sameModeMessages: string[] = []; + const allImages: ImageAttachment[] = []; let mode = firstItem.mode; let isolate = firstItem.isolate ?? false; const targetModeHash = firstItem.modeHash; @@ -260,6 +268,7 @@ export class MessageQueue2 { if (firstItem.isolate) { const item = this.queue.shift()!; sameModeMessages.push(item.message); + if (item.images) allImages.push(...item.images); logger.debug(`[MessageQueue2] Collected isolated message with mode hash: ${targetModeHash}`); } else { // Collect all messages with the same mode until we hit an isolated message @@ -268,6 +277,7 @@ export class MessageQueue2 { !this.queue[0].isolate) { const item = this.queue.shift()!; sameModeMessages.push(item.message); + if (item.images) allImages.push(...item.images); } logger.debug(`[MessageQueue2] Collected batch of ${sameModeMessages.length} messages with mode hash: ${targetModeHash}`); } @@ -279,7 +289,8 @@ export class MessageQueue2 { message: combinedMessage, mode, hash: targetModeHash, - isolate + isolate, + images: allImages.length > 0 ? allImages : undefined, }; } diff --git a/packages/happy-wire/src/legacyProtocol.ts b/packages/happy-wire/src/legacyProtocol.ts index c2f4f29a9..a835244ff 100644 --- a/packages/happy-wire/src/legacyProtocol.ts +++ b/packages/happy-wire/src/legacyProtocol.ts @@ -1,11 +1,17 @@ import * as z from 'zod'; import { MessageMetaSchema } from './messageMeta'; +export const ImageAttachmentSchema = z.object({ + base64: z.string(), + mediaType: z.string(), +}); + export const UserMessageSchema = z.object({ role: z.literal('user'), content: z.object({ type: z.literal('text'), text: z.string(), + images: z.array(ImageAttachmentSchema).optional(), }), localKey: z.string().optional(), meta: MessageMetaSchema.optional(),