ios: fix attached image turn input and paste support (#42)

Co-authored-by: dnakov <3777433+dnakov@users.noreply.github.com>

Author: dnakov

apps/ios/Litter.xcodeproj/project.pbxproj

@@ -265,7 +265,7 @@ struct UserInput: Encodable {
         case text
         case path
         case name
-        case imageURL = "image_url"
+        case url
     }
 
     init(type: String, text: String? = nil, path: String? = nil, name: String? = nil, imageURL: String? = nil) {
@@ -275,6 +275,17 @@ struct UserInput: Encodable {
         self.name = name
         self.imageURL = imageURL
     }
+
+    func encode(to encoder: Encoder) throws {
+        var container = encoder.container(keyedBy: CodingKeys.self)
+        try container.encode(type, forKey: .type)
+        try container.encodeIfPresent(text, forKey: .text)
+        try container.encodeIfPresent(path, forKey: .path)
+        try container.encodeIfPresent(name, forKey: .name)
+        if type == "image" {
+            try container.encodeIfPresent(imageURL, forKey: .url)
+        }
+    }
 }
 
 struct TurnStartParams: Encodable {

apps/ios/Sources/Litter/Bridge/CodexProtocol.swift

@@ -0,0 +1,60 @@
+import Foundation
+import UIKit
+
+struct PreparedImageAttachment {
+    let data: Data
+    let mimeType: String
+
+    var userInput: UserInput {
+        UserInput(type: "image", imageURL: dataURI)
+    }
+
+    var chatImage: ChatImage {
+        ChatImage(data: data)
+    }
+
+    private var dataURI: String {
+        "data:\(mimeType);base64,\(data.base64EncodedString())"
+    }
+}
+
+enum ConversationAttachmentSupport {
+    static func prepareImage(_ image: UIImage) -> PreparedImageAttachment? {
+        guard let encodedImage = encodedImageData(for: image) else { return nil }
+        return PreparedImageAttachment(data: encodedImage.data, mimeType: encodedImage.mimeType)
+    }
+
+    static func buildTurnInputs(text: String, additionalInput: [UserInput]) -> [UserInput] {
+        var inputs: [UserInput] = []
+        if !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
+            inputs.append(UserInput(type: "text", text: text))
+        }
+        inputs.append(contentsOf: additionalInput)
+        return inputs
+    }
+
+    private static func encodedImageData(for image: UIImage) -> (data: Data, mimeType: String)? {
+        if image.litterHasAlpha, let pngData = image.pngData() {
+            return (pngData, "image/png")
+        }
+        if let jpegData = image.jpegData(compressionQuality: 0.85) {
+            return (jpegData, "image/jpeg")
+        }
+        if let pngData = image.pngData() {
+            return (pngData, "image/png")
+        }
+        return nil
+    }
+}
+
+private extension UIImage {
+    var litterHasAlpha: Bool {
+        guard let alphaInfo = cgImage?.alphaInfo else { return false }
+        switch alphaInfo {
+        case .first, .last, .premultipliedFirst, .premultipliedLast:
+            return true
+        default:
+            return false
+        }
+    }
+}

apps/ios/Sources/Litter/Models/ConversationAttachmentSupport.swift

@@ -329,8 +329,14 @@ final class ServerConnection: Identifiable {
         serviceTier: String? = nil,
         additionalInput: [UserInput] = []
     ) async throws -> TurnStartResponse {
-        var inputs: [UserInput] = [UserInput(type: "text", text: text)]
-        inputs.append(contentsOf: additionalInput)
+        let inputs = ConversationAttachmentSupport.buildTurnInputs(text: text, additionalInput: additionalInput)
+        guard !inputs.isEmpty else {
+            throw NSError(
+                domain: "Litter",
+                code: 1020,
+                userInfo: [NSLocalizedDescriptionKey: "Cannot send an empty turn"]
+            )
+        }
         return try await routedSendRequest(
             method: "turn/start",
             params: TurnStartParams(

apps/ios/Sources/Litter/Models/ServerConnection.swift

@@ -1098,6 +1098,7 @@ final class ServerManager {
 
     func send(
         _ text: String,
+        attachmentImage: UIImage? = nil,
         skillMentions: [SkillMentionSelection] = [],
         cwd: String,
         model: String? = nil,
@@ -1106,6 +1107,10 @@ final class ServerManager {
         approvalPolicy: String = "never",
         sandboxMode: String? = nil
     ) async {
+        let preparedAttachment = attachmentImage.flatMap(ConversationAttachmentSupport.prepareImage)
+        let images = preparedAttachment.map { [$0.chatImage] } ?? []
+        let trimmedText = text.trimmingCharacters(in: .whitespacesAndNewlines)
+        guard !trimmedText.isEmpty || !images.isEmpty || !skillMentions.isEmpty else { return }
         var key = activeThreadKey
         if key == nil {
             guard let serverId = connections.values.first(where: { $0.isConnected })?.id else { return }
@@ -1127,7 +1132,7 @@ final class ServerManager {
                     serverName: conn?.server.name ?? "Server",
                     serverSource: conn?.server.source ?? .local
                 )
-                state.items.append(makeUserItem(text: text, sourceTurnId: nil, sourceTurnIndex: nil, isBoundary: true))
+                state.items.append(makeUserItem(text: text, images: images, sourceTurnId: nil, sourceTurnIndex: nil, isBoundary: true))
                 state.items.append(makeErrorItem(message: error.localizedDescription, sourceTurnId: nil, sourceTurnIndex: nil))
                 state.status = .error(error.localizedDescription)
                 threads[errorKey] = state
@@ -1137,15 +1142,23 @@ final class ServerManager {
         }
         guard let key, let thread = threads[key], let conn = connections[key.serverId] else { return }
         let nextTurnIndex = threadTurnCounts[key] ?? inferredTurnCount(from: thread.items)
-        thread.items.append(makeUserItem(text: text, sourceTurnId: nil, sourceTurnIndex: nextTurnIndex, isBoundary: true))
+        thread.items.append(makeUserItem(text: text, images: images, sourceTurnId: nil, sourceTurnIndex: nextTurnIndex, isBoundary: true))
         thread.status = .thinking
         thread.updatedAt = Date()
         requestNotificationPermissionIfNeeded()
-        startLiveActivity(key: key, model: thread.model, cwd: thread.cwd, prompt: text)
+        startLiveActivity(
+            key: key,
+            model: thread.model,
+            cwd: thread.cwd,
+            prompt: !trimmedText.isEmpty ? text : (!images.isEmpty ? "Shared image" : text)
+        )
         do {
-            let skillInputs = skillMentions.map { mention in
+            var additionalInputs = skillMentions.map { mention in
                 UserInput(type: "skill", path: mention.path, name: mention.name)
             }
+            if let preparedAttachment {
+                additionalInputs.append(preparedAttachment.userInput)
+            }
             let resp = try await conn.sendTurn(
                 threadId: key.threadId,
                 text: text,
@@ -1154,7 +1167,7 @@ final class ServerManager {
                 model: model,
                 effort: effort,
                 serviceTier: serviceTier,
-                additionalInput: skillInputs
+                additionalInput: additionalInputs
             )
             NSLog("[send] sendTurn succeeded, turnId=%@", resp.turnId ?? "nil")
             thread.activeTurnId = resp.turnId

apps/ios/Sources/Litter/Models/ServerManager.swift

@@ -0,0 +1,49 @@
+import SwiftUI
+
+struct ConversationComposerAttachSheet: View {
+    let onPickPhotoLibrary: () -> Void
+    let onTakePhoto: () -> Void
+
+    var body: some View {
+        VStack(spacing: 12) {
+            Text("Attach")
+                .litterFont(.headline, weight: .semibold)
+                .foregroundColor(LitterTheme.textPrimary)
+                .frame(maxWidth: .infinity, alignment: .leading)
+
+            Button(action: onPickPhotoLibrary) {
+                sheetButtonLabel("Photo Library", systemImage: "photo.on.rectangle")
+            }
+
+            Button(action: onTakePhoto) {
+                sheetButtonLabel("Take Photo", systemImage: "camera")
+            }
+
+            Spacer(minLength: 0)
+        }
+        .padding(.horizontal, 16)
+        .padding(.top, 12)
+        .padding(.bottom, 20)
+        .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .top)
+        .background(LitterTheme.backgroundGradient.ignoresSafeArea())
+    }
+
+    @ViewBuilder
+    private func sheetButtonLabel(_ title: String, systemImage: String) -> some View {
+        HStack(spacing: 10) {
+            Image(systemName: systemImage)
+                .litterFont(.body, weight: .medium)
+                .foregroundColor(LitterTheme.accent)
+                .frame(width: 20)
+
+            Text(title)
+                .litterFont(.body, weight: .medium)
+                .foregroundColor(LitterTheme.textPrimary)
+
+            Spacer()
+        }
+        .padding(.horizontal, 16)
+        .frame(height: 52)
+        .modifier(GlassRoundedRectModifier(cornerRadius: 18))
+    }
+}

apps/ios/Sources/Litter/Views/ConversationComposerAttachSheet.swift

@@ -8,15 +8,16 @@ struct ConversationComposerContentView: View {
     let contextPercent: Int64?
     let isTurnActive: Bool
     let voiceManager: VoiceTranscriptionManager
+    @Binding var showAttachMenu: Bool
     let onClearAttachment: () -> Void
     let onRespondToPendingUserInput: ([String: [String]]) -> Void
-    let onShowAttachMenu: () -> Void
+    let onPasteImage: (UIImage) -> Void
     let onSendText: () -> Void
     let onStopRecording: () -> Void
     let onStartRecording: () -> Void
     let onInterrupt: () -> Void
     @Binding var inputText: String
-    let isComposerFocused: FocusState<Bool>.Binding
+    @Binding var isComposerFocused: Bool
 
     var body: some View {
         VStack(spacing: 0) {
@@ -52,11 +53,13 @@ struct ConversationComposerContentView: View {
                 }
 
                 ConversationComposerEntryRowView(
+                    showAttachMenu: $showAttachMenu,
                     inputText: $inputText,
-                    isComposerFocused: isComposerFocused,
+                    isComposerFocused: $isComposerFocused,
                     voiceManager: voiceManager,
                     isTurnActive: isTurnActive,
-                    onShowAttachMenu: onShowAttachMenu,
+                    hasAttachment: attachedImage != nil,
+                    onPasteImage: onPasteImage,
                     onSendText: onSendText,
                     onStopRecording: onStopRecording,
                     onStartRecording: onStartRecording,

apps/ios/Sources/Litter/Views/ConversationComposerContentView.swift

@@ -1,11 +1,14 @@
 import SwiftUI
+import UIKit
 
 struct ConversationComposerEntryRowView: View {
+    @Binding var showAttachMenu: Bool
     @Binding var inputText: String
-    let isComposerFocused: FocusState<Bool>.Binding
+    @Binding var isComposerFocused: Bool
     let voiceManager: VoiceTranscriptionManager
     let isTurnActive: Bool
-    let onShowAttachMenu: () -> Void
+    let hasAttachment: Bool
+    let onPasteImage: (UIImage) -> Void
     let onSendText: () -> Void
     let onStopRecording: () -> Void
     let onStartRecording: () -> Void
@@ -15,10 +18,16 @@ struct ConversationComposerEntryRowView: View {
         !inputText.trimmingCharacters(in: .whitespaces).isEmpty
     }
 
+    private var canSend: Bool {
+        hasText || hasAttachment
+    }
+
     var body: some View {
         HStack(alignment: .center, spacing: 8) {
             if !voiceManager.isRecording && !voiceManager.isTranscribing && !isTurnActive {
-                Button(action: onShowAttachMenu) {
+                Button {
+                    showAttachMenu = true
+                } label: {
                     Image(systemName: "plus")
                         .litterFont(.body, weight: .semibold)
                         .foregroundColor(LitterTheme.textPrimary)
@@ -29,17 +38,24 @@ struct ConversationComposerEntryRowView: View {
             }
 
             HStack(spacing: 0) {
-                TextField("Message litter...", text: $inputText, axis: .vertical)
-                    .litterFont(.body)
-                    .foregroundColor(LitterTheme.textPrimary)
-                    .lineLimit(1...5)
-                    .focused(isComposerFocused)
-                    .textInputAutocapitalization(.never)
-                    .autocorrectionDisabled(true)
-                    .padding(.leading, 16)
-                    .padding(.vertical, 10)
+                ZStack(alignment: .topLeading) {
+                    ConversationComposerTextView(
+                        text: $inputText,
+                        isFocused: $isComposerFocused,
+                        onPasteImage: onPasteImage
+                    )
+
+                    if inputText.isEmpty {
+                        Text("Message litter...")
+                            .litterFont(.body)
+                            .foregroundColor(LitterTheme.textMuted)
+                            .padding(.leading, 16)
+                            .padding(.top, 10)
+                            .allowsHitTesting(false)
+                    }
+                }
 
-                if hasText {
+                if canSend {
                     Button(action: onSendText) {
                         Image(systemName: "arrow.up.circle.fill")
                             .litterFont(.title2)

apps/ios/Sources/Litter/Views/ConversationComposerEntryRowView.swift

@@ -61,9 +61,19 @@ struct ConversationComposerModalCoordinator<Content: View>: View {
 
     var body: some View {
         content
-            .confirmationDialog("Attach", isPresented: $showAttachMenu) {
-                Button("Photo Library") { showPhotoPicker = true }
-                Button("Take Photo") { showCamera = true }
+            .sheet(isPresented: $showAttachMenu) {
+                ConversationComposerAttachSheet(
+                    onPickPhotoLibrary: {
+                        showAttachMenu = false
+                        showPhotoPicker = true
+                    },
+                    onTakePhoto: {
+                        showAttachMenu = false
+                        showCamera = true
+                    }
+                )
+                .presentationDetents([.height(210)])
+                .presentationDragIndicator(.visible)
             }
             .photosPicker(isPresented: $showPhotoPicker, selection: $selectedPhoto, matching: .images)
             .onChange(of: selectedPhoto) { _, item in