test codex and extend model selection

Author: justdeko

CLAUDE.md

@@ -266,8 +266,8 @@ When touching the canvas, read kuiver's sources rather than its README:
 unzip ~/.gradle/caches/modules-2/files-2.1/io.github.justdeko/kuiver-jvm/*/*/core-jvm-*-sources.jar
 ```
 
-Material 3's expressive APIs are opted into once for the whole `:shared` source set in its
-`build.gradle.kts`, not per file — the theme is app-wide.
+Material 3's experimental and expressive APIs are opted into once for the whole `:shared` source set
+in its `build.gradle.kts`, not per file — the theme is app-wide.
 
 `.mcp.json` wires up `:desktopApp:hotMcpServer`, which drives a running hot-reload instance —
 screenshots, the semantic tree, clicks. That is how to check a UI change actually looks right rather

README.md

@@ -37,17 +37,15 @@ instead. There's also a CLI which allows you to run zopf workflows headlessly.
 
 You need **macOS on Apple Silicon** and one of these installed and signed in:
 
-| CLI                                                                                  | `provider:` | What zopf runs              |
-|--------------------------------------------------------------------------------------|-------------|-----------------------------|
-| [Claude Code](https://code.claude.com/docs/en/quickstart#step-1-install-claude-code) | `claude`    | `claude -p`, streaming JSON |
-| [Codex](https://learn.chatgpt.com/docs/codex/cli)                                    | `codex`     | `codex exec --json`         |
-| [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness)                  | `dsh`       | `dsh --profile headless`    |
+| CLI                                                                 | `provider:` | What zopf runs              |
+|---------------------------------------------------------------------|-------------|-----------------------------|
+| [Claude Code](https://github.com/anthropics/claude-code)            | `claude`    | `claude -p`, streaming JSON |
+| [Codex](https://github.com/openai/codex)                            | `codex`     | `codex exec --json`         |
+| [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) | `dsh`       | `dsh --profile headless`    |
 
 > [!NOTE]
-> So far, only the Claude Code path was used daily and tested against a real CLI. The codex and dsh paths are built from
-those
-> CLIs' documented flags and covered by unit tests, but they haven't been tested end to end here.
-> Please open an issue if you notice a bug in their implementations.
+> While zopf supports claude code, codex and dsh so far, I've only extensively tested claude code as it is my daily
+driver.
 
 ### The App
 
@@ -168,7 +166,8 @@ defaults:
   repo: self
 ```
 
-The run archive grows, so a chatty build step can run quite large. To cap it, go to Settings → History, or do `zopf prune --keep 50`.
+The run archive grows, so a chatty build step can run quite large. To cap it, go to Settings → History, or do
+`zopf prune --keep 50`.
 
 ## CLI usage
 

core/src/commonMain/kotlin/com/dk/zopf/model/AgentCapabilities.kt

@@ -35,12 +35,12 @@ val AgentProviderId.capabilities: AgentCapabilities
                 AgentCapabilities(
                     followUps = false,
                     inlineApproval = false,
-                    resumeInTerminal = false,
+                    resumeInTerminal = true,
                     reportsCostUsd = false,
                     skills = false,
                     toolPermissions = false,
                     extraDirectories = false,
-                    outputSchema = false,
+                    outputSchema = true,
                     sandbox = true,
                     modelSelection = true,
                 )
@@ -66,7 +66,7 @@ val AgentProviderId.modelOptions: List<String>
     get() =
         when (this) {
             AgentProviderId.CLAUDE -> listOf("opus", "sonnet", "haiku")
-            AgentProviderId.CODEX -> emptyList()
+            AgentProviderId.CODEX -> listOf("gpt-5.6-terra", "gpt-5.6-luna", "gpt-5.5", "gpt-5.4-mini")
             AgentProviderId.DSH -> emptyList()
         }
 

core/src/commonMain/kotlin/com/dk/zopf/model/OutputSchema.kt

@@ -46,13 +46,20 @@ fun List<SchemaField>.toJsonSchema(): JsonObject {
         putJsonObject("properties") {
             fields.forEach { field ->
                 putJsonObject(field.name) {
-                    put("type", field.type.jsonType)
+                    if (field.required) {
+                        put("type", field.type.jsonType)
+                    } else {
+                        putJsonArray("type") {
+                            add(field.type.jsonType)
+                            add("null")
+                        }
+                    }
                     if (field.description.isNotBlank()) put("description", field.description)
                 }
             }
         }
         putJsonArray("required") {
-            fields.filter { it.required }.forEach { add(it.name) }
+            fields.forEach { add(it.name) }
         }
         put("additionalProperties", false)
     }

core/src/commonMain/kotlin/com/dk/zopf/runtime/CodexEvents.kt

@@ -105,6 +105,10 @@ object CodexEvents {
                 AgentEvent.AssistantMessage(threadId, null, listOf(ContentBlock.Thinking(item.text())), obj)
             }
 
+            itemType == "error" && complete -> {
+                AgentEvent.Notice(threadId, "error", item.string("message"), obj)
+            }
+
             itemType in TOOL_ITEMS && type == "item.started" -> {
                 AgentEvent.AssistantMessage(
                     threadId,
@@ -163,7 +167,13 @@ object CodexEvents {
         return (input ?: 0) + (output ?: 0)
     }
 
-    private fun JsonObject.errorMessage(): String? = string("message") ?: this["error"]?.asObject()?.string("message")
+    private fun JsonObject.errorMessage(): String? = (string("message") ?: this["error"]?.asObject()?.string("message"))?.unwrapped()
+
+    private fun String.unwrapped(): String =
+        runCatching { json.parseToJsonElement(this).jsonObject }
+            .getOrNull()
+            ?.errorMessage()
+            ?: this
 
     private fun JsonObject.intOrNull(key: String): Int? = this[key]?.intOrNull()
 }

core/src/commonMain/kotlin/com/dk/zopf/runtime/CodexProvider.kt

@@ -1,6 +1,9 @@
 package com.dk.zopf.runtime
 
 import com.dk.zopf.model.AgentProviderId
+import java.nio.file.Files
+import java.nio.file.Path
+import kotlin.io.path.writeText
 
 object CodexProvider : AgentProvider {
     override val id = AgentProviderId.CODEX
@@ -30,10 +33,20 @@ object CodexProvider : AgentProvider {
             add("exec")
             add("--json")
             add("--skip-git-repo-check")
+            invocation.jsonSchema?.let {
+                add("--output-schema")
+                add(schemaFile(it).toString())
+            }
             add(invocation.prompt)
         }
 
+    internal fun schemaFile(schema: String): Path =
+        Files.createTempFile("zopf-schema-", ".json").also {
+            it.writeText(schema)
+            it.toFile().deleteOnExit()
+        }
+
     override fun parse(line: String): AgentEvent? = CodexEvents.parse(line)
 
-    override fun terminalArgs(sessionId: String?): List<String> = emptyList()
+    override fun terminalArgs(sessionId: String?): List<String> = sessionId?.let { listOf("resume", it) }.orEmpty()
 }

core/src/commonMain/kotlin/com/dk/zopf/runtime/NodeExecutor.kt

@@ -123,6 +123,7 @@ class ProcessNodeExecutor(
         run.live = AgentLive(session)
         run.sessionId = session.sessionId
         run.command = session.command
+        run.model = invocation.model
         run.status = RunStatus.STARTING
         if (sandbox != null) run.notice("Sandbox: ${sandbox.cliValue}")
 
@@ -146,6 +147,7 @@ class ProcessNodeExecutor(
                 },
                 awaitExit = session::awaitExit,
             )
+        if (invocation.jsonSchema != null) run.takeFieldsFromJsonResult()
         if (session.stderrText.isNotBlank() && exit != 0) {
             run.notice(session.stderrText, isWarning = true)
         }

core/src/commonMain/kotlin/com/dk/zopf/runtime/NodeRun.kt

@@ -8,6 +8,8 @@ import androidx.compose.runtime.setValue
 import com.dk.zopf.model.AgentProviderId
 import com.dk.zopf.model.NodeType
 import kotlinx.coroutines.CompletableDeferred
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.jsonObject
 import java.nio.file.Path
 import java.time.Duration
 import java.time.Instant
@@ -209,8 +211,8 @@ class NodeRun(
 
         when (event) {
             is AgentEvent.SystemInit -> {
-                model = event.model
-                notice("Session started · ${event.model ?: "default model"} · ${event.cwd ?: cwd}")
+                model = event.model ?: model
+                notice("Session started · ${model ?: "default model"} · ${event.cwd ?: cwd}")
             }
 
             is AgentEvent.TextDelta -> appendDelta(event.text, event.isThinking)
@@ -248,16 +250,18 @@ class NodeRun(
                     outputBuffer.append(it)
                 }
                 if (event.fields.isNotEmpty()) outputFields = event.fields
-                add(
-                    ConsoleEntry.Summary(
-                        key = nextKey,
-                        text = event.text,
-                        costUsd = event.costUsd,
-                        durationMs = event.durationMs,
-                        isError = event.isError,
-                        tokens = event.tokens,
-                    ),
-                )
+                if (!repeatsLastSummary(event)) {
+                    add(
+                        ConsoleEntry.Summary(
+                            key = nextKey,
+                            text = event.text,
+                            costUsd = event.costUsd,
+                            durationMs = event.durationMs,
+                            isError = event.isError,
+                            tokens = event.tokens,
+                        ),
+                    )
+                }
 
                 status =
                     when {
@@ -268,8 +272,14 @@ class NodeRun(
             }
 
             is AgentEvent.Notice ->
-                if (event.kind == "rate_limit" && event.detail != "allowed") {
-                    notice("Rate limit: ${event.detail}", isWarning = true)
+                when {
+                    event.kind == "rate_limit" && event.detail != "allowed" ->
+                        notice("Rate limit: ${event.detail}", isWarning = true)
+
+                    event.kind == "error" ->
+                        event.detail?.takeIf { it.isNotBlank() }?.let { notice(it, isWarning = true) }
+
+                    else -> Unit
                 }
 
             is AgentEvent.NonJson -> notice(event.line, isWarning = true)
@@ -332,6 +342,20 @@ class NodeRun(
         )
     }
 
+    @Synchronized
+    internal fun takeFieldsFromJsonResult() {
+        if (outputFields.isNotEmpty()) return
+        val answer = outputBuffer.toString().trim()
+        val obj = runCatching { Json.parseToJsonElement(answer).jsonObject }.getOrNull() ?: return
+
+        outputFields = obj.toFields()
+    }
+
+    private fun repeatsLastSummary(event: AgentEvent.Result): Boolean {
+        val last = entries.lastOrNull() as? ConsoleEntry.Summary ?: return false
+        return last.isError && event.isError && last.text == event.text
+    }
+
     @Synchronized
     fun notice(
         text: String,

core/src/commonTest/kotlin/com/dk/zopf/model/WorkflowModelTest.kt

@@ -720,7 +720,7 @@ class WorkflowEditsTest {
     }
 
     @Test
-    fun `the declared fields become a strict JSON schema, and only the required ones are required`() {
+    fun `an optional field is declared nullable, since a partial required list is not a strict schema`() {
         val schema =
             listOf(
                 SchemaField("severity", description = "low, medium or high"),
@@ -729,7 +729,7 @@ class WorkflowEditsTest {
 
         assertEquals(
             """{"type":"object","properties":{"severity":{"type":"string","description":"low, medium or high"},""" +
-                """"notes":{"type":"array"}},"required":["severity"],"additionalProperties":false}""",
+                """"notes":{"type":["array","null"]}},"required":["severity","notes"],"additionalProperties":false}""",
             schema.toString(),
         )
     }

core/src/commonTest/kotlin/com/dk/zopf/runtime/AgentNodeTest.kt

@@ -244,6 +244,28 @@ class CodexEventsTest {
         assertTrue(assertIs<AgentEvent.Result>(error).isError)
     }
 
+    @Test
+    fun `an item that names its type the older way still parses`() {
+        val event = CodexEvents.parse("""{"type":"item.completed","item":{"id":"i","item_type":"agent_message","text":"done"}}""")
+        assertEquals("done", assertIs<AgentEvent.AssistantMessage>(event).text)
+    }
+
+    @Test
+    fun `an error the CLI reports mid-turn is a warning rather than the end of the node`() {
+        val event = CodexEvents.parse("""{"type":"item.completed","item":{"id":"i","type":"error","message":"Model metadata not found."}}""")
+        val notice = assertIs<AgentEvent.Notice>(event)
+
+        assertEquals("error", notice.kind)
+        assertEquals("Model metadata not found.", notice.detail)
+    }
+
+    @Test
+    fun `an error message that is itself JSON is unwrapped to the sentence inside it`() {
+        val line = """{"type":"turn.failed","error":{"message":"{\"type\":\"error\",\"status\":400,\"error\":{\"message\":\"That model is not supported on this account.\"}}"}}"""
+
+        assertEquals("That model is not supported on this account.", assertIs<AgentEvent.Result>(CodexEvents.parse(line)).text)
+    }
+
     @Test
     fun `an item type this version has never seen survives rather than throwing`() {
         val event = CodexEvents.parse("""{"type":"item.completed","item":{"id":"i","item_type":"invented_later"}}""")
@@ -330,13 +352,80 @@ class NodeRunTest {
 
         assertEquals(RunStatus.RUNNING, run.status)
         assertFalse(run.canFollowUp)
-        assertFalse(run.canTakeOver, "an exec session cannot be resumed, so there is nothing to hand over")
+        assertTrue(run.canTakeOver, "the thread id codex reported is enough to reopen it in a terminal")
 
         assertEquals("It printed `hello`.", run.output().result)
         assertNull(run.costUsd)
         assertEquals(2634, run.tokens)
     }
 
+    @Test
+    fun `a codex answer that fills a schema becomes fields, and stays the result`() {
+        val line = """{"type":"item.completed","item":{"id":"i","type":"agent_message","text":"{\"severity\":\"low\",\"count\":2}"}}"""
+        val run =
+            NodeRun("r", "demo", "ask", "Ask", NodeType.AGENT, Paths.get("/tmp"), provider = AgentProviderId.CODEX)
+                .apply {
+                    CodexEvents.parseAll(line).forEach(::consume)
+                    takeFieldsFromJsonResult()
+                }
+
+        val output = run.output()
+
+        assertEquals("low", output.field("severity"))
+        assertEquals("2", output.field("count"))
+        assertEquals("""{"severity":"low","count":2}""", output.result)
+    }
+
+    @Test
+    fun `an answer in prose is left as prose, however the node was asked`() {
+        val run =
+            NodeRun("r", "demo", "ask", "Ask", NodeType.AGENT, Paths.get("/tmp"), provider = AgentProviderId.CODEX)
+                .apply {
+                    CodexEvents
+                        .parseAll("""{"type":"item.completed","item":{"id":"i","type":"agent_message","text":"Nothing looked wrong."}}""")
+                        .forEach(::consume)
+                    takeFieldsFromJsonResult()
+                }
+
+        assertEquals(emptyMap(), run.output().extras)
+        assertEquals("Nothing looked wrong.", run.output().result)
+    }
+
+    @Test
+    fun `a codex failure that arrives as both an error and a failed turn is reported once`() {
+        val stream =
+            """
+            {"type":"error","message":"That model is not supported on this account."}
+            {"type":"turn.failed","error":{"message":"That model is not supported on this account."}}
+            """.trimIndent()
+        val run =
+            NodeRun("r", "demo", "ask", "Ask", NodeType.AGENT, Paths.get("/tmp"), provider = AgentProviderId.CODEX)
+                .apply { CodexEvents.parseAll(stream).forEach(::consume) }
+
+        val summary = run.entries.filterIsInstance<ConsoleEntry.Summary>().single()
+
+        assertEquals("That model is not supported on this account.", summary.text)
+        assertTrue(summary.isError)
+        assertEquals(RunStatus.FAILED, run.status)
+    }
+
+    @Test
+    fun `an error codex reports mid-turn shows as a warning and leaves the node running`() {
+        val run =
+            NodeRun("r", "demo", "ask", "Ask", NodeType.AGENT, Paths.get("/tmp"), provider = AgentProviderId.CODEX)
+                .apply {
+                    CodexEvents
+                        .parseAll("""{"type":"item.completed","item":{"id":"i","type":"error","message":"Model metadata not found."}}""")
+                        .forEach(::consume)
+                }
+
+        val notice = run.entries.filterIsInstance<ConsoleEntry.Notice>().single()
+
+        assertEquals("Model metadata not found.", notice.text)
+        assertTrue(notice.isWarning)
+        assertEquals(RunStatus.RUNNING, run.status)
+    }
+
     @Test
     fun `a dsh run has no stream, so its stdout is the answer and the whole answer`() {
         val text = "Both tests pass.\n\nThe flake was the clock, not the parser.\n"