fix(mcp): guarantee valid UTF-8 in tree-format output

Dry run 30206811293: macos-15-intel release smoke failed B3 with a
visibly correct 'semantic: 50' table — BSD grep returns no-match for
the ENTIRE output when any line carries a NUL, control byte, or
invalid UTF-8 (verified: printf poisoned input -> grep -qE fails), so
one bad byte in one row silently unmatches every anchored check.

Close the class at the emission choke point: control bytes and
invalid UTF-8 force the quoted form; quoting escapes controls as
\u00XX and replaces invalid sequences with U+FFFD (RFC 3629
validation, NUL-safe). Tool output is valid UTF-8 by construction.

Unit test proven RED-on-revert (raw bytes emitted) and green with the
fix. The smoke's B3 failure path now also dumps od -c bytes so a
future red names the exact byte.

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>

Author: DeusData

scripts/smoke-test.sh

@@ -568,7 +568,11 @@ fi
 if echo "$IM_ARR" | grep -qE '^semantic: [0-9]+'; then
   echo "OK B3: ARRAY flag (repeated --semantic-query) → semantic TOON table"
 else
-  echo "FAIL B3: repeated --semantic-query did not produce a semantic table"; echo "$IM_ARR" | head -c 300; exit 1
+  echo "FAIL B3: repeated --semantic-query did not produce a semantic table"; echo "$IM_ARR" | head -c 300
+  # Byte-exact post-mortem: an invisible control/invalid-UTF8 byte anywhere in
+  # the output makes BSD grep treat ALL of it as unmatchable binary, and the
+  # rendered log cannot show which byte — od can.
+  echo; echo "-- B3 first bytes (od) --"; echo "$IM_ARR" | od -c | head -6; exit 1
 fi
 
 # B4: STDIN — piped JSON resolves; this path must NOT emit a deprecation warning.

src/mcp/compact_out.c

@@ -97,16 +97,61 @@ static bool looks_numeric(const char *s) {
     return digit;
 }
 
+/* UTF-8 sequence length starting at p, validating continuation bytes and the
+ * lead-byte ranges (RFC 3629: no overlongs, no surrogates, max U+10FFFF);
+ * 0 = not a valid sequence start. Reads stop at the first invalid byte, so a
+ * terminating NUL is never overrun. */
+static size_t utf8_sequence_length(const unsigned char *p) {
+    unsigned char c = p[0];
+    if (c < 0x80) {
+        return 1;
+    }
+    if (c < 0xC2) {
+        return 0; /* bare continuation byte or overlong lead */
+    }
+    if (c < 0xE0) {
+        return (p[1] & 0xC0) == 0x80 ? 2 : 0;
+    }
+    if (c < 0xF0) {
+        unsigned char lo = c == 0xE0 ? 0xA0 : 0x80;
+        unsigned char hi = c == 0xED ? 0x9F : 0xBF;
+        if (p[1] < lo || p[1] > hi || (p[2] & 0xC0) != 0x80) {
+            return 0;
+        }
+        return 3;
+    }
+    if (c < 0xF5) {
+        unsigned char lo = c == 0xF0 ? 0x90 : 0x80;
+        unsigned char hi = c == 0xF4 ? 0x8F : 0xBF;
+        if (p[1] < lo || p[1] > hi || (p[2] & 0xC0) != 0x80 || (p[3] & 0xC0) != 0x80) {
+            return 0;
+        }
+        return 4;
+    }
+    return 0;
+}
+
 static bool needs_quotes(const char *s) {
     if (!s || !*s) {
         return false; /* empty cells emit as the "-" placeholder, not quotes */
     }
     for (const char *p = s; *p; p++) {
+        unsigned char c = (unsigned char)*p;
         /* Space-delimited rows: any internal whitespace or quote forces
-         * quoting so column positions stay parseable. */
-        if (isspace((unsigned char)*p) || *p == '"' || *p == '\r') {
+         * quoting so column positions stay parseable. Control bytes and
+         * invalid UTF-8 force the quoted path too, which sanitizes them —
+         * one raw byte otherwise makes line-oriented consumers (BSD grep)
+         * treat the ENTIRE tool output as unmatchable binary. */
+        if (isspace(c) || *p == '"' || *p == '\r' || c < 0x20 || c == 0x7f) {
             return true;
         }
+        if (c >= 0x80) {
+            size_t len = utf8_sequence_length((const unsigned char *)p);
+            if (!len) {
+                return true;
+            }
+            p += len - 1;
+        }
     }
     if (strcmp(s, "true") == 0 || strcmp(s, "false") == 0 || strcmp(s, "null") == 0 ||
         strcmp(s, "-") == 0) {
@@ -131,10 +176,29 @@ static void append_quoted(cbm_sb_t *sb, const char *s) {
         case '\r':
             cbm_sb_append_n(sb, "\\r", 2);
             break;
-        default:
-            cbm_sb_append_n(sb, p, 1);
+        default: {
+            unsigned char c = (unsigned char)*p;
+            if (c >= 0x20 && c != 0x7f && c < 0x80) {
+                cbm_sb_append_n(sb, p, 1);
+            } else if (c < 0x20 || c == 0x7f) {
+                /* JSON-style escape: the value stays one printable line. */
+                char esc[8];
+                snprintf(esc, sizeof(esc), "\\u%04x", (unsigned)c);
+                cbm_sb_append(sb, esc);
+            } else {
+                size_t len = utf8_sequence_length((const unsigned char *)p);
+                if (len) {
+                    cbm_sb_append_n(sb, p, len);
+                    p += len - 1;
+                } else {
+                    /* Invalid byte → U+FFFD: output is valid UTF-8 by
+                     * construction, and the corruption stays visible. */
+                    cbm_sb_append_n(sb, "\xEF\xBF\xBD", 3);
+                }
+            }
             break;
         }
+        }
     }
     cbm_sb_append_n(sb, "\"", 1);
 }

tests/test_mcp.c

@@ -10,6 +10,7 @@
 #include "../src/foundation/log.h"
 #include "../src/foundation/platform.h" /* cbm_file_size */
 #include "../src/foundation/subprocess.h"
+#include "../src/mcp/compact_out.h"
 #include "test_framework.h"
 #include "test_helpers.h"
 #include <cli/cli.h>
@@ -550,6 +551,33 @@ static void mcp_replacing_mutation_guard_end(void *context, const char *project)
     }
 }
 
+TEST(tree_cell_sanitizes_control_and_invalid_utf8) {
+    /* One raw control or invalid-UTF8 byte in a cell poisons LINE-ORIENTED
+     * consumers of the ENTIRE output (BSD grep treats all of it as
+     * unmatchable binary — the macos-15-intel release-smoke B3 class), so
+     * cell emission guarantees valid UTF-8: control bytes escape as \u00XX,
+     * invalid sequences become U+FFFD, and both force the quoted form. */
+    cbm_sb_t sb;
+    cbm_sb_init(&sb);
+    cbm_tree_cell_str(&sb,
+                      "evil\x01name\xff"
+                      "end",
+                      true);
+    char *out = cbm_sb_finish(&sb);
+    ASSERT_NOT_NULL(out);
+    ASSERT_STR_EQ(out, "\"evil\\u0001name\xEF\xBF\xBD"
+                       "end\"");
+    free(out);
+
+    cbm_sb_init(&sb);
+    cbm_tree_cell_str(&sb, "b\xC3\xA4r_ok", true);
+    out = cbm_sb_finish(&sb);
+    ASSERT_NOT_NULL(out);
+    ASSERT_STR_EQ(out, "b\xC3\xA4r_ok"); /* valid UTF-8 stays raw + unquoted */
+    free(out);
+    PASS();
+}
+
 /* ══════════════════════════════════════════════════════════════════
  *  JSON-RPC PARSING
  * ══════════════════════════════════════════════════════════════════ */
@@ -9468,6 +9496,7 @@ SUITE(mcp) {
     RUN_TEST(jsonrpc_parse_request);
     RUN_TEST(jsonrpc_parse_notification);
     RUN_TEST(jsonrpc_parse_invalid);
+    RUN_TEST(tree_cell_sanitizes_control_and_invalid_utf8);
     RUN_TEST(jsonrpc_parse_tools_call);
     RUN_TEST(jsonrpc_parse_string_id_issue253);
     RUN_TEST(jsonrpc_format_response_string_id_issue253);