diff --git a/proxy.js b/proxy.js index 9e646a7..696a6c1 100644 --- a/proxy.js +++ b/proxy.js @@ -738,6 +738,259 @@ function reverseMap(text, config) { return r; } +// ─── Raw-string SSE field codec ───────────────────────────────────────────── +// The SSE reverse path must NOT JSON.parse/stringify whole event payloads: a full +// round-trip would normalize bytes the proxy never meant to touch (\uXXXX -> literal, +// \/ -> /, number/whitespace/key-order normalization) across the ENTIRE event, +// breaking thinking-block byte-equality and busting prompt caching. Instead we locate +// just the one string field we must reverse, decode that value, reverse it, and +// re-encode it back in place. +// +// Byte-fidelity therefore holds exactly where it matters: every byte OUTSIDE the +// reversed field passes through untouched, and thinking/redacted_thinking blocks are +// never decoded at all. The reversed value ITSELF is decoded and re-encoded, so within +// that one value \uXXXX/\/ are re-emitted in canonical form (logical content preserved, +// not byte-identical to upstream) — which is fine because it is not a thinking block. + +// Locate a JSON string field's value. Returns {start,end}: CHAR offsets into the JS +// string (used directly with String.prototype.slice — these are UTF-16 char indices, +// NOT byte offsets; do not add byte<->char conversion). `start` is just inside the +// opening quote, `end` is at the closing quote. Escape-aware (skips \" etc.). Null if absent. +function findSseStringField(s, field) { + const key = '"' + field + '":"'; + const k = s.indexOf(key); + if (k === -1) return null; + const start = k + key.length; + let i = start; + while (i < s.length) { + const c = s[i]; + if (c === '\\') { i += 2; continue; } + if (c === '"') return { start, end: i }; + i++; + } + return null; +} + +// Read an integer-valued field (e.g. "index":3) without parsing the object. +function extractSseIntField(s, field) { + const key = '"' + field + '":'; + const k = s.indexOf(key); + if (k === -1) return null; + let i = k + key.length; + while (i < s.length && s[i] === ' ') i++; + const start = i; + if (s[i] === '-') i++; + const digitsAt = i; + while (i < s.length && s[i] >= '0' && s[i] <= '9') i++; + if (i === digitsAt) return null; + return parseInt(s.slice(start, i), 10); +} + +// Decode a JSON string body (the bytes between the quotes) to real characters. +// Each SSE event carries a COMPLETE, valid JSON string, so escape sequences are +// never split within one event (only the logical value is fragmented across +// events), and this never has to handle a truncated escape. +function jsonStringDecode(s) { + if (s.indexOf('\\') === -1) return s; + let out = ''; + for (let i = 0; i < s.length; i++) { + const c = s[i]; + if (c !== '\\') { out += c; continue; } + const n = s[i + 1]; + switch (n) { + case '"': out += '"'; i++; break; + case '\\': out += '\\'; i++; break; + case '/': out += '/'; i++; break; + case 'b': out += '\b'; i++; break; + case 'f': out += '\f'; i++; break; + case 'n': out += '\n'; i++; break; + case 'r': out += '\r'; i++; break; + case 't': out += '\t'; i++; break; + // \uXXXX: 4 hex digits live at i+2..i+6; advance i by 5 (+1 from the loop = past all 6 chars) + case 'u': out += String.fromCharCode(parseInt(s.slice(i + 2, i + 6), 16)); i += 5; break; + default: out += c; // malformed; keep the backslash literally + } + } + return out; +} + +// Re-encode characters into a JSON string body. Matches JSON.stringify escaping +// (escape ", \, control chars; leave / and non-ASCII literal) and additionally +// escapes LONE surrogates as \uXXXX, so a value cut between a surrogate pair still +// survives UTF-8 transport and reassembles correctly when the client concatenates. +function jsonStringEncode(s) { + if (!/[\\"\u0000-\u001F\uD800-\uDFFF]/.test(s)) return s; + let out = ''; + for (let i = 0; i < s.length; i++) { + const c = s[i]; + const code = s.charCodeAt(i); + if (c === '"') { out += '\\"'; continue; } + if (c === '\\') { out += '\\\\'; continue; } + if (code < 0x20) { + switch (c) { + case '\b': out += '\\b'; break; + case '\f': out += '\\f'; break; + case '\n': out += '\\n'; break; + case '\r': out += '\\r'; break; + case '\t': out += '\\t'; break; + default: out += '\\u' + code.toString(16).padStart(4, '0'); + } + continue; + } + if (code >= 0xD800 && code <= 0xDBFF) { // high surrogate + const next = s.charCodeAt(i + 1); + if (next >= 0xDC00 && next <= 0xDFFF) { out += c + s[i + 1]; i++; } // valid pair -> literal + else out += '\\u' + code.toString(16).padStart(4, '0'); // lone high -> escape + continue; + } + if (code >= 0xDC00 && code <= 0xDFFF) { // lone low surrogate -> escape + out += '\\u' + code.toString(16).padStart(4, '0'); + continue; + } + out += c; + } + return out; +} + +function createSseEventTransformer(config) { + let currentBlockIsThinking = false; + const streamState = new Map(); // key "index:field" -> raw, un-emitted buffer + + // Every literal string reverseMap() searches for. reverseMap() rewrites tool + // and property names in BOTH plain ("Name") and JSON-escaped (\"Name\") form, + // so both variants are patterns whose prefix could appear at a buffer tail. + const patterns = []; + for (const [, cc] of config.toolRenames) { + patterns.push('"' + cc + '"', '\\"' + cc + '\\"'); + } + for (const [, renamed] of config.propRenames) { + patterns.push('"' + renamed + '"', '\\"' + renamed + '\\"'); + } + for (const [sanitized] of config.reverseMap) { + patterns.push(sanitized); + } + const maxPatternLen = patterns.reduce((m, p) => Math.max(m, p.length), 1); + + // Streaming reverse-map. reverseMap() only rewrites COMPLETE patterns, so a + // pattern split across SSE delta events (".ocpla" then "tform") would be + // emitted raw before it completes and could never be retracted. Hold back + // the trailing raw bytes that might still grow into a pattern, emit only the + // safe prefix, and carry the rest to the next delta (or the stop flush). + function streamReverse(key, value, isFinal) { + const buf = (streamState.get(key) || '') + value; + let cut; + if (isFinal) { + cut = buf.length; + } else { + // A pattern is at most maxPatternLen long, so anything before this point + // cannot be the start of a still-incomplete pattern. + cut = Math.max(0, buf.length - (maxPatternLen - 1)); + // ...but a *complete* occurrence may straddle that point. Pull the cut + // back to its start so the whole occurrence stays in the carry buffer. + let moved = true; + while (moved) { + moved = false; + for (const p of patterns) { + let idx = buf.indexOf(p); + while (idx !== -1 && idx < cut) { + if (idx + p.length > cut) { cut = idx; moved = true; } + idx = buf.indexOf(p, idx + 1); + } + } + } + } + const carry = buf.slice(cut); + if (carry) streamState.set(key, carry); else streamState.delete(key); + return reverseMap(buf.slice(0, cut), config); + } + + function buildDeltaEvent(index, field, value) { + const inner = field === 'partial_json' + ? '{"type":"input_json_delta","partial_json":"' + jsonStringEncode(value) + '"}' + : '{"type":"text_delta","text":"' + jsonStringEncode(value) + '"}'; + return 'event: content_block_delta\ndata: {"type":"content_block_delta","index":' + + index + ',"delta":' + inner + '}\n\n'; + } + + // Emit whatever is still held for a block as synthetic delta event(s). + // Called on content_block_stop (and flushAll) — the block has no more deltas, + // so the held tail must come out now or it is lost. + function flushBlock(index) { + let out = ''; + for (const field of ['text', 'partial_json']) { + const key = index + ':' + field; + if (!streamState.has(key)) continue; + const rest = streamReverse(key, '', true); + if (rest) out += buildDeltaEvent(index, field, rest); + } + return out; + } + + const transform = (event) => { + let dataIdx = event.startsWith('data: ') ? 0 : event.indexOf('\ndata: '); + if (dataIdx === -1) return reverseMap(event, config); + if (dataIdx > 0) dataIdx += 1; + const dataLineEnd = event.indexOf('\n', dataIdx + 6); + const dataStr = dataLineEnd === -1 + ? event.slice(dataIdx + 6) + : event.slice(dataIdx + 6, dataLineEnd); + + // Raw-string event classification (NO JSON.parse): these markers only ever + // appear unescaped at the envelope level; an escaped occurrence inside a + // string value carries \" and cannot match. + if (dataStr.indexOf('"type":"content_block_start"') !== -1) { + if (dataStr.indexOf('"content_block":{"type":"thinking"') !== -1 || + dataStr.indexOf('"content_block":{"type":"redacted_thinking"') !== -1) { + currentBlockIsThinking = true; + return event; + } + currentBlockIsThinking = false; + return reverseMap(event, config); + } + if (dataStr.indexOf('"type":"content_block_stop"') !== -1) { + const wasThinking = currentBlockIsThinking; + currentBlockIsThinking = false; + if (wasThinking) return event; + const index = extractSseIntField(dataStr, 'index'); + const flushed = index !== null ? flushBlock(index) : ''; + return flushed + reverseMap(event, config); + } + if (currentBlockIsThinking) return event; + + if (dataStr.indexOf('"type":"content_block_delta"') !== -1) { + let field = null; + if (dataStr.indexOf('"type":"input_json_delta"') !== -1) field = 'partial_json'; + else if (dataStr.indexOf('"type":"text_delta"') !== -1) field = 'text'; + if (field === null) return reverseMap(event, config); + + const index = extractSseIntField(dataStr, 'index'); + const loc = findSseStringField(dataStr, field); + if (index === null || loc === null) return reverseMap(event, config); + + // Decode just this field's value, stream-reverse it, re-encode it in place; + // every other byte of the event passes through untouched. + const decoded = jsonStringDecode(dataStr.slice(loc.start, loc.end)); + const reversed = jsonStringEncode(streamReverse(index + ':' + field, decoded, false)); + const newDataStr = dataStr.slice(0, loc.start) + reversed + dataStr.slice(loc.end); + return event.slice(0, dataIdx + 6) + newDataStr + (dataLineEnd === -1 ? '' : event.slice(dataLineEnd)); + } + + return reverseMap(event, config); + }; + + // Flush any buffer left when the stream ends without a content_block_stop + // (malformed stream). Well-formed streams flush per block on stop. + transform.flushAll = () => { + let out = ''; + const indices = new Set(); + for (const key of streamState.keys()) indices.add(Number(key.slice(0, key.indexOf(':')))); + for (const index of indices) out += flushBlock(index); + return out; + }; + + return transform; +} + // ─── Server ───────────────────────────────────────────────────────────────── function startServer(config) { let requestCount = 0; @@ -861,38 +1114,7 @@ function startServer(config) { // don't decode as U+FFFD. const decoder = new StringDecoder('utf8'); let pending = ''; - let currentBlockIsThinking = false; - - const transformEvent = (event) => { - // Locate the data: line (always at the start of an SSE line) - let dataIdx = event.startsWith('data: ') ? 0 : event.indexOf('\ndata: '); - if (dataIdx === -1) return reverseMap(event, config); - if (dataIdx > 0) dataIdx += 1; // skip the leading \n - const dataLineEnd = event.indexOf('\n', dataIdx + 6); - const dataStr = dataLineEnd === -1 - ? event.slice(dataIdx + 6) - : event.slice(dataIdx + 6, dataLineEnd); - - if (dataStr.indexOf('"type":"content_block_start"') !== -1) { - if (dataStr.indexOf('"content_block":{"type":"thinking"') !== -1 || - dataStr.indexOf('"content_block":{"type":"redacted_thinking"') !== -1) { - currentBlockIsThinking = true; - return event; // pass through unchanged - } - currentBlockIsThinking = false; - return reverseMap(event, config); - } - if (dataStr.indexOf('"type":"content_block_stop"') !== -1) { - const wasThinking = currentBlockIsThinking; - currentBlockIsThinking = false; - return wasThinking ? event : reverseMap(event, config); - } - if (currentBlockIsThinking) { - // thinking_delta / signature_delta / etc. inside a thinking block - return event; - } - return reverseMap(event, config); - }; + const transformEvent = createSseEventTransformer(config); upRes.on('data', (chunk) => { pending += decoder.write(chunk); @@ -910,6 +1132,8 @@ function startServer(config) { // well-formed SSE, but flush to avoid silent drops. res.write(transformEvent(pending)); } + // Flush any buffer held for a block that never got a stop event. + res.write(transformEvent.flushAll()); res.end(); }); } else { @@ -974,6 +1198,34 @@ function startServer(config) { process.on('SIGTERM', () => process.exit(0)); } +function applySseReverseMapChunks(chunks, config) { + const decoder = new StringDecoder('utf8'); + let pending = ''; + let out = ''; + const transformEvent = createSseEventTransformer(config); + + for (const chunk of chunks) { + pending += decoder.write(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, 'utf8')); + let sepIdx; + while ((sepIdx = pending.indexOf('\n\n')) !== -1) { + const event = pending.slice(0, sepIdx + 2); + pending = pending.slice(sepIdx + 2); + out += transformEvent(event); + } + } + pending += decoder.end(); + if (pending.length > 0) out += transformEvent(pending); + out += transformEvent.flushAll(); + return out; +} + +module.exports = { + loadConfig, reverseMap, applySseReverseMapChunks, + jsonStringDecode, jsonStringEncode, findSseStringField, extractSseIntField, +}; + // ─── Main ─────────────────────────────────────────────────────────────────── -const config = loadConfig(); -startServer(config); +if (require.main === module) { + const config = loadConfig(); + startServer(config); +} diff --git a/test/sse-reversemap.boundary.test.js b/test/sse-reversemap.boundary.test.js new file mode 100644 index 0000000..af8d679 --- /dev/null +++ b/test/sse-reversemap.boundary.test.js @@ -0,0 +1,304 @@ +// Boundary tests for the raw-string SSE reverse path (companion to +// sse-reversemap.test.js). Pure local logic — no network, no proxy, no API. +// +// Property under test: for ANY way a logical value is fragmented across SSE +// *_delta events, the client-reassembled result equals reverseMap() applied to +// the whole value. Plus: thinking byte-equality, multi-block isolation, flush, +// JSON validity, and the raw-string codec helpers. +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { + reverseMap, applySseReverseMapChunks, + jsonStringDecode, jsonStringEncode, findSseStringField, extractSseIntField, +} = require('../proxy.js'); + +// Deterministic config — independent of the host's config.json / credentials. +const CONFIG = { + reverseMap: [ + ['ocplatform', 'openclaw'], + ['OCPlatform', 'OpenClaw'], + ['skillhub.example.com', 'clawhub.com'], + ['skillhub', 'clawhub'], + ['hb_signal', 'heartbeat'], + ['create_task', 'sessions_spawn'], + ], + toolRenames: [['exec', 'Bash'], ['message', 'SendMessage'], ['create_task', 'TaskCreate']], + propRenames: [['session_id', 'thread_id'], ['wake_at', 'trigger_at']], +}; + +// ── builders ──────────────────────────────────────────────────────────────── +function startEvent(index, kind) { + const cb = kind === 'thinking' ? { type: 'thinking', thinking: '' } + : kind === 'redacted_thinking' ? { type: 'redacted_thinking', data: 'abc' } + : kind === 'tool_use' ? { type: 'tool_use', id: 'tu', name: 'Bash', input: {} } + : { type: 'text', text: '' }; + return 'event: content_block_start\ndata: ' + + JSON.stringify({ type: 'content_block_start', index, content_block: cb }) + '\n\n'; +} +function deltaEvent(index, field, value) { + const delta = field === 'partial_json' ? { type: 'input_json_delta', partial_json: value } + : field === 'thinking' ? { type: 'thinking_delta', thinking: value } + : { type: 'text_delta', text: value }; + return 'event: content_block_delta\ndata: ' + + JSON.stringify({ type: 'content_block_delta', index, delta }) + '\n\n'; +} +function stopEvent(index) { + return 'event: content_block_stop\ndata: ' + + JSON.stringify({ type: 'content_block_stop', index }) + '\n\n'; +} + +// ── reassembly + validity ───────────────────────────────────────────────────── +function dataLines(out) { + return out.split('\n\n').map((e) => e.trim()).filter(Boolean) + .map((e) => e.split('\n').find((l) => l.startsWith('data: '))) + .filter(Boolean).map((l) => l.slice(6)); +} +function assertAllValidJson(out) { + for (const d of dataLines(out)) JSON.parse(d); // throws on any malformed emitted event +} +function reassemble(out) { + const acc = {}; + for (const d of dataLines(out)) { + const p = JSON.parse(d); + if (p.type !== 'content_block_delta' || !p.delta || typeof p.index !== 'number') continue; + const f = p.delta.type === 'input_json_delta' ? 'pj' : p.delta.type === 'text_delta' ? 'tx' + : p.delta.type === 'thinking_delta' ? 'th' : null; + if (!f) continue; + const v = f === 'pj' ? p.delta.partial_json : f === 'tx' ? p.delta.text : p.delta.thinking; + acc[p.index + ':' + f] = (acc[p.index + ':' + f] || '') + v; + } + return acc; +} +function streamOf(field, parts, byteChunkSize) { + const kind = field === 'partial_json' ? 'tool_use' : 'text'; + let sse = startEvent(0, kind); + for (const p of parts) sse += deltaEvent(0, field, p); + sse += stopEvent(0); + let chunks = [sse]; + if (byteChunkSize) { + const buf = Buffer.from(sse, 'utf8'); chunks = []; + for (let i = 0; i < buf.length; i += byteChunkSize) chunks.push(buf.slice(i, i + byteChunkSize)); + } + const out = applySseReverseMapChunks(chunks, CONFIG); + assertAllValidJson(out); + return reassemble(out)['0:' + (field === 'partial_json' ? 'pj' : 'tx')] || ''; +} + +// Assert reconstruct === reverseMap(whole) for EVERY 2-way split, all 3-way +// splits, char-by-char, single event, and byte-level TCP fragmentation. +function everySplit(field, W) { + const expected = reverseMap(W, CONFIG); + for (let i = 1; i < W.length; i++) { + assert.equal(streamOf(field, [W.slice(0, i), W.slice(i)]), expected, `2-way @${i}`); + } + for (let a = 1; a < W.length - 1; a++) for (let b = a + 1; b < W.length; b++) { + assert.equal(streamOf(field, [W.slice(0, a), W.slice(a, b), W.slice(b)]), expected, `3-way @${a},${b}`); + } + assert.equal(streamOf(field, [W]), expected, 'single event'); + assert.equal(streamOf(field, W.split('')), expected, 'char-by-char'); + assert.equal(streamOf(field, [W.slice(0, (W.length / 2) | 0), W.slice((W.length / 2) | 0)], 3), expected, 'byte chunks=3'); +} + +// ── 1. text_delta boundary splits ──────────────────────────────────────────── +test('text: brand/dict targets at every split', () => { + everySplit('text', 'open ~/.ocplatform via skillhub.example.com hb_signal ok'); +}); +test('text: capitalized OCPlatform + multiple targets', () => { + everySplit('text', 'OCPlatform talks to ocplatform through skillhub'); +}); + +// ── 2. input_json_delta, REAL-quote tool input (the common case) ───────────── +test('json (real quotes): C1 property key thread_id -> session_id', () => { + everySplit('partial_json', '{"thread_id":"abc-123","wake_at":17}'); +}); +test('json (real quotes): renamed tool name as a value', () => { + everySplit('partial_json', '{"target_tool":"TaskCreate"}'); +}); +test('json (real quotes): ocplatform path in a command arg', () => { + everySplit('partial_json', '{"command":"ls ~/.ocplatform/cfg"}'); +}); + +// ── 3. input_json_delta, ESCAPED-quote (nested-JSON) form, like #56 ────────── +test('json (escaped quotes): property + tool name', () => { + everySplit('partial_json', '{\\"session_id\\":\\"x\\",\\"message\\":\\"hi\\"}'); +}); + +// ── 4. multi-byte / surrogate boundaries ───────────────────────────────────── +test('text: emoji surrogate pairs adjacent to a target, every split', () => { + everySplit('text', 'hi \u{1F600} run ocplatform \u{1F680} skillhub done'); +}); +test('text: CJK around a target', () => { + everySplit('text', '請執行 ocplatform 然後連到 skillhub.example.com'); +}); + +// ── 5. escaped characters inside the value ─────────────────────────────────── +test('json: escaped quote inside a string value', () => { + everySplit('partial_json', '{"q":"say \\"ocplatform\\" now"}'); +}); +test('text: newline and tab around a target', () => { + everySplit('text', 'line1\nline2\tocplatform\tend'); +}); + +// ── 6. thinking byte-equality (vector B preserved) ─────────────────────────── +test('thinking + redacted_thinking pass through byte-identical (targets NOT reversed)', () => { + const events = [ + startEvent(0, 'thinking'), + deltaEvent(0, 'thinking', 'plan: run ocplatform via skillhub'), + stopEvent(0), + startEvent(1, 'redacted_thinking'), + deltaEvent(1, 'thinking', 'ocplatform secret'), + stopEvent(1), + ]; + const out = applySseReverseMapChunks(events, CONFIG); + assert.equal(out, events.join('')); +}); +test('a text block after a thinking block still reverses (no desync)', () => { + const events = [ + startEvent(0, 'thinking'), deltaEvent(0, 'thinking', 'ocplatform'), stopEvent(0), + startEvent(1, 'text'), deltaEvent(1, 'text', 'now run oc'), deltaEvent(1, 'text', 'platform'), stopEvent(1), + ]; + const out = applySseReverseMapChunks(events, CONFIG); + assert.equal(reassemble(out)['1:tx'], 'now run openclaw'); +}); + +// ── 7. multiple content blocks interleaved (per-index isolation) ───────────── +test('two tool_use blocks streaming concurrently stay isolated', () => { + const events = [ + startEvent(0, 'tool_use'), startEvent(1, 'tool_use'), + deltaEvent(0, 'partial_json', '{"a":"oc'), + deltaEvent(1, 'partial_json', '{"b":"skill'), + deltaEvent(0, 'partial_json', 'platform"}'), + deltaEvent(1, 'partial_json', 'hub"}'), + stopEvent(0), stopEvent(1), + ]; + const out = applySseReverseMapChunks(events, CONFIG); + assertAllValidJson(out); + const r = reassemble(out); + assert.equal(r['0:pj'], '{"a":"openclaw"}'); + assert.equal(r['1:pj'], '{"b":"clawhub"}'); +}); + +// ── 8. flush without content_block_stop ────────────────────────────────────── +test('flushAll emits held tail when stream ends with no stop', () => { + const events = [startEvent(0, 'text'), deltaEvent(0, 'text', 'go to oc'), deltaEvent(0, 'text', 'platform')]; + const out = applySseReverseMapChunks(events, CONFIG); // no stop, no manual flush — applySseReverseMapChunks calls flushAll + assert.equal(reassemble(out)['0:tx'], 'go to openclaw'); +}); + +// ── 9. fake "index"/"partial_json" inside a value must not fool extraction ─── +test('envelope index/field anchoring is escape-aware', () => { + everySplit('text', 'note: "index": 9 and "partial_json" appear; run ocplatform'); +}); + +// ── 10. raw-string codec helpers ───────────────────────────────────────────── +test('jsonStringDecode/Encode round-trip + valid JSON body', () => { + const cases = [ + 'plain', 'q"uote and \\back', 'tab\tnl\ncr\r', 'café 日本語 résumé', + 'emoji \u{1F600}\u{1F680}', 'slash a/b', 'ctrlend', + ]; + for (const s of cases) { + const enc = jsonStringEncode(s); + JSON.parse('"' + enc + '"'); // enc must be a valid JSON string body + assert.equal(jsonStringDecode(enc), s, `round-trip ${JSON.stringify(s)}`); + } +}); +test('jsonStringEncode escapes lone surrogate (transport-safe, reassembles)', () => { + const encHi = jsonStringEncode('\uD83D'); + assert.equal(encHi, '\\ud83d'); + JSON.parse('"' + encHi + '"'); + const encLo = jsonStringEncode('\uDE00'); + assert.equal(encLo, '\\ude00'); + // a valid pair stays literal (not over-escaped) + assert.equal(jsonStringEncode('\u{1F600}'), '\u{1F600}'); +}); +test('jsonStringDecode handles \\uXXXX and escapes', () => { + assert.equal(jsonStringDecode('\\u0041\\n\\t\\"\\\\'), 'A\n\t"\\'); +}); +test('findSseStringField / extractSseIntField anchor on the envelope', () => { + const ds = '{"type":"content_block_delta","index":7,"delta":{"type":"input_json_delta","partial_json":"{\\"k\\":\\"v\\"}"}}'; + assert.equal(extractSseIntField(ds, 'index'), 7); + const loc = findSseStringField(ds, 'partial_json'); + assert.equal(ds.slice(loc.start, loc.end), '{\\"k\\":\\"v\\"}'); + assert.equal(extractSseIntField(ds, 'nope'), null); + assert.equal(findSseStringField(ds, 'nope'), null); +}); + +// ── 11. signature_delta inside a thinking block must stay byte-identical ────── +// (the signature is cryptographically validated on the next turn — any mutation +// breaks the conversation). +test('signature_delta inside a thinking block passes through byte-identical', () => { + const sig = 'event: content_block_delta\ndata: ' + + JSON.stringify({ type: 'content_block_delta', index: 0, delta: { type: 'signature_delta', signature: 'ErUBC+ocplatform/sig+base64==' } }) + '\n\n'; + const events = [startEvent(0, 'thinking'), deltaEvent(0, 'thinking', 'reasoning about ocplatform'), sig, stopEvent(0)]; + const out = applySseReverseMapChunks(events, CONFIG); + assert.equal(out, events.join('')); +}); + +// ── 12. non-content_block events (ping / message_delta) pass through untouched +// and do not desync the carry buffer ───────────────────────────────────── +test('ping / message_delta interleaved pass through and do not desync the stream', () => { + const ping = 'event: ping\ndata: ' + JSON.stringify({ type: 'ping' }) + '\n\n'; + const msgDelta = 'event: message_delta\ndata: ' + + JSON.stringify({ type: 'message_delta', delta: { stop_reason: 'end_turn' }, usage: { output_tokens: 5 } }) + '\n\n'; + const events = [startEvent(0, 'text'), deltaEvent(0, 'text', 'go to oc'), ping, deltaEvent(0, 'text', 'platform'), stopEvent(0), msgDelta]; + const out = applySseReverseMapChunks(events, CONFIG); + assertAllValidJson(out); + assert.equal(reassemble(out)['0:tx'], 'go to openclaw', 'token reverses across the ping-interrupted deltas'); + assert.ok(out.includes(ping), 'ping passed through unchanged'); + assert.ok(out.includes(msgDelta), 'message_delta passed through unchanged'); +}); + +// ── 13. cut landing INSIDE a surrogate pair (each delta carries a lone half) ── +test('split inside a surrogate pair reassembles the astral char intact', () => { + const W = 'x\u{1F600}y ocplatform'; // 😀 = 😀 + const hi = W.indexOf('\uD83D'); + const got = streamOf('text', [W.slice(0, hi + 1), W.slice(hi + 1)]); // cut between high and low + assert.equal(got, reverseMap(W, CONFIG)); + assert.ok(got.includes('\u{1F600}'), 'astral code point intact after reassembly'); +}); + +// ── 14. extractSseIntField numeric edge cases ──────────────────────────────── +test('extractSseIntField: negative, post-colon spaces, non-numeric, no suffix/prefix collision', () => { + assert.equal(extractSseIntField('{"index":-5}', 'index'), -5); + assert.equal(extractSseIntField('{"index": 42}', 'index'), 42); + assert.equal(extractSseIntField('{"index":abc}', 'index'), null); + assert.equal(extractSseIntField('{"index_hint":1,"index":3}', 'index'), 3); + assert.equal(extractSseIntField('{"stop_index":9,"index":7}', 'index'), 7); +}); + +// ── 15. combined: surrogate-pair split + escaped fake envelope in one value ─── +test('combined surrogate + embedded escaped fake-envelope in one text value', () => { + everySplit('text', 'pre \u{1F680} say "index": 9 then ocplatform end'); +}); + +// ── DOCUMENTED LIMITATIONS ─────────────────────────────────────────────────── +// Two properties are inherited from PR #56's streamReverse + the single +// currentBlockIsThinking flag. Both are SAFE under the real default config and the +// Anthropic streaming contract, and are intentionally left unchanged here per the +// maintainer's "keep streamReverse verbatim" direction. We lock the SUPPORTED shape: +// +// (a) Chaining configs — if one entry's OUTPUT can combine with later streamed bytes +// to re-form ANOTHER entry's search key, the streamed result can diverge from +// reverseMap(whole). Default config has no such window-splitting chain. +test('non-chaining multi-entry config round-trips at every split (supported shape)', () => { + const cfg = { reverseMap: [['aaa', 'X'], ['bbb', 'Y']], toolRenames: [], propRenames: [] }; + const W = 'aaa mid bbb end'; + const expected = reverseMap(W, cfg); + for (let i = 1; i < W.length; i++) { + const sse = startEvent(0, 'text') + deltaEvent(0, 'text', W.slice(0, i)) + deltaEvent(0, 'text', W.slice(i)) + stopEvent(0); + assert.equal(reassemble(applySseReverseMapChunks([sse], cfg))['0:tx'], expected, `@${i}`); + } +}); +// +// (b) Thinking isolation — currentBlockIsThinking is a single flag, not per-index. +// The Anthropic stream emits content blocks strictly sequentially (never +// interleaved), so this is safe. We lock the contract-shaped behavior. +test('sequential thinking then text (contract order) reverses correctly', () => { + const events = [ + startEvent(0, 'thinking'), deltaEvent(0, 'thinking', 'ocplatform'), stopEvent(0), + startEvent(1, 'text'), deltaEvent(1, 'text', 'run ocplatform'), stopEvent(1), + ]; + const out = applySseReverseMapChunks(events, CONFIG); + assert.equal(reassemble(out)['0:th'], 'ocplatform', 'thinking token NOT reversed'); + assert.equal(reassemble(out)['1:tx'], 'run openclaw', 'following text block reversed'); +}); diff --git a/test/sse-reversemap.test.js b/test/sse-reversemap.test.js new file mode 100644 index 0000000..ecdaf31 --- /dev/null +++ b/test/sse-reversemap.test.js @@ -0,0 +1,95 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { reverseMap, applySseReverseMapChunks } = require('../proxy.js'); + +// Deterministic config — independent of the host's config.json. +const CONFIG = { + reverseMap: [ + ['ocplatform', 'openclaw'], + ['routing-layer', 'billing-proxy'], + ], + toolRenames: [['message', 'SendMessage']], + propRenames: [['path', 'file_path']], +}; + +function textEvent(index, text) { + return 'event: content_block_delta\ndata: ' + + JSON.stringify({ type: 'content_block_delta', index, delta: { type: 'text_delta', text } }) + '\n\n'; +} +function jsonEvent(index, partial) { + return 'event: content_block_delta\ndata: ' + + JSON.stringify({ type: 'content_block_delta', index, delta: { type: 'input_json_delta', partial_json: partial } }) + '\n\n'; +} +function stopEvent(index) { + return 'data: ' + JSON.stringify({ type: 'content_block_stop', index }) + '\n\n'; +} + +// Reassemble the text_delta / input_json_delta payloads from transformer output. +function reconstruct(out, field) { + return out.split('\n\n').map((e) => e.trim()).filter(Boolean) + .map((e) => e.split('\n').find((l) => l.startsWith('data: '))) + .filter(Boolean) + .map((l) => JSON.parse(l.slice(6))) + .filter((p) => p.type === 'content_block_delta' && p.delta && + (p.delta.type === 'text_delta' || p.delta.type === 'input_json_delta')) + .map((p) => (field in p.delta ? p.delta[field] : '')) + .join(''); +} + +test('text_delta: exact reconstruction at every two-way split offset', () => { + const full = 'open ~/.ocplatform inside the routing-layer please'; + const expected = reverseMap(full, CONFIG); + for (let i = 1; i < full.length; i++) { + const events = [textEvent(0, full.slice(0, i)), textEvent(0, full.slice(i)), stopEvent(0)]; + const got = reconstruct(applySseReverseMapChunks(events, CONFIG), 'text'); + assert.equal(got, expected, `split at offset ${i}`); + } +}); + +test('text_delta: exact reconstruction when split into single characters', () => { + const full = 'cd ocplatform/routing-layer/run'; + const expected = reverseMap(full, CONFIG); + const events = full.split('').map((ch) => textEvent(0, ch)); + events.push(stopEvent(0)); + const got = reconstruct(applySseReverseMapChunks(events, CONFIG), 'text'); + assert.equal(got, expected); +}); + +test('input_json_delta: exact reconstruction at every two-way split offset', () => { + // partial_json carries tool args; inner quotes arrive JSON-escaped. + const full = '{\\"path\\":\\"~/.ocplatform/ws\\",\\"SendMessage\\":\\"hi\\"}'; + const expected = reverseMap(full, CONFIG); + for (let i = 1; i < full.length; i++) { + const events = [jsonEvent(1, full.slice(0, i)), jsonEvent(1, full.slice(i)), stopEvent(1)]; + const got = reconstruct(applySseReverseMapChunks(events, CONFIG), 'partial_json'); + assert.equal(got, expected, `split at offset ${i}`); + } +}); + +test('split token across raw TCP chunks within one SSE event', () => { + const event = textEvent(0, 'cd ~/.ocplatform/workspace') + stopEvent(0); + const splitPoint = event.indexOf('ocplatform') + 'ocpla'.length; + const chunks = [event.slice(0, splitPoint), event.slice(splitPoint)]; + const got = reconstruct(applySseReverseMapChunks(chunks, CONFIG), 'text'); + assert.equal(got, 'cd ~/.openclaw/workspace'); +}); + +test('flushAll emits the held tail when no content_block_stop arrives', () => { + // Token split across events, stream ends abruptly without a stop event. + const events = [textEvent(0, 'go to ~/.ocpla'), textEvent(0, 'tform/ws')]; + const got = reconstruct(applySseReverseMapChunks(events, CONFIG), 'text'); + assert.equal(got, 'go to ~/.openclaw/ws'); +}); + +test('thinking and redacted_thinking blocks pass through byte-identical', () => { + const events = [ + 'data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking"}}\n\n', + 'data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"raw ocplatform routing-layer"}}\n\n', + 'data: {"type":"content_block_stop","index":0}\n\n', + 'data: {"type":"content_block_start","index":1,"content_block":{"type":"redacted_thinking"}}\n\n', + 'data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"ocplatform"}}\n\n', + 'data: {"type":"content_block_stop","index":1}\n\n', + ]; + const out = applySseReverseMapChunks(events, CONFIG); + assert.equal(out, events.join('')); +});