Merge pull request #209 from ranxianglei/2026-09-07_range-array-adjacency

fix: segment compressible ranges by array adjacency, not ref arithmetic (#207)

Author: ranxianglei

src/recommend.ts

@@ -28,11 +28,6 @@ import {
 
 // ─── Helpers ──────────────────────────────────────────────────────────────────
 
-function refNum(ref: string): number {
-  const n = parseInt(ref.slice(1), 10);
-  return Number.isNaN(n) ? -1 : n;
-}
-
 /** Default token estimate (chars/4) used when the caller doesn't inject a
  *  countTokens — preserves the historical behavior for backwards compat. */
 function estimateTextTokens(text: string): number {
@@ -153,15 +148,15 @@ export function buildCompressibleRanges(
 ): ContextRanges {
   const compressibleMsgs: {
     ref: string;
-    refNum: number;
+    gapBefore: boolean;
     tokens: number;
     chars: number;
     isTool: boolean;
     isUser: boolean;
   }[] = [];
   const protectedMsgs: {
     ref: string;
-    refNum: number;
+    gapBefore: boolean;
     tokens: number;
     tools: string[];
   }[] = [];
@@ -170,54 +165,68 @@ export function buildCompressibleRanges(
   // callIds of protected tool-calls first, then protect matching results too.
   const protectedCallIds = collectProtectedToolCallIds(messages, config);
 
+  // Segmentation is array adjacency, never ref arithmetic: surface-replacing
+  // hosts leave holes in the ref map (compressed messages leave the array, refs
+  // stay assigned) and insert mid-array summary nodes with fresh HIGH refs —
+  // ref arithmetic fragments every range there and emits startRef > endRef
+  // pairs. Only a numbered-ref message physically skipped between two entries
+  // interrupts; unrefed/BLOCKED consume no slot. On dense append-only hosts the
+  // two rules coincide, so ranges are byte-identical to the old behavior.
+  let skipSinceCompressible = false;
+  let skipSinceProtected = false;
+
   for (const msg of messages) {
-    if (isSyntheticOrPruned(msg, state)) continue;
     const ref = state.messageRefs.byRaw[msg.id];
     if (!ref || ref === "BLOCKED") continue;
-
-    const rn = refNum(ref);
+    if (isSyntheticOrPruned(msg, state)) {
+      skipSinceCompressible = true;
+      skipSinceProtected = true;
+      continue;
+    }
 
     if (isMessageProtectedWithPairing(msg, config, protectedCallIds)) {
       protectedMsgs.push({
         ref,
-        refNum: rn,
+        gapBefore: skipSinceProtected,
         tokens: countTokens(msg.text ?? ""),
         tools: msg.toolName ? [msg.toolName] : [],
       });
+      skipSinceProtected = false;
+      skipSinceCompressible = true;
       continue;
     }
 
     if (protectedZoneRefs?.has(ref)) {
+      skipSinceCompressible = true;
+      skipSinceProtected = true;
       continue;
     }
 
     compressibleMsgs.push({
       ref,
-      refNum: rn,
+      gapBefore: skipSinceCompressible,
       tokens: countTokens(msg.text ?? ""),
       chars: (msg.text ?? "").length,
       isTool: isToolMessage(msg),
       isUser: msg.role === "user",
     });
+    skipSinceCompressible = false;
+    skipSinceProtected = true;
   }
 
-  // Build compressible groups (contiguous, split at ref gaps and at user
-  // messages once a group has >= 3 messages). Splitting at user boundaries
-  // keeps each compressible range aligned to roughly one user turn, instead
-  // of producing one giant range spanning many turns (or, conversely, a
-  // fragment per message when ref gaps appear). Mirrors opencode-acp's
+  // Build compressible groups (split at real array gaps and at user messages
+  // once a group has >= 3 messages). Splitting at user boundaries keeps each
+  // compressible range aligned to roughly one user turn, instead of producing
+  // one giant range spanning many turns. Mirrors opencode-acp's
   // buildCompressibleRanges condition.
   const compressible: CompressibleRange[] = [];
   let cur: CompressibleRange | null = null;
-  let prevRefNum = -2;
 
   for (const info of compressibleMsgs) {
-    const hasGap = info.refNum > prevRefNum + 1;
-    if (cur && ((info.isUser && cur.count >= 3) || hasGap)) {
+    if (cur && ((info.isUser && cur.count >= 3) || info.gapBefore)) {
       compressible.push(cur);
       cur = null;
     }
-    prevRefNum = info.refNum;
     if (!cur) {
       cur = {
         startRef: info.ref,
@@ -246,15 +255,12 @@ export function buildCompressibleRanges(
   // Build protected groups (contiguous)
   const protectedRanges: ProtectedRange[] = [];
   let pcur: ProtectedRange | null = null;
-  let pPrevRefNum = -2;
 
   for (const info of protectedMsgs) {
-    const hasGap = info.refNum > pPrevRefNum + 1;
-    if (pcur && hasGap) {
+    if (pcur && info.gapBefore) {
       protectedRanges.push(pcur);
       pcur = null;
     }
-    pPrevRefNum = info.refNum;
     if (!pcur) {
       pcur = {
         startRef: info.ref,

tests/recommend.test.ts

@@ -245,6 +245,91 @@ test("buildCompressibleRanges: does NOT split before a group reaches 3 messages"
   assert.equal(ranges.compressible[0]!.count, 3);
 });
 
+// ─── Surface-replacing hosts (#207): array-adjacency segmentation ─────────────
+
+test("buildCompressibleRanges: ref-map holes do NOT fragment ranges (surface-replace host)", () => {
+  // Turn 1 assigns m00001..m00005. A compression then durably replaces b..d on
+  // the surface: they leave the next turn's message array while their refs stay
+  // assigned (append-only map, never pruned). Ref-arithmetic segmentation saw
+  // 5 > 1+1 and emitted two singletons; array adjacency sees a and e as array
+  // neighbors and emits one range.
+  const a = msg("a", "x".repeat(2000));
+  const e = msg("e", "v".repeat(2000));
+  const state = assignAll([a, msg("b", "y"), msg("c", "z"), msg("d", "w"), e]);
+  const ranges = buildCompressibleRanges([a, e], state, config());
+  assert.equal(ranges.compressible.length, 1, "holes in the ref map must not split the range");
+  assert.equal(ranges.compressible[0]!.startRef, "m00001");
+  assert.equal(ranges.compressible[0]!.endRef, "m00005");
+  assert.equal(ranges.compressible[0]!.count, 2);
+});
+
+test("buildCompressibleRanges: mid-array summary node extends the range, never a descending pair", () => {
+  // Surface-replace host inserts the model's summary node at the replaced span's
+  // position. assignRefs gives it a fresh HIGH ref (m00006) while the trailing
+  // message keeps m00005 — ref arithmetic flushed the head and emitted the
+  // nonsense pair m00006..m00005. Array adjacency treats the node as a regular
+  // entry: one ascending range over the whole span.
+  const a = msg("a", "x".repeat(2000), "assistant");
+  const d = msg("d", "w".repeat(2000), "assistant");
+  const e = msg("e", "v".repeat(2000), "assistant");
+  const summary = msg("s", "Summary of the compressed span: did the work.", "assistant");
+  const s1 = assignAll([a, msg("b", "y"), msg("c", "z"), d, e]);
+  const state = assignAll([a, summary, d, e], s1);
+  assert.equal(state.messageRefs.byRaw["s"], "m00006", "summary node gets a fresh high ref");
+  const ranges = buildCompressibleRanges([a, summary, d, e], state, config());
+  assert.equal(ranges.compressible.length, 1, "mid-array summary node must not flush the range");
+  assert.equal(ranges.compressible[0]!.startRef, "m00001");
+  assert.equal(ranges.compressible[0]!.endRef, "m00005");
+  assert.equal(ranges.compressible[0]!.count, 4);
+});
+
+test("buildCompressibleRanges: synthetic (covered) summary node still splits ranges", () => {
+  // A summary node detected via the "[Compressed conversation section]" prefix
+  // (or active-block coverage) is intentionally skipped — a skipped numbered
+  // message between two entries is a real interruption, unlike a numeric hole
+  // where nothing is physically present between them.
+  const a = msg("a", "x".repeat(2000));
+  const e = msg("e", "v".repeat(2000));
+  const synthetic = msg("s", "[Compressed conversation section] earlier work summarized.", "assistant");
+  const s1 = assignAll([a, msg("b", "y"), msg("c", "z"), msg("d", "w"), e]);
+  const state = assignAll([a, synthetic, e], s1);
+  const ranges = buildCompressibleRanges([a, synthetic, e], state, config());
+  assert.equal(ranges.compressible.length, 2, "covered span must not be folded into the new range");
+  assert.equal(ranges.compressible[0]!.startRef, "m00001");
+  assert.equal(ranges.compressible[0]!.endRef, "m00001");
+  assert.equal(ranges.compressible[1]!.startRef, "m00005");
+  assert.equal(ranges.compressible[1]!.endRef, "m00005");
+});
+
+test("buildCompressibleRanges: protected groups segment by array adjacency too", () => {
+  // A compressible message physically between two protected tool-calls splits
+  // them (dense-host behavior preserved); a bare ref-map hole does not.
+  const p1 = toolMsg("p1", "skill");
+  const p2 = toolMsg("p2", "skill");
+  const dense = createInitialState();
+  dense.messageRefs = {
+    byRaw: { p1: "m00001", x: "m00002", p2: "m00003" },
+    byRef: { m00001: "p1", m00002: "x", m00003: "p2" },
+  };
+  const denseRanges = buildCompressibleRanges(
+    [p1, msg("x", "y"), p2],
+    dense,
+    config({ protectedTools: ["skill"] }),
+  );
+  assert.equal(denseRanges.protected.length, 2, "interleaved non-protected message splits protected groups");
+
+  const holed = createInitialState();
+  holed.messageRefs = {
+    byRaw: { p1: "m00001", p2: "m00005" },
+    byRef: { m00001: "p1", m00005: "p2" },
+  };
+  const holedRanges = buildCompressibleRanges([p1, p2], holed, config({ protectedTools: ["skill"] }));
+  assert.equal(holedRanges.protected.length, 1, "hole in the ref map must not split the protected group");
+  assert.equal(holedRanges.protected[0]!.count, 2);
+  assert.equal(holedRanges.protected[0]!.startRef, "m00001");
+  assert.equal(holedRanges.protected[0]!.endRef, "m00005");
+});
+
 // ─── Integration: 19-token bug fix ─────────────────────────────────────────────
 
 test("integration: tiny ranges are suppressed — fixes the 19-token compression bug", () => {