Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions plugins/web-ui/src/composer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1078,6 +1078,10 @@ export function createComposerSurface(ctx: ConvCtx): ComposerSurface {
}

function onComposerKeydown(e: KeyboardEvent, agent: Agent): void {
// During IME composition (Japanese/Chinese/Korean), Enter confirms the
// conversion — it must never send. Safari reports composition Enter with
// keyCode 229 and may fire after compositionend, so check both.
if (e.isComposing || e.keyCode === 229) return;
const slash = currentSlashMenu();
if (slash.open) {
if (e.key === "Escape") {
Expand Down
1 change: 1 addition & 0 deletions plugins/web-ui/src/sessions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -971,6 +971,7 @@ function renameInput(menuKey: string, ariaLabel: string, commit: () => Promise<v
renameDraft = (e.currentTarget as HTMLInputElement).value;
}}
@keydown=${(e: KeyboardEvent) => {
if (e.isComposing || e.keyCode === 229) return;
if (e.key === "Enter") {
e.preventDefault();
void commit();
Expand Down
3 changes: 2 additions & 1 deletion scripts/dev/twin-instance.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
import { fileURLToPath } from "node:url";
import { spawn, spawnSync } from "node:child_process";
import { existsSync, mkdirSync, openSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { provisionTwinEnvironment, teardownTwinEnvironment } from "../../test/live-slack/arga-provision.ts";
import { readEnvFile } from "./lib/util.ts";

const root = new URL("../..", import.meta.url).pathname.replace(/\/$/, "");
const root = fileURLToPath(new URL("../..", import.meta.url)).replace(/\/$/, "");
const dir = join(root, ".twin-instance");
const envFile = join(dir, "env");
const pumpPid = join(dir, "pump.pid");
Expand Down
3 changes: 2 additions & 1 deletion scripts/memory-bench.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { fileURLToPath } from "node:url";
import { mkdtempSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
Expand Down Expand Up @@ -37,7 +38,7 @@ for (const k of kinds) {
}
}

const fixtureDir = new URL("../test/memory-bench/conversations/", import.meta.url).pathname;
const fixtureDir = fileURLToPath(new URL("../test/memory-bench/conversations/", import.meta.url));
const conversations = readdirSync(fixtureDir)
.filter((f) => f.endsWith(".json"))
.sort()
Expand Down
11 changes: 11 additions & 0 deletions src/api/app-turn.ts
Original file line number Diff line number Diff line change
Expand Up @@ -315,11 +315,22 @@ export function createTurnMethods(
return req.async ? { status: "queued", runId: live.id, steered: true } : drive(live.id);
if (decision === "unscreened") injectedText = `${unscreenedNotice("mid-turn message")}\n${steerText}`;
}
// A mid-run message can carry files. They can't be materialized into
// the live turn's inbox, but the run must hear about them — name
// them in the steer (with the message ts so the agent can pull each
// via the surface-file API), and never report a captionless file as
// steered while silently dropping it.
const fileNames = (req.attachments ?? []).map((a) =>
a.sourceId
? `${a.name} (fetch via surface-file, ts ${origin.kind === "human" ? (origin.messageTs ?? origin.entryTs) : origin.entryTs})`
: a.name,
);
const wake: Wake = {
situation: origin.kind === "ambient" ? "ambientUpdate" : "addressed",
ts: String(req.clientSentAt ?? Date.now()),
text: injectedText,
halt: origin.kind === "human" && isHalt(req.text),
...(fileNames.length ? { fileNames } : {}),
};
const route = routeWake(wake, true, resolveTurnOrigin(live.request).kind === "ambient");
if (route.kind === "steer" || route.kind === "drop") {
Expand Down
16 changes: 14 additions & 2 deletions src/memory/strategies/scratch-promote.ts
Original file line number Diff line number Diff line change
Expand Up @@ -183,15 +183,27 @@ export function createScratchPromote(deps: ScratchPromoteDeps): { strategy: Memo
const now = Date.now();
const window = await readLogWindow(scopeId, now, LOG_RETENTION_DAYS);
if (window.length && deps.harness.oneShot) {
const longTerm = stripMarker(await base.read(scopeId));
// Promotion is a read → model round-trip → write. A save that lands
// during the round-trip must not be silently reverted by the write,
// so the write is compare-and-set against the revision we read; on a
// lost race we skip — the next promotion pass will pick everything up.
const head = base.readHead ? await base.readHead(scopeId) : null;
const raw = head ? head.content : await base.read(scopeId);
const longTerm = stripMarker(raw);
const scratch = window.map(({ date, body }) => `## ${date}\n${body}`).join("\n\n");
const out = (
(await deps.harness.oneShot(
PROMOTION_PROMPT,
`Current notebook:\n${longTerm || "(empty)"}\n\nScratch log:\n${scratch}`,
)) ?? ""
).trim();
if (out && !/^none$/i.test(out)) await base.replace(scopeId, out);
if (out && !/^none$/i.test(out)) {
if (head && base.replaceIfRevision) {
await base.replaceIfRevision(scopeId, out, head.revision);
} else {
await base.replace(scopeId, out);
}
}
}
const cutoff = dateStr(now - LOG_RETENTION_DAYS * 86_400_000);
for (const abs of await workspace.list(scopeId)) {
Expand Down
43 changes: 38 additions & 5 deletions src/persistence/durable-map.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,39 @@ export interface DurableMap<T> {
take(id: string): Promise<T | null>;
}

/**
* Serialize for a Postgres jsonb column. jsonb rejects two things a JS string
* happily carries: NUL (\u0000) and unpaired surrogate halves — and qm's own
* truncation helpers can manufacture the latter by slicing mid-emoji. The
* memory map accepts those values, so production diverged from every
* in-memory test. Sanitize at the serialization boundary: drop NULs and
* replace lone surrogates with U+FFFD, recursively, only when a string
* actually needs it.
*/
const LONE_SURROGATE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g;

function jsonbSafeString(s: string): string {
let out = s;
if (out.includes("\u0000")) out = out.replaceAll("\u0000", "");
if (LONE_SURROGATE.test(out)) out = out.replace(LONE_SURROGATE, "\uFFFD");
return out;
}

function jsonbSafe(value: unknown): unknown {
if (typeof value === "string") return jsonbSafeString(value);
if (Array.isArray(value)) return value.map(jsonbSafe);
if (value && typeof value === "object") {
const out: Record<string, unknown> = {};
for (const [k, v] of Object.entries(value)) out[jsonbSafeString(k)] = jsonbSafe(v);
return out;
}
return value;
}

export function jsonbStringify(value: unknown): string {
return JSON.stringify(jsonbSafe(value));
}

function applyPatch<T>(value: T, patch: Partial<T>): T {
const next = { ...value } as Record<string, unknown>;
for (const [k, v] of Object.entries(patch)) {
Expand Down Expand Up @@ -153,7 +186,7 @@ export function createPostgresMap<T>(pg: PgPool, table: string): DurableMap<T> {
client.query(
`INSERT INTO ${table} (id, json) VALUES ($1, $2)
ON CONFLICT (id) DO UPDATE SET json = EXCLUDED.json`,
[id, JSON.stringify(value)],
[id, jsonbStringify(value)],
),
);
},
Expand All @@ -163,7 +196,7 @@ export function createPostgresMap<T>(pg: PgPool, table: string): DurableMap<T> {
`INSERT INTO ${table} (id, json) VALUES ($1, $2)
ON CONFLICT (id) DO UPDATE SET json = ${table}.json
RETURNING json`,
[id, JSON.stringify(value)],
[id, jsonbStringify(value)],
),
);
return res.rows[0]!.json as T;
Expand All @@ -173,7 +206,7 @@ export function createPostgresMap<T>(pg: PgPool, table: string): DurableMap<T> {
client.query(
`INSERT INTO ${table} (id, json) VALUES ($1, $2)
ON CONFLICT (id) DO NOTHING`,
[id, JSON.stringify(value)],
[id, jsonbStringify(value)],
),
);
return (inserted.rowCount ?? 0) > 0;
Expand All @@ -186,7 +219,7 @@ export function createPostgresMap<T>(pg: PgPool, table: string): DurableMap<T> {
client.query(`UPDATE ${table} SET json = (json - $2::text[]) || $3::jsonb WHERE id = $1 RETURNING json`, [
id,
removeKeys,
JSON.stringify(set),
jsonbStringify(set),
]),
);
return res.rows.length ? (res.rows[0]!.json as T) : null;
Expand All @@ -196,7 +229,7 @@ export function createPostgresMap<T>(pg: PgPool, table: string): DurableMap<T> {
const current = await client.query(`SELECT json FROM ${table} WHERE id = $1 FOR UPDATE`, [id]);
if (!current.rows[0]) return null;
const next = fn(current.rows[0].json as T);
await client.query(`UPDATE ${table} SET json = $2 WHERE id = $1`, [id, JSON.stringify(next)]);
await client.query(`UPDATE ${table} SET json = $2 WHERE id = $1`, [id, jsonbStringify(next)]);
return next;
});
},
Expand Down
2 changes: 1 addition & 1 deletion src/sandbox/exec-process-session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
export function redactCommand(command: string, env?: Record<string, string>): string {
return createSecretValueMasker(env)(command)
.replace(/(--?(?:token|password|secret|client[-_]?secret|api[-_]?key)[ =])\S+/gi, "$1<redacted>")
.replace(/(--with-token\b)/gi, "$1")
.replace(/(?:printf|echo)(?:\s+(?:-\w+|"[^"]*"|'[^']*'|\S+))+(\s*\|[^|]*--with-token\b)/gi, "echo <redacted>$1")
.replace(
/(export\s+\w*(?:PASS|PASSWORD|SECRET|TOKEN|KEY|IDENTIFIER|CREDENTIAL|PROXY_USER)\w*=')[^']*'/gi,
"$1<redacted>'",
Expand Down
1 change: 1 addition & 0 deletions src/security/secret-masking.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ export function createSecretValueMasker(env: Record<string, string> | undefined)
const uri = encodeURIComponent(value);
if (uri !== value) variants.push({ needle: uri, label: key });
variants.push({ needle: Buffer.from(value, "utf8").toString("base64").replace(/=+$/, ""), label: key });
variants.push({ needle: Buffer.from(value, "utf8").toString("base64url"), label: key });
}
if (!variants.length) return (text) => text;
variants.sort((a, b) => b.needle.length - a.needle.length);
Expand Down
7 changes: 2 additions & 5 deletions src/slack/mrkdwn.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { safeChunks } from "./safe-cut.ts";
export function decodeSlackEntities(text: string): string {
return text.replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&amp;/g, "&");
}
Expand Down Expand Up @@ -173,9 +174,5 @@ function reformatTables(text: string, keep: (s: string) => string): string {
}

export function slackSectionBlocks(text: string): Array<Record<string, unknown>> {
const blocks: Array<Record<string, unknown>> = [];
for (let offset = 0; offset < text.length; offset += 2_900) {
blocks.push({ type: "section", text: { type: "mrkdwn", text: text.slice(offset, offset + 2_900) } });
}
return blocks;
return safeChunks(text, 2_900).map((chunk) => ({ type: "section", text: { type: "mrkdwn", text: chunk } }));
}
56 changes: 56 additions & 0 deletions src/slack/safe-cut.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
/**
* Cut points that never split what a reader (or Slack's parser) treats as one
* unit: a surrogate pair (emoji), a `<url|label>` entity, or an inline code /
* bold / strike span. Backing off never goes past `floor` characters (a
* pathological single entity longer than the whole budget is cut anyway —
* better a broken link than an empty message).
*/
const FLOOR_FRACTION = 0.25;

function isLowSurrogate(code: number): boolean {
return code >= 0xdc00 && code <= 0xdfff;
}

/** The largest index <= max that is safe to cut at. */
export function safeCutIndex(text: string, max: number): number {
if (text.length <= max) return text.length;
let cut = max;
// never split a surrogate pair
if (isLowSurrogate(text.charCodeAt(cut))) cut--;
const floor = Math.max(1, Math.floor(max * FLOOR_FRACTION));
// never split a <...> entity (links, mentions): back off to before the "<"
const lastOpen = text.lastIndexOf("<", cut - 1);
if (lastOpen >= 0) {
const lastClose = text.lastIndexOf(">", cut - 1);
if (lastOpen > lastClose && lastOpen >= floor) cut = lastOpen;
}
// never leave an unbalanced formatting run open right at the cut
for (const marker of ["`", "*", "~"]) {
let count = 0;
for (let i = 0; i < cut; i++) if (text[i] === marker) count++;
if (count % 2 === 1) {
const back = text.lastIndexOf(marker, cut - 1);
if (back >= floor) cut = Math.min(cut, back);
}
}
if (isLowSurrogate(text.charCodeAt(cut))) cut--;
return Math.max(cut, 1);
}

/** Split text into chunks of at most `max`, cutting only at safe points. */
export function safeChunks(text: string, max: number): string[] {
const chunks: string[] = [];
let rest = text;
while (rest.length > 0) {
const cut = safeCutIndex(rest, max);
chunks.push(rest.slice(0, cut));
rest = rest.slice(cut);
}
return chunks.length ? chunks : [""];
}

/** Truncate to at most `max` characters (plus ellipsis) at a safe point. */
export function safeClip(text: string, max: number): string {
if (text.length <= max) return text;
return `${text.slice(0, safeCutIndex(text, max))}…`;
}
9 changes: 6 additions & 3 deletions src/slack/util.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
import { safeCutIndex, safeClip } from "./safe-cut.ts";

export function inlineCode(text: string): string {
const safe = text.replace(/`/g, "'").slice(0, 2000);
return `\`${safe}${text.length > safe.length ? "..." : ""}\``;
const cleaned = text.replace(/`/g, "'");
const safe = cleaned.slice(0, safeCutIndex(cleaned, 2000));
return `\`${safe}${cleaned.length > safe.length ? "..." : ""}\``;
}

export function clip(text: string, max: number): string {
return text.length > max ? `${text.slice(0, max)}…` : text;
return safeClip(text, max);
}

export function sleep(ms: number): Promise<void> {
Expand Down
7 changes: 5 additions & 2 deletions src/wake/wake.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ export interface Wake {
text?: string;
isSelf?: boolean;
halt?: boolean;
/** Names of files attached to the message — a file with no caption is still a message. */
fileNames?: string[];
}

export type WakeRoute =
Expand All @@ -26,6 +28,7 @@ export function routeWake(wake: Wake, runIsLive: boolean, liveRunGated = false):
if (wake.halt) return { kind: "steer", signal: "abort" };
if (liveRunGated && wake.situation === "addressed") return { kind: "engage" };
const text = wake.text?.trim();
if (!text) return { kind: "drop", reason: "empty-mid-turn" };
return { kind: "steer", signal: "steer", text };
const files = wake.fileNames?.length ? `[files attached mid-run: ${wake.fileNames.join(", ")}]` : "";
if (!text && !files) return { kind: "drop", reason: "empty-mid-turn" };
return { kind: "steer", signal: "steer", text: [text, files].filter(Boolean).join("\n") };
}
25 changes: 25 additions & 0 deletions test/durable-map-jsonb.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { jsonbStringify } from "../src/persistence/durable-map.ts";

test("NUL characters are dropped for jsonb", () => {
const out = jsonbStringify({ name: "a\u0000b" });
assert.equal(out, '{"name":"ab"}');
});

test("a lone high surrogate (emoji cut in half by slice) becomes U+FFFD", () => {
const cut = "prefix 😀".slice(0, 8); // strands the high surrogate
const out = JSON.parse(jsonbStringify({ title: cut })) as { title: string };
assert.ok(!/[\uD800-\uDBFF](?![\uDC00-\uDFFF])/.test(out.title), "no stranded surrogate survives");
assert.ok(out.title.includes("\uFFFD"));
});

test("well-formed strings, keys, arrays, and nesting pass through untouched", () => {
const value = { a: "hello 😀", list: ["x", { deep: "ok" }], n: 3, b: true, z: null };
assert.equal(jsonbStringify(value), JSON.stringify(value));
});

test("keys are sanitized too", () => {
const out = jsonbStringify({ ["k\u0000ey"]: 1 });
assert.equal(out, '{"key":1}');
});
14 changes: 14 additions & 0 deletions test/exec-process-session.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,20 @@ test("redactCommand strips secret-bearing flags", () => {
assert.match(redactCommand("svc --client-secret=shh"), /<redacted>/);
});

test("redactCommand hides a token piped into --with-token", () => {
for (const cmd of [
"echo ghp_secretvalue12345 | gh auth login --with-token",
'echo "ghp_secretvalue12345" | gh auth login --with-token',
"printf ghp_secretvalue12345 | gh auth login --with-token",
"echo -n ghp_secretvalue12345 | gh auth login --hostname x.test --with-token",
]) {
const out = redactCommand(cmd);
assert.ok(!out.includes("ghp_secretvalue12345"), `leaked in: ${out}`);
assert.match(out, /<redacted>/);
assert.match(out, /--with-token/);
}
});

test("redactCommand masks known injected env values the pattern layer misses", () => {
const env = { GITHUB_TOKEN: "ghp_secretvalue12345" };
for (const cmd of [
Expand Down
3 changes: 2 additions & 1 deletion test/memory-bench.test.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { fileURLToPath } from "node:url";
import { test } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, readdirSync, readFileSync } from "node:fs";
Expand All @@ -19,7 +20,7 @@ import {
type BenchResult,
} from "../src/memory/bench.ts";

const FIXTURE_DIR = new URL("./memory-bench/conversations/", import.meta.url).pathname;
const FIXTURE_DIR = fileURLToPath(new URL("./memory-bench/conversations/", import.meta.url));

test("every bench fixture parses and has at least two turns", () => {
const files = readdirSync(FIXTURE_DIR).filter((f) => f.endsWith(".json"));
Expand Down
15 changes: 15 additions & 0 deletions test/memory-strategy-scratch-promote.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -173,3 +173,18 @@ test("strategy wiring: scratch-promote parses, wraps the store, and ships prompt
"per-turn gets the consolidating store — captures by any path trigger the after-N check",
);
});

test("a save landing during promotion is not reverted by the promote write", async () => {
const { base, strategy, memory } = fresh({
oneShot: async () => {
// a user edit lands while the model call is in flight
await base.replace(SCOPE, "# Memory\n\n- (2026-06-10) the newer edit");
return "# Memory\n\n- (2026-06-10) promoted fact";
},
});
await withNow(TODAY, () => memory.capture(SCOPE, ["something recent"], TODAY));
await withNow(TODAY, () => strategy.maintain!(SCOPE));
const after = await base.read(SCOPE);
assert.match(after, /the newer edit/, "the mid-flight edit survives");
assert.doesNotMatch(after, /promoted fact/, "the stale promotion is dropped, not applied");
});
Loading