diff --git a/CHANGELOG.md b/CHANGELOG.md index bf8496376f..48fc86a89a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,11 @@ between releases do not reach them. Remote-PC runtime bundles are versioned sepa grows with a size above 16px (it stays at 16px or more, so iOS still does not zoom). ([#515](https://github.com/devswha/herdr-web-ui/pull/515) by @phirschybar) +### Fixed +- Codex chat hides internal memory citation blocks and pairs the answer with its display record, + so a reply that uses memory appears once without raw citation markup. Quoted code examples + keep their text. ([#514](https://github.com/devswha/herdr-web-ui/pull/514) by @JJLiebig) + ## [0.3.52] - 2026-10-06 ### Added diff --git a/server/codex.test.ts b/server/codex.test.ts index 0a584daa60..21dc8a0383 100644 --- a/server/codex.test.ts +++ b/server/codex.test.ts @@ -64,6 +64,121 @@ describe("CODEX_HOME in a ps -E line", () => { }); describe("Codex conversation records", () => { + it("hides memory citations before pairing display/model answers in either order", () => { + const citation = "\n\nMEMORY.md:1-2|note=[context]\n\n\nthread-id\n\n"; + for (const reverse of [false, true]) { + const pair = [message("assistant", `Done\n${citation}`, "final_answer"), event({ type: "agent_message", message: "Done" })]; + if (reverse) pair.reverse(); + expect(parseCodexTranscript(jsonl(...pair))).toEqual([ + { role: "assistant", ts, end_ts: ts, parts: [{ kind: "text", text: "Done", phase: "final_answer" }] }, + ]); + } + }); + + it("hides multiple and unfinished memory blocks without dropping surrounding prose", () => { + expect(parseCodexTranscript(jsonl( + message("assistant", "Beforeone aftertwo", "final_answer"), + event({ type: "agent_message", message: "Next\nunfinished" }), + message("assistant", "metadata only"), + ))[0]?.parts).toEqual([ + { kind: "text", text: "Before after", phase: "final_answer" }, + { kind: "text", text: "Next" }, + ]); + expect(parseCodexTranscript(jsonl(message("assistant", "unfinished")))).toEqual([]); + }); + + it("preserves memory markup quoted by a user or returned by a tool", () => { + const text = "quoted metadata"; + const turns = parseCodexTranscript(jsonl( + message("user", text), + item({ type: "function_call", call_id: "read", name: "read", arguments: "{}" }), + item({ type: "function_call_output", call_id: "read", output: text }), + )); + expect(turns[0]?.parts).toEqual([{ kind: "text", text }]); + expect(turns[1]?.parts[0]).toMatchObject({ kind: "tool", output: text }); + }); + + it("preserves literal memory tags in inline code and code fences", () => { + const examples = [ + "Match the literal tag `` in the parser.\nKeep this explanatory paragraph.", + "The block is `sample`.", + "Use `` `` `` to quote the tag.", + "A backslash inside code is literal: `\\` and prose follows.", + "Example:\n```xml\nsample\n```\nExplanation follows.", + "Example:\n ~~~~xml\nsample\n ~~~\nStill code.\n ~~~~~\nExplanation follows.", + "Example:\n````xml\nsample\n```\nStill code.\n````\nExplanation follows.", + "An unfinished example:\n```xml\nliteral tag", + "An unfinished example:\n~~~xml\nliteral tag", + ]; + for (const text of examples) { + expect(parseCodexTranscript(jsonl(message("assistant", text, "final_answer")))[0]?.parts).toEqual([ + { kind: "text", text, phase: "final_answer" }, + ]); + } + }); + + it("removes metadata around code examples before pairing their display records", () => { + const example = "Before `` after\n~~~xml\nsample\n~~~\nDone"; + const response = "first" + example + "\nunfinished"; + for (const reverse of [false, true]) { + const pair = [message("assistant", response, "final_answer"), event({ type: "agent_message", message: example })]; + if (reverse) pair.reverse(); + expect(parseCodexTranscript(jsonl(...pair))[0]?.parts).toEqual([ + { kind: "text", text: example, phase: "final_answer" }, + ]); + } + const mixed = "Before\nhidden\n```xml\nliteral\n```\nalso hidden\nAfter"; + expect(parseCodexTranscript(jsonl(message("assistant", mixed)))[0]?.parts).toEqual([ + { kind: "text", text: "Before\n\n```xml\nliteral\n```\n\nAfter" }, + ]); + const payload = "Before\n~~~xml\n`unclosed\nAfter\nhidden too"; + expect(parseCodexTranscript(jsonl(message("assistant", payload)))[0]?.parts).toEqual([{ kind: "text", text: "BeforeAfter" }]); + }); + + it("preserves fences inside quotes and lists only within their containers", () => { + const hidden = "actual metadata"; + const examples = [ + "> ```xml\n> literal tag\n> explanation", + "> > ~~~xml\n> > literal tag\n> > explanation", + "> > ```xml\n> > literal tag\n> > ```\n> Quote continues.", + "- Example:\n ```xml\n literal tag\n explanation", + "- ~~~xml\n literal tag\n ~~~\n Item continues.", + "1. Example:\n - ```xml\n literal tag\n explanation", + "> - Example:\n> ```xml\n> literal tag\n> explanation", + ]; + for (const example of examples) { + const text = `${example}\nOutside\n${hidden}`; + expect(parseCodexTranscript(jsonl(message("assistant", text)))[0]?.parts).toEqual([{ kind: "text", text: `${example}\nOutside` }]); + } + const nested = "> > ```xml\n> > literal tag\n> Parent quote\n> " + hidden; + expect(parseCodexTranscript(jsonl(message("assistant", nested)))[0]?.parts).toEqual([ + { kind: "text", text: "> > ```xml\n> > literal tag\n> Parent quote\n>" }, + ]); + const crlf = "> ```xml\r\n> literal tag\r\nOutside\r\n" + hidden; + expect(parseCodexTranscript(jsonl(message("assistant", crlf)))[0]?.parts).toEqual([ + { kind: "text", text: "> ```xml\r\n> literal tag\r\nOutside" }, + ]); + }); + + it("does not let escaped, unmatched or cross-paragraph backticks hide metadata", () => { + for (const text of [ + "An unmatched ` delimiter.\nhidden", + "An escaped \\` delimiter.\nhidden\nA second \\` delimiter.", + "First `\n\nhidden\n\nSecond `", + ]) { + const expected = text.replace("hidden", "").trimEnd(); + expect(parseCodexTranscript(jsonl(message("assistant", text)))[0]?.parts).toEqual([{ kind: "text", text: expected }]); + } + }); + + it("bounds citation parsing with many unmatched backtick lengths", () => { + const text = "Unmatched delimiters: " + Array.from({ length: 1000 }, (_, i) => "`".repeat(i + 1) + " x ").join(""); + const transcript = jsonl(message("assistant", `${text}hidden`)); + const started = performance.now(); + expect(parseCodexTranscript(transcript)[0]?.parts).toEqual([{ kind: "text", text: text.trimEnd() }]); + expect(performance.now() - started).toBeLessThan(1000); + }); + it("hides injected context, metadata and developer messages while preserving the real request", () => { const turns = parseCodexTranscript(jsonl( { type: "session_meta", payload: { base_instructions: "internal system prompt" } }, @@ -562,6 +677,21 @@ describe("Codex tool calls that failed, and patches", () => { }); describe("incremental Codex records", () => { + it("pairs citation-bearing answers across writes and previews without changing earlier snapshots", async () => { + const { createCodexTranscriptParser } = await import("./codex.ts"); + const parser = createCodexTranscriptParser(); + const text = "Match `` literally.\nThe answer continues."; + parser.write(jsonl(event({ type: "agent_message", message: text }))); + const earlier = parser.snapshot(); + const response = jsonl(message("assistant", `${text}\ninternal metadata`, "final_answer")); + const expected: ReturnType = [{ role: "assistant", ts, end_ts: ts, parts: [{ kind: "text", text, phase: "final_answer" }] }]; + expect(parser.snapshot(response)).toEqual(expected); + expect(earlier[0]?.parts).toEqual([{ kind: "text", text }]); + expect(parser.snapshot()).toEqual(earlier); + parser.write(response); + expect(parser.snapshot()).toEqual(expected); + }); + it("pairs duplicates and tool results across writes without mutating earlier snapshots", async () => { const { createCodexTranscriptParser } = await import("./codex.ts"); const parser = createCodexTranscriptParser(); diff --git a/server/codex.ts b/server/codex.ts index 2a4d993add..c31845bb47 100644 --- a/server/codex.ts +++ b/server/codex.ts @@ -70,6 +70,100 @@ function entries(text: string): RecordValue[] { }); } +/** Memory blocks are metadata unless the answer quotes them in Markdown code. */ +function withoutMemoryCitations(text: string): string { + const opening = ""; + const closing = ""; + if (!text.includes(opening)) return text.trimEnd(); + + type Line = { kind: "line"; index: number; quoteDepth: number; indent: number; listIndent?: number; blank: boolean }; + type Token = Line | { kind: "fence"; index: number; marker: string; info: string; line: Line } + | { kind: "ticks"; index: number; size: number } | { kind: "citation"; index: number }; + const tokens: Token[] = []; + const quote = /[ \t]*>[ \t]?/y; + for (const match of text.matchAll(/([^\r\n]*)(?:\r\n?|\n|$)/g)) { + const source = match[1]!; + let offset = 0; + let quoteDepth = 0; + quote.lastIndex = 0; + while (quote.exec(source)) { offset = quote.lastIndex; quoteDepth++; } + const indent = /^[ \t]*/.exec(source.slice(offset))![0].length; + const content = source.slice(offset + indent); + const list = /^(?:[-+*]|\d+[.)])[ \t]+/.exec(content); + const line: Line = { kind: "line", index: match.index!, quoteDepth, indent, + ...(list ? { listIndent: indent + list[0].length } : {}), blank: content.trim() === "" }; + tokens.push(line); + const body = content.slice(list?.[0].length ?? 0); + const marker = /^(`{3,}|~{3,})/.exec(body)?.[0]; + const info = marker ? body.slice(marker.length) : ""; + // A backtick fence's info string cannot contain backticks. + if (marker && (marker[0] !== "`" || !info.includes("`"))) tokens.push({ kind: "fence", + index: match.index! + offset + indent + (list?.[0].length ?? 0), marker, info, line }); + for (const token of source.matchAll(/`+|/g)) { + tokens.push(token[0] === opening ? { kind: "citation", index: match.index! + token.index! } + : { kind: "ticks", index: match.index! + token.index!, size: token[0].length }); + } + } + const closes = new Map(); + const pairs: number[] = []; + // Index matching runs once: unmatched delimiters must not repeatedly scan the tail. + for (let i = tokens.length - 1; i >= 0; i--) { + const token = tokens[i]!; + if (token.kind !== "ticks") { + if (token.kind === "line") closes.clear(); + continue; + } + let backslashes = 0; + for (let at = token.index - 1; at >= 0 && text[at] === "\\"; at--) backslashes++; + const length = token.size - (backslashes % 2); + pairs[i] = length > 0 ? closes.get(length) ?? -1 : -1; + // Inside a code span a backslash does not escape its closing backticks. + closes.set(token.size, i); + } + const parts: string[] = []; + const listIndents: number[] = []; + let quoteDepth = 0; + let kept = 0; + for (let i = 0; i < tokens.length; i++) { + const token = tokens[i]!; + if (token.kind === "line") { + if (token.quoteDepth !== quoteDepth) { listIndents.length = 0; quoteDepth = token.quoteDepth; } + if (token.blank) continue; + while (listIndents.length > 0 && listIndents.at(-1)! > token.indent) listIndents.pop(); + if (token.listIndent !== undefined) listIndents.push(token.listIndent); + continue; + } + if (token.kind === "fence") { + const within = token.line.listIndent ?? listIndents.at(-1) ?? 0; + const indent = token.line.listIndent === undefined ? token.line.indent - within : 0; + if (indent < 0 || indent > 3) continue; + // An unfinished fence remains code until its quote/list container ends. + for (i++; i < tokens.length; i++) { + const end = tokens[i]!; + if (end.kind === "line" && (end.quoteDepth < token.line.quoteDepth + || (within > 0 && end.quoteDepth === token.line.quoteDepth && !end.blank && end.indent < within))) { i--; break; } + if (end.kind === "fence" && end.line.quoteDepth === token.line.quoteDepth && end.marker[0] === token.marker[0] + && end.marker.length >= token.marker.length && /^[ \t]*$/.test(end.info)) { + // Skip the closing line's raw backtick/citation tokens too. + while (i + 1 < tokens.length && tokens[i + 1]!.kind !== "line") i++; + break; + } + } + continue; + } + const pair = pairs[i]; + if (pair !== undefined && pair >= 0) { i = pair; continue; } + if (token.kind !== "citation") continue; + parts.push(text.slice(kept, token.index)); + const close = text.indexOf(closing, token.index + opening.length); + kept = close === -1 ? text.length : close + closing.length; + // Metadata contents cannot open code spans or fences in the surrounding answer. + while (i + 1 < tokens.length && tokens[i + 1]!.index < kept) i++; + } + parts.push(text.slice(kept)); + return parts.join("").trimEnd(); +} + /** * Whether a Codex tool output says the call failed. Codex records no flag: its command @@ -106,7 +200,8 @@ export function createCodexTranscriptParser(state: CodexParseState = { turns: [] return turn; }; const message = (role: "user" | "assistant", text: string, source: string, ts: string, phase?: "commentary" | "final_answer", images: ConversationPart[] = []): void => { - const body = role === "user" ? questionReply(text) ?? text : text; + const body = role === "user" ? questionReply(text) ?? text + : withoutMemoryCitations(text); if (!body.trim() && images.length === 0) return; const duplicate = messages.slice(-8).reverse().find((other) => !other.paired && other.role === role && other.text === body && other.source !== source && (other.ts === ts || Math.abs(Date.parse(other.ts) - Date.parse(ts)) <= 1000));