fix(pipeline): discriminate post-fold resends from new instances in reconcile-live-ids (#462)

src/compress.ts

@@ -716,11 +716,16 @@ export function createCore(ports: Ports = {}): CompressionCore {
     // message texts receive an <acp> tag.
     const strategy: RenderStrategy = input.renderTags ?? "all";
     const nodes = buildNodes(strategy);
+    // #462: snapshot the raw inbound ids BEFORE any node runs — reconcile-
+    // live-ids consumes the PREVIOUS pass's snapshot (echo continuity), and the
+    // returned state carries THIS pass's snapshot for the next one.
+    const inboundIds = input.messages.map((m) => m.id);
     const result = runPipeline(nodes, initial, ctx);
+    const state = { ...result.state, lastPassIds: inboundIds };
     const ccrEffect = result.effects.ccr as CcrEffect | undefined;
     return {
       messages: result.messages,
-      state: result.state,
+      state,
       nudge: result.effects.nudge,
       terminalEscape: result.effects.terminalEscape,
       truncationSkipped: result.effects.truncationSkipped,

src/instance-reid.ts

@@ -15,38 +15,40 @@ function clusterRoot(id: string): string | null {
 
 /**
  * Re-mint live messages whose id collides with an already-FOLDED copy of the same
- * content (billion-context #1476). Runs as the FIRST pipeline node, before
- * assign-refs and prune, so the re-minted id is what gets ref'd and what prune sees.
+ * content (billion-context #1476), WITHOUT un-covering the folded originals' own
+ * resends (#462). Runs as the FIRST pipeline node, before assign-refs and prune,
+ * so the re-minted id is what gets ref'd and what prune sees.
  *
- * Root cause: deriveMessageId's cluster counter restarts every conversion pass, so
- * text re-sent after its earlier copy was folded re-derives the SAME bare `h_…` id.
- * From that single collision: assignRefs is first-wins (`if (map.byRaw[id]) continue`)
- * and the folded original already owns `byRaw[h_…]`, so the fresh instance gets NO
- * ref; and prune drops any covered id that isn't the pinned first user message, so
- * the fresh user turn silently vanishes upstream.
+ * Two byte-identical shapes carry an exact-covered id and must be told apart:
  *
- * Why "covered base" is the discriminator: a base is covered iff an active block
- * lists it in effectiveMessageIds, i.e. its original is off the wire (replaced by a
- * summary). So any LIVE message carrying a covered id is a new instance, not the
- * original — and must get a distinct instance id. A NON-covered base keeps the
- * converter's numbering untouched: that is the shape where the old copy is still on
- * the wire and un-folded, where the in-pass occurrence count already keeps the pair
- * distinct, and touching the ids would only churn the prefix cache. Do not broaden
- * this to all bases.
+ * - the folded original's ECHO: stateless hosts resend the full raw history
+ *   every turn, so the original re-derives exactly the covered bare id. It was
+ *   present in the previous pass's inbound (`state.lastPassIds`) and must KEEP
+ *   its id — prune then drops it (it is already summarized) and the first-user
+ *   pin keeps a stable ref. Renumbering it (#459) minted a fresh ref and, worse,
+ *   made it escape prune's covered check (baseIdOf strips only "#tail", not
+ *   "_n"), so folded content rejoined the wire beside its summary every turn.
+ * - a genuinely NEW identical instance: absent from the previous pass, present
+ *   now. Renumber it to the first "_k" no folded copy claims so assign-refs
+ *   mints a distinct ref and prune keeps it (#1476's starvation fix).
  *
- * For each conflicting root, live instances are renumbered in arrival order to
- * root_1, root_2, … skipping any number a folded copy already claims. Deterministic
- * for a fixed (state, body): a live message whose root stays covered keeps the same
- * _k every turn (prefix-cache stable) and shifts only when a newer identical
- * instance joins the pass.
+ * Ids that are not exactly covered (converter-numbered "_1" second-occurrence
+ * forms of a covered root, non-h_ ids, roots without a covered copy) keep the
+ * converter's numbering untouched — renumbering those would only churn the
+ * prefix cache.
  */
 export function remintCoveredLiveIds(
   messages: CoreMessage[],
   state: CompressionState,
 ): CoreMessage[] {
-  const covered = new Set<string>();
-  for (const id of coveredMessageIds(state)) covered.add(baseIdOf(id));
-  if (covered.size === 0) return messages;
+  const coveredBases = new Set<string>();
+  for (const id of coveredMessageIds(state)) coveredBases.add(baseIdOf(id));
+  if (coveredBases.size === 0) return messages;
+  // Pre-feature persisted state has no prior-pass snapshot: fall back to
+  // renumber-nothing (0.0.95 semantics) for this one pass. The snapshot is
+  // written below in processTurn, so the next pass discriminates correctly.
+  if (!state.lastPassIds) return messages;
+  const prior = new Set(state.lastPassIds);
 
   const groups = new Map<string, number[]>();
   for (let i = 0; i < messages.length; i++) {
@@ -62,15 +64,35 @@ export function remintCoveredLiveIds(
   for (const [root, idxs] of groups) {
     // Only renumber when some live instance claims an id a folded copy owns;
     // otherwise the converter's numbering is already collision-free here.
-    const conflict = idxs.some((i) => covered.has(baseIdOf(messages[i]!.id)));
+    const conflict = idxs.some((i) =>
+      coveredBases.has(baseIdOf(messages[i]!.id)),
+    );
     if (!conflict) continue;
+    // Renumbered ids must dodge both folded-claimed numbers and ids other
+    // live instances of this root already hold (untouched ones keep theirs).
+    const liveIds = new Set(idxs.map((i) => baseIdOf(messages[i]!.id)));
     let k = 1;
     for (const i of idxs) {
-      while (covered.has(`${root}_${k}`)) k++;
       const id = messages[i]!.id;
+      const exactCovered = coveredBases.has(baseIdOf(id));
+      // Only a genuinely new instance is renumbered: its exact id is claimed
+      // by a folded copy AND it was absent from the previous pass. Everything
+      // else — the folded original's own resend (prior.has), converter-numbered
+      // "_n" forms that no folded copy claims, the live un-folded bare — keeps
+      // the converter's numbering; renumbering those would only churn the
+      // prefix cache and let summarized content escape prune (#462).
+      if (!exactCovered || prior.has(id)) continue;
+      while (
+        coveredBases.has(`${root}_${k}`) ||
+        liveIds.has(`${root}_${k}`) ||
+        prior.has(`${root}_${k}`)
+      )
+        k++;
       const hash = id.indexOf("#");
       const tail = hash > 0 ? id.slice(hash) : "";
-      next[i] = { ...messages[i]!, id: `${root}_${k}${tail}` };
+      const minted = `${root}_${k}${tail}`;
+      liveIds.add(baseIdOf(minted));
+      next[i] = { ...messages[i]!, id: minted };
       k++;
       changed = true;
     }

src/types.ts

@@ -279,6 +279,16 @@ export interface CompressionState {
    *  this set and keeps the clear "cannot be anchored" error. Optional for
    *  pre-feature persisted states. */
   hiddenOrphanRefs?: string[];
+  /** Exact inbound ids of the PREVIOUS processTurn pass (pre-pipeline,
+   * converter-derived). Continuity signal for reconcile-live-ids (#462): an
+   * id that is covered by a fold but was present in the prior pass is the
+   * folded original's RESEND (stateless hosts resend full history every
+   * turn) and must keep its id so prune drops it; the same id absent from
+   * the prior pass is a genuinely new identical instance and gets renumbered
+   * (#1476). Optional: pre-feature persisted states lack it — one pass of
+   * 0.0.95 semantics (renumber nothing), self-heals once written. Written
+   * every processTurn. */
+  lastPassIds?: string[];
   nextBlockId: number;
   nextRunId: number;
 }

tests/issue1476-reid.test.ts

@@ -37,6 +37,7 @@ function folded(ids: string[]): CompressionBlock {
 test("re-mints a live message whose id collides with a folded copy (#1476)", () => {
   const state = createInitialState();
   state.blocks.push(folded([HASH]));
+  state.lastPassIds = [];
   const out = remintCoveredLiveIds([msg(HASH, "都按推荐来")], state);
   assert.equal(out[0].id, `${HASH}_1`);
 });
@@ -52,29 +53,45 @@ test("leaves numbering untouched when the base is not folded (incident 3)", () =
   assert.equal(out, msgs);
 });
 
-test("renumbers every live instance of a conflicting root", () => {
+test("renumbers only the colliding new instance, keeping stable ids (#462)", () => {
   const state = createInitialState();
   state.blocks.push(folded([HASH]));
+  state.lastPassIds = [`${HASH}_1`];
+  // The bare instance is genuinely new (absent last pass); HASH_1 is not
+  // exactly covered, so it keeps the converter's numbering. The renumbered
+  // instance dodges the number the live HASH_1 already holds.
   const out = remintCoveredLiveIds(
     [msg(HASH, "a"), msg(`${HASH}_1`, "b")],
     state,
   );
   assert.deepEqual(
     out.map((m) => m.id),
-    [`${HASH}_1`, `${HASH}_2`],
+    [`${HASH}_2`, `${HASH}_1`],
   );
 });
 
 test("skips instance numbers already claimed by folded copies", () => {
   const state = createInitialState();
   state.blocks.push(folded([HASH, `${HASH}_1`]));
+  state.lastPassIds = [];
+  const out = remintCoveredLiveIds([msg(HASH, "a")], state);
+  assert.equal(out[0].id, `${HASH}_2`);
+});
+
+test("renumber skips numbers claimed by the previous pass inbound (#462)", () => {
+  const state = createInitialState();
+  state.blocks.push(folded([HASH]));
+  // HASH_1 was live in the previous pass (ref'd there) but is no longer in the
+  // resend set; re-minting it would alias its stale ref onto the new instance.
+  state.lastPassIds = [`${HASH}_1`];
   const out = remintCoveredLiveIds([msg(HASH, "a")], state);
   assert.equal(out[0].id, `${HASH}_2`);
 });
 
 test("preserves the sub-id projection tail", () => {
   const state = createInitialState();
   state.blocks.push(folded([HASH]));
+  state.lastPassIds = [];
   const out = remintCoveredLiveIds([msg(`${HASH}#sub`, "a")], state);
   assert.equal(out[0].id, `${HASH}_1#sub`);
 });
@@ -95,31 +112,47 @@ test("ignores non-content-hash ids", () => {
 test("is deterministic for a fixed (state, body)", () => {
   const state = createInitialState();
   state.blocks.push(folded([HASH]));
+  state.lastPassIds = [];
   const a = remintCoveredLiveIds([msg(HASH, "x")], state).map((m) => m.id);
   const b = remintCoveredLiveIds([msg(HASH, "x")], state).map((m) => m.id);
   assert.deepEqual(a, b);
 });
 
-test("fresh later user turn survives prune with a distinct id + ref (#1476 e2e)", () => {
-  const core = createCore();
+function seededFoldState(): ReturnType<typeof createInitialState> {
   const state = createInitialState();
   // Earlier "都按推荐来" was ref'd (m00001) then folded into b1 (off the wire).
   state.messageRefs.byRaw[HASH] = "m00001";
   state.messageRefs.byRef["m00001"] = HASH;
   state.blocks.push(folded([HASH]));
-  // Re-sent history: an earlier user turn (firstUserIndex), the rendered summary,
-  // then the fresh resend whose bare id collides with the folded original.
-  const input = [
-    msg(EARLY, "帮我看看这段代码"),
-    msg("acp_summary_b1", "[Compressed conversation section] …", "system"),
-    msg(HASH, "都按推荐来"),
-  ];
-  const result = core.processTurn({
-    messages: input,
+  return state;
+}
+
+test("fresh later user turn survives prune with a distinct id + ref (#1476 e2e)", () => {
+  const core = createCore();
+  const state = seededFoldState();
+  // Pass A: history WITHOUT any HASH instance — establishes the prior-pass
+  // snapshot the next pass discriminates against.
+  const passA = core.processTurn({
+    messages: [
+      msg(EARLY, "帮我看看这段代码"),
+      msg("acp_summary_b1", "[Compressed conversation section] …", "system"),
+    ],
     state,
     config: defaultConfig(100000),
     tokenCount: 300,
   });
+  // Pass B: the fresh resend whose bare id collides with the folded original
+  // and was absent from pass A.
+  const result = core.processTurn({
+    messages: [
+      msg(EARLY, "帮我看看这段代码"),
+      msg("acp_summary_b1", "[Compressed conversation section] …", "system"),
+      msg(HASH, "都按推荐来"),
+    ],
+    state: passA.state,
+    config: defaultConfig(100000),
+    tokenCount: 300,
+  });
   const fresh = result.messages.find((m) => m.id === `${HASH}_1`);
   assert.ok(fresh, "fresh user turn must survive prune under a distinct id");
   assert.match(fresh!.text ?? "", /都按推荐来/);
@@ -132,3 +165,88 @@ test("fresh later user turn survives prune with a distinct id + ref (#1476 e2e)"
     "folded bare id must not be re-issued",
   );
 });
+
+test("post-fold resend of the folded original stays covered and pruned (#462 e2e)", () => {
+  const core = createCore();
+  const state = seededFoldState();
+  // Pass A: the original "都按推荐来" still live (pre-fold shape).
+  const passA = core.processTurn({
+    messages: [msg(EARLY, "帮我看看这段代码"), msg(HASH, "都按推荐来")],
+    state,
+    config: defaultConfig(100000),
+    tokenCount: 300,
+  });
+  // Pass B: the stateless host resends the SAME raw history. This is the
+  // folded original's echo — #459 renumbered it, prune then missed it, and
+  // the summarized content re-inflated the wire every turn.
+  const result = core.processTurn({
+    messages: [msg(EARLY, "帮我看看这段代码"), msg(HASH, "都按推荐来")],
+    state: passA.state,
+    config: defaultConfig(100000),
+    tokenCount: 300,
+  });
+  assert.ok(
+    !result.messages.some((m) => m.id === HASH || m.id === `${HASH}_1`),
+    "echo of the folded original must stay pruned, not renumbered back onto the wire",
+  );
+  assert.ok(
+    !result.state.messageRefs.byRaw[`${HASH}_1`],
+    "no fresh ref may be minted for the echo",
+  );
+  assert.equal(
+    result.state.messageRefs.byRaw[HASH],
+    "m00001",
+    "the folded original keeps its original ref (no churn)",
+  );
+});
+
+test("first-user pinned echo keeps its id and ref across post-fold passes (#462 e2e)", () => {
+  const core = createCore();
+  const state = createInitialState();
+  state.messageRefs.byRaw[HASH] = "m00001";
+  state.messageRefs.byRef["m00001"] = HASH;
+  state.blocks.push(folded([HASH]));
+  const history = [msg(HASH, "开场白"), msg(EARLY, "后续")];
+  const passA = core.processTurn({
+    messages: history,
+    state,
+    config: defaultConfig(100000),
+    tokenCount: 300,
+  });
+  const passB = core.processTurn({
+    messages: history,
+    state: passA.state,
+    config: defaultConfig(100000),
+    tokenCount: 300,
+  });
+  const pinned = passB.messages.find((m) => (m.text ?? "").includes("开场白"));
+  assert.ok(pinned, "pinned first user message stays on the wire");
+  assert.equal(pinned.id, HASH, "pinned first user message keeps its id");
+  assert.equal(
+    passB.state.messageRefs.byRaw[HASH],
+    "m00001",
+    "pinned first user message keeps its ref (no per-turn churn)",
+  );
+});
+
+test("processTurn writes this pass's inbound ids into state.lastPassIds (#462)", () => {
+  const core = createCore();
+  const out = core.processTurn({
+    messages: [msg(EARLY, "帮我看看这段代码"), msg(HASH, "都按推荐来")],
+    state: createInitialState(),
+    config: defaultConfig(100000),
+    tokenCount: 300,
+  });
+  assert.deepEqual(out.state.lastPassIds, [EARLY, HASH]);
+});
+
+test("missing lastPassIds falls back to renumber-nothing for one pass (#462)", () => {
+  const state = seededFoldState();
+  const out = remintCoveredLiveIds([msg(HASH, "都按推荐来")], state);
+  assert.equal(
+    out[0].id,
+    HASH,
+    "0.0.95 semantics: untouched without a snapshot",
+  );
+  assert.equal(out.length, 1);
+});