fix(recommend): unify effective-gate units to real chars (CJK regression)

src/compress.ts

@@ -907,9 +907,12 @@ function resolveAdaptiveGrowth(
 }
 
 /** Compressible amount for each tier. T1 = EFFECTIVE merged-range tokens —
- *  only ranges whose `tokens*4 >= minCompressRange` count (avoids inflation
- *  from fragmentation); T2 = total summary tokens of all active tier-1 blocks;
- *  T3 = total summary tokens of all active tier-2 blocks. */
+ *  only ranges whose real char count >= minCompressRange count (avoids
+ *  inflation from fragmentation; matches the apply-side gate, which counts
+ *  raw `msg.text.length`, so a nudge never offers a range the kernel would
+ *  atomically reject — see CompressibleRange.chars); T2 = total summary
+ *  tokens of all active tier-1 blocks; T3 = total summary tokens of all
+ *  active tier-2 blocks. */
 function pendingByTier(
   state: CompressionState,
   recommendation: Recommendation | undefined,
@@ -920,7 +923,7 @@ function pendingByTier(
   const merged = recommendation?.recommendedRanges ?? [];
   const effective =
     minCompressRange > 0
-      ? merged.filter((r) => r.tokens * 4 >= minCompressRange)
+      ? merged.filter((r) => (r.chars ?? r.tokens * 4) >= minCompressRange)
       : merged;
   out[1] = { pending: effective.reduce((s, r) => s + r.tokens, 0), targetBlocks: [] };
   const active = activeBlocks(state);
@@ -992,7 +995,7 @@ function decideNudge(input: NudgeInput): NudgeDecision {
   if (pressure) {
     // High pressure: pick the tier with the MAX pending so pressure can route
     // to distillation when that reclaims the most tokens. Gated on effective
-    // pending (tokens*4 >= minCompressRange for T1) so we never offer ranges
+    // pending (real chars >= minCompressRange for T1) so we never offer ranges
     // the kernel would atomically reject. emergency vs over-limit only
     // changes the reason label/voice; truncate.threshold remains the
     // independent last resort when there is genuinely nothing to compress.

src/recommend.ts

@@ -155,6 +155,7 @@ export function buildCompressibleRanges(
     ref: string;
     refNum: number;
     tokens: number;
+    chars: number;
     isTool: boolean;
     isUser: boolean;
   }[] = [];
@@ -194,6 +195,7 @@ export function buildCompressibleRanges(
       ref,
       refNum: rn,
       tokens: countTokens(msg.text ?? ""),
+      chars: (msg.text ?? "").length,
       isTool: isToolMessage(msg),
       isUser: msg.role === "user",
     });
@@ -222,13 +224,15 @@ export function buildCompressibleRanges(
         endRef: info.ref,
         count: 1,
         tokens: info.tokens,
+        chars: info.chars,
         toolPct: info.isTool ? 100 : 0,
         textPct: info.isTool ? 0 : 100,
       };
     } else {
       cur.endRef = info.ref;
       cur.count++;
       cur.tokens += info.tokens;
+      cur.chars = (cur.chars ?? 0) + info.chars;
       if (info.isTool) {
         cur.toolPct = Math.round((cur.toolPct * (cur.count - 1) + 100) / cur.count);
       } else {
@@ -281,6 +285,7 @@ function mergeBatch(batch: CompressibleRange[]): CompressibleRange {
   const last = batch[batch.length - 1]!;
   const count = batch.reduce((s, r) => s + r.count, 0);
   const tokens = batch.reduce((s, r) => s + r.tokens, 0);
+  const chars = batch.reduce((s, r) => s + rangeChars(r), 0);
   const toolPct = Math.round(
     batch.reduce((s, r) => s + r.toolPct * r.count, 0) / count,
   );
@@ -289,6 +294,7 @@ function mergeBatch(batch: CompressibleRange[]): CompressibleRange {
     endRef: last.endRef,
     count,
     tokens,
+    chars,
     toolPct,
     textPct: 100 - toolPct,
   };
@@ -298,19 +304,36 @@ function mergeBatch(batch: CompressibleRange[]): CompressibleRange {
   return merged;
 }
 
+/** Effective size of a range in characters — the unit the apply-side
+ *  minCompressRange gate uses. Falls back to the historical tokens*4
+ *  estimate only for hand-built ranges that predate the `chars` field. */
+function rangeChars(r: CompressibleRange): number {
+  return r.chars ?? r.tokens * 4;
+}
+
+/** Merge adjacent ranges into batches that clear `minChars` of REAL text —
+ *  the same accounting `applyCompression` uses — so a recommended range is
+ *  never below the threshold the kernel would atomically reject. Batching by
+ *  token estimates (tokens*4) instead broke whenever the host injected a
+ *  tokenizer where tokens != chars/4 (CJK-aware estimators are ~1:1, so
+ *  tokens*4 overestimated size ~4x and nudge recommended ranges the apply
+ *  side then refused). A sub-threshold tail batch is still emitted — callers
+ *  filter by effectiveness separately (see pendingByTier). */
 export function mergeRangesToThreshold(
   ranges: CompressibleRange[],
   minChars: number,
 ): CompressibleRange[] {
   if (minChars <= 0 || ranges.length === 0) return ranges;
   const result: CompressibleRange[] = [];
   let batch: CompressibleRange[] = [];
+  let batchChars = 0;
   for (const r of ranges) {
     batch.push(r);
-    const batchTokens = batch.reduce((s, x) => s + x.tokens, 0);
-    if (batchTokens * 4 >= minChars) {
+    batchChars += rangeChars(r);
+    if (batchChars >= minChars) {
       result.push(mergeBatch(batch));
       batch = [];
+      batchChars = 0;
     }
   }
   if (batch.length > 0) {

src/types.ts

@@ -159,6 +159,15 @@ export interface CompressibleRange {
   endRef: string;
   count: number;
   tokens: number;
+  /** Range size in characters (sum of message text lengths). The apply-side
+   *  minCompressRange gate counts raw `msg.text.length`, so recommend-side
+   *  gates must use this field — NOT `tokens` — or the two sides disagree
+   *  whenever a host injects a tokenizer where tokens != chars/4 (e.g. a
+   *  CJK-aware estimator: 1 token per char, making `tokens*4` a ~4x
+   *  overestimate). Always set on ranges produced by buildCompressibleRanges
+   *  / mergeRangesToThreshold; hand-built ranges without it fall back to the
+   *  historical tokens*4 estimate for backwards compat. */
+  chars?: number;
   toolPct: number;
   textPct: number;
   dangerous?: boolean;

tests/effective-gate-chars.test.ts

@@ -0,0 +1,132 @@
+/**
+ * Regression: recommend/nudge gates and the apply-side minCompressRange gate
+ * must use the SAME unit. The apply side counts raw characters
+ * (`msg.text.length`); the recommend side used to approximate chars with
+ * `tokens*4`, which only holds for the default chars/4 estimator. Hosts that
+ * inject a CJK-aware tokenizer (≈1 token per char) made `tokens*4` a ~4x
+ * overestimate, so nudge offered ranges the kernel then atomically rejected
+ * with "Total compressible content too small" — the exact failure mode the
+ * effective gate exists to prevent (see PR fixing #57/#70 regression).
+ */
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { createCore } from "../src/compress.js";
+import { createInitialState } from "../src/state.js";
+import type { Config, CoreMessage } from "../src/types.js";
+import { mergeRangesToThreshold } from "../src/recommend.js";
+import type { CompressibleRange } from "../src/types.js";
+
+function buildConfig(overrides: Partial<Config> = {}): Config {
+  return {
+    tiers: { enabled: true, tier2Trigger: 5, tier3Trigger: 10 },
+    nudge: {
+      maxContextLimitPct: 0.9,
+      minContextLimitPct: 0.45,
+      frequency: 1,
+      iterationThreshold: 15,
+      force: "soft",
+      growthRatio: 0.05,
+      growthFloor: 6000,
+      growthCap: 50000,
+      minGrowthFloor: 5000,
+      minGrowthRatio: 0.45,
+      emergencyThresholdPct: 0.98,
+    },
+    promotionThreshold: 5,
+    truncate: { threshold: 1 },
+    merge: { maxSummaryLength: 3000, minOldGenBlocks: 3 },
+    compress: { minCompressRange: 5000, maxSummaryLength: 3000, minSummaryLength: 100 },
+    protectedTools: [],
+    preserveRecentMessages: 0,
+    preserveRecentTokens: 0,
+    modelContextLimit: 100000,
+    ...overrides,
+  };
+}
+
+/** CJK-aware estimator: 1 token per char for CJK text (the kernel's own
+ *  estimateTokensFast behaves this way for CJK). Identity for our fixtures. */
+const cjkTokenizer = (text: string): number => text.length;
+
+function cjkMessages(charsPerMessage: number, count: number): CoreMessage[] {
+  return Array.from({ length: count }, (_, i) => ({
+    id: `raw-${i}`,
+    role: i % 2 === 0 ? "user" : "assistant",
+    contentType: "text",
+    text: "内容".repeat(charsPerMessage / 2),
+  }));
+}
+
+test("mergeRangesToThreshold: batches by real chars, not tokens*4 (CJK tokenizer)", () => {
+  // Two hand-built ranges WITHOUT chars exercise the legacy fallback;
+  // ranges WITH chars exercise the real accounting.
+  const legacyA: CompressibleRange = {
+    startRef: "m00001", endRef: "m00002", count: 2, tokens: 1250, toolPct: 0, textPct: 100,
+  };
+  const out = mergeRangesToThreshold([legacyA], 5000);
+  assert.equal(out.length, 1, "fallback tokens*4=5000 still clears the 5000 threshold");
+
+  // 3000 chars of CJK = 3000 tokens under a CJK tokenizer. tokens*4 = 12000
+  // used to "clear" 5000; real chars (3000) do not — tail stays sub-threshold
+  // and pendingByTier must not count it as effective.
+  const cjk: CompressibleRange = {
+    startRef: "m00001", endRef: "m00006", count: 6, tokens: 3000, chars: 3000,
+    toolPct: 0, textPct: 100,
+  };
+  assert.equal(cjk.chars < 5000, true);
+  assert.equal(cjk.tokens * 4 >= 5000, true, "pre-fix this range looked effective");
+});
+
+test("nudge: CJK session below minCompressRange chars is NOT offered (apply would reject)", () => {
+  const core = createCore({ countTokens: cjkTokenizer });
+  const config = buildConfig();
+  const messages = cjkMessages(500, 6); // 3000 chars total < 5000 min
+  let state = createInitialState();
+
+  state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state;
+  // usage 95% >= maxContextLimitPct 0.9 → pressure path
+  const turn = core.processTurn({ messages, state, config, tokenCount: 95000 });
+
+  assert.equal(turn.nudge.shouldInject, false, "3000 chars < minCompressRange 5000 — nudge must not offer it");
+  assert.match(
+    turn.nudge.reason,
+    /no tier has effective compressible content/,
+    `reason explains the suppression, got: ${turn.nudge.reason}`,
+  );
+
+  // The apply side agrees: the same range is atomically rejected.
+  const applied = core.applyCompression({
+    ranges: [{ startRef: "m00001", endRef: "m00006", summary: "s", topic: "t" }],
+    messages,
+    state: turn.state,
+    config,
+  });
+  assert.equal(applied.result.blocksCreated, 0);
+  assert.ok(
+    applied.result.errors.some((e) => e.includes("too small")),
+    `apply rejects with too-small gate, got: ${JSON.stringify(applied.result.errors)}`,
+  );
+});
+
+test("nudge: CJK session above minCompressRange chars IS offered (control)", () => {
+  const core = createCore({ countTokens: cjkTokenizer });
+  const config = buildConfig();
+  const messages = cjkMessages(1000, 6); // 6000 chars total >= 5000 min
+  let state = createInitialState();
+
+  state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state;
+  const turn = core.processTurn({ messages, state, config, tokenCount: 95000 });
+
+  assert.equal(turn.nudge.shouldInject, true, "6000 chars >= 5000 — effective T1 pending exists");
+  assert.match(turn.nudge.reason, /T1/);
+
+  // And the apply side accepts the same range — both gates agree on chars.
+  const applied = core.applyCompression({
+    ranges: [{ startRef: "m00001", endRef: "m00006", summary: "总结".repeat(60), topic: "t" }],
+    messages,
+    state: turn.state,
+    config,
+  });
+  assert.equal(applied.result.blocksCreated, 1, "apply accepts: 6000 real chars >= 5000");
+  assert.deepEqual(applied.result.errors, []);
+});