diff --git a/src/compress.ts b/src/compress.ts index ffb4994..51010f7 100644 --- a/src/compress.ts +++ b/src/compress.ts @@ -20,6 +20,7 @@ import { adjustBoundariesForToolPairs } from "./tool-pairs.js"; import { computeProtectedRefs, buildCompressibleRanges, + mergeRangesToThreshold, } from "./recommend.js"; import { runPipeline, @@ -403,7 +404,10 @@ const recommendNode: PipelineNode = { const nothingToCompress = contextRanges.compressible.length === 0; const recommendation: Recommendation = { contextRanges, - recommendedRanges: contextRanges.compressible, + recommendedRanges: mergeRangesToThreshold( + contextRanges.compressible, + ctx.config.compress.minCompressRange, + ), nothingToCompress, }; return { ...io, effects: { ...io.effects, recommendation } }; @@ -812,20 +816,23 @@ function resolveAdaptiveGrowth( ); } -/** Compressible amount for each tier — all tiers use the SAME definition - * ("how much can be compressed at this tier"), so they are handled by one - * unified loop. T1 = compressible raw-message tokens (excludes protected / - * already-covered); T2 = total summary tokens of all active tier-1 blocks; - * T3 = total summary tokens of all active tier-2 blocks. The "zero filter" - * for block tiers is simply that all blocks are compressible by default. */ +/** Compressible amount for each tier. T1 = EFFECTIVE merged-range tokens — + * only ranges whose `tokens*4 >= minCompressRange` count (avoids inflation + * from fragmentation); T2 = total summary tokens of all active tier-1 blocks; + * T3 = total summary tokens of all active tier-2 blocks. */ function pendingByTier( state: CompressionState, recommendation: Recommendation | undefined, countTokens: (t: string) => number, + minCompressRange: number, ): Record { const out: Record = {}; - const compressible = recommendation?.contextRanges.compressible ?? []; - out[1] = { pending: compressible.reduce((s, r) => s + r.tokens, 0), targetBlocks: [] }; + const merged = recommendation?.recommendedRanges ?? []; + const effective = + minCompressRange > 0 + ? merged.filter((r) => r.tokens * 4 >= minCompressRange) + : merged; + out[1] = { pending: effective.reduce((s, r) => s + r.tokens, 0), targetBlocks: [] }; const active = activeBlocks(state); const t1 = active.filter((b) => b.tier === 1); const t2 = active.filter((b) => b.tier === 2); @@ -866,35 +873,78 @@ function decideNudge(input: NudgeInput): NudgeDecision { const growthSinceReference = tokenCount - growthReference; const rec = recommendation; - const tiers = pendingByTier(state, rec, countTokens); + const tiers = pendingByTier( + state, + rec, + countTokens, + config.compress.minCompressRange, + ); - // Unified injection arbitration, priority T1 > T2 > T3 (ascending). Each - // tier has its OWN cadence (lastShownByTier) so firing one doesn't block - // another across turns; within a turn we inject at most the FIRST eligible - // tier and stop, which raises the shared baseline (lastNudgeShownTokens) so - // lower-priority tiers see a higher reference and naturally defer. + // Tier arbitration. Emergency (usage >= emergencyThresholdPct) ignores tier + // priority and picks the tier with the MAX pending. Non-emergency defaults to + // T1; T2 and T3 override when each crossed the shared 1.5x threshold AND + // exceeds the effective pending of every lower tier (T2 > T1 effective; + // T3 > T2 and > T1 effective). + const tier2Threshold = Math.round( + nudgeGrowthTokens * (config.nudge.tier2GrowthMultiplier ?? 1.5), + ); let injectedTier: CompressionTier | null = null; let injectedReason = ""; - // growth acts as the "has accumulated since baseline" signal; pending is the - // "there is enough compressible content" signal. Both must hold for a - // non-emergency injection. This keeps the first turn (growth 0) from firing - // immediately even if a lot is compressible — it just establishes baseline. const growthReady = growthSinceReference >= growthFloor; - if (!emergencyOverride && growthReady) { - for (const tier of [1, 2, 3] as const) { - if (!config.tiers.enabled && tier > 1) break; - const info = tiers[tier]; - if (!info || info.pending < nudgeGrowthTokens) continue; - const lastShown = state.nudge.lastShownByTier[tier] ?? 0; - // Per-tier cadence: even with growth, a tier that fired recently waits - // for its own cadence window. First time (lastShown===0) is always met. - const cadenceMet = lastShown === 0 || tokenCount - lastShown >= growthFloor; - if (!cadenceMet) continue; - injectedTier = tier; - injectedReason = tier === 1 - ? `T1 compressible ${info.pending} >= ${nudgeGrowthTokens}, growth ${growthSinceReference}, usage ${Math.round(usage * 100)}%` - : `T${tier} distill ready: ${info.targetBlocks.length} tier-${tier - 1} blocks (${info.pending} tokens) >= ${nudgeGrowthTokens}, usage ${Math.round(usage * 100)}%`; - break; + const t1Eff = tiers[1]?.pending ?? 0; + const t2Pen = tiers[2]?.pending ?? 0; + const t3Pen = tiers[3]?.pending ?? 0; + + if (emergencyOverride) { + const candidates: CompressionTier[] = [1]; + if (config.tiers.enabled) { + candidates.push(2, 3); + } + let best: CompressionTier | null = null; + let bestPending = 0; + for (const t of candidates) { + const p = tiers[t]?.pending ?? 0; + if (p > bestPending) { + bestPending = p; + best = t; + } + } + if (best !== null) { + injectedTier = best; + injectedReason = + best === 1 + ? `EMERGENCY T1: max effective pending ${bestPending}, usage ${Math.round(usage * 100)}%` + : `EMERGENCY T${best} distill: max pending ${bestPending} (T1 effective ${t1Eff}, T2 ${t2Pen}, T3 ${t3Pen}), usage ${Math.round(usage * 100)}%`; + } + } else if (growthReady) { + if (t1Eff >= nudgeGrowthTokens) { + injectedTier = 1; + injectedReason = `T1 effective ${t1Eff} >= ${nudgeGrowthTokens}, growth ${growthSinceReference}, usage ${Math.round(usage * 100)}%`; + } else if ( + config.tiers.enabled && + t2Pen >= tier2Threshold && + t2Pen > t1Eff + ) { + const lastShown = state.nudge.lastShownByTier[2] ?? 0; + const cadenceMet = + lastShown === 0 || tokenCount - lastShown >= growthFloor; + if (cadenceMet) { + injectedTier = 2; + injectedReason = `T2 distill ready: ${tiers[2]!.targetBlocks.length} tier-1 blocks (${t2Pen} tokens) >= ${tier2Threshold} (1.5x) and > T1 effective ${t1Eff}, usage ${Math.round(usage * 100)}%`; + } + } else if ( + config.tiers.enabled && + t3Pen >= tier2Threshold && + t3Pen > t2Pen && + t3Pen > t1Eff + ) { + const lastShown = state.nudge.lastShownByTier[3] ?? 0; + const cadenceMet = + lastShown === 0 || tokenCount - lastShown >= growthFloor; + if (cadenceMet) { + injectedTier = 3; + injectedReason = `T3 condense ready: ${tiers[3]!.targetBlocks.length} tier-2 blocks (${t3Pen} tokens) >= ${tier2Threshold} (1.5x) and > T2 ${t2Pen} and > T1 effective ${t1Eff}, usage ${Math.round(usage * 100)}%`; + } } } diff --git a/src/config.ts b/src/config.ts index e32cda7..c218685 100644 --- a/src/config.ts +++ b/src/config.ts @@ -18,6 +18,7 @@ export function defaultConfig( minGrowthFloor: 20000, minGrowthRatio: 0.45, emergencyThresholdPct: 0.80, + tier2GrowthMultiplier: 1.5, }, promotionThreshold: 5, truncate: { threshold: 1 }, diff --git a/src/nudge-text.ts b/src/nudge-text.ts index 25f7b50..cf61cd2 100644 --- a/src/nudge-text.ts +++ b/src/nudge-text.ts @@ -118,6 +118,7 @@ export function formatRanges(compressible: CompressibleRange[], protectedRanges: export function renderNudgeText(decision: NudgeDecision): RenderedNudge { const breakdownStr = formatBreakdown(decision.contextBreakdown); const rangesStr = formatRanges(decision.compressibleRanges, decision.protectedRanges ?? []); + const isEmergency = !!decision.breakdown?.emergencyOverride; if (decision.tier !== null && decision.tier >= 2) { const isT2 = decision.tier === 2; @@ -125,14 +126,19 @@ export function renderNudgeText(decision: NudgeDecision): RenderedNudge { const blockList = formatTierTargetBlocks(targets); const startId = targets[0]?.blockId ?? "b1"; const endId = targets[targets.length - 1]?.blockId ?? "b5"; + const header = isEmergency ? EMERGENCY_HEADER : EFFICIENCY_NOTE; + const voice: NudgeVoice = isEmergency ? "emergency" : "gentle"; + const triggerLine = isEmergency + ? `[EMERGENCY — TIER ${decision.tier} ${isT2 ? "DISTILLATION" : "CONDENSATION"}] Context limit reached — distill NOW into a denser summary to reclaim tokens.` + : `[TIER ${decision.tier} ${isT2 ? "DISTILLATION" : "CONDENSATION"} TRIGGER]`; return { - voice: "gentle", + voice, text: [ - EFFICIENCY_NOTE, + header, "", breakdownStr, "", - `[TIER ${decision.tier} ${isT2 ? "DISTILLATION" : "CONDENSATION"} TRIGGER]`, + triggerLine, isT2 ? `Your tier-1 compression summaries have accumulated. Distill them into a single denser tier-2 summary. Use block IDs as boundaries (startId and endId as bN). Any raw (uncompressed) messages sitting between the boundary blocks are absorbed into the tier-2 block as well — apply HOW TO COMPRESS to those raw messages and the TIER 2 distillation rules to the existing summaries, so the whole span is covered and nothing is lost.` : `Your tier-2 compression summaries have accumulated. Condense them further into a tier-3 ultra-condensed summary. Use block IDs as boundaries (startId and endId as bN). Any raw (uncompressed) messages sitting between the boundary blocks are absorbed into the tier-3 block as well — apply HOW TO COMPRESS to those raw messages and the TIER 3 condensation rules to the existing summaries, so the whole span is covered and nothing is lost.`, @@ -146,8 +152,6 @@ export function renderNudgeText(decision: NudgeDecision): RenderedNudge { }; } - const isEmergency = !!decision.breakdown?.emergencyOverride; - if (isEmergency) { return { voice: "emergency", diff --git a/src/recommend.ts b/src/recommend.ts index 09bb016..56dabd0 100644 --- a/src/recommend.ts +++ b/src/recommend.ts @@ -273,3 +273,46 @@ export function buildCompressibleRanges( protected: protectedRanges, }; } + +function mergeBatch(batch: CompressibleRange[]): CompressibleRange { + const first = batch[0]!; + const last = batch[batch.length - 1]!; + const count = batch.reduce((s, r) => s + r.count, 0); + const tokens = batch.reduce((s, r) => s + r.tokens, 0); + const toolPct = Math.round( + batch.reduce((s, r) => s + r.toolPct * r.count, 0) / count, + ); + const merged: CompressibleRange = { + startRef: first.startRef, + endRef: last.endRef, + count, + tokens, + toolPct, + textPct: 100 - toolPct, + }; + if (batch.some((r) => r.dangerous === true)) { + merged.dangerous = true; + } + return merged; +} + +export function mergeRangesToThreshold( + ranges: CompressibleRange[], + minChars: number, +): CompressibleRange[] { + if (minChars <= 0 || ranges.length === 0) return ranges; + const result: CompressibleRange[] = []; + let batch: CompressibleRange[] = []; + for (const r of ranges) { + batch.push(r); + const batchTokens = batch.reduce((s, x) => s + x.tokens, 0); + if (batchTokens * 4 >= minChars) { + result.push(mergeBatch(batch)); + batch = []; + } + } + if (batch.length > 0) { + result.push(mergeBatch(batch)); + } + return result; +} diff --git a/src/types.ts b/src/types.ts index e72c60b..73c5872 100644 --- a/src/types.ts +++ b/src/types.ts @@ -92,6 +92,10 @@ export interface NudgeConfig { minGrowthRatio: number; /** Emergency override: always nudge when usage ≥ this fraction. Default 0.98 (98%). */ emergencyThresholdPct: number; + /** Growth multiplier for the tier-2 trigger: T2 distillation fires when T2 + * pending ≥ nudgeGrowthTokens × this multiplier AND T2 > T1 effective. + * Default 1.5. */ + tier2GrowthMultiplier: number; } export interface TruncateConfig { diff --git a/tests/nudge-text.test.ts b/tests/nudge-text.test.ts index 0a30f91..9d38428 100644 --- a/tests/nudge-text.test.ts +++ b/tests/nudge-text.test.ts @@ -85,6 +85,30 @@ test("tier-3 condensation: guidance warns raw messages in span are absorbed", () assert.ok(result.text.includes("HOW TO COMPRESS"), "should direct raw messages to HOW TO COMPRESS rules"); }); +test("emergency + tier 2: emergency voice with distillation guidance", () => { + const result = renderNudgeText( + makeDecision({ tier: 2, breakdown: { emergencyOverride: 1 } }), + ); + assert.equal(result.voice, "emergency"); + assert.ok( + result.text.toLowerCase().includes("distill"), + "should still carry distillation guidance", + ); + assert.ok(result.text.includes("TIER 2"), "should still name the tier"); +}); + +test("emergency + tier 3: emergency voice with condensation guidance", () => { + const result = renderNudgeText( + makeDecision({ tier: 3, breakdown: { emergencyOverride: 1 } }), + ); + assert.equal(result.voice, "emergency"); + assert.ok( + result.text.toLowerCase().includes("condense"), + "should still carry condensation guidance", + ); + assert.ok(result.text.includes("TIER 3"), "should still name the tier"); +}); + test("both modes include compressible ranges", () => { const gentle = renderNudgeText(makeDecision({ contextUsage: 0.5 })); const emergency = renderNudgeText( diff --git a/tests/nudge.test.ts b/tests/nudge.test.ts index 658e50c..fd863be 100644 --- a/tests/nudge.test.ts +++ b/tests/nudge.test.ts @@ -297,3 +297,254 @@ test("nudge: compressible ranges exclude messages covered by active blocks", () ); } }); + +function t1Blocks(anchorIds: string[][], summaryChars: number) { + return anchorIds.map((effIds, i) => ({ + blockId: `b${i + 1}`, + runId: "r1", + tier: 1 as const, + summary: "x".repeat(summaryChars), + directMessageIds: effIds, + effectiveMessageIds: effIds, + directBlockIds: [], + compressedTokens: summaryChars, + createdAt: Date.now(), + survivedCount: 0, + generation: "young" as const, + active: true, + })); +} + +function t2Blocks(count: number, summaryChars: number, effIds: string[]) { + return Array.from({ length: count }, (_, i) => ({ + blockId: `t2-${i + 1}`, + runId: "r2", + tier: 2 as const, + summary: "x".repeat(summaryChars), + directMessageIds: [], + // syncBlocks deactivates blocks whose effectiveMessageIds are all absent + // from the message list — tier-2 blocks must carry transitive ids to stay + // active. directBlockIds point to fake sources (not real block ids) so the + // tier-1 source blocks are NOT marked consumed/deactivated. + effectiveMessageIds: [...effIds], + directBlockIds: [`t2-src-${i + 1}`], + compressedTokens: summaryChars, + createdAt: Date.now(), + survivedCount: 0, + generation: "young" as const, + active: true, + })); +} + +test("arbitration: non-emergency T1 effective >= threshold → tier 1", () => { + const core = createCore(); + // minCompressRange > 0 so merge + effective filter actually engage — without + // this, pendingByTier bypasses the filter (minCompressRange > 0 ? filter : merged) + // and the test exercises the OLD T1 definition. + const config = buildConfig({ compress: { minCompressRange: 5000, maxSummaryLength: 0, minSummaryLength: 0 } }); + const messages = makeMessages(10); + let state = createInitialState(); + state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; + const turn = core.processTurn({ messages, state, config, tokenCount: 55000 }); + assert.equal(turn.nudge.shouldInject, true); + assert.equal(turn.nudge.tier, 1, "T1 effective (~50K) >= 6000 → tier 1, no T2 blocks present"); +}); + +test("arbitration: non-emergency T2 >= 1.5x threshold AND > T1 effective → tier 2", () => { + const core = createCore(); + // minCompressRange > 0 so the effective filter engages (see test above). + const config = buildConfig({ + compress: { minCompressRange: 5000, maxSummaryLength: 0, minSummaryLength: 0 }, + preserveRecentMessages: 9, + }); + const messages = makeMessages(10); + let state = createInitialState(); + state = core.processTurn({ messages, state, config, tokenCount: 50000 }).state; + // Two T1 blocks anchored to real messages (m1, m2) with ~6K-token summaries + // each → T2 ~12K >= 9000 (1.5x); T1 effective is ~0 (preserve + blocks cover + // all visible messages). + state = { ...state, blocks: t1Blocks([["m1"], ["m2"]], 24000) }; + const turn = core.processTurn({ messages, state, config, tokenCount: 60000 }); + assert.equal(turn.nudge.shouldInject, true); + assert.equal( + turn.nudge.tier, + 2, + "T2 ~12K >= 9000 (1.5x) and > T1 effective → tier 2", + ); +}); + +test("arbitration: non-emergency T2 large but T1 effective >= threshold → tier 1 wins", () => { + const core = createCore(); + // minCompressRange > 0 so the effective filter engages. + const config = buildConfig({ compress: { minCompressRange: 5000, maxSummaryLength: 0, minSummaryLength: 0 } }); + const messages = makeMessages(10); + let state = createInitialState(); + state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; + // Large T2 pending (blocks anchored to m0, m1), but T1 effective (~40K from + // m2..m9) still dominates the threshold. + state = { ...state, blocks: t1Blocks([["m0"], ["m1"]], 40000) }; + const turn = core.processTurn({ messages, state, config, tokenCount: 55000 }); + assert.equal(turn.nudge.tier, 1, "T1 effective >= threshold wins even when T2 is large"); +}); + +test("arbitration: emergency T2 > T1 effective → tier 2 with emergencyOverride", () => { + const core = createCore(); + const config = buildConfig(); + const messages = makeMessages(10); + let state = createInitialState(); + state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; + // One T1 block covering ALL messages → compressible empty (T1 effective 0), + // with a large summary → T2 pending ~10K dominates. + state = { + ...state, + blocks: [ + { + blockId: "b1", + runId: "r1", + tier: 1 as const, + summary: "x".repeat(40000), + directMessageIds: messages.map((m) => m.id), + effectiveMessageIds: messages.map((m) => m.id), + directBlockIds: [], + compressedTokens: 50000, + createdAt: Date.now(), + survivedCount: 0, + generation: "young" as const, + active: true, + }, + ], + }; + const turn = core.processTurn({ messages, state, config, tokenCount: 99000 }); + assert.equal(turn.nudge.shouldInject, true); + assert.equal(turn.nudge.tier, 2, "emergency picks T2 (max pending) when T2 > T1 effective"); + assert.equal(turn.nudge.breakdown.emergencyOverride, 1); +}); + +test("arbitration: emergency T1 effective largest → tier 1", () => { + const core = createCore(); + const config = buildConfig(); + const messages = makeMessages(10); + let state = createInitialState(); + state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; + // No blocks → T2/T3 = 0; T1 effective ~50K is the max. + const turn = core.processTurn({ messages, state, config, tokenCount: 99000 }); + assert.equal(turn.nudge.shouldInject, true); + assert.equal(turn.nudge.tier, 1, "emergency picks T1 when its effective pending is max"); + assert.equal(turn.nudge.breakdown.emergencyOverride, 1); +}); + +test("arbitration: T2 boundary — must NOT fire when pending ∈ [nudgeGrowthTokens, 1.5×)", () => { + const core = createCore(); + // OLD code fired T2 at >= nudgeGrowthTokens; NEW requires >= 1.5× — this + // range (T2 pending ~7500 ∈ [6000, 9000)) proves the boundary. Diverges from + // master, which injected T2 here. + const config = buildConfig({ + compress: { minCompressRange: 5000, maxSummaryLength: 0, minSummaryLength: 0 }, + preserveRecentMessages: 10, + }); + const messages = makeMessages(10); + let state = createInitialState(); + state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; + // preserveRecentMessages:10 → no compressible msgs → T1 effective = 0. + // 1 T1 block, summary 'x'.repeat(30000) → 7500 tokens ∈ [6000, 9000). + state = { ...state, blocks: t1Blocks([["m0"]], 30000) }; + const turn = core.processTurn({ messages, state, config, tokenCount: 35000 }); + assert.equal( + turn.nudge.shouldInject, + false, + "T2 pending 7500 < 9000 (1.5×) and T1 effective 0 < 6000 → no injection", + ); +}); + +test("arbitration: non-emergency T3 >= 1.5× threshold AND > T2 AND > T1 effective → tier 3", () => { + const core = createCore(); + // Validates FIX 1: master had no T3 non-emergency branch → tier would be + // null here. NEW code injects T3. + const config = buildConfig({ + compress: { minCompressRange: 5000, maxSummaryLength: 0, minSummaryLength: 0 }, + preserveRecentMessages: 10, + }); + const messages = makeMessages(10); + let state = createInitialState(); + state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; + // T1 effective = 0 (preserveRecentMessages:10). + // T2 pending: 1 small T1 block, summary 4000 chars → 1000 tokens (< 9000). + // T3 pending: 2 T2 blocks × 20000-char summaries → 5000 tokens each = 10000 (>= 9000). + state = { + ...state, + blocks: [...t1Blocks([["m0"]], 4000), ...t2Blocks(2, 20000, ["m0"])], + }; + const turn = core.processTurn({ messages, state, config, tokenCount: 35000 }); + assert.equal(turn.nudge.shouldInject, true); + assert.equal( + turn.nudge.tier, + 3, + "T3 10000 >= 9000 (1.5×) and > T2 1000 and > T1 effective 0 → tier 3", + ); + assert.match(turn.nudge.reason ?? "", /T3 condense/); +}); + +test("arbitration: emergency argmax picks T3 when T3 > T2 > T1 effective", () => { + const core = createCore(); + const config = buildConfig({ + compress: { minCompressRange: 5000, maxSummaryLength: 0, minSummaryLength: 0 }, + }); + const messages = makeMessages(10); + let state = createInitialState(); + state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; + // T1 block covers ALL messages → compressible empty → T1 effective = 0. + // Its summary 'x'.repeat(20000) → 5000 tokens = T2 pending. + // 3 T2 blocks × 20000-char summaries → 5000 each = T3 pending 15000 (largest). + state = { + ...state, + blocks: [ + { + blockId: "cover", + runId: "r1", + tier: 1 as const, + summary: "x".repeat(20000), + directMessageIds: messages.map((m) => m.id), + effectiveMessageIds: messages.map((m) => m.id), + directBlockIds: [], + compressedTokens: 50000, + createdAt: Date.now(), + survivedCount: 0, + generation: "young" as const, + active: true, + }, + ...t2Blocks(3, 20000, messages.map((m) => m.id)), + ], + }; + const turn = core.processTurn({ messages, state, config, tokenCount: 99000 }); + assert.equal(turn.nudge.shouldInject, true); + assert.equal( + turn.nudge.tier, + 3, + "emergency argmax: T3 15000 > T2 5000 > T1 effective 0", + ); + assert.equal(turn.nudge.breakdown.emergencyOverride, 1); +}); + +test("arbitration: effective filter drops fragmented merge tail (pendingT1 < raw sum)", () => { + const core = createCore(); + // With minCompressRange > 0, pendingByTier applies the effective filter + // (merged ranges whose tokens*4 < minCompressRange are dropped). 10 short + // msgs (800 chars / 200 tokens each) form 3 groups: [m0..m3]=800, + // [m4..m7]=800, [m8..m9]=400 (raw sum 2000). merge → [{m0..m7}=1600, + // {m8..m9}=400]; filter drops the 400-token tail (1600 chars < 5000) → + // pendingT1 = 1600, NOT the raw 2000. + const config = buildConfig({ + compress: { minCompressRange: 5000, maxSummaryLength: 0, minSummaryLength: 0 }, + }); + const messages = Array.from({ length: 10 }, (_, i) => + textMessage(i % 2 === 0 ? "user" : "assistant", `m${i}`, "x".repeat(800)), + ); + let state = createInitialState(); + state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; + const turn = core.processTurn({ messages, state, config, tokenCount: 35000 }); + assert.equal( + turn.nudge.breakdown.pendingT1, + 1600, + "effective filter keeps only the merged m0..m7 range (1600 tokens); drops the m8..m9 tail", + ); +}); diff --git a/tests/recommend-merge.test.ts b/tests/recommend-merge.test.ts new file mode 100644 index 0000000..a899c00 --- /dev/null +++ b/tests/recommend-merge.test.ts @@ -0,0 +1,216 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mergeRangesToThreshold } from "../src/recommend.js"; +import type { CompressibleRange } from "../src/types.js"; + +function makeRange( + overrides: Partial & Pick, +): CompressibleRange { + return { + startRef: "m00001", + endRef: "m00005", + count: 1, + toolPct: 0, + textPct: 100, + ...overrides, + }; +} + +test("empty ranges → []", () => { + assert.deepEqual(mergeRangesToThreshold([], 5000), []); +}); + +test("minChars <= 0 → ranges unchanged (disabled)", () => { + const ranges = [ + makeRange({ tokens: 100, startRef: "m00001", endRef: "m00002" }), + makeRange({ tokens: 200, startRef: "m00003", endRef: "m00004" }), + ]; + assert.equal(mergeRangesToThreshold(ranges, 0), ranges); + assert.equal(mergeRangesToThreshold(ranges, -1), ranges); +}); + +test("single range already >= threshold → returned as one batch unchanged", () => { + const ranges = [ + makeRange({ + tokens: 1250, + count: 4, + toolPct: 50, + textPct: 50, + startRef: "m00001", + endRef: "m00010", + }), + ]; + const out = mergeRangesToThreshold(ranges, 5000); + assert.equal(out.length, 1); + assert.equal(out[0]!.tokens, 1250); + assert.equal(out[0]!.count, 4); + assert.equal(out[0]!.startRef, "m00001"); + assert.equal(out[0]!.endRef, "m00010"); +}); + +test("two small ranges whose sum >= threshold → ONE merged batch", () => { + const ranges = [ + makeRange({ + tokens: 600, + count: 2, + toolPct: 100, + textPct: 0, + startRef: "m00001", + endRef: "m00005", + }), + makeRange({ + tokens: 700, + count: 3, + toolPct: 0, + textPct: 100, + startRef: "m00010", + endRef: "m00020", + }), + ]; + const out = mergeRangesToThreshold(ranges, 5000); + assert.equal(out.length, 1, "should merge into a single batch"); + assert.equal(out[0]!.count, 5); + assert.equal(out[0]!.tokens, 1300); + assert.equal(out[0]!.startRef, "m00001"); + assert.equal(out[0]!.endRef, "m00020", "endRef = second child endRef"); +}); + +test("three ranges: first two >= threshold, third tiny → TWO batches (tail kept)", () => { + const ranges = [ + makeRange({ + tokens: 600, + count: 1, + startRef: "m00001", + endRef: "m00003", + }), + makeRange({ + tokens: 700, + count: 1, + startRef: "m00004", + endRef: "m00006", + }), + makeRange({ + tokens: 50, + count: 1, + startRef: "m00010", + endRef: "m00011", + }), + ]; + const out = mergeRangesToThreshold(ranges, 5000); + assert.equal(out.length, 2, "batch1 (>= threshold) + tail"); + assert.equal(out[0]!.tokens, 1300, "first batch is the merged first two"); + assert.equal(out[0]!.startRef, "m00001"); + assert.equal(out[0]!.endRef, "m00006"); + assert.equal(out[1]!.tokens, 50, "tail is the tiny third range"); + assert.equal(out[1]!.startRef, "m00010"); + assert.equal(out[1]!.endRef, "m00011"); +}); + +test("dangerous: true on a child propagates to merged batch", () => { + const ranges = [ + makeRange({ + tokens: 600, + count: 1, + startRef: "m00001", + endRef: "m00002", + }), + makeRange({ + tokens: 700, + count: 1, + startRef: "m00003", + endRef: "m00004", + dangerous: true, + }), + ]; + const out = mergeRangesToThreshold(ranges, 5000); + assert.equal(out.length, 1); + assert.equal(out[0]!.dangerous, true, "dangerous flag propagates"); +}); + +test("no dangerous children → merged batch omits dangerous", () => { + const ranges = [ + makeRange({ + tokens: 600, + count: 1, + startRef: "m00001", + endRef: "m00002", + }), + makeRange({ + tokens: 700, + count: 1, + startRef: "m00003", + endRef: "m00004", + }), + ]; + const out = mergeRangesToThreshold(ranges, 5000); + assert.equal(out.length, 1); + assert.equal(out[0]!.dangerous, undefined, "no dangerous flag when none set"); +}); + +test("count-weighted toolPct: A(count2,tool100) + B(count3,tool0) → 40 / 60", () => { + const ranges = [ + makeRange({ + tokens: 600, + count: 2, + toolPct: 100, + textPct: 0, + startRef: "m00001", + endRef: "m00002", + }), + makeRange({ + tokens: 700, + count: 3, + toolPct: 0, + textPct: 100, + startRef: "m00003", + endRef: "m00004", + }), + ]; + const out = mergeRangesToThreshold(ranges, 5000); + assert.equal(out.length, 1); + assert.equal( + out[0]!.toolPct, + 40, + "round((100×2 + 0×3)/5) = 40", + ); + assert.equal(out[0]!.textPct, 60, "textPct = 100 - toolPct"); +}); + +test("merges across ref gaps (non-adjacent refs)", () => { + const ranges = [ + makeRange({ + tokens: 600, + count: 1, + startRef: "m00001", + endRef: "m00002", + }), + makeRange({ + tokens: 700, + count: 1, + startRef: "m00050", + endRef: "m00060", + }), + ]; + const out = mergeRangesToThreshold(ranges, 5000); + assert.equal(out.length, 1, "non-adjacent refs still merge"); + assert.equal(out[0]!.startRef, "m00001"); + assert.equal(out[0]!.endRef, "m00060"); +}); + +test("all-tiny tail: three ranges each < threshold → ONE merged range via trailing flush", () => { + // Each range tokens=100 → 100*4=400 chars, never reaches 5000 mid-loop, so + // the batch accumulates all three and is emitted as one range by the + // trailing flush. The merged range spans r1.startRef..r3.endRef with + // count/tokens summed. + const ranges = [ + makeRange({ tokens: 100, count: 1, startRef: "m00001", endRef: "m00002" }), + makeRange({ tokens: 100, count: 1, startRef: "m00003", endRef: "m00004" }), + makeRange({ tokens: 100, count: 1, startRef: "m00005", endRef: "m00006" }), + ]; + const out = mergeRangesToThreshold(ranges, 5000); + assert.equal(out.length, 1, "trailing flush emits a single merged range"); + assert.equal(out[0]!.tokens, 300, "tokens summed"); + assert.equal(out[0]!.count, 3, "count summed"); + assert.equal(out[0]!.startRef, "m00001", "spans first child startRef"); + assert.equal(out[0]!.endRef, "m00006", "spans last child endRef"); +});