From 1f6c77a287eb33c6eaf855b1aa2c3800d57223cc Mon Sep 17 00:00:00 2001 From: ework-agent Date: Sat, 26 Sep 2026 02:16:45 +0800 Subject: [PATCH 1/3] chore: add prettier config and CI format gate --- .github/workflows/ci.yml | 11 +++++++++++ .prettierignore | 12 ++++++++++++ .prettierrc | 11 +++++++++++ 3 files changed, 34 insertions(+) create mode 100644 .prettierignore create mode 100644 .prettierrc diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d086fde..d1ab925 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,17 @@ jobs: - run: npm test - run: npm run build + format: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + registry-url: https://registry.npmjs.org + - run: npm ci + - run: npm run format:check + pr-validation: runs-on: ubuntu-latest if: github.event_name == 'pull_request' diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..96a6494 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,12 @@ +# Build output (prettier would reformat bundled/minified artifacts) +dist/ + +# npm-generated lockfile: npm rewrites it with its own formatting on +# dependency changes, so formatting it here causes permanent churn. +package-lock.json + +# Local scratch directory (never present in CI) +.tmp/ + +# Ignored by prettier by default; listed for clarity +node_modules/ diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..ac0fa89 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,11 @@ +{ + "printWidth": 80, + "tabWidth": 2, + "useTabs": false, + "semi": true, + "singleQuote": false, + "trailingComma": "all", + "bracketSpacing": true, + "arrowParens": "always", + "endOfLine": "lf" +} From dcd8b317378443df81f27efad3138c82e90f17a7 Mon Sep 17 00:00:00 2001 From: ework-daemon Date: Sat, 26 Sep 2026 18:26:49 +0800 Subject: [PATCH 2/3] chore(git): ignore .tmp/ local scratch dir --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 85f5ffa..5894506 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ node_modules/ dist/ tmp/ +.tmp/ *.log .DS_Store coverage/ From 53033aa5b2d7ae58ff883b568ea073ceb6385d78 Mon Sep 17 00:00:00 2001 From: ework-daemon Date: Sat, 26 Sep 2026 18:26:59 +0800 Subject: [PATCH 3/3] style: apply prettier defaults across tree --- .github/workflows/pr-artifact.yml | 208 +-- AGENTS.md | 46 +- AUTO-MERGE-GUARDRAILS.md | 10 +- DESIGN-prompts.md | 29 +- DESIGN.md | 295 ++-- PROVENANCE.md | 84 +- README.md | 73 +- bench/search/bench.ts | 41 +- bench/search/corpus.ts | 258 ++- package.json | 132 +- src/cache-report.ts | 4 +- src/compress-tools.ts | 21 +- src/config.ts | 15 +- src/decompress.ts | 382 ++--- src/filter/apply.ts | 174 +- src/filter/index.ts | 7 +- src/filter/registry.ts | 20 +- src/filter/types.ts | 32 +- src/handoff.ts | 31 +- src/hide-consumed.ts | 357 +++-- src/image-compress.ts | 79 +- src/message-kind.ts | 4 +- src/nudge-text.ts | 123 +- src/packs.ts | 88 +- src/panel/cache.ts | 4 +- src/panel/panel.ts | 62 +- src/panel/topic.ts | 5 +- src/parse-compress-input.ts | 1154 ++++++++------ src/persist/index.ts | 7 +- src/persist/state-merge.ts | 4 +- src/persist/store.ts | 1236 ++++++++------- src/protected.ts | 31 +- src/rebuild.ts | 78 +- src/recommend.ts | 25 +- src/render-refs.ts | 23 +- src/report.ts | 761 +++++---- src/search.ts | 42 +- src/search/SEARCH.md | 61 +- src/search/algorithms/bm25.ts | 63 +- src/search/algorithms/fuzzy.ts | 39 +- src/search/algorithms/hybrid.ts | 32 +- src/search/algorithms/semantic.ts | 116 +- src/search/algorithms/substring.ts | 33 +- src/search/doc-cache.ts | 78 +- src/search/index.ts | 237 +-- src/search/registry.ts | 10 +- src/search/stemmer.ts | 31 +- src/search/tokenizer.ts | 112 +- src/search/types.ts | 114 +- src/surface-config.ts | 15 +- src/tool-pairs.ts | 80 +- src/truncate-tools.ts | 196 +-- src/truncate.ts | 21 +- src/turn-integrity.ts | 5 +- src/wire/anthropic.ts | 430 ++--- src/wire/bili-message.ts | 124 +- src/wire/compress-detect.ts | 173 +- src/wire/demoted-thinking.ts | 56 +- src/wire/formats.ts | 7 +- src/wire/message-id.ts | 30 +- src/wire/mirror.ts | 344 ++-- src/wire/openai.ts | 583 ++++--- src/wire/responses.ts | 1026 +++++++----- src/wire/strip-images.ts | 3 +- src/wire/util.ts | 14 +- tests/block-map.test.ts | 39 +- tests/cache-report.test.ts | 13 +- tests/checkpoint-carrier.test.ts | 158 +- tests/compress-detect.test.ts | 190 ++- tests/compress-schema-contract.test.ts | 127 +- tests/compress-tools.test.ts | 63 +- tests/compress.test.ts | 6 +- tests/decompress-report.test.ts | 513 +++--- tests/doc-cache.test.ts | 84 +- tests/effective-gate-chars.test.ts | 79 +- tests/handoff.test.ts | 265 +++- tests/hide-consumed-stub.test.ts | 329 ++-- tests/hide-merge-rebuild.test.ts | 778 ++++++--- tests/image-compress.test.ts | 247 ++- tests/issue684-turn-integrity.test.ts | 303 ++-- tests/nudge-text.test.ts | 203 ++- tests/nudge.test.ts | 844 ++++++++-- tests/openai-system-hoist.test.ts | 25 +- tests/orphan-fixes.test.ts | 39 +- tests/packs.test.ts | 253 ++- tests/panel.test.ts | 162 +- tests/parse-compress-input.test.ts | 1316 +++++++++------- tests/persist.test.ts | 1749 ++++++++++++--------- tests/prompts.test.ts | 10 +- tests/protect-recent.test.ts | 332 +++- tests/protected-content.test.ts | 231 ++- tests/protected-latest.test.ts | 41 +- tests/recommend-merge.test.ts | 6 +- tests/recommend.test.ts | 220 ++- tests/regression-bugfixes.test.ts | 136 +- tests/regression-empty-rewrap.test.ts | 56 +- tests/regression-features.test.ts | 90 +- tests/report-accounting.test.ts | 100 +- tests/report-tool-accounting.test.ts | 238 +-- tests/report-truncation.test.ts | 223 +-- tests/search.test.ts | 955 +++++++---- tests/state-prune.test.ts | 15 +- tests/status-surface-meta.test.ts | 173 +- tests/surface-config.test.ts | 69 +- tests/sync-config.test.ts | 39 +- tests/tier-scale-separation.test.ts | 84 +- tests/token-snapshot.test.ts | 70 +- tests/tool-pairs.test.ts | 231 ++- tests/truncate-keep-filter.test.ts | 286 ++-- tests/truncate-surrogates.test.ts | 368 +++-- tests/validation.test.ts | 84 +- tests/viable.test.ts | 5 +- tests/wire-bili-message-roundtrip.test.ts | 1580 ++++++++++++------- tests/wire-conversation-signal.test.ts | 96 +- tests/wire-demoted-thinking.test.ts | 309 ++-- tests/wire-formats.test.ts | 5 +- tests/wire-mirror.test.ts | 509 +++--- tests/wire-proxy-message-id.test.ts | 216 +-- tests/wire-reasoning-echo.test.ts | 317 ++-- tests/wire-schema-top-level.test.ts | 73 +- tests/wire-subagent-namespace.test.ts | 92 +- tests/wire-tool-result-toolname.test.ts | 225 ++- tsconfig.json | 46 +- tsup.config.ts | 17 +- 124 files changed, 16008 insertions(+), 9302 deletions(-) diff --git a/.github/workflows/pr-artifact.yml b/.github/workflows/pr-artifact.yml index 4e26093..46e2d49 100644 --- a/.github/workflows/pr-artifact.yml +++ b/.github/workflows/pr-artifact.yml @@ -1,123 +1,123 @@ name: PR Build Artifact on: - pull_request: - branches: [master] + pull_request: + branches: [master] permissions: - contents: read - pull-requests: write - issues: write + contents: read + pull-requests: write + issues: write jobs: - build-artifact: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 + build-artifact: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 22 - cache: npm - registry-url: https://registry.npmjs.org + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + registry-url: https://registry.npmjs.org - - run: npm ci + - run: npm ci - - run: npm run build + - run: npm run build - - name: Publish to npm with PR tag - if: github.event.pull_request.head.repo.full_name == github.repository - run: | - PR_NUMBER="${{ github.event.pull_request.number }}" - RUN_NUMBER="${{ github.run_number }}" - BASE_VERSION=$(node -p "require('./package.json').version") - # Strip any existing prerelease suffix - BASE_VERSION=$(echo "$BASE_VERSION" | sed 's/-.*//') - PR_VERSION="${BASE_VERSION}-pr.${PR_NUMBER}.${RUN_NUMBER}" + - name: Publish to npm with PR tag + if: github.event.pull_request.head.repo.full_name == github.repository + run: | + PR_NUMBER="${{ github.event.pull_request.number }}" + RUN_NUMBER="${{ github.run_number }}" + BASE_VERSION=$(node -p "require('./package.json').version") + # Strip any existing prerelease suffix + BASE_VERSION=$(echo "$BASE_VERSION" | sed 's/-.*//') + PR_VERSION="${BASE_VERSION}-pr.${PR_NUMBER}.${RUN_NUMBER}" - echo "Publishing version $PR_VERSION with tag pr-${PR_NUMBER}" - npm version "$PR_VERSION" --no-git-tag-version - npm publish --tag "pr-${PR_NUMBER}" --access public --ignore-scripts - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + echo "Publishing version $PR_VERSION with tag pr-${PR_NUMBER}" + npm version "$PR_VERSION" --no-git-tag-version + npm publish --tag "pr-${PR_NUMBER}" --access public --ignore-scripts + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - - name: Create installable tarball - run: | - npm pack - PR_NUM="${{ github.event.pull_request.number }}" - TARBALL=$(ls acp-kernel-*.tgz | head -1) - if [ -n "$PR_NUM" ] && [ -n "$TARBALL" ]; then - cp "$TARBALL" "acp-kernel-pr${PR_NUM}.tgz" - fi + - name: Create installable tarball + run: | + npm pack + PR_NUM="${{ github.event.pull_request.number }}" + TARBALL=$(ls acp-kernel-*.tgz | head -1) + if [ -n "$PR_NUM" ] && [ -n "$TARBALL" ]; then + cp "$TARBALL" "acp-kernel-pr${PR_NUM}.tgz" + fi - - name: Upload artifact - uses: actions/upload-artifact@v4 - with: - name: acp-kernel-pr-${{ github.event.pull_request.number }} - path: | - acp-kernel-pr*.tgz - dist/ - retention-days: 30 + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: acp-kernel-pr-${{ github.event.pull_request.number }} + path: | + acp-kernel-pr*.tgz + dist/ + retention-days: 30 - - name: Comment on PR with install instructions - continue-on-error: true - uses: actions/github-script@v7 - with: - script: | - const prNumber = context.payload.pull_request.number; - const sha = context.payload.pull_request.head.sha.slice(0, 7); - const branch = context.payload.pull_request.head.ref; - const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + - name: Comment on PR with install instructions + continue-on-error: true + uses: actions/github-script@v7 + with: + script: | + const prNumber = context.payload.pull_request.number; + const sha = context.payload.pull_request.head.sha.slice(0, 7); + const branch = context.payload.pull_request.head.ref; + const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - const body = [ - '## 📦 Built Package Artifact', - '', - `**Branch:** \`${branch}\` (${sha})`, - '', - '### Option A — Install from npm PR tag (recommended)', - '', - 'In your adapter project:', - '', - '```bash', - 'npm install acp-kernel@pr-' + prNumber, - '```', - '', - 'Each push to this PR publishes a new version under the `pr-' + prNumber + '` npm tag.', - '', - '### Option B — Download artifact', - '', - `1. Download the artifact from the [Actions run](${runUrl})`, - '2. Extract the tarball and install:', - '', - '```bash', - 'tar xzf acp-kernel-pr' + prNumber + '.tgz', - 'npm install ./package', - '```', - '', - '---', - '*This comment is automatically updated on each push.*', - ].join('\n'); + const body = [ + '## 📦 Built Package Artifact', + '', + `**Branch:** \`${branch}\` (${sha})`, + '', + '### Option A — Install from npm PR tag (recommended)', + '', + 'In your adapter project:', + '', + '```bash', + 'npm install acp-kernel@pr-' + prNumber, + '```', + '', + 'Each push to this PR publishes a new version under the `pr-' + prNumber + '` npm tag.', + '', + '### Option B — Download artifact', + '', + `1. Download the artifact from the [Actions run](${runUrl})`, + '2. Extract the tarball and install:', + '', + '```bash', + 'tar xzf acp-kernel-pr' + prNumber + '.tgz', + 'npm install ./package', + '```', + '', + '---', + '*This comment is automatically updated on each push.*', + ].join('\n'); - // Find and update existing comment, or create new one - const comments = await github.rest.issues.listComments({ - ...context.repo, - issue_number: prNumber, - }); + // Find and update existing comment, or create new one + const comments = await github.rest.issues.listComments({ + ...context.repo, + issue_number: prNumber, + }); - const existing = comments.data.find(c => - c.user.type === 'Bot' && c.body.includes('📦 Built Package Artifact') - ); + const existing = comments.data.find(c => + c.user.type === 'Bot' && c.body.includes('📦 Built Package Artifact') + ); - if (existing) { - await github.rest.issues.updateComment({ - ...context.repo, - comment_id: existing.id, - body, - }); - } else { - await github.rest.issues.createComment({ - ...context.repo, - issue_number: prNumber, - body, - }); - } + if (existing) { + await github.rest.issues.updateComment({ + ...context.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + ...context.repo, + issue_number: prNumber, + body, + }); + } diff --git a/AGENTS.md b/AGENTS.md index 4136e1c..4615101 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,21 +10,21 @@ Consumed by adapters: `pai-acp` (Pi), and future adapters for other agent platfo ### Tech Stack -| Category | Technology | -|----------|-----------| -| Language | TypeScript (strict, ESM) | -| Build | tsup (bundling) + tsc --emitDeclarationOnly | -| Test | Node.js built-in: `node --import tsx --test tests/*.test.ts` | -| Package Manager | npm | -| Zero Runtime Dependencies | `dependencies: {}` in package.json | +| Category | Technology | +| ------------------------- | ------------------------------------------------------------ | +| Language | TypeScript (strict, ESM) | +| Build | tsup (bundling) + tsc --emitDeclarationOnly | +| Test | Node.js built-in: `node --import tsx --test tests/*.test.ts` | +| Package Manager | npm | +| Zero Runtime Dependencies | `dependencies: {}` in package.json | ### Repository Info -| Field | Value | -|-------|-------| -| npm package | `acp-kernel` | -| GitHub | https://github.com/ranxianglei/acp-kernel | -| License | MIT | +| Field | Value | +| ----------- | ----------------------------------------- | +| npm package | `acp-kernel` | +| GitHub | https://github.com/ranxianglei/acp-kernel | +| License | MIT | ## 2. Architecture @@ -121,12 +121,12 @@ npm run format:check # Check formatting ## 4. Git Safety Rules (MANDATORY) -| Rule | Enforcement | -|------|-------------| -| **NEVER force-push to `master`** | Under no circumstances | -| **NEVER merge PRs** | PR merges are human-only. The Agent MUST NEVER merge. | -| **Branch naming** | `YYYY-MM-DD_short-title` | -| **NEVER modify `version` on non-release branches** | Version bumps only on release branches | +| Rule | Enforcement | +| -------------------------------------------------- | ----------------------------------------------------- | +| **NEVER force-push to `master`** | Under no circumstances | +| **NEVER merge PRs** | PR merges are human-only. The Agent MUST NEVER merge. | +| **Branch naming** | `YYYY-MM-DD_short-title` | +| **NEVER modify `version` on non-release branches** | Version bumps only on release branches | ### PR Merge — Absolute Prohibition @@ -206,18 +206,22 @@ All source changes require review by **at least 2 separate agents** before merge > Distilled from a full-history audit of this repo + siblings ([billion-context#801](https://github.com/ranxianglei/billion-context/issues/801)). Full cited material: [AUTO-MERGE-GUARDRAILS.md](./AUTO-MERGE-GUARDRAILS.md). This is the CORE library: it OWNS the compression-block model and the wire-artifact format every adapter depends on, so its auto-merge surface is the narrowest of the three repos. Complements §6 Code Review (≥2 agents) by defining when a bugfix may skip the human round-trip. ### 7.1 Before you start + - **Duplicate screening first** (open AND closed). Link existing work, don't start parallel work. - **One issue = one scope.** Never bundle unrelated changes or mass whitespace/reformatting. - **Open a PR, never just push a branch.** ### 7.2 Review discipline + - **Rebase to CURRENT master** before claiming mergeable; after rebase re-run typecheck + test + build + format:check. - **Pipeline nodes are shared hotspots** — if another open PR rewrites the same node/module, resolve by union of intent, then prove with tests. - **Done = evidence**; tests stay pure (no I/O / network / mocks of kernel internals). - **Deterministic tests** — no env/port luck. ### 7.3 Kernel-owned contracts (first-class; ALWAYS human-gated) + The invariants every adapter relies on. Changing ANY of these is cross-repo + prefix-cache-affecting by construction and MUST stay human (and follow cross-repo sequencing — kernel ships first): + 1. **Message-id / ref immutability** (§2 principle 6) — no slot reclamation/recycling; widen `REF_WIDTH`. A re-issued number silently misattributes on decompress. 2. **Wire-artifact format** — the ACP compression tag wrapping `mNNNNN`, the `acp_summary` structure, block refs, and rendered-tag token counts being SNAPSHOTS frozen at first render. 3. **Single-owner content** (§2 principle 3) — `assignRefsNode` is the sole writer of message content. @@ -230,7 +234,9 @@ The invariants every adapter relies on. Changing ANY of these is cross-repo + pr Plus structural constraints that stay human: zero runtime deps (§2 principle 1 — no new dep without sign-off), platform-agnostic (§2 principle 5 — no host API / file I/O / network creep), and the state schema (explicit in/out; changing its shape is cross-repo). ### 7.4 Auto-merge gate (narrowest of the three repos) + A bugfix may **auto-merge** only if ALL hold: + 1. Scoped to ONE pipeline node or standalone module; no change to node ordering or pipeline shape. 2. A pure regression test reproduces the bug and now passes. 3. Green on the rebased head (typecheck + test + build + format:check). @@ -242,7 +248,9 @@ A bugfix may **auto-merge** only if ALL hold: **Must stay human:** any §7.3 contract, any structural constraint, wire/message-shape changes, config/schema, state format, cross-repo dependencies, feat/refactor/architecture, security, or any default-value change (a product decision). Because a contract change is cross-repo by construction, it ALSO follows the manual cross-repo rule: release acp-kernel first, verify `npm view acp-kernel version`, then bump the adapters. ### 7.5 Reviewer focus — the "重灾区" + Of merged PRs that drew review, ≈25% needed a 2nd+ round. Dominant drivers: + 1. **Stale-base / concurrent-file churn** on shared pipeline nodes. 2. **Incomplete first pass** — the repro passes but round-trip fidelity, tool-pair atomicity, or a doc-vs-default drift is missed. -Also check explicitly: does the fix preserve lossless round-trip and tool-pair atomicity? Do docs still match the actual code defaults (#163)? + Also check explicitly: does the fix preserve lossless round-trip and tool-pair atomicity? Do docs still match the actual code defaults (#163)? diff --git a/AUTO-MERGE-GUARDRAILS.md b/AUTO-MERGE-GUARDRAILS.md index 74e8214..68f37b8 100644 --- a/AUTO-MERGE-GUARDRAILS.md +++ b/AUTO-MERGE-GUARDRAILS.md @@ -24,13 +24,15 @@ ## 2. Rules humans actually enforce (evidence) -> Rule text lives in `AGENTS.md` §7.1–§7.3; below is *why* each exists. +> Rule text lives in `AGENTS.md` §7.1–§7.3; below is _why_ each exists. ### 2.A Already codified in AGENTS.md + §2 Key Design Principles (esp. #3 single-owner content, #6 message-id immutability), §4 Git Safety, §5 Release Workflow, §6 Code Review ("at least 2 separate agents"). ### 2.B Implicit rules observed in review threads + - **Never split a tool call from its result** in prune/compress/fold ([#286](https://github.com/ranxianglei/acp-kernel/issues/286) / [#287](https://github.com/ranxianglei/acp-kernel/pull/287) / @@ -54,8 +56,10 @@ Safety, §5 Release Workflow, §6 Code Review ("at least 2 separate agents"). ## 3. What AI cannot reliably self-judge (blind spots) ### 3.1 Kernel-owned contracts — the incident proving each is load-bearing -> Definitions live in `AGENTS.md` §7.3 (always human-gated). Below is only *why* each is + +> Definitions live in `AGENTS.md` §7.3 (always human-gated). Below is only _why_ each is > non-negotiable: + 1. **Message-id / ref immutability** — a re-issued number silently misattributes on decompress (**wrong content, not an error**); #176 was reverted for exactly this. 2. **Wire-artifact format** (tag / `acp_summary` / token-snapshot freezing) — changing shape breaks @@ -70,11 +74,13 @@ Safety, §5 Release Workflow, §6 Code Review ("at least 2 separate agents"). reintroduces feedback-loop over-compression. ### 3.2 Structural constraints that stay human + Zero runtime deps (§2 principle 1 — a new dep lands inside every adapter's inline bundle), platform-agnostic (§2 principle 5 — no host API / file I/O / network creep into the core), and the state schema (explicit in/out; changing its shape is cross-repo). ## Appendix + - **Gate + must-stay-human list + reviewer checklist:** see `AGENTS.md` §7.4 / §7.5 (single source of truth). Because a contract change is cross-repo by construction, it also follows the manual cross-repo rule: release acp-kernel first, verify `npm view acp-kernel version`, then bump adapters. diff --git a/DESIGN-prompts.md b/DESIGN-prompts.md index 27dc2d8..d34ec47 100644 --- a/DESIGN-prompts.md +++ b/DESIGN-prompts.md @@ -16,10 +16,10 @@ drops the "KEEP VERBATIM file paths / signatures / decisions" guidance will quietly lose the strings a later `decompress` or `grep` needs. So customization must distinguish two classes of text: -| Class | Examples | Override risk | -|-------|----------|---------------| +| Class | Examples | Override risk | +| ---------------- | ----------------------------------------------------------------------------------------------------------------------- | ------------------------------------------- | | **Load-bearing** | the 4 compression rules (`COMPRESS_PHILOSOPHY`, `HOW_TO_COMPRESS_RULES`, `TIER2_DISTILL_RULES`, `TIER3_CONDENSE_RULES`) | High — degrades summary quality / retrieval | -| **Surface** | summary section header, status-report chrome, tool descriptions, nudge tone | Low — presentation only | +| **Surface** | summary section header, status-report chrome, tool descriptions, nudge tone | Low — presentation only | Load-bearing overrides require explicit risk acknowledgement; surface overrides are free. @@ -32,7 +32,7 @@ consistent: - **acp-kernel** owns the load-bearing rules as exported constants (`src/compression-rules.ts`) and the nudge renderer (`renderNudgeText` in `src/nudge-text.ts`) that embeds them. Crucially, - `processTurn` returns a *structured* `NudgeDecision`, not text — the host + `processTurn` returns a _structured_ `NudgeDecision`, not text — the host renders it, so nudge wording is already host-controllable. - **Adapters** (billion-context-pi, opencode-acp, …) own the system prompt and tool descriptions, and they inline the kernel's rule constants when composing @@ -63,7 +63,7 @@ export interface Prompts { tier3CondenseRules: string; } -export const defaultPrompts: Prompts; // == the verbatim rule constants +export const defaultPrompts: Prompts; // == the verbatim rule constants export function resolvePrompts( overrides?: Partial, @@ -107,7 +107,7 @@ systemPrompt += prompts.compressPhilosophy + prompts.howToCompressRules; ## Layer 1 — prompt/tool surface overrides (shipped) -Scope: open all *surface* text — standing-prompt sections, tool descriptions, +Scope: open all _surface_ text — standing-prompt sections, tool descriptions, parameter descriptions — to free-form override. No risk gate: surface text is presentation, and defaults stay byte-identical when no override is given (regression-tested against 0.0.46 fixtures). @@ -148,7 +148,7 @@ export function applyAcpToolOverrides( (header + body), `null` removes it, omitted keeps the default. Keys that do not belong to a given builder are ignored. - `cloneWithDescriptions` deep-clones a JSON tool schema and replaces the - `description` of every property whose *name* matches, at any nesting depth. + `description` of every property whose _name_ matches, at any nesting depth. Names and structure are fixed — only human-readable text moves. - `applyAcpToolOverrides` applies per-tool `description` + `paramDescriptions` to all three wire shapes (anthropic `input_schema`, openai @@ -176,6 +176,7 @@ tier3.md ``` Principles: + - **Partial override + inheritance** — a set specifies only what it changes; everything else inherits kernel defaults. Forward-compatible: a new kernel prompt field does not break existing sets. @@ -193,7 +194,9 @@ For billion-context-pi, `user-config.ts` (already reads `~/.pi/agent/acp.json`) gains: ```json -{ "prompts": { "preset": "acp-prompt-pack-zen", "overrides": { "..." : "..." } } } +{ + "prompts": { "preset": "acp-prompt-pack-zen", "overrides": { "...": "..." } } +} ``` The same format works for every downstream (opencode-acp, omp-acp, …), so a @@ -213,12 +216,12 @@ The kernel gains a `registerPromptSet` registry, matching the existing ## Risks & mitigations -| Risk | Mitigation | -|------|------------| -| Compression-quality regression from bad rules | `acknowledgeRisk` gate; defaults stay the strongly-recommended path | +| Risk | Mitigation | +| ---------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | +| Compression-quality regression from bad rules | `acknowledgeRisk` gate; defaults stay the strongly-recommended path | | Version drift (kernel changes `NudgeDecision` shape) | `Prompts` is data-only (no function fields), so it is stable across refactors; future renderer overrides would carry a version stamp | -| Prompt injection via third-party packs | Prompt packs are trusted code (same trust level as any npm dep); documented in the trust model | -| Tool-description / schema mismatch | Tool schemas stay kernel-owned and fixed; only human-readable descriptions are customizable | +| Prompt injection via third-party packs | Prompt packs are trusted code (same trust level as any npm dep); documented in the trust model | +| Tool-description / schema mismatch | Tool schemas stay kernel-owned and fixed; only human-readable descriptions are customizable | ## Migration / rollout diff --git a/DESIGN.md b/DESIGN.md index af24cbe..e6e2eba 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -9,7 +9,7 @@ Framework-agnostic, model-driven context-compression engine. This document is th ACP-style compression is **not** like zip in one essential way: **the model writes the summaries; this library orchestrates everything around them.** - **zip**: computes the compressed output itself (`bytes → bytes`). -- **acp-kernel**: the *summary text* is produced externally (by an LLM) and passed in as an argument. The core decides *when* to compress, *what range* to compress, tracks *state*, applies a compress *decision*, prunes ranges, and supports decompress/search. **It never calls a model.** +- **acp-kernel**: the _summary text_ is produced externally (by an LLM) and passed in as an argument. The core decides _when_ to compress, _what range_ to compress, tracks _state_, applies a compress _decision_, prunes ranges, and supports decompress/search. **It never calls a model.** This is precisely why the core can be a pure library: the one external dependency (the summarizer) is reduced to a string input. @@ -48,60 +48,60 @@ Portable, host-agnostic. Adapters translate host-native messages into `CoreMessa ```ts type CoreMessage = { - id: string; - role: "user" | "assistant" | "system" | "tool"; - contentType: "text" | "tool-call" | "tool-result" | "reasoning"; - text?: string; - toolName?: string; // for protected-tool filtering - toolCallId?: string; // links tool-call ↔ tool-result + id: string; + role: "user" | "assistant" | "system" | "tool"; + contentType: "text" | "tool-call" | "tool-result" | "reasoning"; + text?: string; + toolName?: string; // for protected-tool filtering + toolCallId?: string; // links tool-call ↔ tool-result }; type CompressionBlock = { - blockId: string; // "b0", "b1", ... - runId: string; - tier: 1 | 2 | 3; - topic?: string; - summary: string; // produced by the model - directMessageIds: string[]; - effectiveMessageIds: string[]; - directBlockIds: string[]; // nested blocks consumed - createdAt: number; - survivedCount: number; - generation: "young" | "old"; - active: boolean; + blockId: string; // "b0", "b1", ... + runId: string; + tier: 1 | 2 | 3; + topic?: string; + summary: string; // produced by the model + directMessageIds: string[]; + effectiveMessageIds: string[]; + directBlockIds: string[]; // nested blocks consumed + createdAt: number; + survivedCount: number; + generation: "young" | "old"; + active: boolean; }; type CompressionState = { - blocks: CompressionBlock[]; - messageRefs: { byRaw: Record; byRef: Record }; // raw ↔ mNNNNN - nudge: { - lastPerMessageNudgeTokens: number; - lastNudgeShownTokens: number; - baselineTokens: number; - anchors: Record; - }; - stats: { tokensCompressed: number; compressionCount: number }; - nextBlockId: number; - nextRunId: number; + blocks: CompressionBlock[]; + messageRefs: { byRaw: Record; byRef: Record }; // raw ↔ mNNNNN + nudge: { + lastPerMessageNudgeTokens: number; + lastNudgeShownTokens: number; + baselineTokens: number; + anchors: Record; + }; + stats: { tokensCompressed: number; compressionCount: number }; + nextBlockId: number; + nextRunId: number; }; type Config = { - tiers: { enabled: boolean; tier2Trigger: number; tier3Trigger: number }; - nudge: { - maxContextLimitPct: number; // e.g. 0.55 (currently advisory; threshold gate uses minContextLimitPct) - minContextLimitPct: number; // e.g. 0.45 — nudge threshold gate - frequency: number; // advisory (reserved for future turn-frequency gating) - iterationThreshold: number; // advisory (reserved for future iteration gating) - force: "soft" | "strong"; - }; - // young→old promotion after N survivals (drives the merge-blocks node). - // NOTE: there is no GC — no age-based deactivation, no summary truncation. - promotionThreshold: number; - truncate: { threshold: number }; // emergency tool-output truncation node (LAST safety valve); 1.0 = 100% - merge: { maxSummaryLength: number; minOldGenBlocks: number }; // batch-merge old-gen blocks into one summary - protectedTools: string[]; - preserveRecentMessages: number; - modelContextLimit: number; + tiers: { enabled: boolean; tier2Trigger: number; tier3Trigger: number }; + nudge: { + maxContextLimitPct: number; // e.g. 0.55 (currently advisory; threshold gate uses minContextLimitPct) + minContextLimitPct: number; // e.g. 0.45 — nudge threshold gate + frequency: number; // advisory (reserved for future turn-frequency gating) + iterationThreshold: number; // advisory (reserved for future iteration gating) + force: "soft" | "strong"; + }; + // young→old promotion after N survivals (drives the merge-blocks node). + // NOTE: there is no GC — no age-based deactivation, no summary truncation. + promotionThreshold: number; + truncate: { threshold: number }; // emergency tool-output truncation node (LAST safety valve); 1.0 = 100% + merge: { maxSummaryLength: number; minOldGenBlocks: number }; // batch-merge old-gen blocks into one summary + protectedTools: string[]; + preserveRecentMessages: number; + modelContextLimit: number; }; ``` @@ -111,78 +111,99 @@ type Config = { ```ts interface CompressionCore { - // Per-turn node pipeline (replaces the message-transform hook's algorithm part). - // Runs every turn (canonical order): assign-refs → sync-blocks → merge-blocks → - // prune → ccr-store → filter → hide-compress-calls → nudge-inject → - // emergency-truncate → render-refs. Survives/promotes blocks via advanceSurvival - // (no age-based deactivation). Returns transformed messages + updated state + - // nudge decision + the (possibly grown) content store. - processTurn(input: { - messages: CoreMessage[]; - state: CompressionState; - config: Config; - tokenCount: number; - /** Per-session CCR content store from the previous turn (optional; an - * empty store is created when omitted). */ - contentStore?: MessageContentStore; - }): { - messages: CoreMessage[]; - state: CompressionState; - nudge?: NudgeDecision; - /** Always present; pass back as input.contentStore next turn. */ - contentStore: MessageContentStore; + // Per-turn node pipeline (replaces the message-transform hook's algorithm part). + // Runs every turn (canonical order): assign-refs → sync-blocks → merge-blocks → + // prune → ccr-store → filter → hide-compress-calls → nudge-inject → + // emergency-truncate → render-refs. Survives/promotes blocks via advanceSurvival + // (no age-based deactivation). Returns transformed messages + updated state + + // nudge decision + the (possibly grown) content store. + processTurn(input: { + messages: CoreMessage[]; + state: CompressionState; + config: Config; + tokenCount: number; + /** Per-session CCR content store from the previous turn (optional; an + * empty store is created when omitted). */ + contentStore?: MessageContentStore; + }): { + messages: CoreMessage[]; + state: CompressionState; + nudge?: NudgeDecision; + /** Always present; pass back as input.contentStore next turn. */ + contentStore: MessageContentStore; + }; + + // When the model calls acp_retrieve. Pure lookup over the host-provided store + // plus an ephemeral injection message (id prefix acp_retrieved_*) that consumes + // no ref and never enters fold space. Hallucinated refs → not-found ack. + retrieve(store: MessageContentStore, ref: string): ApplyRetrieveResult; + + // When the model calls compress. `ranges[].summary` is model-produced text. + // Allocates block(s), deactivates consumed blocks, updates indices, resets the + // nudge growth baseline on success (§5.7 feedback-loop fix). + applyCompression(input: { + ranges: { + startRef: string; + endRef: string; + summary: string; + topic?: string; + }[]; + messages: CoreMessage[]; + state: CompressionState; + config: Config; + }): { + state: CompressionState; + result: { + blocksCreated: number; + tokensCompressed: number; + errors: string[]; }; - - // When the model calls acp_retrieve. Pure lookup over the host-provided store - // plus an ephemeral injection message (id prefix acp_retrieved_*) that consumes - // no ref and never enters fold space. Hallucinated refs → not-found ack. - retrieve(store: MessageContentStore, ref: string): ApplyRetrieveResult; - - // When the model calls compress. `ranges[].summary` is model-produced text. - // Allocates block(s), deactivates consumed blocks, updates indices, resets the - // nudge growth baseline on success (§5.7 feedback-loop fix). - applyCompression(input: { - ranges: { startRef: string; endRef: string; summary: string; topic?: string }[]; - messages: CoreMessage[]; - state: CompressionState; - config: Config; - }): { - state: CompressionState; - result: { blocksCreated: number; tokensCompressed: number; errors: string[] }; - }; - - resolveBoundaries(input: { - startRef: string; - endRef: string; - messages: CoreMessage[]; - state: CompressionState; - }): { startIndex: number; endIndex: number; protectedGaps: number[] }; - // protectedGaps is reserved (currently always []); protected-tool hard-exclusion - // is intentionally NOT implemented in the core — see README "Known limitation". - - decompress(blockId: string, state: CompressionState): CompressionBlock | undefined; - - search(query: string, state: CompressionState): CompressionBlock[]; - - status(state: CompressionState, tokenCount: number, config: Config): StatusReport; - - // No gc(): age-based deactivation was removed (it caused memory-loss upstream). - // Block promotion (young→old) still happens via advanceSurvival in sync-blocks, - // and old-gen blocks are batch-merged by the merge-blocks node — not dropped. + }; + + resolveBoundaries(input: { + startRef: string; + endRef: string; + messages: CoreMessage[]; + state: CompressionState; + }): { startIndex: number; endIndex: number; protectedGaps: number[] }; + // protectedGaps is reserved (currently always []); protected-tool hard-exclusion + // is intentionally NOT implemented in the core — see README "Known limitation". + + decompress( + blockId: string, + state: CompressionState, + ): CompressionBlock | undefined; + + search(query: string, state: CompressionState): CompressionBlock[]; + + status( + state: CompressionState, + tokenCount: number, + config: Config, + ): StatusReport; + + // No gc(): age-based deactivation was removed (it caused memory-loss upstream). + // Block promotion (young→old) still happens via advanceSurvival in sync-blocks, + // and old-gen blocks are batch-merged by the merge-blocks node — not dropped. } type CompressCall = { - mode: "range" | "message"; - ranges: { startRef: string; endRef: string; summary: string; topic?: string }[]; + mode: "range" | "message"; + ranges: { + startRef: string; + endRef: string; + summary: string; + topic?: string; + }[]; }; type NudgeDecision = { - shouldInject: boolean; - reason: string; - compressibleRanges: { startRef: string; endRef: string; tokens: number }[]; - contextUsage: number; // 0..1 - tier: 1 | 2 | 3 | null; // multi-tier trigger, if any - breakdown: Record; + shouldInject: boolean; + reason: string; + compressibleRanges: { startRef: string; endRef: string; tokens: number }[]; + contextUsage: number; // 0..1 + tier: 1 | 2 | 3 | null; // multi-tier trigger, if any + breakdown: Record; }; ``` @@ -194,9 +215,9 @@ The core needs two capabilities from the host, both injected (the core never imp ```ts interface Ports { - countTokens(text: string): number; // default impl ships in core - // messages are PASSED IN to each call (never fetched) → no MessageStore port - // state persistence is the host's job (state is plain data) → no storage port + countTokens(text: string): number; // default impl ships in core + // messages are PASSED IN to each call (never fetched) → no MessageStore port + // state persistence is the host's job (state is plain data) → no storage port } ``` @@ -206,32 +227,32 @@ A default `countTokens` ships with the core (word-level + unicode CJK tokenizer, ## 6. Concept Mapping (nothing is lost) -| ACP concept | Destination in acp-kernel | -|---|---| -| message-id ↔ ref mapping | **core** `processTurn` (pure) | -| prune (range → summary block) | **core** `processTurn` (pure) | -| boundary resolution / search | **core** `resolveBoundaries` (pure) | -| block allocation / state mutation / tiers | **core** `applyCompression` (pure) | -| compress **argument parsing** (lenient: fences, trailing commas, truncated-array salvage; field-name variants) | **core** `parseCompressArgs` (pure; diagnostics are data — adapters emit them) | -| young→old promotion / batch merge | **core** `sync-blocks` node (`advanceSurvival`) + `merge-blocks` node (pure) | -| emergency truncation (context near full) | **core** `emergency-truncate` node — the LAST safety valve; no age-based GC | -| protected-tools filtering logic | **core** (pure: message + config → bool) | -| `inject` **decision** (shouldNudge / growth / threshold) | **core** `decideNudge` (pure) | -| `inject` **text rendering** (nudge → message string) | **adapter** (host message format) | -| prompts (system / nudge text templates) | rules as **structured data** in core; text rendering in **adapter** | -| compress/decompress/search/status **tool registration** | **adapter** (calls core pure fns) | -| `/acp` commands | **adapter** | -| opencode hooks | **OpenCode adapter** | -| output-steering **decisions** (turn classification / verbosity level / effort clamp, #355) | **core** `decideOutputSteering` (pure; operates on a structural summary — role + block kinds, no content) | -| output-steering **landing** (wire field write-back, system-prompt carrier selection, numeric budget floors) | **adapter** | -| config three-layer merge | **adapter** (core only consumes its own `Config`) | -| logger / auth / persistence / update | **adapter** (core does zero I/O) | +| ACP concept | Destination in acp-kernel | +| -------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| message-id ↔ ref mapping | **core** `processTurn` (pure) | +| prune (range → summary block) | **core** `processTurn` (pure) | +| boundary resolution / search | **core** `resolveBoundaries` (pure) | +| block allocation / state mutation / tiers | **core** `applyCompression` (pure) | +| compress **argument parsing** (lenient: fences, trailing commas, truncated-array salvage; field-name variants) | **core** `parseCompressArgs` (pure; diagnostics are data — adapters emit them) | +| young→old promotion / batch merge | **core** `sync-blocks` node (`advanceSurvival`) + `merge-blocks` node (pure) | +| emergency truncation (context near full) | **core** `emergency-truncate` node — the LAST safety valve; no age-based GC | +| protected-tools filtering logic | **core** (pure: message + config → bool) | +| `inject` **decision** (shouldNudge / growth / threshold) | **core** `decideNudge` (pure) | +| `inject` **text rendering** (nudge → message string) | **adapter** (host message format) | +| prompts (system / nudge text templates) | rules as **structured data** in core; text rendering in **adapter** | +| compress/decompress/search/status **tool registration** | **adapter** (calls core pure fns) | +| `/acp` commands | **adapter** | +| opencode hooks | **OpenCode adapter** | +| output-steering **decisions** (turn classification / verbosity level / effort clamp, #355) | **core** `decideOutputSteering` (pure; operates on a structural summary — role + block kinds, no content) | +| output-steering **landing** (wire field write-back, system-prompt carrier selection, numeric budget floors) | **adapter** | +| config three-layer merge | **adapter** (core only consumes its own `Config`) | +| logger / auth / persistence / update | **adapter** (core does zero I/O) | --- ## 7. Why the algorithm, but not the DCP-derived code, comes here -Copyright protects *expression*, not ideas, methods, or algorithms (17 USC §102(b)). The compression *methods* (3-tier, growth cadence, protected filtering) are the author's. This core reimplements them in **fresh expression** — it is not a copy or refactor of DCP-derived files. See [PROVENANCE.md](./PROVENANCE.md) for the per-module origin classification (original-bring / DCP-derived-reimplement / adapter-only). +Copyright protects _expression_, not ideas, methods, or algorithms (17 USC §102(b)). The compression _methods_ (3-tier, growth cadence, protected filtering) are the author's. This core reimplements them in **fresh expression** — it is not a copy or refactor of DCP-derived files. See [PROVENANCE.md](./PROVENANCE.md) for the per-module origin classification (original-bring / DCP-derived-reimplement / adapter-only). --- @@ -241,10 +262,10 @@ Copyright protects *expression*, not ideas, methods, or algorithms (17 USC §102 ### 8.1 First-user-message pin -The session's **first user message survives prune unconditionally** — even when it is covered by an active block. In `rebuildMessages` (`src/prune.ts`) the pin check runs *before* the covered-by-active-block check, so after compressing a range that includes the first user message, the rebuilt wire contains **both** the rendered summary **and** the first user message verbatim; all other covered messages drop as usual. Only the *first* user message is pinned — later covered user messages are pruned normally. +The session's **first user message survives prune unconditionally** — even when it is covered by an active block. In `rebuildMessages` (`src/prune.ts`) the pin check runs _before_ the covered-by-active-block check, so after compressing a range that includes the first user message, the rebuilt wire contains **both** the rendered summary **and** the first user message verbatim; all other covered messages drop as usual. Only the _first_ user message is pinned — later covered user messages are pruned normally. - **Why:** strict providers reject conversations with no user message (e.g., Anthropic requires the conversation to start from the user role). Pinning guarantees the rebuilt wire retains at least one user message whenever the input had one. Deliberate since prune's first implementation (v0.0.2) — the ordering of the two checks is part of the contract, not incidental. Regression-tested by `tests/state-prune.test.ts` ("prune preserves first user message even when covered") and `tests/orphan-fixes.test.ts` ("prune: first user message pruned when covered (no duplication)"). - **Consequences for consumers:** - Consumers cannot assert byte-level disappearance of the first user message after compression; post-compress wire assertions must expect the rendered summary **plus** the pinned message verbatim. - Token-size and prefix-cache stability estimates must account for the pinned message staying verbatim (it is often the task description and can be long). - - Boundary resolution compensates: `blockVisibleInRange` (`src/boundaries.ts`) treats a block as present in a range via its rendered summary **or** its earliest surviving raw — required precisely because the summary anchors *before* the pinned raw. + - Boundary resolution compensates: `blockVisibleInRange` (`src/boundaries.ts`) treats a block as present in a range via its rendered summary **or** its earliest surviving raw — required precisely because the summary anchors _before_ the pinned raw. diff --git a/PROVENANCE.md b/PROVENANCE.md index 14e5627..ff0e894 100644 --- a/PROVENANCE.md +++ b/PROVENANCE.md @@ -2,17 +2,17 @@ This document classifies **every** source file in `opencode-acp/lib/` (the donor) into one of three buckets, to determine what can be carried into the MIT `acp-kernel` and what must be reimplemented in fresh expression. -**Method**: exact path comparison of `opencode-acp` (the ACP fork, 82 `lib/*.ts` files) against the upstream DCP repository (66 `lib/*.ts` files). A file that has **no DCP equivalent** is original work (bucket A). A file with a **DCP equivalent at the same path** is a DCP derivative — its *expression* is AGPL-bound regardless of how much it was later changed (derivation is judged by whether a file was created by transforming the original, not by % changed; per copyright law expression is protected, not ideas/methods/algorithms). +**Method**: exact path comparison of `opencode-acp` (the ACP fork, 82 `lib/*.ts` files) against the upstream DCP repository (66 `lib/*.ts` files). A file that has **no DCP equivalent** is original work (bucket A). A file with a **DCP equivalent at the same path** is a DCP derivative — its _expression_ is AGPL-bound regardless of how much it was later changed (derivation is judged by whether a file was created by transforming the original, not by % changed; per copyright law expression is protected, not ideas/methods/algorithms). --- ## Legend -| Bucket | Meaning | Action in acp-kernel | -|---|---|---| -| **A** | No DCP equivalent → original work of ranxianglei | **Bring verbatim** (the author's own code, MIT-safe). License header rewritten to MIT. | -| **B** | DCP-derived file (same path exists upstream) → derivative expression is AGPL | **Reimplement in fresh expression** using the author's algorithm; do NOT copy/refactor the file's code. | -| **N/A** | Adapter-only (framework-specific: opencode hooks, I/O, auth, commands, UI) | **Excluded** from the pure core. Stays in host adapters. Origin irrelevant. | +| Bucket | Meaning | Action in acp-kernel | +| ------- | ---------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | +| **A** | No DCP equivalent → original work of ranxianglei | **Bring verbatim** (the author's own code, MIT-safe). License header rewritten to MIT. | +| **B** | DCP-derived file (same path exists upstream) → derivative expression is AGPL | **Reimplement in fresh expression** using the author's algorithm; do NOT copy/refactor the file's code. | +| **N/A** | Adapter-only (framework-specific: opencode hooks, I/O, auth, commands, UI) | **Excluded** from the pure core. Stays in host adapters. Origin irrelevant. | --- @@ -21,26 +21,28 @@ This document classifies **every** source file in `opencode-acp/lib/` (the donor Quality gate intentionally omitted from acp-kernel (may be added later if needed). ### `compress/` -| File | Notes | -|---|---| -| `compress/decompress.ts` | ACP-original (v1.11+); DCP has no decompress | -| `compress/decompress-logic.ts` | ACP-original | -| `compress/hide-consumed.ts` | ACP-original (v1.14+) | -| `compress/hide-failed.ts` | ACP-original | -| `compress/keep-markers.ts` | ACP-original (v1.12+) | -| `compress/parts.ts` | ACP-original | -| `compress/recap.ts` | ACP-original (v1.12.1) | -| `compress/status.ts` | ACP-original (v1.11+) | + +| File | Notes | +| ------------------------------ | -------------------------------------------- | +| `compress/decompress.ts` | ACP-original (v1.11+); DCP has no decompress | +| `compress/decompress-logic.ts` | ACP-original | +| `compress/hide-consumed.ts` | ACP-original (v1.14+) | +| `compress/hide-failed.ts` | ACP-original | +| `compress/keep-markers.ts` | ACP-original (v1.12+) | +| `compress/parts.ts` | ACP-original | +| `compress/recap.ts` | ACP-original (v1.12.1) | +| `compress/status.ts` | ACP-original (v1.11+) | ### other -| File | Notes | -|---|---| -| `config-validation.ts` | ACP-extracted for testability; no DCP equivalent | -| `gc/merge.ts` | ACP-original (DCP has no `gc/` dir) | -| `messages/truncate-tools.ts` | ACP-original (v1.14.5, replaced DCP's `gc/truncate.ts`) | -| `messages/filter/*` (9 files) | ACP-original filtering subsystem | + +| File | Notes | +| ------------------------------------ | --------------------------------------------------------------- | +| `config-validation.ts` | ACP-extracted for testability; no DCP equivalent | +| `gc/merge.ts` | ACP-original (DCP has no `gc/` dir) | +| `messages/truncate-tools.ts` | ACP-original (v1.14.5, replaced DCP's `gc/truncate.ts`) | +| `messages/filter/*` (9 files) | ACP-original filtering subsystem | | `messages/inject/policy/*` (3 files) | ACP-original (v1.13.1) — inject **policy** logic, not rendering | -| `state/rebuild.ts` | ACP-original (v1.11+) | +| `state/rebuild.ts` | ACP-original (v1.11+) | --- @@ -49,22 +51,25 @@ Quality gate intentionally omitted from acp-kernel (may be added later if needed Only the **algorithmic** subset is reimplemented into the pure core; the rest are adapter-only (persistence, config-merge, prompts-rendering) and excluded from the core. Listed by what they become. ### Reimplemented into acp-kernel (fresh expression, ~9 substantial files) -| DCP-derived file | Becomes | What to reimplement | -|---|---|---| -| `compress/range.ts` | `core/applyCompression` (range mode) | block allocation, nested-block handling, boundary resolution | -| `compress/search.ts` | `core/resolveBoundaries` | ref→index mapping, reversed-boundary swap, protected-gap detection | -| `compress/state.ts` | `core/state` (mutation) | block id/run allocation, deactivation, byMessageId index | -| `compress/pipeline.ts` | `core/processTurn` prep/finalize | permission (host-side), fetch (host-side), state wrap | -| `messages/prune.ts` | `core/prune` | replace compressed ranges with summary blocks | -| `messages/sync.ts` | `core/sync` | deactivate orphaned blocks when messages deleted | -| `message-ids.ts` | `core/refs` | raw↔mNNNNN bidirectional map | -| `messages/inject/inject.ts` + `inject/utils.ts` | `core/decideNudge` | **decision only** — shouldNudge, growth baseline, threshold, compressible ranges | -| `config.ts` (core subset) | `core/Config` defaults | defaults + validation (use A-class `config-validation.ts`) | + +| DCP-derived file | Becomes | What to reimplement | +| ----------------------------------------------- | ------------------------------------ | -------------------------------------------------------------------------------- | +| `compress/range.ts` | `core/applyCompression` (range mode) | block allocation, nested-block handling, boundary resolution | +| `compress/search.ts` | `core/resolveBoundaries` | ref→index mapping, reversed-boundary swap, protected-gap detection | +| `compress/state.ts` | `core/state` (mutation) | block id/run allocation, deactivation, byMessageId index | +| `compress/pipeline.ts` | `core/processTurn` prep/finalize | permission (host-side), fetch (host-side), state wrap | +| `messages/prune.ts` | `core/prune` | replace compressed ranges with summary blocks | +| `messages/sync.ts` | `core/sync` | deactivate orphaned blocks when messages deleted | +| `message-ids.ts` | `core/refs` | raw↔mNNNNN bidirectional map | +| `messages/inject/inject.ts` + `inject/utils.ts` | `core/decideNudge` | **decision only** — shouldNudge, growth baseline, threshold, compressible ranges | +| `config.ts` (core subset) | `core/Config` defaults | defaults + validation (use A-class `config-validation.ts`) | ### Supporting types/barrels (trivial, write fresh) + `compress/{index,types,timing,range-utils}.ts`, `messages/{index,priority,query,reasoning-strip,shape,utils}.ts`, `state/{index,types,utils}.ts`, `token-utils.ts` (wrap `cc-alg` tokenizer), `protected-patterns.ts`, `compress/protected-content.ts`. ### Excluded from core (adapter-only despite B lineage) + `state/persistence.ts` (filesystem I/O), `prompts/*` (text rendering → adapter), `compress-permission.ts` (permission = host concern). --- @@ -79,14 +84,13 @@ All framework-specific; irrelevant to the pure core. Stay in the OpenCode adapte ## Summary -| Bucket | Files | Disposition | -|---|---|---| -| **B** DCP-derived | 40 | ~9 substantial algorithms reimplemented fresh + ~12 types/barrels rewritten fresh; rest excluded (adapter) | -| **N/A** adapter | 12 | excluded from core | - +| Bucket | Files | Disposition | +| ----------------- | ----- | ---------------------------------------------------------------------------------------------------------- | +| **B** DCP-derived | 40 | ~9 substantial algorithms reimplemented fresh + ~12 types/barrels rewritten fresh; rest excluded (adapter) | +| **N/A** adapter | 12 | excluded from core | --- ## Compliance note -This audit exists to ensure acp-kernel is **genuinely MIT**, not "MIT-labeled but AGPL-tainted." The rule applied throughout: an idea/algorithm is free regardless of source; a *file's code expression* is bound to the license of the file it descends from. A-class files descend from no DCP file. B-class files' code is NOT carried — the algorithms are reimplemented in new expression. Should any contributor question a classification, the comparison data above (and the upstream DCP tree) allow independent verification. +This audit exists to ensure acp-kernel is **genuinely MIT**, not "MIT-labeled but AGPL-tainted." The rule applied throughout: an idea/algorithm is free regardless of source; a _file's code expression_ is bound to the license of the file it descends from. A-class files descend from no DCP file. B-class files' code is NOT carried — the algorithms are reimplemented in new expression. Should any contributor question a classification, the comparison data above (and the upstream DCP tree) allow independent verification. diff --git a/README.md b/README.md index 0c59134..5094069 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Framework-agnostic, model-driven context-compression engine. Pure TypeScript cor `acp-kernel` is a **host-agnostic, model-driven context-compression engine**: 3-tier LSM-tree context compression, growth-based nudge policy, protected-content filtering. Its compression algorithms and pipeline architecture (`PipelineNode` / `processTurn` / `CompressionCore`) are **original work by the ACP authors** — an independent reimplementation, not a port of any existing codebase. -The key design principle: **the model writes the summaries; this library orchestrates everything around them.** The core decides *when* to compress, *what range* to compress, tracks *state* (blocks, message-id mapping, tiers), applies a compress *decision*, prunes compressed ranges, and supports decompress/search. It never calls a model. +The key design principle: **the model writes the summaries; this library orchestrates everything around them.** The core decides _when_ to compress, _what range_ to compress, tracks _state_ (blocks, message-id mapping, tiers), applies a compress _decision_, prunes compressed ranges, and supports decompress/search. It never calls a model. ## Why a separate library @@ -32,7 +32,7 @@ See [DESIGN.md](./DESIGN.md) for the full contract and [PROVENANCE.md](./PROVENA ```ts import { createCore, createInitialState, defaultConfig } from "acp-kernel"; -const core = createCore(); // optional: { countTokens } +const core = createCore(); // optional: { countTokens } const state = createInitialState(); const config = defaultConfig(200000); // modelContextLimit (positional); optional overrides as 2nd arg @@ -40,20 +40,30 @@ const config = defaultConfig(200000); // modelContextLimit (positional); optiona // assign-refs → sync-blocks → prune → ccr-store → absorb-hide → crush → // absorb-prompt → filter → hide-compress-calls → recommend → nudge-inject // → emergency-truncate → render-refs -const { messages, state: nextState, nudge, contentStore } = core.processTurn({ - messages, state, config, tokenCount, +const { + messages, + state: nextState, + nudge, + contentStore, +} = core.processTurn({ + messages, + state, + config, + tokenCount, contentStore, // previous turn's content store (optional; see CCR below) }); // When the model emits a compress decision (summary written by the model): const { state: compressed, result } = core.applyCompression({ ranges: [{ startRef: "m00005", endRef: "m00020", summary: "..." }], - messages, state: nextState, config, + messages, + state: nextState, + config, }); -core.decompress("b3", compressed); // look up a block -core.search("auth token", compressed); // relevance-ranked block search -core.status(compressed, tokenCount, config); // context-usage report +core.decompress("b3", compressed); // look up a block +core.search("auth token", compressed); // relevance-ranked block search +core.status(compressed, tokenCount, config); // context-usage report ``` #### `renderTags` — host-side rendering strategy @@ -103,7 +113,10 @@ const config = defaultConfig(200000, { ccr: { enabled: true } }); // opt-in // Turn N: pass the previous turn's store back in. const { messages, state, contentStore } = core.processTurn({ - messages, state, config, tokenCount, + messages, + state, + config, + tokenCount, contentStore, // persisted from turn N-1 (host owns persistence) }); @@ -159,25 +172,25 @@ right after `applyCompression` so pruned originals stay retrievable by ref. `` attribute is frozen at the tag's first render (the per-message `tokenSnapshot` in state) and is not recomputed when the message text is later filtered, truncated, or edited by the host — the number always -describes what the model originally saw. Nudge/pressure *decisions* are +describes what the model originally saw. Nudge/pressure _decisions_ are unaffected: they recount live text every turn. If you need the legacy live-recomputed tags, use `renderVisibleRefs` directly. ### Standalone modules -| Module | Purpose | -|--------|---------| -| `truncateLargeToolOutputs` | Emergency context-threshold-gated truncation of large visible tool outputs (last-resort safety valve; summaries are never touched) | -| `hideConsumedCompressCalls` | Hide historical compress tool-calls whose block is inactive | -| `buildStatusReport` / `buildRecap` | Context-usage report + block recap | -| `mergeMarkedBlocks` / `collectOldGenBlocks` | Batch merge old-gen blocks into one summary | -| `rebuildCompressionState` | Fork-recovery: replay historical compress calls | -| `MessageContentStore` / `storeLargeResults` / `retrieveByRef` / `storeCoveredOriginals` | CCR content store: per-session, content-addressed dedup of tool-result originals + on-demand retrieval (see "CCR" above) | -| `applyMessageFilters` | Pluggable message-filter framework | -| `resolveTransformChannel` | Channel-selection policy: an explicit preference wins; the default is the wire channel only when the caller reports it viable | -| `decideOutputSteering` / `classifyTurn` / `clampEffortToFloor` / `applySteeringToPrompt` | Output-side steering *decisions* (#355): structural turn classification (`new_user_ask` / `mechanical_continuation` / `error_continuation` / `unknown`), byte-stable verbosity directives L0–L4 (default L2), and clamp-only effort routing. The kernel decides; adapters land decisions onto wire fields. Default OFF | -| `applySectionOverrides` / `cloneWithDescriptions` / `applyAcpToolOverrides` | Prompt/tool *surface* customization (see below) | -| `sanitizePackSurface` / `createPackResolver` / `defaultPackSources` | Prompt packs: named, swappable surface presets resolved over pluggable sources (see below) | +| Module | Purpose | +| ---------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `truncateLargeToolOutputs` | Emergency context-threshold-gated truncation of large visible tool outputs (last-resort safety valve; summaries are never touched) | +| `hideConsumedCompressCalls` | Hide historical compress tool-calls whose block is inactive | +| `buildStatusReport` / `buildRecap` | Context-usage report + block recap | +| `mergeMarkedBlocks` / `collectOldGenBlocks` | Batch merge old-gen blocks into one summary | +| `rebuildCompressionState` | Fork-recovery: replay historical compress calls | +| `MessageContentStore` / `storeLargeResults` / `retrieveByRef` / `storeCoveredOriginals` | CCR content store: per-session, content-addressed dedup of tool-result originals + on-demand retrieval (see "CCR" above) | +| `applyMessageFilters` | Pluggable message-filter framework | +| `resolveTransformChannel` | Channel-selection policy: an explicit preference wins; the default is the wire channel only when the caller reports it viable | +| `decideOutputSteering` / `classifyTurn` / `clampEffortToFloor` / `applySteeringToPrompt` | Output-side steering _decisions_ (#355): structural turn classification (`new_user_ask` / `mechanical_continuation` / `error_continuation` / `unknown`), byte-stable verbosity directives L0–L4 (default L2), and clamp-only effort routing. The kernel decides; adapters land decisions onto wire fields. Default OFF | +| `applySectionOverrides` / `cloneWithDescriptions` / `applyAcpToolOverrides` | Prompt/tool _surface_ customization (see below) | +| `sanitizePackSurface` / `createPackResolver` / `defaultPackSources` | Prompt packs: named, swappable surface presets resolved over pluggable sources (see below) | ### Prompt/tool surface configuration @@ -221,12 +234,12 @@ Named, swappable surface configurations layered on top of the primitives above: - `createDirPackSource(id, dir)` serves `/.json` files; `defaultPackSources({ projectDir, userDirs })` assembles project > user > builtin. Directory paths are host policy. -- Pack *selection* (which name is active per config cascade) stays adapter-side; +- Pack _selection_ (which name is active per config cascade) stays adapter-side; the kernel only resolves names. ### Nudge system -The nudge system tells the model *when* to compress. It implements: +The nudge system tells the model _when_ to compress. It implements: - **First-sight mass bypass** (#194): a session that arrives with ready compressible mass ≥ threshold gets its first nudge without waiting for new growth, provided usage ≥ `nudge.minContextLimitPct`. Steady-state nudges are gated on growth/token-mass, not percentage (#379). - **Growth-gating**: a repeat nudge requires positive growth since the baseline (prevents re-firing every turn). `"strong"` force relaxes this. @@ -244,11 +257,11 @@ Deterministic first tier of the two-tier absorb gate. When a tool result qualifi Built-in strategies form an ordered plugin registry (hosts can add/replace/disable via `registerCrushPlugin` / config): -| Strategy | Kind | Lossy | What it does | -|----------|------|-------|--------------| -| `json-fold` | json | no | constant-field hoisting + identical row/run collapse into annotated envelopes | -| `code-trim` | code | yes | comment/docstring elision behind a string-literal-aware scanner; template literals and assigned triple-quoted strings survive verbatim; unterminated constructs fail open | -| `log-select` | log | yes | level-classified line selection; **every distinct ERROR/FAIL line survives verbatim** (kernel-enforced invariant — output dropping one is rejected); warnings deduped, stack frames collapsed, honest `[N lines omitted]` footer | +| Strategy | Kind | Lossy | What it does | +| ------------ | ---- | ----- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `json-fold` | json | no | constant-field hoisting + identical row/run collapse into annotated envelopes | +| `code-trim` | code | yes | comment/docstring elision behind a string-literal-aware scanner; template literals and assigned triple-quoted strings survive verbatim; unterminated constructs fail open | +| `log-select` | log | yes | level-classified line selection; **every distinct ERROR/FAIL line survives verbatim** (kernel-enforced invariant — output dropping one is rejected); warnings deduped, stack frames collapsed, honest `[N lines omitted]` footer | Kernel-enforced guarantees: dispatch guards keep each payload kind on its own strategies; plugins must be pure and deterministic (same payload + config ⇒ same bytes, prefix-cache friendly); any throw, sub-`minReduction` result, or invariant violation fails open to the next candidate or pass-through. **Default off** (`crush.enabled: false`) and requires `absorb.enabled: true`; when disabled the pipeline is byte-identical to pre-crush behavior. diff --git a/bench/search/bench.ts b/bench/search/bench.ts index fad3a04..d1c73c3 100644 --- a/bench/search/bench.ts +++ b/bench/search/bench.ts @@ -16,16 +16,25 @@ import { CORPUS, QUERIES } from "./corpus.js"; function toState(): CompressionState { const blocks: CompressionBlock[] = CORPUS.map((b) => ({ - blockId: b.blockId, runId: "r", tier: 1, active: true, - topic: b.topic, summary: b.summary, - directMessageIds: [], effectiveMessageIds: [], survivedCount: 0, createdAt: Date.now(), + blockId: b.blockId, + runId: "r", + tier: 1, + active: true, + topic: b.topic, + summary: b.summary, + directMessageIds: [], + effectiveMessageIds: [], + survivedCount: 0, + createdAt: Date.now(), })); return { ...createInitialState(), blocks }; } function evaluate(algorithm: string | undefined) { const state = toState(); - let rr = 0, c1 = 0, it3 = 0; + let rr = 0, + c1 = 0, + it3 = 0; const fails: string[] = []; for (const q of QUERIES) { const r = searchBlocks(blockDocs(state), q.query, { algorithm }); @@ -36,18 +45,32 @@ function evaluate(algorithm: string | undefined) { if (got === q.expectFirst) c1++; if (top3.includes(q.expectFirst)) it3++; if (got !== q.expectFirst) - fails.push(` ✗ [${q.note}] "${q.query}" want ${q.expectFirst} got ${got} ${idx >= 0 ? "#" + (idx + 1) : "ABSENT"}`); + fails.push( + ` ✗ [${q.note}] "${q.query}" want ${q.expectFirst} got ${got} ${idx >= 0 ? "#" + (idx + 1) : "ABSENT"}`, + ); } const n = QUERIES.length; - return { label: algorithm ?? "hybrid (default)", mrr: rr / n, r1: c1 / n, r3: it3 / n, fails }; + return { + label: algorithm ?? "hybrid (default)", + mrr: rr / n, + r1: c1 / n, + r3: it3 / n, + fails, + }; } -console.log(`\n=== Search benchmark: ${CORPUS.length} blocks, ${QUERIES.length} queries ===\n`); +console.log( + `\n=== Search benchmark: ${CORPUS.length} blocks, ${QUERIES.length} queries ===\n`, +); const algos = ["hybrid", "bm25", "fuzzy", "substring"]; const results = algos.map(evaluate); for (const r of results) { - console.log(` ${r.label.padEnd(16)} MRR=${r.mrr.toFixed(3)} R@1=${r.r1.toFixed(3)} R@3=${r.r3.toFixed(3)}`); + console.log( + ` ${r.label.padEnd(16)} MRR=${r.mrr.toFixed(3)} R@1=${r.r1.toFixed(3)} R@3=${r.r3.toFixed(3)}`, + ); } console.log("\n--- default (hybrid) failures ---"); for (const f of results[0].fails) console.log(f); -console.log(`\n (${results[0].fails.length}/${QUERIES.length} not #1 — remaining failures are pure-semantic:\n synonyms / cross-language, solvable only with embeddings via the optional 'semantic' algorithm)`); +console.log( + `\n (${results[0].fails.length}/${QUERIES.length} not #1 — remaining failures are pure-semantic:\n synonyms / cross-language, solvable only with embeddings via the optional 'semantic' algorithm)`, +); diff --git a/bench/search/corpus.ts b/bench/search/corpus.ts index a6852c5..981239f 100644 --- a/bench/search/corpus.ts +++ b/bench/search/corpus.ts @@ -15,38 +15,196 @@ export interface BenchBlock { } export const CORPUS: BenchBlock[] = [ - { blockId: "b1", topic: "auth-token-refresh", summary: "Implemented JWT auth token refresh in src/auth/token.ts. Token refresh logic uses refreshToken() to call /api/auth/refresh endpoint. Fixed 401 expired token handling. Access token stored in localStorage, refresh token in httpOnly cookie. Added interceptor for automatic refresh on 401 responses." }, - { blockId: "b2", topic: "database-connection-pool", summary: "Database connection pooling for PostgreSQL. Configured pg Pool with max 20 connections, idle timeout 30s. Connection pool lives in src/db/pool.ts. Resolved connection leak where queries weren't releasing clients back to pool. Added pool.end() on graceful shutdown." }, - { blockId: "b3", topic: "decompress-default-to-file", summary: "Changed decompress tool to write restored content to a file by default instead of returning inline. Prevents context bloat when decompressing large blocks. New API: decompress({blockId}) writes to ~/.cache/pi/acp-decompress/, decompress({blockId, inline:true}) for explicit inline." }, - { blockId: "b4", topic: "compression-protected-zone", summary: "Soft-protect recent zone: filter protected refs instead of failing the whole compression call. NEVER_PRESERVE_RECENT_TOOLS excludes decompress results from the preserve set. applySingleRange now filters and warns rather than throwing on partial overlap." }, - { blockId: "b5", topic: "用户认证模块", summary: "实现了用户登录认证流程，包含密码哈希（bcrypt）、session 管理、权限校验中间件。验证逻辑覆盖登录验证、token 验证、二次验证。登录接口 POST /api/login 返回 JWT token。身份验证失败返回 401。支持 OAuth2 第三方登录（Google、GitHub）。token 过期自动刷新机制。" }, - { blockId: "b6", topic: "npm-publish-workflow", summary: "CI release workflow: on merge of release-v* branch, CI runs typecheck + test + build, creates git tag, publishes to npm. Prerelease versions (containing -) published with --tag dev. NPM_TOKEN secret required." }, - { blockId: "b7", topic: "context-token-estimation", summary: "Token estimation uses chars/4 heuristic. Real tokenizer differs — the gap lands in 'Framework' category in breakdown. displayTotal reflects real context size, not classified sum. systemPromptTokens measured separately from message categories." }, - { blockId: "b8", topic: "review-fixes", summary: "Code review findings: config 150K context cap fix, token counting precision, protect-recent zone edge cases. Reviewer approved after addressing warnings field backward compatibility. Merged to master." }, - { blockId: "b9", topic: "自动注入子agent工具", summary: "session_start 钩子自动把 compress/decompress/search_context/acp_status 注入到 9 个 builtin pi-subagents 的工具白名单。幂等检查，备份 settings.json.acp-bak，原子写 + mtime 乐观锁防并发。只追加不删除用户自定义工具。" }, - { blockId: "b10", topic: "webpack-build-config", summary: "Webpack 5 build configuration: code splitting with SplitChunksPlugin, tree shaking enabled, source maps in production. Configured babel-loader for TypeScript and JSX. Bundle analyzer via webpack-bundle-analyzer plugin." }, - { blockId: "b11", topic: "api-rate-limiting", summary: "Implemented API rate limiting with token bucket algorithm. Redis-based sliding window counter. Rate limit middleware applies to /api/* routes. 429 Too Many Requests response with Retry-After header. Per-IP and per-user limits configurable." }, - { blockId: "b12", topic: "测试框架搭建", summary: "使用 node --test 内置测试运行器。tsx 处理 TypeScript。测试文件放在 tests/ 目录。覆盖率用 c8。Mock 用 node:test 的 mock module。集成测试和单元测试分离。" }, - { blockId: "b13", topic: "websocket-realtime", summary: "WebSocket realtime updates via ws library. Server pushes events on data change. Client reconnect with exponential backoff. Heartbeat ping/pong every 30s to detect dead connections. Message queue for offline clients." }, - { blockId: "b14", topic: "error-handling-middleware", summary: "Global error handling middleware. Custom AppError class with statusCode and code field. Error formatter sanitizes stack traces in production. Async error wrapping with catchAsync wrapper. Sentry integration for production error tracking." }, - { blockId: "b15", topic: "国际化和i18n", summary: "i18n internationalization setup with i18next. Language detection from Accept-Language header and cookie. Translation JSON files in locales/ dir. Supports zh-CN, en-US, ja-JP. Pluralization rules and date/number formatting via Intl API." }, - { blockId: "b16", topic: "git-rebase-conflict", summary: "Resolved git rebase conflicts on feature/auth branch. Cherry-picked commits from upstream. Force-pushed after rewriting history. merge conflict markers in auth.ts and pool.ts. Used git reflog to recover lost commit." }, - { blockId: "b17", topic: "docker-container-setup", summary: "Dockerized the app with multi-stage build. Alpine base image, distroless final stage. Volume mounts for /data and /logs. docker-compose for local dev with postgres and redis services. Healthcheck endpoint /healthz." }, - { blockId: "b18", topic: "css-grid-layout", summary: "Refactored layout from flexbox to CSS Grid. Responsive 12-column grid with auto-fit minmax. Subgrid support for nested layouts. Fixed Safari grid gap bug. Replaced media queries with container queries in sidebar component." }, - { blockId: "b19", topic: "缓存策略redis", summary: "Redis 缓存层，LRU 淘汰策略，TTL 30 分钟。缓存击穿用互斥锁，缓存穿透用布隆过滤器。热点 key 永不过期，后台异步刷新。序列化用 MessagePack 替代 JSON 减少体积。" }, - { blockId: "b20", topic: "graphql-schema-stitching", summary: "GraphQL schema stitching to merge multiple microservice schemas. DataLoader for N+1 query batching. Federation v2 with @key directive for entity resolution. Persisted queries to reduce request size and block malicious queries." }, - { blockId: "b21", topic: "csp-content-security-policy", summary: "Content Security Policy headers. nonce-based script-src to allow inline. report-uri for violation reporting. Upgraded insecure-requests. Blocked eval and inline styles. CSP violation reports aggregated in dashboard." }, - { blockId: "b22", topic: "migration-sql-scripts", summary: "Database migration scripts with Knex. Up/down migrations in migrations/ dir. Added NOT NULL constraint with default backfill. Renamed column user_name to username. Transaction-wrapped for safety. Seed data separate from migrations." }, - { blockId: "b23", topic: "日志收集elk", summary: "ELK 日志栈：Filebeat 采集 → Logstash 过滤 → Elasticsearch 存储 → Kibana 可视化。结构化日志 JSON 格式，含 request_id trace_id。日志级别 DEBUG/INFO/WARN/ERROR。按服务名分索引。" }, - { blockId: "b24", topic: "performance-profiling", summary: "CPU profiling with clinic.js. Flame graph showed hot loop in parser. Memoized expensive computation. Reduced 47MB allocation per request to 3MB. p99 latency dropped from 340ms to 89ms after optimizing regex compilation." }, - { blockId: "b25", topic: "circuit-breaker", summary: "Circuit breaker pattern for downstream service calls. Half-open state after 5 failures. Exponential backoff retry. Hystrix-style fallback response. Breaker state shared via Redis for multi-instance sync." }, - { blockId: "b26", topic: "payment-integration", summary: "Stripe payment integration. Checkout session for one-time, subscription via price ID. Webhook handler verifies signature with raw body. Idempotency key prevents double charge. Refund flow with partial amounts. PCI compliance: no card data on our servers." }, - { blockId: "b27", topic: "权限管理rbac", summary: "RBAC 角色权限模型。用户-角色-权限三层。admin 角色拥有所有权限，editor 可编辑内容，viewer 只读。权限注解 @RequirePermission('post:write')。casbin 策略引擎。资源级细粒度授权。" }, - { blockId: "b28", topic: "ci-cache-optimization", summary: "GitHub Actions cache optimization. Cached node_modules and .turbo. Reduced CI time from 4min to 90s. Cache key includes lockfile hash. Used actions/cache@v3 with restore-keys fallback. Matrix build parallelized across node versions." }, - { blockId: "b29", topic: "memory-leak-debug", summary: "Debugged memory leak in worker pool. Event listeners accumulated — removed on disconnect. WeakRef for cached objects. Heap snapshot comparison found detached DOM nodes. Set maxOldSpaceSize to 4096. Used --inspect to trace retention path." }, - { blockId: "b30", topic: "文件上传oss", summary: "文件上传到阿里云 OSS。分片上传大文件（>5MB），断点续传。预签名 URL 直传前端，减轻服务端压力。图片自动压缩生成缩略图（sharp）。CDN 加速静态资源分发。" }, - { blockId: "b31", topic: "试验记录", summary: "试验记录表：记录三次批次的数据结果。" }, - { blockId: "b32", topic: "图表工具", summary: "图表组件用于绘制折线图和柱状图。" }, + { + blockId: "b1", + topic: "auth-token-refresh", + summary: + "Implemented JWT auth token refresh in src/auth/token.ts. Token refresh logic uses refreshToken() to call /api/auth/refresh endpoint. Fixed 401 expired token handling. Access token stored in localStorage, refresh token in httpOnly cookie. Added interceptor for automatic refresh on 401 responses.", + }, + { + blockId: "b2", + topic: "database-connection-pool", + summary: + "Database connection pooling for PostgreSQL. Configured pg Pool with max 20 connections, idle timeout 30s. Connection pool lives in src/db/pool.ts. Resolved connection leak where queries weren't releasing clients back to pool. Added pool.end() on graceful shutdown.", + }, + { + blockId: "b3", + topic: "decompress-default-to-file", + summary: + "Changed decompress tool to write restored content to a file by default instead of returning inline. Prevents context bloat when decompressing large blocks. New API: decompress({blockId}) writes to ~/.cache/pi/acp-decompress/, decompress({blockId, inline:true}) for explicit inline.", + }, + { + blockId: "b4", + topic: "compression-protected-zone", + summary: + "Soft-protect recent zone: filter protected refs instead of failing the whole compression call. NEVER_PRESERVE_RECENT_TOOLS excludes decompress results from the preserve set. applySingleRange now filters and warns rather than throwing on partial overlap.", + }, + { + blockId: "b5", + topic: "用户认证模块", + summary: + "实现了用户登录认证流程，包含密码哈希（bcrypt）、session 管理、权限校验中间件。验证逻辑覆盖登录验证、token 验证、二次验证。登录接口 POST /api/login 返回 JWT token。身份验证失败返回 401。支持 OAuth2 第三方登录（Google、GitHub）。token 过期自动刷新机制。", + }, + { + blockId: "b6", + topic: "npm-publish-workflow", + summary: + "CI release workflow: on merge of release-v* branch, CI runs typecheck + test + build, creates git tag, publishes to npm. Prerelease versions (containing -) published with --tag dev. NPM_TOKEN secret required.", + }, + { + blockId: "b7", + topic: "context-token-estimation", + summary: + "Token estimation uses chars/4 heuristic. Real tokenizer differs — the gap lands in 'Framework' category in breakdown. displayTotal reflects real context size, not classified sum. systemPromptTokens measured separately from message categories.", + }, + { + blockId: "b8", + topic: "review-fixes", + summary: + "Code review findings: config 150K context cap fix, token counting precision, protect-recent zone edge cases. Reviewer approved after addressing warnings field backward compatibility. Merged to master.", + }, + { + blockId: "b9", + topic: "自动注入子agent工具", + summary: + "session_start 钩子自动把 compress/decompress/search_context/acp_status 注入到 9 个 builtin pi-subagents 的工具白名单。幂等检查，备份 settings.json.acp-bak，原子写 + mtime 乐观锁防并发。只追加不删除用户自定义工具。", + }, + { + blockId: "b10", + topic: "webpack-build-config", + summary: + "Webpack 5 build configuration: code splitting with SplitChunksPlugin, tree shaking enabled, source maps in production. Configured babel-loader for TypeScript and JSX. Bundle analyzer via webpack-bundle-analyzer plugin.", + }, + { + blockId: "b11", + topic: "api-rate-limiting", + summary: + "Implemented API rate limiting with token bucket algorithm. Redis-based sliding window counter. Rate limit middleware applies to /api/* routes. 429 Too Many Requests response with Retry-After header. Per-IP and per-user limits configurable.", + }, + { + blockId: "b12", + topic: "测试框架搭建", + summary: + "使用 node --test 内置测试运行器。tsx 处理 TypeScript。测试文件放在 tests/ 目录。覆盖率用 c8。Mock 用 node:test 的 mock module。集成测试和单元测试分离。", + }, + { + blockId: "b13", + topic: "websocket-realtime", + summary: + "WebSocket realtime updates via ws library. Server pushes events on data change. Client reconnect with exponential backoff. Heartbeat ping/pong every 30s to detect dead connections. Message queue for offline clients.", + }, + { + blockId: "b14", + topic: "error-handling-middleware", + summary: + "Global error handling middleware. Custom AppError class with statusCode and code field. Error formatter sanitizes stack traces in production. Async error wrapping with catchAsync wrapper. Sentry integration for production error tracking.", + }, + { + blockId: "b15", + topic: "国际化和i18n", + summary: + "i18n internationalization setup with i18next. Language detection from Accept-Language header and cookie. Translation JSON files in locales/ dir. Supports zh-CN, en-US, ja-JP. Pluralization rules and date/number formatting via Intl API.", + }, + { + blockId: "b16", + topic: "git-rebase-conflict", + summary: + "Resolved git rebase conflicts on feature/auth branch. Cherry-picked commits from upstream. Force-pushed after rewriting history. merge conflict markers in auth.ts and pool.ts. Used git reflog to recover lost commit.", + }, + { + blockId: "b17", + topic: "docker-container-setup", + summary: + "Dockerized the app with multi-stage build. Alpine base image, distroless final stage. Volume mounts for /data and /logs. docker-compose for local dev with postgres and redis services. Healthcheck endpoint /healthz.", + }, + { + blockId: "b18", + topic: "css-grid-layout", + summary: + "Refactored layout from flexbox to CSS Grid. Responsive 12-column grid with auto-fit minmax. Subgrid support for nested layouts. Fixed Safari grid gap bug. Replaced media queries with container queries in sidebar component.", + }, + { + blockId: "b19", + topic: "缓存策略redis", + summary: + "Redis 缓存层，LRU 淘汰策略，TTL 30 分钟。缓存击穿用互斥锁，缓存穿透用布隆过滤器。热点 key 永不过期，后台异步刷新。序列化用 MessagePack 替代 JSON 减少体积。", + }, + { + blockId: "b20", + topic: "graphql-schema-stitching", + summary: + "GraphQL schema stitching to merge multiple microservice schemas. DataLoader for N+1 query batching. Federation v2 with @key directive for entity resolution. Persisted queries to reduce request size and block malicious queries.", + }, + { + blockId: "b21", + topic: "csp-content-security-policy", + summary: + "Content Security Policy headers. nonce-based script-src to allow inline. report-uri for violation reporting. Upgraded insecure-requests. Blocked eval and inline styles. CSP violation reports aggregated in dashboard.", + }, + { + blockId: "b22", + topic: "migration-sql-scripts", + summary: + "Database migration scripts with Knex. Up/down migrations in migrations/ dir. Added NOT NULL constraint with default backfill. Renamed column user_name to username. Transaction-wrapped for safety. Seed data separate from migrations.", + }, + { + blockId: "b23", + topic: "日志收集elk", + summary: + "ELK 日志栈：Filebeat 采集 → Logstash 过滤 → Elasticsearch 存储 → Kibana 可视化。结构化日志 JSON 格式，含 request_id trace_id。日志级别 DEBUG/INFO/WARN/ERROR。按服务名分索引。", + }, + { + blockId: "b24", + topic: "performance-profiling", + summary: + "CPU profiling with clinic.js. Flame graph showed hot loop in parser. Memoized expensive computation. Reduced 47MB allocation per request to 3MB. p99 latency dropped from 340ms to 89ms after optimizing regex compilation.", + }, + { + blockId: "b25", + topic: "circuit-breaker", + summary: + "Circuit breaker pattern for downstream service calls. Half-open state after 5 failures. Exponential backoff retry. Hystrix-style fallback response. Breaker state shared via Redis for multi-instance sync.", + }, + { + blockId: "b26", + topic: "payment-integration", + summary: + "Stripe payment integration. Checkout session for one-time, subscription via price ID. Webhook handler verifies signature with raw body. Idempotency key prevents double charge. Refund flow with partial amounts. PCI compliance: no card data on our servers.", + }, + { + blockId: "b27", + topic: "权限管理rbac", + summary: + "RBAC 角色权限模型。用户-角色-权限三层。admin 角色拥有所有权限，editor 可编辑内容，viewer 只读。权限注解 @RequirePermission('post:write')。casbin 策略引擎。资源级细粒度授权。", + }, + { + blockId: "b28", + topic: "ci-cache-optimization", + summary: + "GitHub Actions cache optimization. Cached node_modules and .turbo. Reduced CI time from 4min to 90s. Cache key includes lockfile hash. Used actions/cache@v3 with restore-keys fallback. Matrix build parallelized across node versions.", + }, + { + blockId: "b29", + topic: "memory-leak-debug", + summary: + "Debugged memory leak in worker pool. Event listeners accumulated — removed on disconnect. WeakRef for cached objects. Heap snapshot comparison found detached DOM nodes. Set maxOldSpaceSize to 4096. Used --inspect to trace retention path.", + }, + { + blockId: "b30", + topic: "文件上传oss", + summary: + "文件上传到阿里云 OSS。分片上传大文件（>5MB），断点续传。预签名 URL 直传前端，减轻服务端压力。图片自动压缩生成缩略图（sharp）。CDN 加速静态资源分发。", + }, + { + blockId: "b31", + topic: "试验记录", + summary: "试验记录表：记录三次批次的数据结果。", + }, + { + blockId: "b32", + topic: "图表工具", + summary: "图表组件用于绘制折线图和柱状图。", + }, ]; export interface BenchQuery { @@ -57,11 +215,19 @@ export interface BenchQuery { export const QUERIES: BenchQuery[] = [ { query: "auth token", expectFirst: "b1", note: "exact multi-term" }, - { query: "authentication", expectFirst: "b1", note: "morphology (auth→authentication)" }, + { + query: "authentication", + expectFirst: "b1", + note: "morphology (auth→authentication)", + }, { query: "登录", expectFirst: "b5", note: "CJK exact" }, { query: "身份验证", expectFirst: "b5", note: "CJK synonym" }, { query: "login", expectFirst: "b5", note: "cross-lang synonym" }, - { query: "credentials", expectFirst: "b5", note: "deep synonym (credentials≈auth)" }, + { + query: "credentials", + expectFirst: "b5", + note: "deep synonym (credentials≈auth)", + }, { query: "signin", expectFirst: "b5", note: "synonym signin≈login" }, { query: "database pool", expectFirst: "b2", note: "exact multi-term" }, { query: "postgres", expectFirst: "b2", note: "abbreviation" }, @@ -74,8 +240,16 @@ export const QUERIES: BenchQuery[] = [ { query: "protected zone", expectFirst: "b4", note: "exact" }, { query: "token bucket", expectFirst: "b11", note: "disambiguation" }, { query: "tokan", expectFirst: "b1", note: "typo" }, - { query: "authentication tokan", expectFirst: "b1", note: "multi-word typo+morph" }, - { query: "redis", expectFirst: "b19", note: "ambiguous — b19 is the actual Redis cache block (bigram bloat had buried it under b11)" }, + { + query: "authentication tokan", + expectFirst: "b1", + note: "multi-word typo+morph", + }, + { + query: "redis", + expectFirst: "b19", + note: "ambiguous — b19 is the actual Redis cache block (bigram bloat had buried it under b11)", + }, { query: "缓存", expectFirst: "b19", note: "CJK exact (redis cache)" }, { query: "cache", expectFirst: "b19", note: "synonym (缓存=cache)" }, { query: "webpack", expectFirst: "b10", note: "exact" }, @@ -102,6 +276,14 @@ export const QUERIES: BenchQuery[] = [ { query: "日志", expectFirst: "b23", note: "CJK exact" }, { query: "elk", expectFirst: "b23", note: "abbreviation" }, { query: "security header", expectFirst: "b21", note: "phrase (csp)" }, - { query: "试验证明", expectFirst: "b31", note: "CJK anti-fragment (must not match 验证 in b5)" }, - { query: "图表可视化", expectFirst: "b32", note: "CJK phrase must not match b23 可视化 char-run" }, + { + query: "试验证明", + expectFirst: "b31", + note: "CJK anti-fragment (must not match 验证 in b5)", + }, + { + query: "图表可视化", + expectFirst: "b32", + note: "CJK phrase must not match b23 可视化 char-run", + }, ]; diff --git a/package.json b/package.json index 7a17868..7624169 100644 --- a/package.json +++ b/package.json @@ -1,72 +1,72 @@ { - "name": "acp-kernel", - "version": "0.0.94", - "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", - "license": "MIT", - "author": "ranxianglei", - "type": "module", - "homepage": "https://github.com/ranxianglei/acp-kernel#readme", - "repository": { - "type": "git", - "url": "git+https://github.com/ranxianglei/acp-kernel.git" + "name": "acp-kernel", + "version": "0.0.94", + "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", + "license": "MIT", + "author": "ranxianglei", + "type": "module", + "homepage": "https://github.com/ranxianglei/acp-kernel#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ranxianglei/acp-kernel.git" + }, + "bugs": { + "url": "https://github.com/ranxianglei/acp-kernel/issues" + }, + "keywords": [ + "acp", + "context", + "compression", + "context-management", + "llm", + "context-window", + "summarization" + ], + "main": "./dist/index.js", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" }, - "bugs": { - "url": "https://github.com/ranxianglei/acp-kernel/issues" + "./wire": { + "types": "./dist/wire/index.d.ts", + "import": "./dist/wire/index.js" }, - "keywords": [ - "acp", - "context", - "compression", - "context-management", - "llm", - "context-window", - "summarization" - ], - "main": "./dist/index.js", - "module": "./dist/index.js", - "types": "./dist/index.d.ts", - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - }, - "./wire": { - "types": "./dist/wire/index.d.ts", - "import": "./dist/wire/index.js" - }, - "./panel": { - "types": "./dist/panel/index.d.ts", - "import": "./dist/panel/index.js" - }, - "./persist": { - "types": "./dist/persist/index.d.ts", - "import": "./dist/persist/index.js" - } + "./panel": { + "types": "./dist/panel/index.d.ts", + "import": "./dist/panel/index.js" }, - "files": [ - "dist", - "README.md", - "LICENSE", - "DESIGN.md", - "PROVENANCE.md" - ], - "sideEffects": false, - "scripts": { - "clean": "rm -rf dist", - "build": "npm run clean && tsup && tsc --emitDeclarationOnly", - "typecheck": "tsc --noEmit", - "test": "node --import tsx --test tests/*.test.ts", - "format": "prettier --write .", - "format:check": "prettier --check ." - }, - "devDependencies": { - "@types/node": "^22.0.0", - "prettier": "^3.3.0", - "tsup": "^8.3.0", - "tsx": "^4.19.0", - "typescript": "^5.6.0" - }, - "engines": { - "node": ">=20" + "./persist": { + "types": "./dist/persist/index.d.ts", + "import": "./dist/persist/index.js" } + }, + "files": [ + "dist", + "README.md", + "LICENSE", + "DESIGN.md", + "PROVENANCE.md" + ], + "sideEffects": false, + "scripts": { + "clean": "rm -rf dist", + "build": "npm run clean && tsup && tsc --emitDeclarationOnly", + "typecheck": "tsc --noEmit", + "test": "node --import tsx --test tests/*.test.ts", + "format": "prettier --write .", + "format:check": "prettier --check ." + }, + "devDependencies": { + "@types/node": "^22.0.0", + "prettier": "^3.3.0", + "tsup": "^8.3.0", + "tsx": "^4.19.0", + "typescript": "^5.6.0" + }, + "engines": { + "node": ">=20" + } } diff --git a/src/cache-report.ts b/src/cache-report.ts index 744cd4d..2f09c65 100644 --- a/src/cache-report.ts +++ b/src/cache-report.ts @@ -653,7 +653,9 @@ function formatCacheReportSummary(report: CacheReport): string { }); const last = report.lines.length - 1; if (!shownIdx.includes(last)) shownIdx.push(last); - const kept = shownIdx.slice(Math.max(0, shownIdx.length - SUMMARY_ANOMALY_CAP)); + const kept = shownIdx.slice( + Math.max(0, shownIdx.length - SUMMARY_ANOMALY_CAP), + ); const anomalies = kept.filter((i) => { const l = report.lines[i]; if (!l) return false; diff --git a/src/compress-tools.ts b/src/compress-tools.ts index 18a307b..de0d244 100644 --- a/src/compress-tools.ts +++ b/src/compress-tools.ts @@ -107,8 +107,14 @@ export const COMPRESS_PARAMETERS = { description: "Line form: first line 'm00150–m00220 optional topic', remaining lines the summary markdown, verbatim (no JSON escaping)", }, - { ...COMPRESS_RANGE_OBJECT, required: ["startId", "endId", "summary"] }, - { ...COMPRESS_RANGE_OBJECT, required: ["startRef", "endRef", "summary"] }, + { + ...COMPRESS_RANGE_OBJECT, + required: ["startId", "endId", "summary"], + }, + { + ...COMPRESS_RANGE_OBJECT, + required: ["startRef", "endRef", "summary"], + }, ], }, }, @@ -117,11 +123,13 @@ export const COMPRESS_PARAMETERS = { }, startId: { type: "string", - description: "Flat single-range form (no content): mNNNNN ref at the start of the range", + description: + "Flat single-range form (no content): mNNNNN ref at the start of the range", }, endId: { type: "string", - description: "Flat single-range form (no content): mNNNNN ref at the end of the range", + description: + "Flat single-range form (no content): mNNNNN ref at the end of the range", }, startRef: { type: "string", @@ -133,7 +141,8 @@ export const COMPRESS_PARAMETERS = { }, summary: { type: "string", - description: "Flat single-range form (no content): self-contained summary replacing the range", + description: + "Flat single-range form (no content): self-contained summary replacing the range", }, }, }; @@ -449,7 +458,7 @@ export const ACP_STATUS_TOOL_OPENAI = { }; export const ACP_CACHE_TOOL_DESCRIPTION = - "Prompt-cache reconciliation: grand ledger (total input/cached/output, overall hit rate) with every request's miss split into new content / compression re-pay / upstream-ttl-or-client-rewrite (unattributed stable-prefix misses), plus per-fold economics (breakeven turns vs measured cadence). Defaults to a compact summary (totals + verdicts + anomalies only); pass detail=\"full\" for every fold and line item. Read-only. Call when asked about cache hits, cache invalidation, or what compression costs."; + 'Prompt-cache reconciliation: grand ledger (total input/cached/output, overall hit rate) with every request\'s miss split into new content / compression re-pay / upstream-ttl-or-client-rewrite (unattributed stable-prefix misses), plus per-fold economics (breakeven turns vs measured cadence). Defaults to a compact summary (totals + verdicts + anomalies only); pass detail="full" for every fold and line item. Read-only. Call when asked about cache hits, cache invalidation, or what compression costs.'; export const ACP_CACHE_TOOL_OPENAI = { type: "function" as const, diff --git a/src/config.ts b/src/config.ts index af03db2..5306539 100644 --- a/src/config.ts +++ b/src/config.ts @@ -199,13 +199,22 @@ export function validateConfig(config: Config): string[] { } if (config.imageCompression) { const ic = config.imageCompression; - if (ic.minTokens !== undefined && (!Number.isFinite(ic.minTokens) || ic.minTokens < 0)) { + if ( + ic.minTokens !== undefined && + (!Number.isFinite(ic.minTokens) || ic.minTokens < 0) + ) { errors.push("imageCompression.minTokens must be finite and >= 0"); } - if (ic.maxDimension !== undefined && (!Number.isInteger(ic.maxDimension) || ic.maxDimension < 16)) { + if ( + ic.maxDimension !== undefined && + (!Number.isInteger(ic.maxDimension) || ic.maxDimension < 16) + ) { errors.push("imageCompression.maxDimension must be an integer >= 16"); } - if (ic.quality !== undefined && (!Number.isFinite(ic.quality) || ic.quality < 1 || ic.quality > 100)) { + if ( + ic.quality !== undefined && + (!Number.isFinite(ic.quality) || ic.quality < 1 || ic.quality > 100) + ) { errors.push("imageCompression.quality must be in [1, 100]"); } if ( diff --git a/src/decompress.ts b/src/decompress.ts index 359ea52..d76021c 100644 --- a/src/decompress.ts +++ b/src/decompress.ts @@ -1,146 +1,158 @@ import { resolveBlockSpan } from "./block-map.js"; import { SUMMARY_HEADER } from "./prune.js"; import { clampPrefix } from "./truncate.js"; -import type { CompressionBlock, CompressionState, CoreMessage } from "./types.js"; +import type { + CompressionBlock, + CompressionState, + CoreMessage, +} from "./types.js"; export function parseBlockIdArg(arg: string): string | null { - const normalized = arg.trim().toLowerCase(); - const refMatch = /^b0*(\d+)$/.exec(normalized); - if (refMatch && refMatch[1] !== undefined) return `b${refMatch[1]}`; - const numMatch = /^(\d+)$/.exec(normalized); - if (numMatch && numMatch[1] !== undefined) return `b${numMatch[1]}`; - return null; + const normalized = arg.trim().toLowerCase(); + const refMatch = /^b0*(\d+)$/.exec(normalized); + if (refMatch && refMatch[1] !== undefined) return `b${refMatch[1]}`; + const numMatch = /^(\d+)$/.exec(normalized); + if (numMatch && numMatch[1] !== undefined) return `b${numMatch[1]}`; + return null; } export function findBlocksOverlappingMessages( - state: CompressionState, - messageIds: Set, + state: CompressionState, + messageIds: Set, ): CompressionBlock[] { - if (messageIds.size === 0) return []; - const matched: CompressionBlock[] = []; - for (const block of state.blocks) { - if (!block.active) continue; - if (block.effectiveMessageIds.some((id) => messageIds.has(id))) { - matched.push(block); - } + if (messageIds.size === 0) return []; + const matched: CompressionBlock[] = []; + for (const block of state.blocks) { + if (!block.active) continue; + if (block.effectiveMessageIds.some((id) => messageIds.has(id))) { + matched.push(block); } - return matched.sort((a, b) => numericPart(a.blockId) - numericPart(b.blockId)); + } + return matched.sort( + (a, b) => numericPart(a.blockId) - numericPart(b.blockId), + ); } -export function findActiveAncestor(state: CompressionState, blockId: string): string | null { - const start = state.blocks.find((b) => b.blockId === blockId); - if (!start) return null; - const queue: string[] = [...start.directBlockIds]; - const visited = new Set(); - while (queue.length > 0) { - const currentId = queue.shift()!; - if (visited.has(currentId)) continue; - visited.add(currentId); - const current = state.blocks.find((b) => b.blockId === currentId); - if (!current) continue; - if (current.active) return current.blockId; - for (const ancestorId of current.directBlockIds) { - if (!visited.has(ancestorId)) queue.push(ancestorId); - } +export function findActiveAncestor( + state: CompressionState, + blockId: string, +): string | null { + const start = state.blocks.find((b) => b.blockId === blockId); + if (!start) return null; + const queue: string[] = [...start.directBlockIds]; + const visited = new Set(); + while (queue.length > 0) { + const currentId = queue.shift()!; + if (visited.has(currentId)) continue; + visited.add(currentId); + const current = state.blocks.find((b) => b.blockId === currentId); + if (!current) continue; + if (current.active) return current.blockId; + for (const ancestorId of current.directBlockIds) { + if (!visited.has(ancestorId)) queue.push(ancestorId); } - return null; + } + return null; } export interface DeactivateOptions { - deep?: boolean; + deep?: boolean; } export function deactivateBlock( - state: CompressionState, - blockIds: string[], - options: DeactivateOptions = {}, + state: CompressionState, + blockIds: string[], + options: DeactivateOptions = {}, ): CompressionState { - const targets = new Set(blockIds); + const targets = new Set(blockIds); - const updated = state.blocks.map((block) => { - if (!targets.has(block.blockId) || !block.active) return block; - return { - ...block, - active: false, - durationMs: block.durationMs, - createdAt: block.createdAt, - }; - }); + const updated = state.blocks.map((block) => { + if (!targets.has(block.blockId) || !block.active) return block; + return { + ...block, + active: false, + durationMs: block.durationMs, + createdAt: block.createdAt, + }; + }); - let final = updated; - if (options.deep) { - const visited = new Set(); - const queue: string[] = []; - for (const id of blockIds) { - const block = updated.find((b) => b.blockId === id); - if (block) queue.push(...block.directBlockIds); - } - while (queue.length > 0) { - const id = queue.shift()!; - if (visited.has(id)) continue; - visited.add(id); - final = final.map((block) => { - if (block.blockId !== id) return block; - queue.push(...block.directBlockIds); - return block.active ? { ...block, active: false } : block; - }); - } + let final = updated; + if (options.deep) { + const visited = new Set(); + const queue: string[] = []; + for (const id of blockIds) { + const block = updated.find((b) => b.blockId === id); + if (block) queue.push(...block.directBlockIds); } + while (queue.length > 0) { + const id = queue.shift()!; + if (visited.has(id)) continue; + visited.add(id); + final = final.map((block) => { + if (block.blockId !== id) return block; + queue.push(...block.directBlockIds); + return block.active ? { ...block, active: false } : block; + }); + } + } - return { ...state, blocks: final }; + return { ...state, blocks: final }; } export interface RestoredPreviewResult { - preview: string; - restoredCount: number; + preview: string; + restoredCount: number; } export function buildRestoredContentPreview( - messages: CoreMessage[], - beforeActiveMessageIds: Set, - state: CompressionState, + messages: CoreMessage[], + beforeActiveMessageIds: Set, + state: CompressionState, ): RestoredPreviewResult { - const restored: CoreMessage[] = []; - for (const message of messages) { - if (!beforeActiveMessageIds.has(message.id)) continue; - const stillCovered = state.blocks.some( - (b) => b.active && b.effectiveMessageIds.includes(message.id), - ); - if (!stillCovered) restored.push(message); - } + const restored: CoreMessage[] = []; + for (const message of messages) { + if (!beforeActiveMessageIds.has(message.id)) continue; + const stillCovered = state.blocks.some( + (b) => b.active && b.effectiveMessageIds.includes(message.id), + ); + if (!stillCovered) restored.push(message); + } - if (restored.length === 0) return { preview: "", restoredCount: 0 }; + if (restored.length === 0) return { preview: "", restoredCount: 0 }; - const lines: string[] = []; - let totalLength = 0; - const MAX_PREVIEW = 2000; - const MAX_PER_MESSAGE = 200; + const lines: string[] = []; + let totalLength = 0; + const MAX_PREVIEW = 2000; + const MAX_PER_MESSAGE = 200; - for (const message of restored) { - if (totalLength >= MAX_PREVIEW) break; - const text = message.text ?? ""; - const truncated = text.length > MAX_PER_MESSAGE ? clampPrefix(text, MAX_PER_MESSAGE) + "..." : text; - const label = - message.toolName && message.contentType !== "text" - ? `${message.toolName}: ${truncated}` - : `[${message.role}] ${truncated}`; - lines.push(label); - totalLength += label.length + 1; - } + for (const message of restored) { + if (totalLength >= MAX_PREVIEW) break; + const text = message.text ?? ""; + const truncated = + text.length > MAX_PER_MESSAGE + ? clampPrefix(text, MAX_PER_MESSAGE) + "..." + : text; + const label = + message.toolName && message.contentType !== "text" + ? `${message.toolName}: ${truncated}` + : `[${message.role}] ${truncated}`; + lines.push(label); + totalLength += label.length + 1; + } - return { preview: lines.join("\n"), restoredCount: restored.length }; + return { preview: lines.join("\n"), restoredCount: restored.length }; } export interface CollectedContentResult { - /** Rendered, human-readable content string (empty when count is 0). */ - text: string; - /** Number of items rendered: direct messages + nested summaries (full=false) or all messages (full=true). */ - count: number; + /** Rendered, human-readable content string (empty when count is 0). */ + text: string; + /** Number of items rendered: direct messages + nested summaries (full=false) or all messages (full=true). */ + count: number; } export interface CollectContentOptions { - /** When true, recurse through all nested tiers to original messages. Default: false (one tier up — nested active children stay folded, their summaries shown). */ - full?: boolean; + /** When true, recurse through all nested tiers to original messages. Default: false (one tier up — nested active children stay folded, their summaries shown). */ + full?: boolean; } /** @@ -162,72 +174,74 @@ export interface CollectContentOptions { * Returns { text: "", count: 0 } when the block covers no messages. */ export function collectBlockContent( - state: CompressionState, - block: CompressionBlock, - messages: CoreMessage[], - options: CollectContentOptions = {}, + state: CompressionState, + block: CompressionBlock, + messages: CoreMessage[], + options: CollectContentOptions = {}, ): CollectedContentResult { - const full = options.full ?? false; - const targetIds = new Set(block.effectiveMessageIds); + const full = options.full ?? false; + const targetIds = new Set(block.effectiveMessageIds); - if (full) { - const msgs = messages.filter((m) => targetIds.has(m.id)); - if (msgs.length === 0) return { text: "", count: 0 }; - return { text: msgs.map(formatMessage).join("\n\n"), count: msgs.length }; - } + if (full) { + const msgs = messages.filter((m) => targetIds.has(m.id)); + if (msgs.length === 0) return { text: "", count: 0 }; + return { text: msgs.map(formatMessage).join("\n\n"), count: msgs.length }; + } - // One tier up: messages covered by nested ACTIVE children stay folded - // (their summaries shown); the block's own direct messages shown in full. - const nestedChildren: CompressionBlock[] = []; - const nestedCovered = new Set(); - for (const childId of block.directBlockIds) { - const child = state.blocks.find((b) => b.blockId === childId); - if (!child?.active) continue; - nestedChildren.push(child); - for (const id of child.effectiveMessageIds) nestedCovered.add(id); - } + // One tier up: messages covered by nested ACTIVE children stay folded + // (their summaries shown); the block's own direct messages shown in full. + const nestedChildren: CompressionBlock[] = []; + const nestedCovered = new Set(); + for (const childId of block.directBlockIds) { + const child = state.blocks.find((b) => b.blockId === childId); + if (!child?.active) continue; + nestedChildren.push(child); + for (const id of child.effectiveMessageIds) nestedCovered.add(id); + } - const parts: string[] = []; - for (const child of nestedChildren) { - const label = child.topic ? `${child.blockId}: ${child.topic}` : child.blockId; - parts.push(`${SUMMARY_HEADER} — ${label}\n${child.summary}`); - } + const parts: string[] = []; + for (const child of nestedChildren) { + const label = child.topic + ? `${child.blockId}: ${child.topic}` + : child.blockId; + parts.push(`${SUMMARY_HEADER} — ${label}\n${child.summary}`); + } - let directCount = 0; - for (const m of messages) { - if (targetIds.has(m.id) && !nestedCovered.has(m.id)) { - parts.push(formatMessage(m)); - directCount++; - } + let directCount = 0; + for (const m of messages) { + if (targetIds.has(m.id) && !nestedCovered.has(m.id)) { + parts.push(formatMessage(m)); + directCount++; } + } - const count = directCount + nestedChildren.length; - if (count === 0) return { text: "", count: 0 }; - return { text: parts.join("\n\n"), count }; + const count = directCount + nestedChildren.length; + if (count === 0) return { text: "", count: 0 }; + return { text: parts.join("\n\n"), count }; } function formatMessage(message: CoreMessage): string { - const text = message.text ?? ""; - if (message.toolName && message.contentType !== "text") { - return `[${message.role} • ${message.toolName}]\n${text}`; - } - return `[${message.role}]\n${text}`; + const text = message.text ?? ""; + if (message.toolName && message.contentType !== "text") { + return `[${message.role} • ${message.toolName}]\n${text}`; + } + return `[${message.role}]\n${text}`; } function numericPart(blockId: string): number { - const match = /^b(\d+)$/.exec(blockId); - return match && match[1] !== undefined ? Number(match[1]) : 0; + const match = /^b(\d+)$/.exec(blockId); + return match && match[1] !== undefined ? Number(match[1]) : 0; } export interface InlineRestoreResult { - restored: true; - blockId: string; - /** m-ref of the block span's first message; omitted when not derivable - * (e.g. a multi-segment restore), in which case the adapter degrades to a - * generic re-fold hint. */ - restoredStartRef?: string; - /** m-ref of the block span's last message (see restoredStartRef). */ - restoredEndRef?: string; + restored: true; + blockId: string; + /** m-ref of the block span's first message; omitted when not derivable + * (e.g. a multi-segment restore), in which case the adapter degrades to a + * generic re-fold hint. */ + restoredStartRef?: string; + /** m-ref of the block span's last message (see restoredStartRef). */ + restoredEndRef?: string; } /** Inline-decompress marker for refold-in-place (#398): flags a block whose @@ -236,23 +250,23 @@ export interface InlineRestoreResult { * flag flips, so ref immutability holds. Returns metadata for the host's * re-fold guidance; `result` is null when the block does not exist. */ export function markBlockRestoredInline( - state: CompressionState, - blockId: string, + state: CompressionState, + blockId: string, ): { state: CompressionState; result: InlineRestoreResult | null } { - const existing = state.blocks.find((b) => b.blockId === blockId); - if (!existing) return { state, result: null }; - const blocks = state.blocks.map((block) => - block.blockId === blockId ? { ...block, restoredInline: true } : block, - ); - const span = resolveBlockSpan(existing, state.messageRefs.byRaw); - const result: InlineRestoreResult = { - restored: true, - blockId, - ...(span - ? { restoredStartRef: span.startRef, restoredEndRef: span.endRef } - : {}), - }; - return { state: { ...state, blocks }, result }; + const existing = state.blocks.find((b) => b.blockId === blockId); + if (!existing) return { state, result: null }; + const blocks = state.blocks.map((block) => + block.blockId === blockId ? { ...block, restoredInline: true } : block, + ); + const span = resolveBlockSpan(existing, state.messageRefs.byRaw); + const result: InlineRestoreResult = { + restored: true, + blockId, + ...(span + ? { restoredStartRef: span.startRef, restoredEndRef: span.endRef } + : {}), + }; + return { state: { ...state, blocks }, result }; } /** All ACTIVE ancestors of a block (#398), nearest generation first: a block @@ -261,22 +275,22 @@ export function markBlockRestoredInline( * parents. Used to enforce the refold rule that no unrecovered folded tier * sits above a restored block. */ export function activeAncestorIds( - state: CompressionState, - blockId: string, + state: CompressionState, + blockId: string, ): string[] { - const out: string[] = []; - const visited = new Set([blockId]); - let frontier: string[] = [blockId]; - while (frontier.length > 0) { - const next: string[] = []; - for (const block of state.blocks) { - if (visited.has(block.blockId)) continue; - if (!block.directBlockIds.some((id) => frontier.includes(id))) continue; - visited.add(block.blockId); - if (block.active) out.push(block.blockId); - next.push(block.blockId); - } - frontier = next; + const out: string[] = []; + const visited = new Set([blockId]); + let frontier: string[] = [blockId]; + while (frontier.length > 0) { + const next: string[] = []; + for (const block of state.blocks) { + if (visited.has(block.blockId)) continue; + if (!block.directBlockIds.some((id) => frontier.includes(id))) continue; + visited.add(block.blockId); + if (block.active) out.push(block.blockId); + next.push(block.blockId); } - return out; + frontier = next; + } + return out; } diff --git a/src/filter/apply.ts b/src/filter/apply.ts index f4a4a68..2214506 100644 --- a/src/filter/apply.ts +++ b/src/filter/apply.ts @@ -1,102 +1,106 @@ import { listMessageFilters } from "./registry.js"; import type { CoreMessage } from "../types.js"; -import type { FilterResult, MessageFilterContext, MessageFiltersConfig } from "./types.js"; +import type { + FilterResult, + MessageFilterContext, + MessageFiltersConfig, +} from "./types.js"; export interface ApplyResult { - messages: CoreMessage[]; - partsFiltered: number; - partsDropped: number; - partsModified: number; + messages: CoreMessage[]; + partsFiltered: number; + partsDropped: number; + partsModified: number; } export function applyMessageFilters( - messages: CoreMessage[], - config: MessageFiltersConfig | undefined, + messages: CoreMessage[], + config: MessageFiltersConfig | undefined, ): ApplyResult { - if (!config?.enabled) { - return { messages, partsFiltered: 0, partsDropped: 0, partsModified: 0 }; - } + if (!config?.enabled) { + return { messages, partsFiltered: 0, partsDropped: 0, partsModified: 0 }; + } - const active = listMessageFilters().filter( - (filter) => config.filters?.[filter.name]?.enabled !== false, - ); - if (active.length === 0) { - return { messages, partsFiltered: 0, partsDropped: 0, partsModified: 0 }; - } + const active = listMessageFilters().filter( + (filter) => config.filters?.[filter.name]?.enabled !== false, + ); + if (active.length === 0) { + return { messages, partsFiltered: 0, partsDropped: 0, partsModified: 0 }; + } - let working = messages.map((message) => ({ ...message })); - const tally = { partsFiltered: 0, partsDropped: 0, partsModified: 0 }; - const total = working.length; + let working = messages.map((message) => ({ ...message })); + const tally = { partsFiltered: 0, partsDropped: 0, partsModified: 0 }; + const total = working.length; - const immediate = active.filter((filter) => !filter.keepLastOnly); - for (let index = 0; index < working.length; index++) { - const message = working[index]!; - const text = message.text ?? ""; - if (text.length === 0) continue; - let current = text; - const baseCtx: MessageFilterContext = { - text: current, - role: message.role, - messageIndex: index, - totalMessages: total, - toolName: message.toolName, - }; - for (const filter of immediate) { - let decision: FilterResult; - try { - decision = filter.filter(baseCtx); - } catch { - continue; - } - if (decision.action === "keep") continue; - tally.partsFiltered++; - if (decision.action === "drop") { - current = ""; - tally.partsDropped++; - } else if (decision.action === "modify" && decision.text !== undefined) { - current = decision.text; - tally.partsModified++; - } - baseCtx.text = current; - } - if (current !== text) working[index] = { ...message, text: current }; + const immediate = active.filter((filter) => !filter.keepLastOnly); + for (let index = 0; index < working.length; index++) { + const message = working[index]!; + const text = message.text ?? ""; + if (text.length === 0) continue; + let current = text; + const baseCtx: MessageFilterContext = { + text: current, + role: message.role, + messageIndex: index, + totalMessages: total, + toolName: message.toolName, + }; + for (const filter of immediate) { + let decision: FilterResult; + try { + decision = filter.filter(baseCtx); + } catch { + continue; + } + if (decision.action === "keep") continue; + tally.partsFiltered++; + if (decision.action === "drop") { + current = ""; + tally.partsDropped++; + } else if (decision.action === "modify" && decision.text !== undefined) { + current = decision.text; + tally.partsModified++; + } + baseCtx.text = current; } + if (current !== text) working[index] = { ...message, text: current }; + } - const keepLast = active.filter((filter) => filter.keepLastOnly); - for (const filter of keepLast) { - let foundLast = false; - for (let index = working.length - 1; index >= 0; index--) { - const message = working[index]!; - const text = message.text ?? ""; - if (text.length === 0) continue; - const ctx: MessageFilterContext = { - text, - role: message.role, - messageIndex: index, - totalMessages: total, - toolName: message.toolName, - }; - let decision: FilterResult; - try { - decision = filter.filter(ctx); - } catch { - continue; - } - if (decision.action !== "drop" && decision.action !== "modify") continue; - if (foundLast) { - tally.partsFiltered++; - tally.partsDropped++; - working[index] = { ...message, text: "" }; - } else { - foundLast = true; - if (decision.action === "modify" && decision.text !== undefined) { - tally.partsFiltered++; - tally.partsModified++; - working[index] = { ...message, text: decision.text }; - } - } + const keepLast = active.filter((filter) => filter.keepLastOnly); + for (const filter of keepLast) { + let foundLast = false; + for (let index = working.length - 1; index >= 0; index--) { + const message = working[index]!; + const text = message.text ?? ""; + if (text.length === 0) continue; + const ctx: MessageFilterContext = { + text, + role: message.role, + messageIndex: index, + totalMessages: total, + toolName: message.toolName, + }; + let decision: FilterResult; + try { + decision = filter.filter(ctx); + } catch { + continue; + } + if (decision.action !== "drop" && decision.action !== "modify") continue; + if (foundLast) { + tally.partsFiltered++; + tally.partsDropped++; + working[index] = { ...message, text: "" }; + } else { + foundLast = true; + if (decision.action === "modify" && decision.text !== undefined) { + tally.partsFiltered++; + tally.partsModified++; + working[index] = { ...message, text: decision.text }; } + } } + } - return { messages: working, ...tally }; + return { messages: working, ...tally }; } diff --git a/src/filter/index.ts b/src/filter/index.ts index 66a393b..a6b6d90 100644 --- a/src/filter/index.ts +++ b/src/filter/index.ts @@ -1,4 +1,9 @@ export * from "./types.js"; -export { registerMessageFilter, getMessageFilter, listMessageFilters, clearMessageFilters } from "./registry.js"; +export { + registerMessageFilter, + getMessageFilter, + listMessageFilters, + clearMessageFilters, +} from "./registry.js"; export { applyMessageFilters } from "./apply.js"; export type { ApplyResult } from "./apply.js"; diff --git a/src/filter/registry.ts b/src/filter/registry.ts index e987cc4..0e13629 100644 --- a/src/filter/registry.ts +++ b/src/filter/registry.ts @@ -3,23 +3,23 @@ import type { MessageFilter } from "./types.js"; const registry = new Map(); export function registerMessageFilter(filter: MessageFilter): void { - const existing = registry.get(filter.name); - if (existing && existing.version !== filter.version) { - throw new Error( - `Message filter "${filter.name}" already registered with version ${existing.version}, cannot register version ${filter.version}.`, - ); - } - registry.set(filter.name, filter); + const existing = registry.get(filter.name); + if (existing && existing.version !== filter.version) { + throw new Error( + `Message filter "${filter.name}" already registered with version ${existing.version}, cannot register version ${filter.version}.`, + ); + } + registry.set(filter.name, filter); } export function getMessageFilter(name: string): MessageFilter | undefined { - return registry.get(name); + return registry.get(name); } export function listMessageFilters(): MessageFilter[] { - return [...registry.values()]; + return [...registry.values()]; } export function clearMessageFilters(): void { - registry.clear(); + registry.clear(); } diff --git a/src/filter/types.ts b/src/filter/types.ts index 5c36323..8f5dfec 100644 --- a/src/filter/types.ts +++ b/src/filter/types.ts @@ -1,29 +1,29 @@ export interface MessageFilterContext { - text: string; - role: string; - messageIndex: number; - totalMessages: number; - toolName?: string; - modelContextLimit?: number; + text: string; + role: string; + messageIndex: number; + totalMessages: number; + toolName?: string; + modelContextLimit?: number; } export interface FilterResult { - action: "keep" | "modify" | "drop"; - text?: string; - reason?: string; + action: "keep" | "modify" | "drop"; + text?: string; + reason?: string; } export interface MessageFilter { - name: string; - version: string; - description: string; - filter(ctx: MessageFilterContext): FilterResult; - keepLastOnly?: boolean; + name: string; + version: string; + description: string; + filter(ctx: MessageFilterContext): FilterResult; + keepLastOnly?: boolean; } export type MessageFilterConfig = Record; export interface MessageFiltersConfig { - enabled: boolean; - filters: MessageFilterConfig; + enabled: boolean; + filters: MessageFilterConfig; } diff --git a/src/handoff.ts b/src/handoff.ts index 23e7202..58cc7d3 100644 --- a/src/handoff.ts +++ b/src/handoff.ts @@ -39,10 +39,14 @@ export function renderMessage(m: CoreMessage): string { parts.push(m.text ?? ""); break; case "tool-call": - parts.push(`\`${m.toolName ?? "?"}(${m.toolCallId ?? ""})\` args: ${m.text ?? ""}`); + parts.push( + `\`${m.toolName ?? "?"}(${m.toolCallId ?? ""})\` args: ${m.text ?? ""}`, + ); break; case "tool-result": - parts.push(`\`${m.toolName ?? "?"}(${m.toolCallId ?? ""})\` → ${m.text ?? ""}`); + parts.push( + `\`${m.toolName ?? "?"}(${m.toolCallId ?? ""})\` → ${m.text ?? ""}`, + ); break; case "reasoning": parts.push(`_reasoning_: ${m.text ?? ""}`); @@ -61,16 +65,21 @@ export function renderHandoff(input: HandoffInput): string { if (meta.label) lines.push(`- label: ${meta.label}`); lines.push(`- session id: ${meta.sessionId}`); for (const bullet of meta.extraBullets ?? []) lines.push(bullet); - if (meta.contextTokens) lines.push(`- last context tokens: ~${meta.contextTokens}`); - lines.push(`- compression blocks: ${state.blocks.length} (active ${state.blocks.filter((b) => b.active).length})`); + if (meta.contextTokens) + lines.push(`- last context tokens: ~${meta.contextTokens}`); + lines.push( + `- compression blocks: ${state.blocks.length} (active ${state.blocks.filter((b) => b.active).length})`, + ); lines.push(""); const folded = input.folded === true; const view = full || folded ? coreMessages : prune(coreMessages, state); - lines.push(full && !folded - ? `## Full conversation (${coreMessages.length} messages)` - : folded - ? `## Conversation (persisted folded snapshot, ${coreMessages.length} messages)` - : `## Conversation (folded view as the model saw it, ${coreMessages.length} client messages)`); + lines.push( + full && !folded + ? `## Full conversation (${coreMessages.length} messages)` + : folded + ? `## Conversation (persisted folded snapshot, ${coreMessages.length} messages)` + : `## Conversation (folded view as the model saw it, ${coreMessages.length} client messages)`, + ); lines.push(""); if (view.length === 0) { lines.push("No conversation messages to export."); @@ -108,5 +117,7 @@ export function matchSession( if (exact.length > 0) return exact; const byLabel = sessions.filter((s) => labelOf(s) === selector); if (byLabel.length > 0) return byLabel; - return sessions.filter((s) => s.id.startsWith(selector) || (labelOf(s) ?? "").startsWith(selector)); + return sessions.filter( + (s) => s.id.startsWith(selector) || (labelOf(s) ?? "").startsWith(selector), + ); } diff --git a/src/hide-consumed.ts b/src/hide-consumed.ts index 0452ccd..4091508 100644 --- a/src/hide-consumed.ts +++ b/src/hide-consumed.ts @@ -13,70 +13,88 @@ import type { CompressionState, CoreMessage } from "./types.js"; const KEEP_LAST_ORPHANED = 2; export interface HideConsumedResult { - messages: CoreMessage[]; - hidden: number; - /** Padded refs of the orphan compress call/result messages hidden by this - * pass (orphans beyond the keep window). Consumed by boundary resolution to - * snap a range endpoint onto a hidden-but-intact orphan (#396). */ - hiddenOrphanRefs: string[]; + messages: CoreMessage[]; + hidden: number; + /** Padded refs of the orphan compress call/result messages hidden by this + * pass (orphans beyond the keep window). Consumed by boundary resolution to + * snap a range endpoint onto a hidden-but-intact orphan (#396). */ + hiddenOrphanRefs: string[]; } function rangeKey(startRef: string, endRef: string): string { - return `${startRef}::${endRef}`; + return `${startRef}::${endRef}`; } // Adapters (pi) persist the rendered ref tag in front of the tool-call text, // so the JSON args no longer start at index 0. Locate the first "{" instead of // parsing the raw text — the prefix is preserved on output. -function parseCallText(text: string | undefined): { prefix: string; obj: Record; content: unknown[]; contentWasString: boolean } | null { - const raw = text ?? ""; - const start = raw.indexOf("{"); - if (start < 0) return null; - let parsed: unknown; +function parseCallText(text: string | undefined): { + prefix: string; + obj: Record; + content: unknown[]; + contentWasString: boolean; +} | null { + const raw = text ?? ""; + const start = raw.indexOf("{"); + if (start < 0) return null; + let parsed: unknown; + try { + parsed = JSON.parse(raw.slice(start)); + } catch { + return null; + } + if (!parsed || typeof parsed !== "object") return null; + const obj = parsed as Record; + let content: unknown[] | null = null; + let contentWasString = false; + if (Array.isArray(obj.content)) { + content = obj.content; + } else if (typeof obj.content === "string") { + // Non-strict-tool providers (qwen etc.) sometimes stringify the content + // array inside the JSON args; the compress tool accepts it, so the + // rewrite must too. Measured: ALL 52 calls in the billion-context-pi + // #336 storm session used this form (#230). + contentWasString = true; try { - parsed = JSON.parse(raw.slice(start)); + const inner: unknown = JSON.parse(obj.content); + if (Array.isArray(inner)) content = inner; } catch { - return null; + content = null; } - if (!parsed || typeof parsed !== "object") return null; - const obj = parsed as Record; - let content: unknown[] | null = null; - let contentWasString = false; - if (Array.isArray(obj.content)) { - content = obj.content; - } else if (typeof obj.content === "string") { - // Non-strict-tool providers (qwen etc.) sometimes stringify the content - // array inside the JSON args; the compress tool accepts it, so the - // rewrite must too. Measured: ALL 52 calls in the billion-context-pi - // #336 storm session used this form (#230). - contentWasString = true; - try { - const inner: unknown = JSON.parse(obj.content); - if (Array.isArray(inner)) content = inner; - } catch { - content = null; - } - } - if (!content || content.length === 0) return null; - return { prefix: raw.slice(0, start), obj, content, contentWasString }; + } + if (!content || content.length === 0) return null; + return { prefix: raw.slice(0, start), obj, content, contentWasString }; } -function rewriteCompressText(text: string | undefined, liveKeys: Set): string | null { - const parsed = parseCallText(text); - if (!parsed) return null; - const { prefix, obj, content, contentWasString } = parsed; +function rewriteCompressText( + text: string | undefined, + liveKeys: Set, +): string | null { + const parsed = parseCallText(text); + if (!parsed) return null; + const { prefix, obj, content, contentWasString } = parsed; - const kept = content.filter((entry): entry is Record => { - if (!entry || typeof entry !== "object") return false; - const e = entry as Record; - const s = typeof e.startId === "string" ? e.startId : typeof e.messageId === "string" ? e.messageId : ""; - const end = typeof e.endId === "string" ? e.endId : typeof e.messageId === "string" ? e.messageId : ""; - return liveKeys.has(rangeKey(s, end)); - }); + const kept = content.filter((entry): entry is Record => { + if (!entry || typeof entry !== "object") return false; + const e = entry as Record; + const s = + typeof e.startId === "string" + ? e.startId + : typeof e.messageId === "string" + ? e.messageId + : ""; + const end = + typeof e.endId === "string" + ? e.endId + : typeof e.messageId === "string" + ? e.messageId + : ""; + return liveKeys.has(rangeKey(s, end)); + }); - if (kept.length === 0) return null; + if (kept.length === 0) return null; - return prefix + serializeCompacted(obj, kept, contentWasString).text; + return prefix + serializeCompacted(obj, kept, contentWasString).text; } // Live compress-call args duplicate every range's full summary text while the @@ -86,134 +104,155 @@ function rewriteCompressText(text: string | undefined, liveKeys: Set): s const SUMMARY_STUB_CHARS = 200; function compactEntry(entry: unknown): unknown { - if (!entry || typeof entry !== "object") return entry; - const e = entry as Record; - if (typeof e.summary !== "string" || e.summary.length <= SUMMARY_STUB_CHARS) return entry; - return { ...e, summary: `${clampPrefix(e.summary, SUMMARY_STUB_CHARS - 1)}…` }; + if (!entry || typeof entry !== "object") return entry; + const e = entry as Record; + if (typeof e.summary !== "string" || e.summary.length <= SUMMARY_STUB_CHARS) + return entry; + return { + ...e, + summary: `${clampPrefix(e.summary, SUMMARY_STUB_CHARS - 1)}…`, + }; } -function serializeCompacted(obj: Record, content: unknown[], contentWasString: boolean): { text: string; changed: boolean } { - let changed = false; - const compacted = content.map((entry) => { - const out = compactEntry(entry); - if (out !== entry) changed = true; - return out; - }); - // Preserve the original shape: a stringified content array stays a string - // so downstream text comparisons and replays are unaffected. - const outContent = contentWasString ? JSON.stringify(compacted) : compacted; - return { text: JSON.stringify({ ...obj, content: outContent }), changed }; +function serializeCompacted( + obj: Record, + content: unknown[], + contentWasString: boolean, +): { text: string; changed: boolean } { + let changed = false; + const compacted = content.map((entry) => { + const out = compactEntry(entry); + if (out !== entry) changed = true; + return out; + }); + // Preserve the original shape: a stringified content array stays a string + // so downstream text comparisons and replays are unaffected. + const outContent = contentWasString ? JSON.stringify(compacted) : compacted; + return { text: JSON.stringify({ ...obj, content: outContent }), changed }; } function compactCompressText(text: string | undefined): string | null { - const parsed = parseCallText(text); - if (!parsed) return null; - const { prefix, obj, content, contentWasString } = parsed; - const { text: out, changed } = serializeCompacted(obj, content, contentWasString); - return changed ? prefix + out : null; + const parsed = parseCallText(text); + if (!parsed) return null; + const { prefix, obj, content, contentWasString } = parsed; + const { text: out, changed } = serializeCompacted( + obj, + content, + contentWasString, + ); + return changed ? prefix + out : null; } export function hideConsumedCompressCalls( - state: CompressionState, - messages: CoreMessage[], + state: CompressionState, + messages: CoreMessage[], ): HideConsumedResult { - const allBlockCallIds = new Set(); - const activeCallIds = new Set(); - const liveRangeKeysByCallId = new Map>(); - const legacyLiveByCallId = new Set(); - for (const block of state.blocks) { - if (!block.compressCallId) continue; - allBlockCallIds.add(block.compressCallId); - if (!block.active) continue; - activeCallIds.add(block.compressCallId); - if (block.startRef === undefined || block.endRef === undefined) { - legacyLiveByCallId.add(block.compressCallId); - continue; - } - let keys = liveRangeKeysByCallId.get(block.compressCallId); - if (!keys) { - keys = new Set(); - liveRangeKeysByCallId.set(block.compressCallId, keys); - } - keys.add(rangeKey(block.startRef, block.endRef)); + const allBlockCallIds = new Set(); + const activeCallIds = new Set(); + const liveRangeKeysByCallId = new Map>(); + const legacyLiveByCallId = new Set(); + for (const block of state.blocks) { + if (!block.compressCallId) continue; + allBlockCallIds.add(block.compressCallId); + if (!block.active) continue; + activeCallIds.add(block.compressCallId); + if (block.startRef === undefined || block.endRef === undefined) { + legacyLiveByCallId.add(block.compressCallId); + continue; + } + let keys = liveRangeKeysByCallId.get(block.compressCallId); + if (!keys) { + keys = new Set(); + liveRangeKeysByCallId.set(block.compressCallId, keys); } + keys.add(rangeKey(block.startRef, block.endRef)); + } - const lastOrphanedCallIds: string[] = []; - for (let i = messages.length - 1; i >= 0 && lastOrphanedCallIds.length < KEEP_LAST_ORPHANED; i--) { - const message = messages[i]!; - if (message.toolName !== "compress" || message.contentType !== "tool-call") continue; - const callId = message.toolCallId; - if (callId && !allBlockCallIds.has(callId)) { - lastOrphanedCallIds.push(callId); - } + const lastOrphanedCallIds: string[] = []; + for ( + let i = messages.length - 1; + i >= 0 && lastOrphanedCallIds.length < KEEP_LAST_ORPHANED; + i-- + ) { + const message = messages[i]!; + if (message.toolName !== "compress" || message.contentType !== "tool-call") + continue; + const callId = message.toolCallId; + if (callId && !allBlockCallIds.has(callId)) { + lastOrphanedCallIds.push(callId); } + } - const keepCallIds = new Set([...activeCallIds, ...lastOrphanedCallIds]); + const keepCallIds = new Set([...activeCallIds, ...lastOrphanedCallIds]); - const hiddenCallIds = new Set(); - for (const message of messages) { - if ( - message.toolName === "compress" && - message.contentType === "tool-call" && - (!message.toolCallId || !keepCallIds.has(message.toolCallId)) - ) { - if (message.toolCallId) hiddenCallIds.add(message.toolCallId); - } + const hiddenCallIds = new Set(); + for (const message of messages) { + if ( + message.toolName === "compress" && + message.contentType === "tool-call" && + (!message.toolCallId || !keepCallIds.has(message.toolCallId)) + ) { + if (message.toolCallId) hiddenCallIds.add(message.toolCallId); } + } - let hidden = 0; - const hiddenOrphanRefs: string[] = []; - const rememberHiddenRef = (message: CoreMessage): void => { - if (message.toolCallId && allBlockCallIds.has(message.toolCallId)) { - return; - } - const ref = state.messageRefs.byRaw[message.id]; - if (ref && ref !== BLOCKED_REF && !hiddenOrphanRefs.includes(ref)) { - hiddenOrphanRefs.push(ref); - } - }; - const result: CoreMessage[] = []; - for (const message of messages) { - if ( - message.toolName === "compress" && - message.contentType === "tool-call" && - (!message.toolCallId || !keepCallIds.has(message.toolCallId)) - ) { - hidden++; - rememberHiddenRef(message); - continue; - } - if ( - message.contentType === "tool-result" && - message.toolCallId && - hiddenCallIds.has(message.toolCallId) - ) { - hidden++; - rememberHiddenRef(message); - continue; - } - if ( - message.toolName === "compress" && - message.contentType === "tool-call" && - message.toolCallId && - keepCallIds.has(message.toolCallId) - ) { - const liveKeys = liveRangeKeysByCallId.get(message.toolCallId); - if (liveKeys && liveKeys.size > 0 && !legacyLiveByCallId.has(message.toolCallId)) { - const rewritten = rewriteCompressText(message.text, liveKeys); - if (rewritten !== null) { - result.push({ ...message, text: rewritten }); - continue; - } - } - const compacted = compactCompressText(message.text); - if (compacted !== null) { - result.push({ ...message, text: compacted }); - continue; - } + let hidden = 0; + const hiddenOrphanRefs: string[] = []; + const rememberHiddenRef = (message: CoreMessage): void => { + if (message.toolCallId && allBlockCallIds.has(message.toolCallId)) { + return; + } + const ref = state.messageRefs.byRaw[message.id]; + if (ref && ref !== BLOCKED_REF && !hiddenOrphanRefs.includes(ref)) { + hiddenOrphanRefs.push(ref); + } + }; + const result: CoreMessage[] = []; + for (const message of messages) { + if ( + message.toolName === "compress" && + message.contentType === "tool-call" && + (!message.toolCallId || !keepCallIds.has(message.toolCallId)) + ) { + hidden++; + rememberHiddenRef(message); + continue; + } + if ( + message.contentType === "tool-result" && + message.toolCallId && + hiddenCallIds.has(message.toolCallId) + ) { + hidden++; + rememberHiddenRef(message); + continue; + } + if ( + message.toolName === "compress" && + message.contentType === "tool-call" && + message.toolCallId && + keepCallIds.has(message.toolCallId) + ) { + const liveKeys = liveRangeKeysByCallId.get(message.toolCallId); + if ( + liveKeys && + liveKeys.size > 0 && + !legacyLiveByCallId.has(message.toolCallId) + ) { + const rewritten = rewriteCompressText(message.text, liveKeys); + if (rewritten !== null) { + result.push({ ...message, text: rewritten }); + continue; } - result.push(message); + } + const compacted = compactCompressText(message.text); + if (compacted !== null) { + result.push({ ...message, text: compacted }); + continue; + } } + result.push(message); + } - return { messages: result, hidden, hiddenOrphanRefs }; + return { messages: result, hidden, hiddenOrphanRefs }; } diff --git a/src/image-compress.ts b/src/image-compress.ts index be2fa99..a49f2f9 100644 --- a/src/image-compress.ts +++ b/src/image-compress.ts @@ -34,20 +34,24 @@ import type { * #1097's placeholder determinism). */ -export const DEFAULT_IMAGE_COMPRESSION_CONFIG: Required = { - enabled: false, - minTokens: 512, - maxDimension: 1280, - quality: 80, - format: "webp", -}; +export const DEFAULT_IMAGE_COMPRESSION_CONFIG: Required = + { + enabled: false, + minTokens: 512, + maxDimension: 1280, + quality: 80, + format: "webp", + }; /** Sub-field merge over defaults; hosts layer global → provider → model * exactly like absorb (three levels). */ export function resolveImageCompressionConfig( config: Config, ): Required { - return { ...DEFAULT_IMAGE_COMPRESSION_CONFIG, ...(config.imageCompression ?? {}) }; + return { + ...DEFAULT_IMAGE_COMPRESSION_CONFIG, + ...(config.imageCompression ?? {}), + }; } // --------------------------------------------------------------------------- @@ -85,11 +89,12 @@ function u16le(b: Uint8Array, off: number): number { function u32be(b: Uint8Array, off: number): number { return ( - (byteAt(b, off) << 24) | - (byteAt(b, off + 1) << 16) | - (byteAt(b, off + 2) << 8) | - byteAt(b, off + 3) - ) >>> 0; + ((byteAt(b, off) << 24) | + (byteAt(b, off + 1) << 16) | + (byteAt(b, off + 2) << 8) | + byteAt(b, off + 3)) >>> + 0 + ); } function i32le(b: Uint8Array, off: number): number { @@ -154,9 +159,11 @@ export function parseImageDimensions( // VP8L: signature byte 0x2F, then a 4-byte pack: width-1 (14 bits), // height-1 (14 bits), version (4 bits). if (b[20] !== 0x2f) return undefined; - const w = ((byteAt(b, 22) & 0x3f) << 8 | byteAt(b, 21)) + 1; + const w = (((byteAt(b, 22) & 0x3f) << 8) | byteAt(b, 21)) + 1; const h = - ((byteAt(b, 24) & 0x0f) << 10) | (byteAt(b, 23) << 2) | ((byteAt(b, 22) & 0xc0) >> 6); + ((byteAt(b, 24) & 0x0f) << 10) | + (byteAt(b, 23) << 2) | + ((byteAt(b, 22) & 0xc0) >> 6); return w > 0 && h > 0 ? { width: w, height: h + 1 } : undefined; } if (b[12] === 0x56 && b[13] === 0x50 && b[14] === 0x38 && b[15] === 0x20) { @@ -179,7 +186,11 @@ export function parseImageDimensions( continue; } const marker = byteAt(b, off + 1); - if (marker === 0xd8 || marker === 0x01 || (marker >= 0xd0 && marker <= 0xd9)) { + if ( + marker === 0xd8 || + marker === 0x01 || + (marker >= 0xd0 && marker <= 0xd9) + ) { off += 2; continue; } @@ -215,10 +226,14 @@ export function parseImageDimensionsFromBase64( b64: string, ): ImageDimensions | undefined { if (typeof b64 !== "string" || b64.length === 0) return undefined; - const head = parseImageDimensions(Buffer.from(b64.slice(0, HEADER_SCAN_CHARS), "base64")); + const head = parseImageDimensions( + Buffer.from(b64.slice(0, HEADER_SCAN_CHARS), "base64"), + ); if (head) return head; if (b64.startsWith("/9j/") && b64.length > HEADER_SCAN_CHARS) { - return parseImageDimensions(Buffer.from(b64.slice(0, JPEG_SCAN_CHARS), "base64")); + return parseImageDimensions( + Buffer.from(b64.slice(0, JPEG_SCAN_CHARS), "base64"), + ); } return undefined; } @@ -227,7 +242,12 @@ export function parseImageDimensionsFromBase64( * (providers bill small images enlarged), cap the long side at 2048, then * 85 + 170 per 512×512 tile. Bounds ≈ [765, 2805]. */ export function pixelTileEstimate(width: number, height: number): number { - if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { + if ( + !Number.isFinite(width) || + !Number.isFinite(height) || + width <= 0 || + height <= 0 + ) { return PIXEL_IMAGE_FALLBACK_TOKENS; } let sw = width; @@ -359,7 +379,8 @@ export function decideImageRoute( ): ImageRoutingDecision { const cfg = resolveImageCompressionConfig(config); const estimatedTokens = estimateImageTokens(meta); - if (!cfg.enabled) return { action: "pass", reason: "disabled", estimatedTokens }; + if (!cfg.enabled) + return { action: "pass", reason: "disabled", estimatedTokens }; if (!meta.mediaType.startsWith("image/")) { return { action: "pass", reason: "not-an-image", estimatedTokens }; } @@ -373,7 +394,11 @@ export function decideImageRoute( action: "downsample", reason: "downsample", estimatedTokens, - recipe: { maxDimension: cfg.maxDimension, quality: cfg.quality, format: cfg.format }, + recipe: { + maxDimension: cfg.maxDimension, + quality: cfg.quality, + format: cfg.format, + }, }; } @@ -420,7 +445,8 @@ export function parseImageFullInput( if (typeof input === "string") { try { const parsed: unknown = JSON.parse(input); - if (parsed && typeof parsed === "object") obj = parsed as Record; + if (parsed && typeof parsed === "object") + obj = parsed as Record; } catch { obj = null; } @@ -428,7 +454,9 @@ export function parseImageFullInput( obj = input as Record; } if (!obj) { - onWarn?.(`[acp-image-full-input] rejected: not an object (${typeof input})`); + onWarn?.( + `[acp-image-full-input] rejected: not an object (${typeof input})`, + ); return null; } let ref: string | undefined; @@ -472,7 +500,10 @@ export function recordImageShrink( }; } -export function isImageFullRestored(state: CompressionState, ref: string): boolean { +export function isImageFullRestored( + state: CompressionState, + ref: string, +): boolean { return (state.imageFullRestored ?? []).includes(ref); } diff --git a/src/message-kind.ts b/src/message-kind.ts index 76525a2..10b75aa 100644 --- a/src/message-kind.ts +++ b/src/message-kind.ts @@ -9,5 +9,7 @@ import type { CoreMessage } from "./types.js"; * (issue #390). */ export function isToolMessage(message: CoreMessage): boolean { - return message.contentType === "tool-call" || message.contentType === "tool-result"; + return ( + message.contentType === "tool-call" || message.contentType === "tool-result" + ); } diff --git a/src/nudge-text.ts b/src/nudge-text.ts index 82116ef..cf7c321 100644 --- a/src/nudge-text.ts +++ b/src/nudge-text.ts @@ -1,4 +1,11 @@ -import type { NudgeDecision, CompressibleRange, ProtectedRange, ContextBreakdown, CompressionBlock, BlockSpan } from "./types.js"; +import type { + NudgeDecision, + CompressibleRange, + ProtectedRange, + ContextBreakdown, + CompressionBlock, + BlockSpan, +} from "./types.js"; import { defaultPrompts } from "./prompts.js"; import type { Prompts } from "./prompts.js"; @@ -16,12 +23,18 @@ export interface RenderedNudge { text: string; } -function efficiencyNote(prompts: Prompts, sections: NudgePromptSections): string | null { +function efficiencyNote( + prompts: Prompts, + sections: NudgePromptSections, +): string | null { if (sections.efficiencyNote !== undefined) return sections.efficiencyNote; return `This is an efficiency nudge to compress early and keep context lean — not an overflow warning. A separate, stronger alert will appear if the context is actually full.\n\n${prompts.compressPhilosophy}`; } -function emergencyHeader(prompts: Prompts, sections: NudgePromptSections): string | null { +function emergencyHeader( + prompts: Prompts, + sections: NudgePromptSections, +): string | null { if (sections.emergencyHeader !== undefined) return sections.emergencyHeader; return `⚠️ Context limit reached — compress now. Prioritize consumed tool outputs.\n\n${prompts.compressPhilosophy}`; } @@ -39,12 +52,11 @@ function formatBreakdown(bd?: ContextBreakdown): string { if (bd.summaries > 0) parts.push(`${formatK(bd.summaries)} summaries`); if (bd.code > 0) parts.push(`${formatK(bd.code)} code`); if (bd.text > 0) parts.push(`${formatK(bd.text)} text`); - const growth = bd.growth > 0 ? `\n+${formatK(bd.growth)} since last nudge` : ""; + const growth = + bd.growth > 0 ? `\n+${formatK(bd.growth)} since last nudge` : ""; return `Context breakdown: ${parts.join(" | ")}${growth}`; } - - function formatTierTargetBlocks(blocks: CompressionBlock[]): string { if (blocks.length === 0) { return "Target blocks: (none — no tier blocks found)"; @@ -64,13 +76,17 @@ function formatBlockMap(spans: BlockSpan[]): string { const hidden = Math.max(0, spans.length - BLOCK_MAP_MAX_SHOWN); const shown = hidden > 0 ? spans.slice(-BLOCK_MAP_MAX_SHOWN) : spans; const items = shown.map( - (s) => `${s.blockId}=${s.startRef}–${s.endRef}${s.tier > 1 ? ` t${s.tier}` : ""}`, + (s) => + `${s.blockId}=${s.startRef}–${s.endRef}${s.tier > 1 ? ` t${s.tier}` : ""}`, ); const prefix = hidden > 0 ? `…+${hidden} older · ` : ""; return `Active blocks (${spans.length}): ${prefix}${items.join(" · ")}`; } -export function formatRanges(compressible: CompressibleRange[], protectedRanges: ProtectedRange[]): string { +export function formatRanges( + compressible: CompressibleRange[], + protectedRanges: ProtectedRange[], +): string { if (compressible.length === 0 && protectedRanges.length === 0) { return "[No specific ranges detected — compress any consumed content.]"; } @@ -80,12 +96,23 @@ export function formatRanges(compressible: CompressibleRange[], protectedRanges: // lost the time order and hid overlaps; a range can be partly compressible // and partly protected, which only the merged view shows correctly. interface Merged { - startRef: string; endRef: string; startNum: number; endNum: number; - startPos: number; endPos: number; - count: number; tokens: number; userMsgs: number; - compressibleTokens: number; compressibleCount: number; - protectedTokens: number; protectedCount: number; protectedTools: string[]; - toolPct: number; textPct: number; dangerous: boolean; + startRef: string; + endRef: string; + startNum: number; + endNum: number; + startPos: number; + endPos: number; + count: number; + tokens: number; + userMsgs: number; + compressibleTokens: number; + compressibleCount: number; + protectedTokens: number; + protectedCount: number; + protectedTools: string[]; + toolPct: number; + textPct: number; + dangerous: boolean; } const refNum = (ref: string): number => { const m = ref.match(/\d+/); @@ -94,20 +121,44 @@ export function formatRanges(compressible: CompressibleRange[], protectedRanges: const entries: Merged[] = []; for (const r of compressible) { entries.push({ - startRef: r.startRef, endRef: r.endRef, startNum: refNum(r.startRef), endNum: refNum(r.endRef), - startPos: r.startIndex ?? refNum(r.startRef), endPos: r.endIndex ?? refNum(r.endRef), - count: r.count, tokens: r.tokens, userMsgs: r.userMsgs ?? 0, toolPct: r.toolPct, textPct: r.textPct, - compressibleTokens: r.tokens, compressibleCount: r.count, - protectedTokens: 0, protectedCount: 0, protectedTools: [], dangerous: r.dangerous ?? false, + startRef: r.startRef, + endRef: r.endRef, + startNum: refNum(r.startRef), + endNum: refNum(r.endRef), + startPos: r.startIndex ?? refNum(r.startRef), + endPos: r.endIndex ?? refNum(r.endRef), + count: r.count, + tokens: r.tokens, + userMsgs: r.userMsgs ?? 0, + toolPct: r.toolPct, + textPct: r.textPct, + compressibleTokens: r.tokens, + compressibleCount: r.count, + protectedTokens: 0, + protectedCount: 0, + protectedTools: [], + dangerous: r.dangerous ?? false, }); } for (const r of protectedRanges) { entries.push({ - startRef: r.startRef, endRef: r.endRef, startNum: refNum(r.startRef), endNum: refNum(r.endRef), - startPos: r.startIndex ?? refNum(r.startRef), endPos: r.endIndex ?? refNum(r.endRef), - count: r.count, tokens: r.tokens, userMsgs: 0, toolPct: 0, textPct: 0, - compressibleTokens: 0, compressibleCount: 0, - protectedTokens: r.tokens, protectedCount: r.count, protectedTools: [...r.tools], dangerous: false, + startRef: r.startRef, + endRef: r.endRef, + startNum: refNum(r.startRef), + endNum: refNum(r.endRef), + startPos: r.startIndex ?? refNum(r.startRef), + endPos: r.endIndex ?? refNum(r.endRef), + count: r.count, + tokens: r.tokens, + userMsgs: 0, + toolPct: 0, + textPct: 0, + compressibleTokens: 0, + compressibleCount: 0, + protectedTokens: r.tokens, + protectedCount: r.count, + protectedTools: [...r.tools], + dangerous: false, }); } // Order/merge by POSITION, never ref number: refs can be non-monotonic vs array @@ -140,7 +191,10 @@ export function formatRanges(compressible: CompressibleRange[], protectedRanges: const userNote = (n: number): string => n > 0 ? ` · ${n} user msg${n > 1 ? "s" : ""}` : ""; const lines = merged.map((e) => { - const suffix = e.dangerous && e.compressibleTokens > 0 ? " ⚠️ NOT recommended unless you are certain." : ""; + const suffix = + e.dangerous && e.compressibleTokens > 0 + ? " ⚠️ NOT recommended unless you are certain." + : ""; if (e.protectedTokens > 0 && e.compressibleTokens === 0) { return ` ${e.startRef}–${e.endRef} ${e.count} msgs ${formatK(e.tokens)} [PROTECTED: ${e.protectedTools.join(", ")} — not compressible]${suffix}`; } @@ -156,7 +210,10 @@ const DEFAULT_T2_GUIDANCE = `Your tier-1 compression summaries have accumulated. const DEFAULT_T3_GUIDANCE = `Your tier-2 compression summaries have accumulated. Condense them further into a tier-3 ultra-condensed summary. Use block IDs as boundaries (startId and endId as bN). Any raw (uncompressed) messages sitting between the boundary blocks are absorbed into the tier-3 block as well — apply HOW TO COMPRESS to those raw messages and the TIER 3 condensation rules to the existing summaries, so the whole span is covered and nothing is lost.`; -function tierGuidance(tier: 2 | 3, sections: NudgePromptSections): string | null { +function tierGuidance( + tier: 2 | 3, + sections: NudgePromptSections, +): string | null { const value = tier === 2 ? sections.t2Guidance : sections.t3Guidance; if (value !== undefined) return value; return tier === 2 ? DEFAULT_T2_GUIDANCE : DEFAULT_T3_GUIDANCE; @@ -167,11 +224,19 @@ function compact(parts: string[]): string[] { return parts; } -export function renderNudgeText(decision: NudgeDecision, prompts: Prompts = defaultPrompts, sections: NudgePromptSections = {}): RenderedNudge { +export function renderNudgeText( + decision: NudgeDecision, + prompts: Prompts = defaultPrompts, + sections: NudgePromptSections = {}, +): RenderedNudge { const breakdownStr = formatBreakdown(decision.contextBreakdown); - const rangesStr = formatRanges(decision.compressibleRanges, decision.protectedRanges ?? []); + const rangesStr = formatRanges( + decision.compressibleRanges, + decision.protectedRanges ?? [], + ); const blockMapStr = formatBlockMap(decision.activeBlockSpans ?? []); - const isEmergency = !!decision.breakdown?.emergencyOverride || !!decision.breakdown?.overLimit; + const isEmergency = + !!decision.breakdown?.emergencyOverride || !!decision.breakdown?.overLimit; if (decision.tier !== null && decision.tier >= 2) { const isT2 = decision.tier === 2; diff --git a/src/packs.ts b/src/packs.ts index 66180b3..0a7b039 100644 --- a/src/packs.ts +++ b/src/packs.ts @@ -68,15 +68,35 @@ export interface PackResolver { listPacks(): Pack[]; } -const PROMPT_RULE_KEYS = ["compressPhilosophy", "howToCompressRules", "tier2DistillRules", "tier3CondenseRules"] as const; -const COMPRESS_SECTION_KEYS = ["acpTags", "tools", "summariesInContext", "textProtocol", "textTools", "functionTools"] as const; -const NUDGE_SECTION_KEYS = ["efficiencyNote", "emergencyHeader", "t2Guidance", "t3Guidance"] as const; +const PROMPT_RULE_KEYS = [ + "compressPhilosophy", + "howToCompressRules", + "tier2DistillRules", + "tier3CondenseRules", +] as const; +const COMPRESS_SECTION_KEYS = [ + "acpTags", + "tools", + "summariesInContext", + "textProtocol", + "textTools", + "functionTools", +] as const; +const NUDGE_SECTION_KEYS = [ + "efficiencyNote", + "emergencyHeader", + "t2Guidance", + "t3Guidance", +] as const; export function isValidPackName(name: string): boolean { return /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(name) && !name.includes(".."); } -function triStateSection(raw: unknown, keys: readonly string[]): Record { +function triStateSection( + raw: unknown, + keys: readonly string[], +): Record { const out: Record = {}; if (!raw || typeof raw !== "object" || Array.isArray(raw)) return out; for (const key of keys) { @@ -90,11 +110,17 @@ function triStateSection(raw: unknown, keys: readonly string[]): Record = {}; const rawPrompts = raw.prompts as Record | undefined; - if (rawPrompts && typeof rawPrompts === "object" && !Array.isArray(rawPrompts)) { + if ( + rawPrompts && + typeof rawPrompts === "object" && + !Array.isArray(rawPrompts) + ) { for (const k of PROMPT_RULE_KEYS) { const v = rawPrompts[k]; if (typeof v === "string") (prompts as Record)[k] = v; @@ -106,11 +132,20 @@ export function sanitizePackSurface(raw: PromptPackFile | null | undefined): Pac for (const [name, value] of Object.entries(rawTools)) { if (!value || typeof value !== "object" || Array.isArray(value)) continue; const ov = value as Record; - const out: { description?: string; paramDescriptions?: Record } = {}; + const out: { + description?: string; + paramDescriptions?: Record; + } = {}; if (typeof ov.description === "string") out.description = ov.description; - if (ov.paramDescriptions && typeof ov.paramDescriptions === "object" && !Array.isArray(ov.paramDescriptions)) { + if ( + ov.paramDescriptions && + typeof ov.paramDescriptions === "object" && + !Array.isArray(ov.paramDescriptions) + ) { const params: Record = {}; - for (const [p, d] of Object.entries(ov.paramDescriptions as Record)) { + for (const [p, d] of Object.entries( + ov.paramDescriptions as Record, + )) { if (typeof d === "string") params[p] = d; } if (Object.keys(params).length > 0) out.paramDescriptions = params; @@ -142,9 +177,11 @@ export const defaultPack: Pack = { const LEAN_TOOL_PROMPTS: ToolPrompts = { compress: { - description: "Replace consumed conversation ranges with self-contained summaries using mNNNNN or bN refs.", + description: + "Replace consumed conversation ranges with self-contained summaries using mNNNNN or bN refs.", paramDescriptions: { - content: "One string per range: first line 'm00150–m00220 optional topic', remaining lines the summary markdown. Object form also accepted.", + content: + "One string per range: first line 'm00150–m00220 optional topic', remaining lines the summary markdown. Object form also accepted.", startId: "Inclusive first mNNNNN or bN ref.", endId: "Inclusive last mNNNNN or bN ref.", summary: "Self-contained replacement preserving exact technical details.", @@ -157,10 +194,12 @@ const LEAN_TOOL_PROMPTS: ToolPrompts = { "Restore compressed content by block id (b5) or message ref; block mode writes to a file by default, inline: true returns small content inline.", }, search_context: { - description: "Search compressed summaries and historical messages by keyword; returns refs, sizes, previews.", + description: + "Search compressed summaries and historical messages by keyword; returns refs, sizes, previews.", }, acp_status: { - description: "Context usage overview, compressible ranges, block drilldown.", + description: + "Context usage overview, compressible ranges, block drilldown.", }, }; @@ -273,7 +312,9 @@ export const builtinSource: PackSource = { function readPackFile(file: string): PromptPackFile | null { try { const parsed: unknown = JSON.parse(readFileSync(file, "utf8")); - return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? (parsed as PromptPackFile) : null; + return parsed && typeof parsed === "object" && !Array.isArray(parsed) + ? (parsed as PromptPackFile) + : null; } catch { return null; } @@ -294,7 +335,8 @@ export function createDirPackSource(id: string, dir: string): PackSource { return { name, version: typeof raw.version === "string" ? raw.version : undefined, - description: typeof raw.description === "string" ? raw.description : undefined, + description: + typeof raw.description === "string" ? raw.description : undefined, surface: sanitizePackSurface(raw), source: `file:${file}`, }; @@ -320,7 +362,9 @@ export function createDirPackSource(id: string, dir: string): PackSource { } /** Ordered resolution over the given sources; the first non-null wins. */ -export function createPackResolver(sources: readonly PackSource[]): PackResolver { +export function createPackResolver( + sources: readonly PackSource[], +): PackResolver { return { sources, resolve(name: string): Pack | null { @@ -350,9 +394,15 @@ export function createPackResolver(sources: readonly PackSource[]): PackResolver /** Default source chain: project pack dir, then any user pack dirs, then the * builtin registry. Directory paths are host policy — the kernel only * assembles the chain. */ -export function defaultPackSources(opts: { projectDir: string; userDirs?: readonly string[] }): PackSource[] { - const sources: PackSource[] = [createDirPackSource("project", opts.projectDir)]; - for (const dir of opts.userDirs ?? []) sources.push(createDirPackSource("user", dir)); +export function defaultPackSources(opts: { + projectDir: string; + userDirs?: readonly string[]; +}): PackSource[] { + const sources: PackSource[] = [ + createDirPackSource("project", opts.projectDir), + ]; + for (const dir of opts.userDirs ?? []) + sources.push(createDirPackSource("user", dir)); sources.push(builtinSource); return sources; } diff --git a/src/panel/cache.ts b/src/panel/cache.ts index 107ac7a..e79777c 100644 --- a/src/panel/cache.ts +++ b/src/panel/cache.ts @@ -36,7 +36,9 @@ function num(v: unknown): number { } /** Aggregate per-request cache usage into panel display numbers. */ -export function cacheHitStats(usages: readonly CacheUsageSample[]): CacheHitSummary { +export function cacheHitStats( + usages: readonly CacheUsageSample[], +): CacheHitSummary { let cacheRead = 0; let billedPrompt = 0; let requests = 0; diff --git a/src/panel/panel.ts b/src/panel/panel.ts index f285968..ccb3650 100644 --- a/src/panel/panel.ts +++ b/src/panel/panel.ts @@ -4,7 +4,11 @@ import type { CompressionState, NudgeDecision } from "../types.js"; import { formatCompactTokens } from "./format.js"; import { topicFallback } from "./topic.js"; import { viableRanges } from "../viable.js"; -import { cacheHitStats, formatHitRate, type CacheUsageSample } from "./cache.js"; +import { + cacheHitStats, + formatHitRate, + type CacheUsageSample, +} from "./cache.js"; export interface StatusPanelInput { /** Adapter identifier for the header, e.g. "billion-context-omp@0.1.6". @@ -44,7 +48,10 @@ export interface StatusPanelInput { function bar(value: number, total: number, width: number = 20): string { if (total === 0) return ""; - const filled = Math.max(0, Math.min(width, Math.round((value / total) * width))); + const filled = Math.max( + 0, + Math.min(width, Math.round((value / total) * width)), + ); return "█".repeat(filled) + "░".repeat(width - filled); } @@ -69,14 +76,19 @@ export function buildStatusPanel(input: StatusPanelInput): string { const fmt = input.fmtTokens ?? formatCompactTokens; const bd = nudge?.contextBreakdown; const limit = modelContextLimit; - const classified = bd ? bd.system + bd.tool + bd.summaries + bd.code + bd.text : 0; + const classified = bd + ? bd.system + bd.tool + bd.summaries + bd.code + bd.text + : 0; const systemPromptTokens = input.systemPromptTokens; const sentTotal = classified + systemPromptTokens; // Same-scale derivation only: both sides use the core's countTokens // estimate (kernel default = CJK-aware defaultCountTokens). The host // footer's tokenCount (provider-anchored, session-tree) is displayed as // its own line and never fed into an arithmetic difference with these. - const sessionOnly = input.unprunedTokens !== undefined ? Math.max(0, input.unprunedTokens - sentTotal) : 0; + const sessionOnly = + input.unprunedTokens !== undefined + ? Math.max(0, input.unprunedTokens - sentTotal) + : 0; const displayTotal = tokenCount; const displayPct = limit > 0 ? Math.round((displayTotal / limit) * 100) : 0; const sentPct = limit > 0 ? Math.round((sentTotal / limit) * 100) : 0; @@ -90,7 +102,9 @@ export function buildStatusPanel(input: StatusPanelInput): string { lines.push("╰─────────────────────────────────────────────╯"); if (input.version) lines.push(input.version); lines.push(""); - lines.push(`Context (session accounting, host footer scale): ${displayPct}% (${fmt(displayTotal)} / ${fmt(limit)}) — includes compressed originals; shrinks slower than the sent view`); + lines.push( + `Context (session accounting, host footer scale): ${displayPct}% (${fmt(displayTotal)} / ${fmt(limit)}) — includes compressed originals; shrinks slower than the sent view`, + ); if (nudge && bd) { const growth = bd.growth; @@ -98,9 +112,13 @@ export function buildStatusPanel(input: StatusPanelInput): string { lines.push(`Growth: +${fmt(growth)} since last nudge`); } lines.push(""); - lines.push(`Sent to LLM (after compression, est.): ${fmt(sentTotal)}${limit > 0 ? ` (${sentPct}% of limit)` : ""}`); + lines.push( + `Sent to LLM (after compression, est.): ${fmt(sentTotal)}${limit > 0 ? ` (${sentPct}% of limit)` : ""}`, + ); if (input.unprunedTokens !== undefined && sessionOnly > 0) { - lines.push(`Session-only (compressed originals, est.): ${fmt(sessionOnly)} — pruned from every request; the footer/nudge still count them`); + lines.push( + `Session-only (compressed originals, est.): ${fmt(sessionOnly)} — pruned from every request; the footer/nudge still count them`, + ); } lines.push(""); lines.push("Token Breakdown (sent view):"); @@ -117,13 +135,19 @@ export function buildStatusPanel(input: StatusPanelInput): string { if (cat.value <= 0) continue; const pct = sentTotal > 0 ? Math.round((cat.value / sentTotal) * 100) : 0; const b = bar(cat.value, sentTotal); - lines.push(` ${cat.label.padEnd(10)} ${b} ${String(pct).padStart(3)}% ${fmt(cat.value)}`); + lines.push( + ` ${cat.label.padEnd(10)} ${b} ${String(pct).padStart(3)}% ${fmt(cat.value)}`, + ); } } if (input.cacheUsages) { const cache = cacheHitStats(input.cacheUsages); - if (cache.requests > 0 && cache.session !== undefined && cache.last !== undefined) { + if ( + cache.requests > 0 && + cache.session !== undefined && + cache.last !== undefined + ) { lines.push(""); lines.push( `Prompt cache (provider-reported): ${formatHitRate(cache.last)} last · ${formatHitRate(cache.session)} session avg — ${fmt(cache.cacheRead)} of ${fmt(cache.billedPrompt)} billed prompt tokens served from cache (${cache.requests} req)`, @@ -151,23 +175,33 @@ export function buildStatusPanel(input: StatusPanelInput): string { if (activeBlocksList.length > 0) { lines.push(""); - lines.push(`Blocks: ${activeBlocksList.length} active / ${totalBlocksList.length} total (${fmt(state.stats.tokensCompressed)} tokens compressed, cumulative)`); + lines.push( + `Blocks: ${activeBlocksList.length} active / ${totalBlocksList.length} total (${fmt(state.stats.tokensCompressed)} tokens compressed, cumulative)`, + ); for (const b of activeBlocksList) { - const topic = b.topic ? `: ${b.topic}` : `: ${topicFallback(b.summary || "")}`; + const topic = b.topic + ? `: ${b.topic}` + : `: ${topicFallback(b.summary || "")}`; const summaryTok = defaultCountTokens(b.summary || ""); const origTok = b.compressedTokens > 0 ? b.compressedTokens : summaryTok; - lines.push(` [${b.blockId}] T${b.tier} ${fmt(origTok)}→${fmt(summaryTok)}${topic}`); + lines.push( + ` [${b.blockId}] T${b.tier} ${fmt(origTok)}→${fmt(summaryTok)}${topic}`, + ); } } else if (totalBlocksList.length > 0) { lines.push(""); - lines.push(`Blocks: 0 active / ${totalBlocksList.length} total (${fmt(state.stats.tokensCompressed)} tokens compressed, cumulative)`); + lines.push( + `Blocks: 0 active / ${totalBlocksList.length} total (${fmt(state.stats.tokensCompressed)} tokens compressed, cumulative)`, + ); } else { lines.push(""); lines.push("Blocks: none (nothing compressed yet)"); } lines.push(""); - lines.push("Tag visibility: tags injected to LLM only (deep copy), not persisted in session, not shown in terminal."); + lines.push( + "Tag visibility: tags injected to LLM only (deep copy), not persisted in session, not shown in terminal.", + ); return lines.join("\n"); } diff --git a/src/panel/topic.ts b/src/panel/topic.ts index 3fef057..0030cec 100644 --- a/src/panel/topic.ts +++ b/src/panel/topic.ts @@ -5,6 +5,9 @@ import { clampPrefix } from "../truncate.js"; export function topicFallback(summary: string): string { const first = summary.split(/[.\n]/)[0] ?? ""; - const t = first.trim().replace(/^["'`]+/, "").trim(); + const t = first + .trim() + .replace(/^["'`]+/, "") + .trim(); return t.length <= 30 ? t : `${clampPrefix(t, 30).trimEnd()}…`; } diff --git a/src/parse-compress-input.ts b/src/parse-compress-input.ts index 3785c36..ae49a5d 100644 --- a/src/parse-compress-input.ts +++ b/src/parse-compress-input.ts @@ -26,37 +26,37 @@ import { clampPrefix } from "./truncate.js"; import type { CompressRangeSpec } from "./types.js"; export type CompressParseKind = - | "ok" - | "empty-input" - | "not-object" - | "missing-content" - | "content-not-array" - | "malformed-json" - | "truncated" - | "no-valid-ranges"; + | "ok" + | "empty-input" + | "not-object" + | "missing-content" + | "content-not-array" + | "malformed-json" + | "truncated" + | "no-valid-ranges"; export interface CompressParseDiagnostics { - /** true only when at least one range was recovered. */ - ok: boolean; - /** Why the input parsed the way it did. */ - kind: CompressParseKind; - /** True when the winning parse came from single→double quote repair. */ - quoteSalvage?: boolean; - /** First 800 chars of the raw string input (string inputs only). */ - rawPrefix?: string; - /** Raw string input length (string inputs only). */ - length?: number; - /** Top-level keys of the parsed object — catches `content` vs `ranges` drift. */ - keys?: string[]; - /** Entries dropped because they were not valid ranges. */ - invalidItems: number; - /** One human-readable reason per dropped entry (index-prefixed). */ - invalidReasons?: string[]; + /** true only when at least one range was recovered. */ + ok: boolean; + /** Why the input parsed the way it did. */ + kind: CompressParseKind; + /** True when the winning parse came from single→double quote repair. */ + quoteSalvage?: boolean; + /** First 800 chars of the raw string input (string inputs only). */ + rawPrefix?: string; + /** Raw string input length (string inputs only). */ + length?: number; + /** Top-level keys of the parsed object — catches `content` vs `ranges` drift. */ + keys?: string[]; + /** Entries dropped because they were not valid ranges. */ + invalidItems: number; + /** One human-readable reason per dropped entry (index-prefixed). */ + invalidReasons?: string[]; } export interface ParsedCompressInput { - ranges: CompressRangeSpec[]; - diagnostics: CompressParseDiagnostics; + ranges: CompressRangeSpec[]; + diagnostics: CompressParseDiagnostics; } /** @@ -67,190 +67,236 @@ export interface ParsedCompressInput { * mode). Invalid entries are skipped, never fatal; the reason is in * `diagnostics.kind`. */ -export function parseCompressArgs(input: unknown, opts?: { callId?: string }): ParsedCompressInput { - const callId = opts?.callId; - const diag: CompressParseDiagnostics = { ok: false, kind: "ok", invalidItems: 0 }; - - if (input === null || input === undefined) { - diag.kind = "empty-input"; - return finish([], diag); - } - - if (typeof input === "string") { - return parseStringInput(input, callId, diag); - } - - if (typeof input !== "object" || Array.isArray(input)) { - diag.kind = "not-object"; - return finish([], diag); - } - - return parseObjectValue(input as Record, callId, diag); +export function parseCompressArgs( + input: unknown, + opts?: { callId?: string }, +): ParsedCompressInput { + const callId = opts?.callId; + const diag: CompressParseDiagnostics = { + ok: false, + kind: "ok", + invalidItems: 0, + }; + + if (input === null || input === undefined) { + diag.kind = "empty-input"; + return finish([], diag); + } + + if (typeof input === "string") { + return parseStringInput(input, callId, diag); + } + + if (typeof input !== "object" || Array.isArray(input)) { + diag.kind = "not-object"; + return finish([], diag); + } + + return parseObjectValue(input as Record, callId, diag); } -function parseStringInput(raw: string, callId: string | undefined, diag: CompressParseDiagnostics): ParsedCompressInput { - diag.rawPrefix = clampPrefix(raw, 800); - diag.length = raw.length; - const cleaned = stripFence(raw.trim()); - const first = parseStringCore(cleaned, callId, diag); - // Weak local models emit single-quoted args ({'content': [...]}). The - // salvage regex only recognizes double-quoted "content", so a truncated - // single-quoted prefix recovers nothing on the first pass. Retry once - // with the quotes normalized; the retry wins only if it recovers strictly - // more ranges, so valid input is never rewritten. - if (first.ranges.length === 0 || first.diagnostics.invalidItems > 0) { - const normalized = normalizeSingleQuotes(cleaned); - if (normalized !== undefined) { - const retryDiag: CompressParseDiagnostics = { ok: false, kind: "ok", invalidItems: 0 }; - retryDiag.rawPrefix = diag.rawPrefix; - retryDiag.length = diag.length; - const retry = parseStringCore(normalized, callId, retryDiag); - if (retry.ranges.length > first.ranges.length) { - retryDiag.quoteSalvage = true; - return retry; - } - } +function parseStringInput( + raw: string, + callId: string | undefined, + diag: CompressParseDiagnostics, +): ParsedCompressInput { + diag.rawPrefix = clampPrefix(raw, 800); + diag.length = raw.length; + const cleaned = stripFence(raw.trim()); + const first = parseStringCore(cleaned, callId, diag); + // Weak local models emit single-quoted args ({'content': [...]}). The + // salvage regex only recognizes double-quoted "content", so a truncated + // single-quoted prefix recovers nothing on the first pass. Retry once + // with the quotes normalized; the retry wins only if it recovers strictly + // more ranges, so valid input is never rewritten. + if (first.ranges.length === 0 || first.diagnostics.invalidItems > 0) { + const normalized = normalizeSingleQuotes(cleaned); + if (normalized !== undefined) { + const retryDiag: CompressParseDiagnostics = { + ok: false, + kind: "ok", + invalidItems: 0, + }; + retryDiag.rawPrefix = diag.rawPrefix; + retryDiag.length = diag.length; + const retry = parseStringCore(normalized, callId, retryDiag); + if (retry.ranges.length > first.ranges.length) { + retryDiag.quoteSalvage = true; + return retry; + } } - return first; + } + return first; } -function parseStringCore(cleaned: string, callId: string | undefined, diag: CompressParseDiagnostics): ParsedCompressInput { - if (cleaned === "") { - diag.kind = "empty-input"; - return finish([], diag); - } - - let value: unknown = tryParseLenient(cleaned); - // One level of double-stringification: the host wrapped an already - // stringified argument in another JSON string. - if (typeof value === "string") { - const inner = tryParseLenient(stripFence(value)); - if (inner !== undefined) value = inner; - } - - if (value !== null && typeof value === "object" && !Array.isArray(value)) { - return parseObjectValue(value as Record, callId, diag); - } - if (value !== undefined) { - // Parsed, but not to an object: bare array, number, boolean, null. - diag.kind = "not-object"; - return finish([], diag); - } - - // Unparseable prefix: salvage the complete content-array entries. - const entries = salvageContentEntries(cleaned); - return finishSalvage(entries, callId, diag, looksTruncated(cleaned)); +function parseStringCore( + cleaned: string, + callId: string | undefined, + diag: CompressParseDiagnostics, +): ParsedCompressInput { + if (cleaned === "") { + diag.kind = "empty-input"; + return finish([], diag); + } + + let value: unknown = tryParseLenient(cleaned); + // One level of double-stringification: the host wrapped an already + // stringified argument in another JSON string. + if (typeof value === "string") { + const inner = tryParseLenient(stripFence(value)); + if (inner !== undefined) value = inner; + } + + if (value !== null && typeof value === "object" && !Array.isArray(value)) { + return parseObjectValue(value as Record, callId, diag); + } + if (value !== undefined) { + // Parsed, but not to an object: bare array, number, boolean, null. + diag.kind = "not-object"; + return finish([], diag); + } + + // Unparseable prefix: salvage the complete content-array entries. + const entries = salvageContentEntries(cleaned); + return finishSalvage(entries, callId, diag, looksTruncated(cleaned)); } -function parseObjectValue(value: Record, callId: string | undefined, diag: CompressParseDiagnostics): ParsedCompressInput { - diag.keys = Object.keys(value); - const content = value["content"]; - if (content === undefined) { - // Model drift: a single range at the top level (no content array). - // The proxy defends against this shape; the kernel now owns it. - const single = validateEntry(value, callId); - if ("range" in single) { - diag.kind = "ok"; - return finish([single.range], diag); - } - diag.kind = "missing-content"; - return finish([], diag); +function parseObjectValue( + value: Record, + callId: string | undefined, + diag: CompressParseDiagnostics, +): ParsedCompressInput { + diag.keys = Object.keys(value); + const content = value["content"]; + if (content === undefined) { + // Model drift: a single range at the top level (no content array). + // The proxy defends against this shape; the kernel now owns it. + const single = validateEntry(value, callId); + if ("range" in single) { + diag.kind = "ok"; + return finish([single.range], diag); } - - let entries: unknown[]; - let salvaged = false; - - if (Array.isArray(content)) { - entries = content; - } else if (typeof content === "string") { - // Stringified content array: vLLM-style gateways stringify nested - // arrays, so `content` arrives as a JSON string of the array. - const parsed = parseContentArray(content); - if (parsed === null || (parsed.entries.length === 0 && content.trim().length > 0)) { - // Not a JSON array (or nothing salvageable): bare line-form text. - // Strip JSON-wrapper residue (stringified ["…"] with broken - // escaping) — the line form needs structure only in the refs - // header line, so the summary body never has to parse. - const bare = stripJsonWrapperResidue(content); - const entries = splitLineEntries(bare); - const { ranges, invalid, reasons } = validateEntries(entries, callId); - if (ranges.length > 0) { - diag.invalidItems = invalid; - if (reasons.length > 0) diag.invalidReasons = reasons; - diag.kind = "ok"; - return finish(ranges, diag); - } - diag.kind = "content-not-array"; - return finish([], diag); - } - entries = parsed.entries; - salvaged = parsed.salvaged; - if (parsed.quoteRepaired) diag.quoteSalvage = true; - } else { - diag.kind = "content-not-array"; - return finish([], diag); + diag.kind = "missing-content"; + return finish([], diag); + } + + let entries: unknown[]; + let salvaged = false; + + if (Array.isArray(content)) { + entries = content; + } else if (typeof content === "string") { + // Stringified content array: vLLM-style gateways stringify nested + // arrays, so `content` arrives as a JSON string of the array. + const parsed = parseContentArray(content); + if ( + parsed === null || + (parsed.entries.length === 0 && content.trim().length > 0) + ) { + // Not a JSON array (or nothing salvageable): bare line-form text. + // Strip JSON-wrapper residue (stringified ["…"] with broken + // escaping) — the line form needs structure only in the refs + // header line, so the summary body never has to parse. + const bare = stripJsonWrapperResidue(content); + const entries = splitLineEntries(bare); + const { ranges, invalid, reasons } = validateEntries(entries, callId); + if (ranges.length > 0) { + diag.invalidItems = invalid; + if (reasons.length > 0) diag.invalidReasons = reasons; + diag.kind = "ok"; + return finish(ranges, diag); + } + diag.kind = "content-not-array"; + return finish([], diag); } - - const { ranges, invalid, reasons } = validateEntries(entries, callId); - // Top-level fallbacks: topic and summaryMaxChars apply to every range - // that does not specify its own (omp/pi schemas define both at the top - // level; per-entry values win). - const topTopic = stringOr(value["topic"]); - const topMaxChars = value["summaryMaxChars"]; - const hasTopMaxChars = typeof topMaxChars === "number" && Number.isFinite(topMaxChars); - if (topTopic !== undefined || hasTopMaxChars) { - for (const r of ranges) { - if (r.topic === undefined && topTopic !== undefined) r.topic = topTopic; - if (r.summaryMaxChars === undefined && hasTopMaxChars) r.summaryMaxChars = topMaxChars; - } + entries = parsed.entries; + salvaged = parsed.salvaged; + if (parsed.quoteRepaired) diag.quoteSalvage = true; + } else { + diag.kind = "content-not-array"; + return finish([], diag); + } + + const { ranges, invalid, reasons } = validateEntries(entries, callId); + // Top-level fallbacks: topic and summaryMaxChars apply to every range + // that does not specify its own (omp/pi schemas define both at the top + // level; per-entry values win). + const topTopic = stringOr(value["topic"]); + const topMaxChars = value["summaryMaxChars"]; + const hasTopMaxChars = + typeof topMaxChars === "number" && Number.isFinite(topMaxChars); + if (topTopic !== undefined || hasTopMaxChars) { + for (const r of ranges) { + if (r.topic === undefined && topTopic !== undefined) r.topic = topTopic; + if (r.summaryMaxChars === undefined && hasTopMaxChars) + r.summaryMaxChars = topMaxChars; } - diag.invalidItems = invalid; - if (reasons.length > 0) diag.invalidReasons = reasons; - diag.kind = salvaged ? "truncated" : ranges.length > 0 ? "ok" : "no-valid-ranges"; - return finish(ranges, diag); + } + diag.invalidItems = invalid; + if (reasons.length > 0) diag.invalidReasons = reasons; + diag.kind = salvaged + ? "truncated" + : ranges.length > 0 + ? "ok" + : "no-valid-ranges"; + return finish(ranges, diag); } -function parseContentArray(s: string): { entries: unknown[]; salvaged: boolean; quoteRepaired?: boolean } | null { - const cleaned = stripFence(s.trim()); - const direct = parseContentArrayCore(cleaned); - if (direct !== null && direct.entries.length > 0) return direct; - // Single-quoted array (weak local models): normalize once and re-run; - // the retry is adopted only if it recovers strictly more entries. - const normalized = normalizeSingleQuotes(cleaned); - if (normalized !== undefined) { - const retry = parseContentArrayCore(normalized); - if (retry !== null && retry.entries.length > 0) return { ...retry, quoteRepaired: true }; - } - return direct; +function parseContentArray( + s: string, +): { entries: unknown[]; salvaged: boolean; quoteRepaired?: boolean } | null { + const cleaned = stripFence(s.trim()); + const direct = parseContentArrayCore(cleaned); + if (direct !== null && direct.entries.length > 0) return direct; + // Single-quoted array (weak local models): normalize once and re-run; + // the retry is adopted only if it recovers strictly more entries. + const normalized = normalizeSingleQuotes(cleaned); + if (normalized !== undefined) { + const retry = parseContentArrayCore(normalized); + if (retry !== null && retry.entries.length > 0) + return { ...retry, quoteRepaired: true }; + } + return direct; } -function parseContentArrayCore(s: string): { entries: unknown[]; salvaged: boolean } | null { - let value: unknown = s === "" ? undefined : tryParseLenient(s); - if (typeof value === "string") { - value = tryParseLenient(stripFence(value)); - } - if (Array.isArray(value)) { - return { entries: value, salvaged: false }; - } - if (value === undefined) { - // Unparseable (usually truncated): recover the complete entries. - const entries = salvageContentEntries('{"content": ' + s); - return { entries, salvaged: entries.length > 0 }; - } - return null; +function parseContentArrayCore( + s: string, +): { entries: unknown[]; salvaged: boolean } | null { + let value: unknown = s === "" ? undefined : tryParseLenient(s); + if (typeof value === "string") { + value = tryParseLenient(stripFence(value)); + } + if (Array.isArray(value)) { + return { entries: value, salvaged: false }; + } + if (value === undefined) { + // Unparseable (usually truncated): recover the complete entries. + const entries = salvageContentEntries('{"content": ' + s); + return { entries, salvaged: entries.length > 0 }; + } + return null; } -function finish(ranges: CompressRangeSpec[], diag: CompressParseDiagnostics): ParsedCompressInput { - diag.ok = ranges.length > 0; - return { ranges, diagnostics: diag }; +function finish( + ranges: CompressRangeSpec[], + diag: CompressParseDiagnostics, +): ParsedCompressInput { + diag.ok = ranges.length > 0; + return { ranges, diagnostics: diag }; } -function finishSalvage(entries: unknown[], callId: string | undefined, diag: CompressParseDiagnostics, truncatedShape: boolean): ParsedCompressInput { - const { ranges, invalid, reasons } = validateEntries(entries, callId); - diag.invalidItems = invalid; - if (reasons.length > 0) diag.invalidReasons = reasons; - diag.kind = entries.length > 0 || truncatedShape ? "truncated" : "malformed-json"; - return finish(ranges, diag); +function finishSalvage( + entries: unknown[], + callId: string | undefined, + diag: CompressParseDiagnostics, + truncatedShape: boolean, +): ParsedCompressInput { + const { ranges, invalid, reasons } = validateEntries(entries, callId); + diag.invalidItems = invalid; + if (reasons.length > 0) diag.invalidReasons = reasons; + diag.kind = + entries.length > 0 || truncatedShape ? "truncated" : "malformed-json"; + return finish(ranges, diag); } // #1001 (problem 3): models routinely split the documented line form across @@ -263,41 +309,45 @@ function finishSalvage(entries: unknown[], callId: string | undefined, diag: Com // (header + \n + body in one string) and object entries reset the pairing — // a stray refless string after a complete entry stays invalid, as today. function coalesceLineEntries(entries: unknown[]): unknown[] { - const out: unknown[] = []; - let headerOnlyIdx = -1; - for (const e of entries) { - if (typeof e === "string") { - const nl = e.indexOf("\n"); - const head = (nl === -1 ? e : e.slice(0, nl)).trim(); - const hasRef = REF_PAIR_IN_LINE.test(head) || SINGLE_REF_IN_LINE.test(head); - if (!hasRef && headerOnlyIdx >= 0) { - out[headerOnlyIdx] = `${out[headerOnlyIdx] as string}\n${e}`; - continue; - } - out.push(e); - headerOnlyIdx = hasRef && nl === -1 ? out.length - 1 : -1; - } else { - out.push(e); - headerOnlyIdx = -1; - } + const out: unknown[] = []; + let headerOnlyIdx = -1; + for (const e of entries) { + if (typeof e === "string") { + const nl = e.indexOf("\n"); + const head = (nl === -1 ? e : e.slice(0, nl)).trim(); + const hasRef = + REF_PAIR_IN_LINE.test(head) || SINGLE_REF_IN_LINE.test(head); + if (!hasRef && headerOnlyIdx >= 0) { + out[headerOnlyIdx] = `${out[headerOnlyIdx] as string}\n${e}`; + continue; + } + out.push(e); + headerOnlyIdx = hasRef && nl === -1 ? out.length - 1 : -1; + } else { + out.push(e); + headerOnlyIdx = -1; } - return out; + } + return out; } -function validateEntries(entries: unknown[], callId: string | undefined): { ranges: CompressRangeSpec[]; invalid: number; reasons: string[] } { - const ranges: CompressRangeSpec[] = []; - const reasons: string[] = []; - let invalid = 0; - const items = coalesceLineEntries(entries); - for (let i = 0; i < items.length; i++) { - const outcome = validateEntry(items[i], callId); - if ("range" in outcome) ranges.push(outcome.range); - else { - invalid++; - reasons.push(`entry ${i}: ${outcome.reason}`); - } +function validateEntries( + entries: unknown[], + callId: string | undefined, +): { ranges: CompressRangeSpec[]; invalid: number; reasons: string[] } { + const ranges: CompressRangeSpec[] = []; + const reasons: string[] = []; + let invalid = 0; + const items = coalesceLineEntries(entries); + for (let i = 0; i < items.length; i++) { + const outcome = validateEntry(items[i], callId); + if ("range" in outcome) ranges.push(outcome.range); + else { + invalid++; + reasons.push(`entry ${i}: ${outcome.reason}`); } - return { ranges, invalid, reasons }; + } + return { ranges, invalid, reasons }; } type EntryOutcome = { range: CompressRangeSpec } | { reason: string }; @@ -308,44 +358,53 @@ type EntryOutcome = { range: CompressRangeSpec } | { reason: string }; // hard requirement is refs at the START of the first line. Both patterns are // anchored (^): summaries cite mNNNNN/bN refs routinely, so an unanchored // scan would let a citation inside a headerless entry supply fake bounds. -const REF_PAIR_IN_LINE = /^([mb]\d{1,7})\s*(?:[-\u2013\u2014\u2026~]|\.\.\.|to)\s*([mb]\d{1,7})\b/i; +const REF_PAIR_IN_LINE = + /^([mb]\d{1,7})\s*(?:[-\u2013\u2014\u2026~]|\.\.\.|to)\s*([mb]\d{1,7})\b/i; const SINGLE_REF_IN_LINE = /^([mb]\d{1,7})\b/i; function normalizeLineRef(raw: string): string { - const lower = raw.toLowerCase(); - const digits = lower.slice(1); - return lower[0]! === "b" ? `b${digits}` : `m${digits.padStart(5, "0")}`; + const lower = raw.toLowerCase(); + const digits = lower.slice(1); + return lower[0]! === "b" ? `b${digits}` : `m${digits.padStart(5, "0")}`; } -function parseLineEntry(entry: string, callId: string | undefined): EntryOutcome { - const nl = entry.indexOf("\n"); - const head = (nl === -1 ? entry : entry.slice(0, nl)).trim(); - const summary = (nl === -1 ? "" : entry.slice(nl + 1)).trim(); - const pair = REF_PAIR_IN_LINE.exec(head); - let startRef: string; - let endRef: string; - let afterRefs: number; - if (pair !== null) { - startRef = normalizeLineRef(pair[1]!); - endRef = normalizeLineRef(pair[2]!); - afterRefs = pair.index + pair[0].length; - } else { - const single = SINGLE_REF_IN_LINE.exec(head); - if (single === null) return { reason: "line entry: no mNNNNN/bN refs in header" }; - startRef = normalizeLineRef(single[1]!); - endRef = startRef; - afterRefs = single.index + single[0].length; - } - if (summary.length === 0) return { reason: "line entry: missing summary after the refs header line" }; - const explicitTopic = head.slice(afterRefs).trim(); - const range: CompressRangeSpec = { - startRef, - endRef, - summary, - topic: explicitTopic.length > 0 ? explicitTopic : deriveTopicFromSummary(summary), - }; - if (callId !== undefined) range.compressCallId = callId; - return { range }; +function parseLineEntry( + entry: string, + callId: string | undefined, +): EntryOutcome { + const nl = entry.indexOf("\n"); + const head = (nl === -1 ? entry : entry.slice(0, nl)).trim(); + const summary = (nl === -1 ? "" : entry.slice(nl + 1)).trim(); + const pair = REF_PAIR_IN_LINE.exec(head); + let startRef: string; + let endRef: string; + let afterRefs: number; + if (pair !== null) { + startRef = normalizeLineRef(pair[1]!); + endRef = normalizeLineRef(pair[2]!); + afterRefs = pair.index + pair[0].length; + } else { + const single = SINGLE_REF_IN_LINE.exec(head); + if (single === null) + return { reason: "line entry: no mNNNNN/bN refs in header" }; + startRef = normalizeLineRef(single[1]!); + endRef = startRef; + afterRefs = single.index + single[0].length; + } + if (summary.length === 0) + return { reason: "line entry: missing summary after the refs header line" }; + const explicitTopic = head.slice(afterRefs).trim(); + const range: CompressRangeSpec = { + startRef, + endRef, + summary, + topic: + explicitTopic.length > 0 + ? explicitTopic + : deriveTopicFromSummary(summary), + }; + if (callId !== undefined) range.compressCallId = callId; + return { range }; } /** Display/retrieval label for line-form entries without an explicit topic: @@ -353,11 +412,14 @@ function parseLineEntry(entry: string, callId: string | undefined): EntryOutcome * truncated. The topic is metadata (panels, T2 source headers, search) — * never load-bearing for correctness. */ export function deriveTopicFromSummary(summary: string): string | undefined { - const heading = /^#{1,6}\s+(.+)$/m.exec(summary); - const source = heading !== null ? heading[1]! : (summary.split("\n").find((l) => l.trim().length > 0) ?? ""); - const text = source.trim().replace(/^[\u2022\-*]\s+/, ""); - if (text.length === 0) return undefined; - return text.length > 60 ? clampPrefix(text, 60).trimEnd() : text; + const heading = /^#{1,6}\s+(.+)$/m.exec(summary); + const source = + heading !== null + ? heading[1]! + : (summary.split("\n").find((l) => l.trim().length > 0) ?? ""); + const text = source.trim().replace(/^[\u2022\-*]\s+/, ""); + if (text.length === 0) return undefined; + return text.length > 60 ? clampPrefix(text, 60).trimEnd() : text; } /** Split a bare string content (not a JSON array) into line-form entries: a @@ -368,11 +430,18 @@ export function deriveTopicFromSummary(summary: string): string | undefined { * on their headers. Fallback path only — arrays of strings are the primary * line-form transport. */ function splitLineEntries(content: string): unknown[] { - const parts = content - .split(/\n(?=\s*(?:["']\s*,\s*)?["']?\s*[mb]\d{1,7}\s*(?:[-\u2013\u2014\u2026~]|\.\.\.|to)\s*[mb]\d{1,7}\b)/i) - .map((p) => p.trim().replace(/^(?:["']\s*,?\s*)+/, "").replace(/["']\s*$/, "")) - .filter((p) => p.length > 0); - return parts.length > 0 ? parts : [content.trim()]; + const parts = content + .split( + /\n(?=\s*(?:["']\s*,\s*)?["']?\s*[mb]\d{1,7}\s*(?:[-\u2013\u2014\u2026~]|\.\.\.|to)\s*[mb]\d{1,7}\b)/i, + ) + .map((p) => + p + .trim() + .replace(/^(?:["']\s*,?\s*)+/, "") + .replace(/["']\s*$/, ""), + ) + .filter((p) => p.length > 0); + return parts.length > 0 ? parts : [content.trim()]; } // Remove the leftover shell of a stringified-but-unescaped array/element: @@ -380,79 +449,90 @@ function splitLineEntries(content: string): unknown[] { // runs after JSON.parse has already failed, so these chars are residue, not // content. Anything deeper and the line form stops being the right tool. function stripJsonWrapperResidue(s: string): string { - let t = s.trim(); - if (t.startsWith("[")) t = t.replace(/^\[+\s*/, ""); - t = t.replace(/^"/, ""); - if (t.endsWith("]")) t = t.replace(/\s*\]+$/, ""); - t = t.replace(/"$/, ""); - return t.trim(); + let t = s.trim(); + if (t.startsWith("[")) t = t.replace(/^\[+\s*/, ""); + t = t.replace(/^"/, ""); + if (t.endsWith("]")) t = t.replace(/\s*\]+$/, ""); + t = t.replace(/"$/, ""); + return t.trim(); } -function validateEntry(entry: unknown, callId: string | undefined): EntryOutcome { - if (typeof entry === "string") return parseLineEntry(entry, callId); - if (entry === null || typeof entry !== "object" || Array.isArray(entry)) return { reason: "not an object" }; - const e = entry as Record; - // Field-name variants: startRef/endRef are canonical; startId/endId is - // model drift (and the legacy rebuild.ts spelling); messageId is the - // startId-less messageRef from the historical API. - const start = stringOr(e["startRef"]) ?? stringOr(e["startId"]) ?? stringOr(e["messageId"]); - const end = stringOr(e["endRef"]) ?? stringOr(e["endId"]) ?? stringOr(e["messageId"]); - if (start === undefined || end === undefined) { - return { reason: "missing range bounds (need startRef/startId and endRef/endId)" }; - } - const summary = stringOr(e["summary"]); - if (summary === undefined) return { reason: "missing summary" }; - const range: CompressRangeSpec = { startRef: start, endRef: end, summary }; - const topic = stringOr(e["topic"]); - if (topic !== undefined) range.topic = topic; - const maxChars = e["summaryMaxChars"]; - if (typeof maxChars === "number" && Number.isFinite(maxChars)) range.summaryMaxChars = maxChars; - if (callId !== undefined) range.compressCallId = callId; - return { range }; +function validateEntry( + entry: unknown, + callId: string | undefined, +): EntryOutcome { + if (typeof entry === "string") return parseLineEntry(entry, callId); + if (entry === null || typeof entry !== "object" || Array.isArray(entry)) + return { reason: "not an object" }; + const e = entry as Record; + // Field-name variants: startRef/endRef are canonical; startId/endId is + // model drift (and the legacy rebuild.ts spelling); messageId is the + // startId-less messageRef from the historical API. + const start = + stringOr(e["startRef"]) ?? + stringOr(e["startId"]) ?? + stringOr(e["messageId"]); + const end = + stringOr(e["endRef"]) ?? stringOr(e["endId"]) ?? stringOr(e["messageId"]); + if (start === undefined || end === undefined) { + return { + reason: "missing range bounds (need startRef/startId and endRef/endId)", + }; + } + const summary = stringOr(e["summary"]); + if (summary === undefined) return { reason: "missing summary" }; + const range: CompressRangeSpec = { startRef: start, endRef: end, summary }; + const topic = stringOr(e["topic"]); + if (topic !== undefined) range.topic = topic; + const maxChars = e["summaryMaxChars"]; + if (typeof maxChars === "number" && Number.isFinite(maxChars)) + range.summaryMaxChars = maxChars; + if (callId !== undefined) range.compressCallId = callId; + return { range }; } function stringOr(value: unknown): string | undefined { - return typeof value === "string" && value.length > 0 ? value : undefined; + return typeof value === "string" && value.length > 0 ? value : undefined; } // --- tolerant JSON parsing ------------------------------------------------- function tryParseLenient(s: string): unknown { - if (s === "") return undefined; + if (s === "") return undefined; + try { + return JSON.parse(s); + } catch { + // keep trying the repaired variants below + } + const noTrailingCommas = stripTrailingCommas(s); + if (noTrailingCommas !== s) { try { - return JSON.parse(s); + return JSON.parse(noTrailingCommas); } catch { - // keep trying the repaired variants below - } - const noTrailingCommas = stripTrailingCommas(s); - if (noTrailingCommas !== s) { - try { - return JSON.parse(noTrailingCommas); - } catch { - // keep trying - } + // keep trying } - const fixed = escapeRawNewlinesInStrings(noTrailingCommas); - if (fixed !== noTrailingCommas) { - try { - return JSON.parse(fixed); - } catch { - // fall through to salvage - } + } + const fixed = escapeRawNewlinesInStrings(noTrailingCommas); + if (fixed !== noTrailingCommas) { + try { + return JSON.parse(fixed); + } catch { + // fall through to salvage } - // Unescaped inner quotes and markdown backslash escapes (\`, \-) defeat - // the string state machines above; the parser's own error position - // pinpoints the real offending characters. Repair exactly where it says - // and iterate. - const ctrlFixed = repairAtParserPositions(noTrailingCommas); - if (ctrlFixed !== noTrailingCommas) { - try { - return JSON.parse(ctrlFixed); - } catch { - // fall through to salvage - } + } + // Unescaped inner quotes and markdown backslash escapes (\`, \-) defeat + // the string state machines above; the parser's own error position + // pinpoints the real offending characters. Repair exactly where it says + // and iterate. + const ctrlFixed = repairAtParserPositions(noTrailingCommas); + if (ctrlFixed !== noTrailingCommas) { + try { + return JSON.parse(ctrlFixed); + } catch { + // fall through to salvage } - return undefined; + } + return undefined; } // State machine that converts single-quoted strings to double-quoted ones. @@ -463,140 +543,160 @@ function tryParseLenient(s: string): unknown { // it never invents structure, so anything it cannot make into valid JSON // simply stays unrecovered. function normalizeSingleQuotes(raw: string): string | undefined { - if (!raw.includes("'") || (!raw.includes("{") && !raw.includes("["))) return undefined; - let out = ""; - let changed = false; - let inDouble = false; - let inSingle = false; - for (let i = 0; i < raw.length; i++) { - const ch = raw.charAt(i); - if (inDouble) { - out += ch; - if (ch === "\\" && i + 1 < raw.length) { - out += raw.charAt(i + 1); - i++; - } else if (ch === '"') { - inDouble = false; - } - continue; - } - if (inSingle) { - if (ch === "\\" && i + 1 < raw.length) { - const next = raw.charAt(i + 1); - out += next === "'" ? "'" : "\\" + next; - i++; - continue; - } - if (ch === "'") { - out += '"'; - inSingle = false; - changed = true; - continue; - } - if (ch === '"') { - out += '\\"'; - continue; - } - if (ch === "\n") { - out += "\\n"; - continue; - } - if (ch === "\r") { - out += "\\r"; - continue; - } - if (ch === "\t") { - out += "\\t"; - continue; - } - out += ch; - continue; - } - if (ch === '"') { - inDouble = true; - out += ch; - continue; - } - if (ch === "'") { - inSingle = true; - out += '"'; - changed = true; - continue; - } - out += ch; + if (!raw.includes("'") || (!raw.includes("{") && !raw.includes("["))) + return undefined; + let out = ""; + let changed = false; + let inDouble = false; + let inSingle = false; + for (let i = 0; i < raw.length; i++) { + const ch = raw.charAt(i); + if (inDouble) { + out += ch; + if (ch === "\\" && i + 1 < raw.length) { + out += raw.charAt(i + 1); + i++; + } else if (ch === '"') { + inDouble = false; + } + continue; + } + if (inSingle) { + if (ch === "\\" && i + 1 < raw.length) { + const next = raw.charAt(i + 1); + out += next === "'" ? "'" : "\\" + next; + i++; + continue; + } + if (ch === "'") { + out += '"'; + inSingle = false; + changed = true; + continue; + } + if (ch === '"') { + out += '\\"'; + continue; + } + if (ch === "\n") { + out += "\\n"; + continue; + } + if (ch === "\r") { + out += "\\r"; + continue; + } + if (ch === "\t") { + out += "\\t"; + continue; + } + out += ch; + continue; } - return changed ? out : undefined; + if (ch === '"') { + inDouble = true; + out += ch; + continue; + } + if (ch === "'") { + inSingle = true; + out += '"'; + changed = true; + continue; + } + out += ch; + } + return changed ? out : undefined; } function stripFence(s: string): string { - if (!s.startsWith("```")) return s; - const firstNewline = s.indexOf("\n"); - if (firstNewline === -1) return s; - const bodyStart = firstNewline + 1; - const end = s.lastIndexOf("```"); - return end > bodyStart ? s.slice(bodyStart, end).trim() : s.slice(bodyStart).trim(); + if (!s.startsWith("```")) return s; + const firstNewline = s.indexOf("\n"); + if (firstNewline === -1) return s; + const bodyStart = firstNewline + 1; + const end = s.lastIndexOf("```"); + return end > bodyStart + ? s.slice(bodyStart, end).trim() + : s.slice(bodyStart).trim(); } // Drop commas that sit immediately before a closing brace/bracket // (outside string literals). function stripTrailingCommas(s: string): string { - let out = ""; - let inString = false; - let escaped = false; - for (let i = 0; i < s.length; i++) { - const ch = s.charAt(i); - if (inString) { - out += ch; - if (escaped) escaped = false; - else if (ch === "\\") escaped = true; - else if (ch === '"') inString = false; - continue; - } - if (ch === '"') { - inString = true; - out += ch; - continue; - } - if (ch === ",") { - let j = i + 1; - while (j < s.length && (s.charAt(j) === " " || s.charAt(j) === "\t" || s.charAt(j) === "\n" || s.charAt(j) === "\r")) j++; - if (j < s.length && (s.charAt(j) === "}" || s.charAt(j) === "]")) continue; - } - out += ch; + let out = ""; + let inString = false; + let escaped = false; + for (let i = 0; i < s.length; i++) { + const ch = s.charAt(i); + if (inString) { + out += ch; + if (escaped) escaped = false; + else if (ch === "\\") escaped = true; + else if (ch === '"') inString = false; + continue; + } + if (ch === '"') { + inString = true; + out += ch; + continue; } - return out; + if (ch === ",") { + let j = i + 1; + while ( + j < s.length && + (s.charAt(j) === " " || + s.charAt(j) === "\t" || + s.charAt(j) === "\n" || + s.charAt(j) === "\r") + ) + j++; + if (j < s.length && (s.charAt(j) === "}" || s.charAt(j) === "]")) + continue; + } + out += ch; + } + return out; } // Raw \n, \r, \t inside string literals are invalid JSON; escape them. // Outside strings they are legal whitespace and left alone. function escapeRawNewlinesInStrings(s: string): string { - let out = ""; - let inString = false; - let escaped = false; - for (let i = 0; i < s.length; i++) { - const ch = s.charAt(i); - if (!inString) { - if (ch === '"') inString = true; - out += ch; - continue; - } - if (escaped) { - out += ch; - escaped = false; - continue; - } - if (ch === "\\") { - out += ch; - escaped = true; - continue; - } - if (ch === "\n") { out += "\\n"; continue; } - if (ch === "\r") { out += "\\r"; continue; } - if (ch === "\t") { out += "\\t"; continue; } - if (ch === '"') inString = false; - out += ch; + let out = ""; + let inString = false; + let escaped = false; + for (let i = 0; i < s.length; i++) { + const ch = s.charAt(i); + if (!inString) { + if (ch === '"') inString = true; + out += ch; + continue; + } + if (escaped) { + out += ch; + escaped = false; + continue; + } + if (ch === "\\") { + out += ch; + escaped = true; + continue; + } + if (ch === "\n") { + out += "\\n"; + continue; } - return out; + if (ch === "\r") { + out += "\\r"; + continue; + } + if (ch === "\t") { + out += "\\t"; + continue; + } + if (ch === '"') inString = false; + out += ch; + } + return out; } // Repair control characters and invalid escapes exactly where JSON.parse @@ -610,59 +710,68 @@ function escapeRawNewlinesInStrings(s: string): string { // it. Compound damage (raw newlines AND \` escapes in one string) needs // several rounds; the cap bounds pathological loops. function repairAtParserPositions(s: string): string { - let cur = s; - for (let i = 0; i < 500; i++) { - let err: unknown; - try { - JSON.parse(cur); - return cur; - } catch (e) { - err = e; - } - const msg = err instanceof Error ? err.message : String(err); - const posMatch = /position (\d+)/.exec(msg); - if (posMatch === null) return cur; - const pos = Number(posMatch[1]); - if (!Number.isInteger(pos) || pos < 0 || pos >= cur.length) return cur; - if (/control character/i.test(msg)) { - const ch = cur.charAt(pos); - let esc: string | undefined; - if (ch === "\n") esc = "\\n"; - else if (ch === "\r") esc = "\\r"; - else if (ch === "\t") esc = "\\t"; - else if (ch.charCodeAt(0) < 32) esc = "\\u" + ch.charCodeAt(0).toString(16).padStart(4, "0"); - if (esc === undefined) return cur; - cur = cur.slice(0, pos) + esc + cur.slice(pos + 1); - } else if (/Bad escaped character/i.test(msg)) { - const b = cur.charAt(pos - 1) === "\\" ? pos - 1 : cur.charAt(pos) === "\\" ? pos : -1; - if (b < 0) return cur; - cur = cur.slice(0, b) + "\\\\" + cur.slice(b + 1); - } else { - return cur; - } + let cur = s; + for (let i = 0; i < 500; i++) { + let err: unknown; + try { + JSON.parse(cur); + return cur; + } catch (e) { + err = e; } - return cur; + const msg = err instanceof Error ? err.message : String(err); + const posMatch = /position (\d+)/.exec(msg); + if (posMatch === null) return cur; + const pos = Number(posMatch[1]); + if (!Number.isInteger(pos) || pos < 0 || pos >= cur.length) return cur; + if (/control character/i.test(msg)) { + const ch = cur.charAt(pos); + let esc: string | undefined; + if (ch === "\n") esc = "\\n"; + else if (ch === "\r") esc = "\\r"; + else if (ch === "\t") esc = "\\t"; + else if (ch.charCodeAt(0) < 32) + esc = "\\u" + ch.charCodeAt(0).toString(16).padStart(4, "0"); + if (esc === undefined) return cur; + cur = cur.slice(0, pos) + esc + cur.slice(pos + 1); + } else if (/Bad escaped character/i.test(msg)) { + const b = + cur.charAt(pos - 1) === "\\" + ? pos - 1 + : cur.charAt(pos) === "\\" + ? pos + : -1; + if (b < 0) return cur; + cur = cur.slice(0, b) + "\\\\" + cur.slice(b + 1); + } else { + return cur; + } + } + return cur; } // Unbalanced brackets or an unterminated string at end of input is the // signature of a mid-stream cutoff (as opposed to balanced garbage). function looksTruncated(s: string): boolean { - let depth = 0; - let inString = false; - let escaped = false; - for (let i = 0; i < s.length; i++) { - const ch = s.charAt(i); - if (inString) { - if (escaped) escaped = false; - else if (ch === "\\") escaped = true; - else if (ch === '"') inString = false; - continue; - } - if (ch === '"') { inString = true; continue; } - if (ch === "{" || ch === "[") depth++; - else if (ch === "}" || ch === "]") depth--; + let depth = 0; + let inString = false; + let escaped = false; + for (let i = 0; i < s.length; i++) { + const ch = s.charAt(i); + if (inString) { + if (escaped) escaped = false; + else if (ch === "\\") escaped = true; + else if (ch === '"') inString = false; + continue; + } + if (ch === '"') { + inString = true; + continue; } - return depth > 0 || inString; + if (ch === "{" || ch === "[") depth++; + else if (ch === "}" || ch === "]") depth--; + } + return depth > 0 || inString; } /** @@ -673,38 +782,41 @@ function looksTruncated(s: string): boolean { * dropped — the parser never invents model content. */ function salvageContentEntries(raw: string): unknown[] { - const match = /"content"\s*:\s*\[/.exec(raw); - if (match === null) return []; - const arrayStart = match.index + match[0].length - 1; - const entries: unknown[] = []; - let depth = 0; // brackets nested inside the content array - let inString = false; - let escaped = false; - let entryStart = -1; - for (let i = arrayStart + 1; i < raw.length; i++) { - const ch = raw.charAt(i); - if (inString) { - if (escaped) escaped = false; - else if (ch === "\\") escaped = true; - else if (ch === '"') inString = false; - continue; - } - if (ch === '"') { inString = true; continue; } - if (ch === "{" || ch === "[") { - if (depth === 0 && ch === "{" && entryStart === -1) entryStart = i; - depth++; - continue; - } - if (ch === "}" || ch === "]") { - depth--; - if (depth < 0) break; // the content array itself closed - if (depth === 0 && entryStart !== -1) { - const entrySlice = raw.slice(entryStart, i + 1); - entryStart = -1; - const parsed = tryParseLenient(entrySlice); - if (parsed !== undefined) entries.push(parsed); - } - } + const match = /"content"\s*:\s*\[/.exec(raw); + if (match === null) return []; + const arrayStart = match.index + match[0].length - 1; + const entries: unknown[] = []; + let depth = 0; // brackets nested inside the content array + let inString = false; + let escaped = false; + let entryStart = -1; + for (let i = arrayStart + 1; i < raw.length; i++) { + const ch = raw.charAt(i); + if (inString) { + if (escaped) escaped = false; + else if (ch === "\\") escaped = true; + else if (ch === '"') inString = false; + continue; + } + if (ch === '"') { + inString = true; + continue; + } + if (ch === "{" || ch === "[") { + if (depth === 0 && ch === "{" && entryStart === -1) entryStart = i; + depth++; + continue; + } + if (ch === "}" || ch === "]") { + depth--; + if (depth < 0) break; // the content array itself closed + if (depth === 0 && entryStart !== -1) { + const entrySlice = raw.slice(entryStart, i + 1); + entryStart = -1; + const parsed = tryParseLenient(entrySlice); + if (parsed !== undefined) entries.push(parsed); + } } - return entries; + } + return entries; } diff --git a/src/persist/index.ts b/src/persist/index.ts index a637871..6ac73fa 100644 --- a/src/persist/index.ts +++ b/src/persist/index.ts @@ -14,5 +14,10 @@ * - the store never deletes files; cleanup is a downstream decision */ export { StateStore, flatFileNameFor } from "./store.js"; -export type { PersistedEnvelope, PersistLogger, StateStoreCodec, StateStoreOptions } from "./store.js"; +export type { + PersistedEnvelope, + PersistLogger, + StateStoreCodec, + StateStoreOptions, +} from "./store.js"; export { mergeCompressionState } from "./state-merge.js"; diff --git a/src/persist/state-merge.ts b/src/persist/state-merge.ts index 454808d..456c26b 100644 --- a/src/persist/state-merge.ts +++ b/src/persist/state-merge.ts @@ -9,7 +9,9 @@ import { createInitialState } from "../state.js"; * * Lifted verbatim in behavior from billion-context's proxy mergeState. */ -export function mergeCompressionState(parsed: CompressionState): CompressionState { +export function mergeCompressionState( + parsed: CompressionState, +): CompressionState { const fresh = createInitialState(); return { blocks: parsed.blocks ?? fresh.blocks, diff --git a/src/persist/store.ts b/src/persist/store.ts index f9f3725..783a703 100644 --- a/src/persist/store.ts +++ b/src/persist/store.ts @@ -4,7 +4,10 @@ import fsp from "node:fs/promises"; import path from "node:path"; /** Logger callback. Downstream routes it wherever it wants (file, stderr). */ -export type PersistLogger = (level: "info" | "warn" | "error", msg: string) => void; +export type PersistLogger = ( + level: "info" | "warn" | "error", + msg: string, +) => void; /** * On-disk record shape. The store owns the envelope (version stamp, save @@ -12,19 +15,19 @@ export type PersistLogger = (level: "info" | "warn" | "error", msg: string) => v * is entirely downstream-owned. `payload` must be JSON-serializable. */ export interface PersistedEnvelope { - version: number; - savedAt: number; - id: string; - payload: T; + version: number; + savedAt: number; + id: string; + payload: T; } /** A legacy (pre-envelope) record adopted on load. `version`/`savedAt` * preserve the source record's own stamps when present. */ export interface LegacyAdoption { - id: string; - payload: T; - version?: number; - savedAt?: number; + id: string; + payload: T; + version?: number; + savedAt?: number; } /** @@ -38,79 +41,79 @@ export interface LegacyAdoption { * so mixed trees of legacy plaintext and encoded files load fine. */ export interface StateStoreCodec { - encode(data: string): Buffer | string; - decode(buf: Buffer): string; + encode(data: string): Buffer | string; + decode(buf: Buffer): string; } export interface StateStoreOptions { - /** - * Storage root. The kernel deliberately has NO default location: the - * downstream decides where state lives (CLI dir, XDG data dir, plugin - * dir, temp dir in tests). All records live under this directory. - */ - dir: string; - /** - * Schema version stamped into every envelope. Owned by the downstream; - * bump it when the payload shape changes. The store itself never - * rejects a record over its version — migration policy belongs to the - * reader. - */ - version: number; - /** Debounce window for scheduleSave, ms. Default 500. */ - debounceMs?: number; - /** Default true. When false, all writes are silent no-ops and loads - * return empty results. */ - enabled?: boolean; - log?: PersistLogger; - /** - * Relative path (under `dir`) for a record, possibly namespaced into - * subdirectories (e.g. `openai/host-hash.json`). Default: flat - * `.json`. - * - * Because the path may depend on payload fields the store only learns - * at write time, single-record loads resolve namespaced files only - * after loadAll() has discovered them (or the store itself wrote - * them). The flat default name is always checked as a fallback, so - * downstreams using a custom relPath should call loadAll() at boot. - */ - relPath?: (id: string, payload: T) => string; - /** - * Adopt records written by an older, pre-envelope schema. Receives the - * parsed JSON of any file that failed the envelope-shape check; return - * an adoption to load it as an envelope, or null to skip it. Adopted - * records are re-persisted in the current envelope format on the next - * dirty write — files migrate organically, and old files keep loading - * (same policy billion-context's proxy used for its v1→v3 migration). - */ - legacy?: (parsed: unknown) => LegacyAdoption | null; - /** - * Payload validation on load. Return false to skip a record (foreign - * schema, corrupt content). Default: envelope-shape check only - * (string id, non-null payload). - */ - validate?: (envelope: PersistedEnvelope) => boolean; - /** - * Optional codec applied around every write (canonical + spill) and - * read. See StateStoreCodec. Default: none — files are plain UTF-8 JSON. - */ - codec?: StateStoreCodec; - /** - * Transient-write retry policy (Windows + hostile temp environments). - * The whole write cycle — mkdir, temp write, rename — is retried with - * exponential backoff when it fails with a TRANSIENT code: EPERM/EBUSY/ - * EACCES (a lock held by AV scan, search indexer, SMB) or ENOENT/ENOTDIR - * (the temp file or its directory vanished between create and rename — - * observed on CI Windows runners where a sweeper deletes freshly - * created files under %TEMP%). The delay after attempt i is - * min(retryBaseMs * 2^i, retryMaxMs). Defaults (6 attempts, 50ms base, - * 1600ms cap) give a ~1.5s window per write path — long enough for most - * locks to release, short enough not to stall a sync flush. When the - * window is exhausted the write spills to a side file (see spillPath) - * instead of dropping the data. - */ - retryAttempts?: number; - retryBaseMs?: number; - retryMaxMs?: number; + /** + * Storage root. The kernel deliberately has NO default location: the + * downstream decides where state lives (CLI dir, XDG data dir, plugin + * dir, temp dir in tests). All records live under this directory. + */ + dir: string; + /** + * Schema version stamped into every envelope. Owned by the downstream; + * bump it when the payload shape changes. The store itself never + * rejects a record over its version — migration policy belongs to the + * reader. + */ + version: number; + /** Debounce window for scheduleSave, ms. Default 500. */ + debounceMs?: number; + /** Default true. When false, all writes are silent no-ops and loads + * return empty results. */ + enabled?: boolean; + log?: PersistLogger; + /** + * Relative path (under `dir`) for a record, possibly namespaced into + * subdirectories (e.g. `openai/host-hash.json`). Default: flat + * `.json`. + * + * Because the path may depend on payload fields the store only learns + * at write time, single-record loads resolve namespaced files only + * after loadAll() has discovered them (or the store itself wrote + * them). The flat default name is always checked as a fallback, so + * downstreams using a custom relPath should call loadAll() at boot. + */ + relPath?: (id: string, payload: T) => string; + /** + * Adopt records written by an older, pre-envelope schema. Receives the + * parsed JSON of any file that failed the envelope-shape check; return + * an adoption to load it as an envelope, or null to skip it. Adopted + * records are re-persisted in the current envelope format on the next + * dirty write — files migrate organically, and old files keep loading + * (same policy billion-context's proxy used for its v1→v3 migration). + */ + legacy?: (parsed: unknown) => LegacyAdoption | null; + /** + * Payload validation on load. Return false to skip a record (foreign + * schema, corrupt content). Default: envelope-shape check only + * (string id, non-null payload). + */ + validate?: (envelope: PersistedEnvelope) => boolean; + /** + * Optional codec applied around every write (canonical + spill) and + * read. See StateStoreCodec. Default: none — files are plain UTF-8 JSON. + */ + codec?: StateStoreCodec; + /** + * Transient-write retry policy (Windows + hostile temp environments). + * The whole write cycle — mkdir, temp write, rename — is retried with + * exponential backoff when it fails with a TRANSIENT code: EPERM/EBUSY/ + * EACCES (a lock held by AV scan, search indexer, SMB) or ENOENT/ENOTDIR + * (the temp file or its directory vanished between create and rename — + * observed on CI Windows runners where a sweeper deletes freshly + * created files under %TEMP%). The delay after attempt i is + * min(retryBaseMs * 2^i, retryMaxMs). Defaults (6 attempts, 50ms base, + * 1600ms cap) give a ~1.5s window per write path — long enough for most + * locks to release, short enough not to stall a sync flush. When the + * window is exhausted the write spills to a side file (see spillPath) + * instead of dropping the data. + */ + retryAttempts?: number; + retryBaseMs?: number; + retryMaxMs?: number; } /** @@ -141,542 +144,602 @@ export interface StateStoreOptions { * state should not be deleted opportunistically). */ export class StateStore { - readonly enabled: boolean; - private readonly dir: string; - private readonly version: number; - private readonly debounceMs: number; - private readonly log: PersistLogger; - private readonly legacyFn?: (parsed: unknown) => LegacyAdoption | null; - private readonly relPathFn?: (id: string, payload: T) => string; - private readonly validateFn: (envelope: PersistedEnvelope) => boolean; - private readonly codec?: StateStoreCodec; - private readonly retryAttempts: number; - private readonly retryBaseMs: number; - private readonly retryMaxMs: number; - private readonly timers = new Map(); - private readonly pending = new Map T>(); - private readonly writeChains = new Map>(); - /** id → absolute path, populated by writes and loadAll. */ - private readonly discovered = new Map(); - /** id → seq of the last COMMITTED write (sync or async). Async writers - * capture the seq at snapshot time and re-check it inside their - * synchronous commit block; a flushSync (or a newer committed write) - * bumping the seq makes the stale in-flight write abort its commit - * instead of clobbering the newer on-disk payload. */ - private readonly commitSeq = new Map(); - /** id → cumulative write-failure count, for rate-limited alerting. */ - private readonly failCounts = new Map(); - /** Monotonic counter for unique temp filenames within a process. */ - private tmpSeq = 0; - - constructor(opts: StateStoreOptions) { - this.dir = opts.dir; - this.version = opts.version; - this.debounceMs = opts.debounceMs ?? 500; - this.enabled = opts.enabled ?? true; - this.log = opts.log ?? ((_level, _msg) => {}); - this.relPathFn = opts.relPath; - this.legacyFn = opts.legacy; - this.validateFn = opts.validate ?? defaultValidate; - this.codec = opts.codec; - this.retryAttempts = Math.max(1, opts.retryAttempts ?? 6); - this.retryBaseMs = Math.max(1, opts.retryBaseMs ?? 50); - this.retryMaxMs = Math.max(this.retryBaseMs, opts.retryMaxMs ?? 1600); + readonly enabled: boolean; + private readonly dir: string; + private readonly version: number; + private readonly debounceMs: number; + private readonly log: PersistLogger; + private readonly legacyFn?: (parsed: unknown) => LegacyAdoption | null; + private readonly relPathFn?: (id: string, payload: T) => string; + private readonly validateFn: (envelope: PersistedEnvelope) => boolean; + private readonly codec?: StateStoreCodec; + private readonly retryAttempts: number; + private readonly retryBaseMs: number; + private readonly retryMaxMs: number; + private readonly timers = new Map(); + private readonly pending = new Map T>(); + private readonly writeChains = new Map>(); + /** id → absolute path, populated by writes and loadAll. */ + private readonly discovered = new Map(); + /** id → seq of the last COMMITTED write (sync or async). Async writers + * capture the seq at snapshot time and re-check it inside their + * synchronous commit block; a flushSync (or a newer committed write) + * bumping the seq makes the stale in-flight write abort its commit + * instead of clobbering the newer on-disk payload. */ + private readonly commitSeq = new Map(); + /** id → cumulative write-failure count, for rate-limited alerting. */ + private readonly failCounts = new Map(); + /** Monotonic counter for unique temp filenames within a process. */ + private tmpSeq = 0; + + constructor(opts: StateStoreOptions) { + this.dir = opts.dir; + this.version = opts.version; + this.debounceMs = opts.debounceMs ?? 500; + this.enabled = opts.enabled ?? true; + this.log = opts.log ?? ((_level, _msg) => {}); + this.relPathFn = opts.relPath; + this.legacyFn = opts.legacy; + this.validateFn = opts.validate ?? defaultValidate; + this.codec = opts.codec; + this.retryAttempts = Math.max(1, opts.retryAttempts ?? 6); + this.retryBaseMs = Math.max(1, opts.retryBaseMs ?? 50); + this.retryMaxMs = Math.max(this.retryBaseMs, opts.retryMaxMs ?? 1600); + } + + /** Debounced save. Coalesces bursts; the builder runs at write time, so + * the freshest state is always persisted. Never throws. */ + scheduleSave(id: string, build: () => T): void { + if (!this.enabled) return; + this.pending.set(id, build); + const existing = this.timers.get(id); + if (existing) return; + const timer = setTimeout(() => { + this.timers.delete(id); + const builder = this.pending.get(id); + this.pending.delete(id); + if (!builder) return; + // Errors are logged inside writeInner; swallowing here keeps a + // failed timer write from becoming an unhandledRejection. + void this.writeNow(id, builder).catch(() => {}); + }, this.debounceMs); + timer.unref?.(); + this.timers.set(id, timer); + } + + /** Immediate save, serialized per id. Rejects on write failure; a + * failing write never breaks the chain for the next one. */ + async writeNow(id: string, build: () => T): Promise { + if (!this.enabled) return; + // The previous promise may reject (disk full, EPERM) — catch so our + // chain doesn't break, then run our own write. + const prev = this.writeChains.get(id) ?? Promise.resolve(); + const next = prev.catch(() => {}).then(() => this.writeInner(id, build)); + this.writeChains.set(id, next); + // Clean up the chain entry once settled so the Map doesn't grow. The + // .catch() is load-bearing: .finally() returns a derived promise that + // nobody else holds — when the chain rejects it would surface as an + // unhandledRejection and crash the host on default Node settings. + next + .finally(() => { + if (this.writeChains.get(id) === next) this.writeChains.delete(id); + }) + .catch(() => {}); + return next; + } + + /** Synchronous flush for one id. Used where the caller cannot await + * (memory eviction, sync shutdown paths). Cancels any pending debounce + * timer. Returns true on success, false on failure — callers that use + * the result to drop in-memory state must NOT drop it on failure. + * Committing bumps the per-id fence so a stale in-flight async write + * cannot land its older payload after this returns. */ + flushSync(id: string, build: () => T): boolean { + if (!this.enabled) return true; + const timer = this.timers.get(id); + if (timer) { + clearTimeout(timer); + this.timers.delete(id); + this.pending.delete(id); } - - /** Debounced save. Coalesces bursts; the builder runs at write time, so - * the freshest state is always persisted. Never throws. */ - scheduleSave(id: string, build: () => T): void { - if (!this.enabled) return; - this.pending.set(id, build); - const existing = this.timers.get(id); - if (existing) return; - const timer = setTimeout(() => { - this.timers.delete(id); - const builder = this.pending.get(id); - this.pending.delete(id); - if (!builder) return; - // Errors are logged inside writeInner; swallowing here keeps a - // failed timer write from becoming an unhandledRejection. - void this.writeNow(id, builder).catch(() => {}); - }, this.debounceMs); - timer.unref?.(); - this.timers.set(id, timer); - } - - /** Immediate save, serialized per id. Rejects on write failure; a - * failing write never breaks the chain for the next one. */ - async writeNow(id: string, build: () => T): Promise { - if (!this.enabled) return; - // The previous promise may reject (disk full, EPERM) — catch so our - // chain doesn't break, then run our own write. - const prev = this.writeChains.get(id) ?? Promise.resolve(); - const next = prev.catch(() => {}).then(() => this.writeInner(id, build)); - this.writeChains.set(id, next); - // Clean up the chain entry once settled so the Map doesn't grow. The - // .catch() is load-bearing: .finally() returns a derived promise that - // nobody else holds — when the chain rejects it would surface as an - // unhandledRejection and crash the host on default Node settings. - next - .finally(() => { - if (this.writeChains.get(id) === next) this.writeChains.delete(id); - }) - .catch(() => {}); - return next; + let payload: T; + try { + payload = build(); + } catch (e) { + this.log("error", `[persist] builder failed for ${id}: ${errText(e)}`); + return false; } - - /** Synchronous flush for one id. Used where the caller cannot await - * (memory eviction, sync shutdown paths). Cancels any pending debounce - * timer. Returns true on success, false on failure — callers that use - * the result to drop in-memory state must NOT drop it on failure. - * Committing bumps the per-id fence so a stale in-flight async write - * cannot land its older payload after this returns. */ - flushSync(id: string, build: () => T): boolean { - if (!this.enabled) return true; - const timer = this.timers.get(id); - if (timer) { - clearTimeout(timer); - this.timers.delete(id); - this.pending.delete(id); - } - let payload: T; + const file = this.resolvePath(id, payload); + const data = this.serialize(id, payload); + let lastErr: unknown; + for (let attempt = 0; attempt < this.retryAttempts; attempt++) { + // Fresh temp name + mkdir per attempt — same reasoning as the + // async writeInner loop (delete-pending tombstones, swept dirs). + const tmp = this.tempPath(file); + try { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(tmp, data); + fs.renameSync(tmp, file); + lastErr = undefined; + break; + } catch (e) { + lastErr = e; try { - payload = build(); - } catch (e) { - this.log("error", `[persist] builder failed for ${id}: ${errText(e)}`); - return false; - } - const file = this.resolvePath(id, payload); - const data = this.serialize(id, payload); - let lastErr: unknown; - for (let attempt = 0; attempt < this.retryAttempts; attempt++) { - // Fresh temp name + mkdir per attempt — same reasoning as the - // async writeInner loop (delete-pending tombstones, swept dirs). - const tmp = this.tempPath(file); - try { - fs.mkdirSync(path.dirname(file), { recursive: true }); - fs.writeFileSync(tmp, data); - fs.renameSync(tmp, file); - lastErr = undefined; - break; - } catch (e) { - lastErr = e; - try { - fs.unlinkSync(tmp); - } catch { - // temp never created or already gone - } - if (!isTransientFsError(e) || attempt === this.retryAttempts - 1) break; - // sync sleep (Atomics.wait) — backoff grows per attempt - Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, this.backoffMs(attempt)); - } - } - if (!lastErr) { - this.discovered.set(id, file); - this.clearFailure(id); - this.bumpCommit(id); - this.removeSpillSync(file); - return true; - } - // The canonical write outlived the retry window (Windows lock held by - // AV/indexer/SMB). Spill to a side file so the data still lands on - // disk instead of being dropped — loadAll picks the freshest of the - // canonical and the spill, with its own transient retry window. - const spill = this.spillPathFor(file); - let spillPath: string | null = null; - for (let attempt = 0; attempt < this.retryAttempts && spillPath === null; attempt++) { - try { - fs.mkdirSync(path.dirname(spill), { recursive: true }); - fs.writeFileSync(spill, data); - spillPath = spill; - } catch (e) { - if (!isTransientFsError(e) || attempt === this.retryAttempts - 1) break; - Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, this.backoffMs(attempt)); - } - } - if (spillPath) this.discovered.set(id, spillPath); - if (spillPath) this.bumpCommit(id); - this.recordFailure(id, lastErr, spillPath); - return spillPath !== null; - } - - /** Load one record. Checks the discovered path (from a prior - * write/loadAll), an optional relative-path hint, and the flat default - * name. Returns null when absent, disabled, corrupt, or rejected by - * validate. The hint covers namespaced records the store has not - * discovered (e.g. an evicted session re-requested with its meta, - * where the path depends on data the store cannot reconstruct from - * the id alone). */ - loadSync(id: string, hint?: string): PersistedEnvelope | null { - if (!this.enabled) return null; - const candidates = [ - this.discovered.get(id), - hint ? path.join(this.dir, hint) : undefined, - path.join(this.dir, flatFileNameFor(id)), - ]; - for (const file of candidates) { - if (!file) continue; - const envelope = this.readEnvelope(file); - if (envelope && envelope.id === id) return envelope; + fs.unlinkSync(tmp); + } catch { + // temp never created or already gone } - return null; + if (!isTransientFsError(e) || attempt === this.retryAttempts - 1) break; + // sync sleep (Atomics.wait) — backoff grows per attempt + Atomics.wait( + new Int32Array(new SharedArrayBuffer(4)), + 0, + 0, + this.backoffMs(attempt), + ); + } } - - /** Load every record under dir. Populates the discovery map (enables - * loadSync for namespaced relPaths). Skips corrupt files, `.tmp-*` - * orphans, and records whose filename does not match their id — one - * bad file never blocks boot. Never throws. */ - async loadAll(): Promise>> { - const out = new Map>(); - if (!this.enabled) return out; - const files = await this.walkJsonFiles(this.dir); - for (const file of files) { - const envelope = this.readEnvelope(file); - if (!envelope) continue; - const base = path.basename(file); - const relBase = path.basename(this.relPathOf(envelope.id, envelope.payload)); - const flatBase = flatFileNameFor(envelope.id); - // A file is a valid record for its envelope id when its name is the - // canonical name OR a spill (`.fb`) of the canonical name. This lets - // a spilled record (written while the canonical was locked) be - // discovered and reconciled against the canonical by savedAt. - let owner: string | null = null; - if (base === relBase || base === flatBase) { - owner = envelope.id; - } else if (base.endsWith(".fb.json")) { - const canonicalBase = `${base.slice(0, -".fb.json".length)}.json`; - if (canonicalBase === relBase || canonicalBase === flatBase) { - owner = envelope.id; - } - } - if (owner === null) { - this.log("warn", `[persist] skipping ${rel(file, this.dir)}: filename does not match record id`); - continue; - } - const existing = out.get(owner); - if (!existing || envelope.savedAt >= existing.savedAt) { - out.set(owner, envelope); - this.discovered.set(owner, file); - } - } - return out; + if (!lastErr) { + this.discovered.set(id, file); + this.clearFailure(id); + this.bumpCommit(id); + this.removeSpillSync(file); + return true; } - - /** Whether a debounced write is pending for an id. */ - hasPending(id: string): boolean { - return this.timers.has(id); + // The canonical write outlived the retry window (Windows lock held by + // AV/indexer/SMB). Spill to a side file so the data still lands on + // disk instead of being dropped — loadAll picks the freshest of the + // canonical and the spill, with its own transient retry window. + const spill = this.spillPathFor(file); + let spillPath: string | null = null; + for ( + let attempt = 0; + attempt < this.retryAttempts && spillPath === null; + attempt++ + ) { + try { + fs.mkdirSync(path.dirname(spill), { recursive: true }); + fs.writeFileSync(spill, data); + spillPath = spill; + } catch (e) { + if (!isTransientFsError(e) || attempt === this.retryAttempts - 1) break; + Atomics.wait( + new Int32Array(new SharedArrayBuffer(4)), + 0, + 0, + this.backoffMs(attempt), + ); + } } - - /** Ids with a pending debounced write. */ - pendingIds(): string[] { - return [...this.timers.keys()]; + if (spillPath) this.discovered.set(id, spillPath); + if (spillPath) this.bumpCommit(id); + this.recordFailure(id, lastErr, spillPath); + return spillPath !== null; + } + + /** Load one record. Checks the discovered path (from a prior + * write/loadAll), an optional relative-path hint, and the flat default + * name. Returns null when absent, disabled, corrupt, or rejected by + * validate. The hint covers namespaced records the store has not + * discovered (e.g. an evicted session re-requested with its meta, + * where the path depends on data the store cannot reconstruct from + * the id alone). */ + loadSync(id: string, hint?: string): PersistedEnvelope | null { + if (!this.enabled) return null; + const candidates = [ + this.discovered.get(id), + hint ? path.join(this.dir, hint) : undefined, + path.join(this.dir, flatFileNameFor(id)), + ]; + for (const file of candidates) { + if (!file) continue; + const envelope = this.readEnvelope(file); + if (envelope && envelope.id === id) return envelope; } - - /** Flush every pending debounced write immediately, then drain in-flight - * writes. For graceful shutdown (SIGTERM/SIGINT). Never rejects. */ - async flushAll(): Promise { - const ids = this.pendingIds(); - for (const timer of this.timers.values()) clearTimeout(timer); - this.timers.clear(); - const builders = new Map T>(); - for (const id of ids) { - const build = this.pending.get(id); - if (build) builders.set(id, build); - this.pending.delete(id); + return null; + } + + /** Load every record under dir. Populates the discovery map (enables + * loadSync for namespaced relPaths). Skips corrupt files, `.tmp-*` + * orphans, and records whose filename does not match their id — one + * bad file never blocks boot. Never throws. */ + async loadAll(): Promise>> { + const out = new Map>(); + if (!this.enabled) return out; + const files = await this.walkJsonFiles(this.dir); + for (const file of files) { + const envelope = this.readEnvelope(file); + if (!envelope) continue; + const base = path.basename(file); + const relBase = path.basename( + this.relPathOf(envelope.id, envelope.payload), + ); + const flatBase = flatFileNameFor(envelope.id); + // A file is a valid record for its envelope id when its name is the + // canonical name OR a spill (`.fb`) of the canonical name. This lets + // a spilled record (written while the canonical was locked) be + // discovered and reconciled against the canonical by savedAt. + let owner: string | null = null; + if (base === relBase || base === flatBase) { + owner = envelope.id; + } else if (base.endsWith(".fb.json")) { + const canonicalBase = `${base.slice(0, -".fb.json".length)}.json`; + if (canonicalBase === relBase || canonicalBase === flatBase) { + owner = envelope.id; } - await Promise.all( - [...builders.entries()].map(([id, build]) => - this.writeNow(id, build).catch((e) => { - this.log("error", `[persist] shutdown flush failed for ${id}: ${errText(e)}`); - }), - ), + } + if (owner === null) { + this.log( + "warn", + `[persist] skipping ${rel(file, this.dir)}: filename does not match record id`, ); - // Drain writes whose timer fired earlier but are still mid-flight. - await Promise.allSettled([...this.writeChains.values()]); + continue; + } + const existing = out.get(owner); + if (!existing || envelope.savedAt >= existing.savedAt) { + out.set(owner, envelope); + this.discovered.set(owner, file); + } } - - /** Cancel all pending debounced writes without flushing (tests). */ - cancelAll(): void { - for (const timer of this.timers.values()) clearTimeout(timer); - this.timers.clear(); - this.pending.clear(); + return out; + } + + /** Whether a debounced write is pending for an id. */ + hasPending(id: string): boolean { + return this.timers.has(id); + } + + /** Ids with a pending debounced write. */ + pendingIds(): string[] { + return [...this.timers.keys()]; + } + + /** Flush every pending debounced write immediately, then drain in-flight + * writes. For graceful shutdown (SIGTERM/SIGINT). Never rejects. */ + async flushAll(): Promise { + const ids = this.pendingIds(); + for (const timer of this.timers.values()) clearTimeout(timer); + this.timers.clear(); + const builders = new Map T>(); + for (const id of ids) { + const build = this.pending.get(id); + if (build) builders.set(id, build); + this.pending.delete(id); } - - private async writeInner(id: string, build: () => T): Promise { - // Fence point, captured BEFORE build(): from here on, any write that - // commits (a flushSync re-entering from inside build(), or a newer - // chained write) bumps the seq and this snapshot is stale. - const baseSeq = this.commitSeq.get(id) ?? 0; - let payload: T; + await Promise.all( + [...builders.entries()].map(([id, build]) => + this.writeNow(id, build).catch((e) => { + this.log( + "error", + `[persist] shutdown flush failed for ${id}: ${errText(e)}`, + ); + }), + ), + ); + // Drain writes whose timer fired earlier but are still mid-flight. + await Promise.allSettled([...this.writeChains.values()]); + } + + /** Cancel all pending debounced writes without flushing (tests). */ + cancelAll(): void { + for (const timer of this.timers.values()) clearTimeout(timer); + this.timers.clear(); + this.pending.clear(); + } + + private async writeInner(id: string, build: () => T): Promise { + // Fence point, captured BEFORE build(): from here on, any write that + // commits (a flushSync re-entering from inside build(), or a newer + // chained write) bumps the seq and this snapshot is stale. + const baseSeq = this.commitSeq.get(id) ?? 0; + let payload: T; + try { + payload = build(); + } catch (e) { + this.log("error", `[persist] builder failed for ${id}: ${errText(e)}`); + throw e; + } + const file = this.resolvePath(id, payload); + const data = this.serialize(id, payload); + let lastErr: unknown; + for (let attempt = 0; attempt < this.retryAttempts; attempt++) { + // Fresh temp name per attempt: on Windows a failed cleanup can + // leave the previous name delete-pending, and creating over a + // tombstoned name fails immediately — reusing it would burn every + // remaining attempt before the write even happens. mkdir runs per + // attempt too: a sweeper may have deleted the directory after the + // previous cycle created it. + const tmp = this.tempPath(file); + try { + await fsp.mkdir(path.dirname(file), { recursive: true }); + await fsp.writeFile(tmp, data); + // Commit block: fence check + rename in one synchronous run + // — the event loop cannot interleave a flushSync between the + // check and the rename, so a stale payload can never land + // after a newer committed write. + if ((this.commitSeq.get(id) ?? 0) !== baseSeq) { + await fsp.unlink(tmp).catch(() => {}); + return; + } try { - payload = build(); + fs.renameSync(tmp, file); } catch (e) { - this.log("error", `[persist] builder failed for ${id}: ${errText(e)}`); - throw e; + lastErr = e; + await fsp.unlink(tmp).catch(() => {}); + if (!isTransientFsError(e) || attempt === this.retryAttempts - 1) + break; + const { promise, resolve } = Promise.withResolvers(); + setTimeout(resolve, this.backoffMs(attempt)); + await promise; + continue; } - const file = this.resolvePath(id, payload); - const data = this.serialize(id, payload); - let lastErr: unknown; - for (let attempt = 0; attempt < this.retryAttempts; attempt++) { - // Fresh temp name per attempt: on Windows a failed cleanup can - // leave the previous name delete-pending, and creating over a - // tombstoned name fails immediately — reusing it would burn every - // remaining attempt before the write even happens. mkdir runs per - // attempt too: a sweeper may have deleted the directory after the - // previous cycle created it. - const tmp = this.tempPath(file); - try { - await fsp.mkdir(path.dirname(file), { recursive: true }); - await fsp.writeFile(tmp, data); - // Commit block: fence check + rename in one synchronous run - // — the event loop cannot interleave a flushSync between the - // check and the rename, so a stale payload can never land - // after a newer committed write. - if ((this.commitSeq.get(id) ?? 0) !== baseSeq) { - await fsp.unlink(tmp).catch(() => {}); - return; - } - try { - fs.renameSync(tmp, file); - } catch (e) { - lastErr = e; - await fsp.unlink(tmp).catch(() => {}); - if (!isTransientFsError(e) || attempt === this.retryAttempts - 1) break; - const { promise, resolve } = Promise.withResolvers(); - setTimeout(resolve, this.backoffMs(attempt)); - await promise; - continue; - } - this.discovered.set(id, file); - this.clearFailure(id); - this.commitSeq.set(id, baseSeq + 1); - await this.removeSpill(file); - return; - } catch (e) { - lastErr = e; - // Clean up the .tmp orphan so the next write doesn't collide - // with it. - await fsp.unlink(tmp).catch(() => {}); - if (!isTransientFsError(e) || attempt === this.retryAttempts - 1) break; - const { promise, resolve } = Promise.withResolvers(); - setTimeout(resolve, this.backoffMs(attempt)); - await promise; - } - } - // Fenced off before spilling: a newer committed write already owns - // the disk record for this id — the stale payload is dropped, not - // spilled (its spill would clobber the newer spill on the same slot). - if ((this.commitSeq.get(id) ?? 0) !== baseSeq) return; - // The canonical write outlived the retry window (Windows lock held - // by AV/indexer/SMB, or a sweeper keeps deleting the fresh temp). - // Spill to a side file so the data still lands on disk instead of - // being dropped — loadAll picks the freshest of the canonical and - // the spill. The spill write gets its own transient retry window for - // the same reason (observed failing with ENOENT alongside the - // canonical on CI Windows runners). - const spill = this.spillPathFor(file); - let spillPath: string | null = null; - for (let attempt = 0; attempt < this.retryAttempts && spillPath === null; attempt++) { - try { - await fsp.mkdir(path.dirname(spill), { recursive: true }); - // Fence + write synchronously: the spill slot is shared per - // id, so a late-landing stale spill would overwrite a newer - // spill exactly like the canonical case above. - if ((this.commitSeq.get(id) ?? 0) !== baseSeq) return; - fs.writeFileSync(spill, data); - spillPath = spill; - } catch (e) { - if (!isTransientFsError(e) || attempt === this.retryAttempts - 1) break; - const { promise, resolve } = Promise.withResolvers(); - setTimeout(resolve, this.backoffMs(attempt)); - await promise; - } - } - if (spillPath) { - this.discovered.set(id, spillPath); - this.commitSeq.set(id, baseSeq + 1); - } - this.recordFailure(id, lastErr, spillPath); - throw lastErr; - } - - private envelope(id: string, payload: T): PersistedEnvelope { - return { version: this.version, savedAt: Date.now(), id, payload }; - } - - /** Serialize an envelope for disk, applying the optional codec. Strings - * are written as UTF-8 (the fs default); Buffer results pass through as - * raw bytes. */ - private serialize(id: string, payload: T): Buffer | string { - const json = JSON.stringify(this.envelope(id, payload)); - return this.codec ? this.codec.encode(json) : json; + this.discovered.set(id, file); + this.clearFailure(id); + this.commitSeq.set(id, baseSeq + 1); + await this.removeSpill(file); + return; + } catch (e) { + lastErr = e; + // Clean up the .tmp orphan so the next write doesn't collide + // with it. + await fsp.unlink(tmp).catch(() => {}); + if (!isTransientFsError(e) || attempt === this.retryAttempts - 1) break; + const { promise, resolve } = Promise.withResolvers(); + setTimeout(resolve, this.backoffMs(attempt)); + await promise; + } } - - /** Absolute path for a record: custom relPath (guarded against path - * escape) or the flat hash default. */ - private resolvePath(id: string, payload: T): string { - return path.join(this.dir, this.relPathOf(id, payload)); - } - - private relPathOf(id: string, payload: T): string { - const custom = this.relPathFn?.(id, payload); - if (!custom) return flatFileNameFor(id); - const rel = path.normalize(custom); - if (path.isAbsolute(rel) || rel.split(/[\\/]+/).includes("..")) { - this.log("warn", `[persist] relPath for ${id} escapes dir; using flat name`); - return flatFileNameFor(id); - } - return rel; - } - - /** Unique temp file next to the destination (same dir ⇒ same volume ⇒ - * rename is atomic). Prefixed `.tmp-` so loadAll skips orphans. */ - private tempPath(dest: string): string { - const seq = this.tmpSeq++; - return path.join(path.dirname(dest), `.tmp-${path.basename(dest, ".json")}-${process.pid}-${seq}`); - } - - private backoffMs(attempt: number): number { - return Math.min(this.retryBaseMs * 2 ** attempt, this.retryMaxMs); - } - - /** Side file for a record whose canonical write keeps failing: the - * canonical name with a `.fb` (fallback) infix, e.g. `a.json` → - * `a.fb.json`. One slot per id, overwritten on each spill, so a stuck - * lock never accumulates files. Ends in `.json` so loadAll discovers it. */ - private spillPathFor(file: string): string { - const base = path.basename(file); - const dot = base.lastIndexOf("."); - const stem = dot > 0 ? base.slice(0, dot) : base; - const ext = dot > 0 ? base.slice(dot) : ".json"; - return path.join(path.dirname(file), `${stem}.fb${ext}`); - } - - /** Remove a stale spill after a successful canonical write (best-effort). */ - private async removeSpill(file: string): Promise { - await fsp.unlink(this.spillPathFor(file)).catch(() => {}); + // Fenced off before spilling: a newer committed write already owns + // the disk record for this id — the stale payload is dropped, not + // spilled (its spill would clobber the newer spill on the same slot). + if ((this.commitSeq.get(id) ?? 0) !== baseSeq) return; + // The canonical write outlived the retry window (Windows lock held + // by AV/indexer/SMB, or a sweeper keeps deleting the fresh temp). + // Spill to a side file so the data still lands on disk instead of + // being dropped — loadAll picks the freshest of the canonical and + // the spill. The spill write gets its own transient retry window for + // the same reason (observed failing with ENOENT alongside the + // canonical on CI Windows runners). + const spill = this.spillPathFor(file); + let spillPath: string | null = null; + for ( + let attempt = 0; + attempt < this.retryAttempts && spillPath === null; + attempt++ + ) { + try { + await fsp.mkdir(path.dirname(spill), { recursive: true }); + // Fence + write synchronously: the spill slot is shared per + // id, so a late-landing stale spill would overwrite a newer + // spill exactly like the canonical case above. + if ((this.commitSeq.get(id) ?? 0) !== baseSeq) return; + fs.writeFileSync(spill, data); + spillPath = spill; + } catch (e) { + if (!isTransientFsError(e) || attempt === this.retryAttempts - 1) break; + const { promise, resolve } = Promise.withResolvers(); + setTimeout(resolve, this.backoffMs(attempt)); + await promise; + } } - - /** Advance the per-id commit fence after a successful synchronous - * commit, fencing off any in-flight async write holding an older - * snapshot. */ - private bumpCommit(id: string): void { - this.commitSeq.set(id, (this.commitSeq.get(id) ?? 0) + 1); + if (spillPath) { + this.discovered.set(id, spillPath); + this.commitSeq.set(id, baseSeq + 1); } - - private removeSpillSync(file: string): void { - try { - fs.unlinkSync(this.spillPathFor(file)); - } catch { - // no spill or still locked — loadAll reconciles by savedAt - } + this.recordFailure(id, lastErr, spillPath); + throw lastErr; + } + + private envelope(id: string, payload: T): PersistedEnvelope { + return { version: this.version, savedAt: Date.now(), id, payload }; + } + + /** Serialize an envelope for disk, applying the optional codec. Strings + * are written as UTF-8 (the fs default); Buffer results pass through as + * raw bytes. */ + private serialize(id: string, payload: T): Buffer | string { + const json = JSON.stringify(this.envelope(id, payload)); + return this.codec ? this.codec.encode(json) : json; + } + + /** Absolute path for a record: custom relPath (guarded against path + * escape) or the flat hash default. */ + private resolvePath(id: string, payload: T): string { + return path.join(this.dir, this.relPathOf(id, payload)); + } + + private relPathOf(id: string, payload: T): string { + const custom = this.relPathFn?.(id, payload); + if (!custom) return flatFileNameFor(id); + const rel = path.normalize(custom); + if (path.isAbsolute(rel) || rel.split(/[\\/]+/).includes("..")) { + this.log( + "warn", + `[persist] relPath for ${id} escapes dir; using flat name`, + ); + return flatFileNameFor(id); } - - /** Rate-limited failure alerting: log on the first failure and at each - * power-of-two count (1,2,4,8,…), so a long lock yields ~log2(N) lines - * instead of one per write. Includes the spill path so the operator can - * see where the data landed. */ - private recordFailure(id: string, err: unknown, spillPath: string | null): void { - const count = (this.failCounts.get(id) ?? 0) + 1; - this.failCounts.set(id, count); - const alertAt = count === 1 || (count >= 2 && (count & (count - 1)) === 0); - if (!alertAt) return; - const where = spillPath - ? `; data spilled to ${rel(spillPath, this.dir)}` - : "; SPILLOVER ALSO FAILED — data at risk"; - this.log(count === 1 ? "error" : "warn", `[persist] write failed for ${id} (total ${count}x): ${errText(err)}${where}`); + return rel; + } + + /** Unique temp file next to the destination (same dir ⇒ same volume ⇒ + * rename is atomic). Prefixed `.tmp-` so loadAll skips orphans. */ + private tempPath(dest: string): string { + const seq = this.tmpSeq++; + return path.join( + path.dirname(dest), + `.tmp-${path.basename(dest, ".json")}-${process.pid}-${seq}`, + ); + } + + private backoffMs(attempt: number): number { + return Math.min(this.retryBaseMs * 2 ** attempt, this.retryMaxMs); + } + + /** Side file for a record whose canonical write keeps failing: the + * canonical name with a `.fb` (fallback) infix, e.g. `a.json` → + * `a.fb.json`. One slot per id, overwritten on each spill, so a stuck + * lock never accumulates files. Ends in `.json` so loadAll discovers it. */ + private spillPathFor(file: string): string { + const base = path.basename(file); + const dot = base.lastIndexOf("."); + const stem = dot > 0 ? base.slice(0, dot) : base; + const ext = dot > 0 ? base.slice(dot) : ".json"; + return path.join(path.dirname(file), `${stem}.fb${ext}`); + } + + /** Remove a stale spill after a successful canonical write (best-effort). */ + private async removeSpill(file: string): Promise { + await fsp.unlink(this.spillPathFor(file)).catch(() => {}); + } + + /** Advance the per-id commit fence after a successful synchronous + * commit, fencing off any in-flight async write holding an older + * snapshot. */ + private bumpCommit(id: string): void { + this.commitSeq.set(id, (this.commitSeq.get(id) ?? 0) + 1); + } + + private removeSpillSync(file: string): void { + try { + fs.unlinkSync(this.spillPathFor(file)); + } catch { + // no spill or still locked — loadAll reconciles by savedAt } - - private clearFailure(id: string): void { - this.failCounts.delete(id); + } + + /** Rate-limited failure alerting: log on the first failure and at each + * power-of-two count (1,2,4,8,…), so a long lock yields ~log2(N) lines + * instead of one per write. Includes the spill path so the operator can + * see where the data landed. */ + private recordFailure( + id: string, + err: unknown, + spillPath: string | null, + ): void { + const count = (this.failCounts.get(id) ?? 0) + 1; + this.failCounts.set(id, count); + const alertAt = count === 1 || (count >= 2 && (count & (count - 1)) === 0); + if (!alertAt) return; + const where = spillPath + ? `; data spilled to ${rel(spillPath, this.dir)}` + : "; SPILLOVER ALSO FAILED — data at risk"; + this.log( + count === 1 ? "error" : "warn", + `[persist] write failed for ${id} (total ${count}x): ${errText(err)}${where}`, + ); + } + + private clearFailure(id: string): void { + this.failCounts.delete(id); + } + + /** Parse + validate one file. Corrupt or invalid records return null + * (logged) instead of throwing — load paths must never block boot. */ + private readEnvelope(file: string): PersistedEnvelope | null { + let parsed: unknown; + try { + const buf = fs.readFileSync(file); + const text = this.codec ? this.codec.decode(buf) : buf.toString("utf8"); + parsed = JSON.parse(text); + } catch (e) { + // A missing candidate path is an expected miss (loadSync probes); + // only genuinely unreadable/corrupt files get logged. + if ((e as NodeJS.ErrnoException).code !== "ENOENT") { + this.log( + "warn", + `[persist] skipping corrupt file ${rel(file, this.dir)}: ${errText(e)}`, + ); + } + return null; } - - /** Parse + validate one file. Corrupt or invalid records return null - * (logged) instead of throwing — load paths must never block boot. */ - private readEnvelope(file: string): PersistedEnvelope | null { - let parsed: unknown; - try { - const buf = fs.readFileSync(file); - const text = this.codec ? this.codec.decode(buf) : buf.toString("utf8"); - parsed = JSON.parse(text); - } catch (e) { - // A missing candidate path is an expected miss (loadSync probes); - // only genuinely unreadable/corrupt files get logged. - if ((e as NodeJS.ErrnoException).code !== "ENOENT") { - this.log("warn", `[persist] skipping corrupt file ${rel(file, this.dir)}: ${errText(e)}`); - } - return null; - } - if (isEnvelopeLike(parsed)) { - if (!this.validateFn(parsed as PersistedEnvelope)) { - this.log("warn", `[persist] skipping invalid record ${rel(file, this.dir)}`); - return null; - } - return parsed as PersistedEnvelope; - } - const adopted = this.adoptLegacy(parsed); - if (adopted) return adopted; - this.log("warn", `[persist] skipping invalid record ${rel(file, this.dir)}`); + if (isEnvelopeLike(parsed)) { + if (!this.validateFn(parsed as PersistedEnvelope)) { + this.log( + "warn", + `[persist] skipping invalid record ${rel(file, this.dir)}`, + ); return null; + } + return parsed as PersistedEnvelope; } - - /** Wrap a legacy (pre-envelope) record as an envelope via the `legacy` - * hook, then validate the adopted payload like any other. */ - private adoptLegacy(parsed: unknown): PersistedEnvelope | null { - const adoption = this.legacyFn ? this.legacyFn(parsed) : null; - if (!adoption || typeof adoption.id !== "string" || adoption.id.length === 0 || adoption.payload == null) { - return null; - } - const source = parsed as Partial>; - const envelope: PersistedEnvelope = { - version: adoption.version ?? source.version ?? this.version, - savedAt: adoption.savedAt ?? source.savedAt ?? 0, - id: adoption.id, - payload: adoption.payload, - }; - if (!this.validateFn(envelope)) return null; - return envelope; + const adopted = this.adoptLegacy(parsed); + if (adopted) return adopted; + this.log( + "warn", + `[persist] skipping invalid record ${rel(file, this.dir)}`, + ); + return null; + } + + /** Wrap a legacy (pre-envelope) record as an envelope via the `legacy` + * hook, then validate the adopted payload like any other. */ + private adoptLegacy(parsed: unknown): PersistedEnvelope | null { + const adoption = this.legacyFn ? this.legacyFn(parsed) : null; + if ( + !adoption || + typeof adoption.id !== "string" || + adoption.id.length === 0 || + adoption.payload == null + ) { + return null; } - - /** Iterative recursive walk (no readdir-recursive dependency), skipping - * `.tmp-*` names and non-.json files. */ - private async walkJsonFiles(root: string): Promise { - const out: string[] = []; - const queue: string[] = [root]; - while (queue.length > 0) { - const dir = queue.pop()!; - let dirents: fs.Dirent[]; - try { - dirents = await fsp.readdir(dir, { withFileTypes: true }); - } catch { - continue; // missing dir → no records yet - } - for (const d of dirents) { - if (d.name.startsWith(".tmp-")) continue; - const full = path.join(dir, d.name); - if (d.isDirectory()) queue.push(full); - else if (d.isFile() && d.name.endsWith(".json")) out.push(full); - } - } - return out; + const source = parsed as Partial>; + const envelope: PersistedEnvelope = { + version: adoption.version ?? source.version ?? this.version, + savedAt: adoption.savedAt ?? source.savedAt ?? 0, + id: adoption.id, + payload: adoption.payload, + }; + if (!this.validateFn(envelope)) return null; + return envelope; + } + + /** Iterative recursive walk (no readdir-recursive dependency), skipping + * `.tmp-*` names and non-.json files. */ + private async walkJsonFiles(root: string): Promise { + const out: string[] = []; + const queue: string[] = [root]; + while (queue.length > 0) { + const dir = queue.pop()!; + let dirents: fs.Dirent[]; + try { + dirents = await fsp.readdir(dir, { withFileTypes: true }); + } catch { + continue; // missing dir → no records yet + } + for (const d of dirents) { + if (d.name.startsWith(".tmp-")) continue; + const full = path.join(dir, d.name); + if (d.isDirectory()) queue.push(full); + else if (d.isFile() && d.name.endsWith(".json")) out.push(full); + } } + return out; + } } function defaultValidate(envelope: PersistedEnvelope): boolean { - return typeof envelope.id === "string" && envelope.id.length > 0 && envelope.payload != null; + return ( + typeof envelope.id === "string" && + envelope.id.length > 0 && + envelope.payload != null + ); } function isEnvelopeLike(value: unknown): value is PersistedEnvelope { - if (!value || typeof value !== "object") return false; - const v = value as Partial>; - return typeof v.id === "string" && "payload" in v; + if (!value || typeof value !== "object") return false; + const v = value as Partial>; + return typeof v.id === "string" && "payload" in v; } /** Deterministic flat filename: `.json`. Truncated hash — * 96 bits keeps collisions unreachable for realistic id counts, and short * names stay greppable. */ export function flatFileNameFor(id: string): string { - return createHash("sha256").update(id, "utf8").digest("hex").slice(0, 24) + ".json"; + return ( + createHash("sha256").update(id, "utf8").digest("hex").slice(0, 24) + ".json" + ); } /** Transient fs failure codes worth retrying the whole write cycle for: @@ -687,16 +750,21 @@ export function flatFileNameFor(id: string): string { * on CI Windows runners), or a delete-pending tombstone from the same * sweep. Recreating the dir + rewriting a fresh temp heals it. */ function isTransientFsError(e: unknown): boolean { - const code = (e as NodeJS.ErrnoException).code; - return code === "EPERM" || code === "EBUSY" || code === "EACCES" || code === "ENOENT" || code === "ENOTDIR"; + const code = (e as NodeJS.ErrnoException).code; + return ( + code === "EPERM" || + code === "EBUSY" || + code === "EACCES" || + code === "ENOENT" || + code === "ENOTDIR" + ); } function errText(e: unknown): string { - return e instanceof Error ? e.message : String(e); + return e instanceof Error ? e.message : String(e); } - function rel(p: string, base: string): string { - const r = path.relative(base, p); - return r && !r.startsWith("..") ? r : p; + const r = path.relative(base, p); + return r && !r.startsWith("..") ? r : p; } diff --git a/src/protected.ts b/src/protected.ts index 06dfda9..f0e9acd 100644 --- a/src/protected.ts +++ b/src/protected.ts @@ -59,11 +59,15 @@ export function isNeverPreserveRecent( } if (!msg.toolName) return false; const base = - patterns === undefined ? (NEVER_PRESERVE_RECENT_TOOLS as readonly string[]) : patterns; + patterns === undefined + ? (NEVER_PRESERVE_RECENT_TOOLS as readonly string[]) + : patterns; const list = preservePatterns === undefined || preservePatterns.length === 0 ? base - : base.filter((tool) => !preservePatterns.some((p) => matchToolPattern(tool, p))); + : base.filter( + (tool) => !preservePatterns.some((p) => matchToolPattern(tool, p)), + ); for (const pattern of list) { if (matchToolPattern(msg.toolName, pattern)) return true; } @@ -114,7 +118,11 @@ export function collectProtectedToolCallIds( ): Set { const ids = new Set(); for (const m of messages) { - if (m.contentType === "tool-call" && m.toolCallId && isMessageProtected(m, config)) { + if ( + m.contentType === "tool-call" && + m.toolCallId && + isMessageProtected(m, config) + ) { ids.add(m.toolCallId); } } @@ -214,9 +222,13 @@ interface MediaSidecar { export function hasMediaPayload(msg: CoreMessage): boolean { const m = msg as CoreMessage & MediaSidecar; - if (typeof m.imageBase64 === "string" && m.imageBase64.length > 0) return true; + if (typeof m.imageBase64 === "string" && m.imageBase64.length > 0) + return true; if (m.rawOpenaiContent != null) return true; - if (Array.isArray(m.rawOpenaiContentParts) && m.rawOpenaiContentParts.length > 0) + if ( + Array.isArray(m.rawOpenaiContentParts) && + m.rawOpenaiContentParts.length > 0 + ) return true; const ab = m.rawAnthropicBlock; if (isObjWith(ab, "type", "image")) return true; @@ -225,7 +237,8 @@ export function hasMediaPayload(msg: CoreMessage): boolean { // bytes outside msg.text. if (isObjWith(ab, "type", "tool_result")) { const content = (ab as { content?: unknown }).content; - if (Array.isArray(content)) return content.some((p) => !isObjWith(p, "type", "text")); + if (Array.isArray(content)) + return content.some((p) => !isObjWith(p, "type", "text")); } const item = m.rawResponsesItem; if (isObjWith(item, "type", "input_image")) return true; @@ -239,5 +252,9 @@ export function hasMediaPayload(msg: CoreMessage): boolean { } function isObjWith(v: unknown, key: string, value: unknown): boolean { - return typeof v === "object" && v !== null && (v as Record)[key] === value; + return ( + typeof v === "object" && + v !== null && + (v as Record)[key] === value + ); } diff --git a/src/rebuild.ts b/src/rebuild.ts index e8fd205..e1c88ca 100644 --- a/src/rebuild.ts +++ b/src/rebuild.ts @@ -6,12 +6,12 @@ import { defaultCountTokens } from "./tokenize.js"; import type { CompressionState, CoreMessage } from "./types.js"; export interface RebuildResult { - state: CompressionState; - blocksRebuilt: number; + state: CompressionState; + blocksRebuilt: number; } export interface RebuildPorts { - countTokens?: (text: string) => number; + countTokens?: (text: string) => number; } /** @@ -25,43 +25,55 @@ export interface RebuildPorts { * inputs are salvaged instead of silently dropped. */ export function rebuildCompressionState( - state: CompressionState, - messages: CoreMessage[], - config: import("./types.js").Config, - ports: RebuildPorts = {}, + state: CompressionState, + messages: CoreMessage[], + config: import("./types.js").Config, + ports: RebuildPorts = {}, ): RebuildResult { - const core = createCore({ countTokens: ports.countTokens ?? defaultCountTokens }); - const refResult = assignRefs(messages, { - existing: state.messageRefs, - nextIndex: highestUsedIndex(state.messageRefs) + 1, - shouldSkip: (m) => m.id.startsWith(RETRIEVED_ID_PREFIX), - }); - let working: CompressionState = { ...state, messageRefs: refResult.map }; + const core = createCore({ + countTokens: ports.countTokens ?? defaultCountTokens, + }); + const refResult = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: highestUsedIndex(state.messageRefs) + 1, + shouldSkip: (m) => m.id.startsWith(RETRIEVED_ID_PREFIX), + }); + let working: CompressionState = { ...state, messageRefs: refResult.map }; - const invocations = collectCompressInvocations(messages); - let blocksRebuilt = 0; + const invocations = collectCompressInvocations(messages); + let blocksRebuilt = 0; - for (const invocation of invocations) { - const { ranges } = parseCompressArgs(invocation.raw, { callId: invocation.callId }); - if (ranges.length === 0) continue; - const result = core.applyCompression({ ranges, messages, state: working, config }); - working = result.state; - blocksRebuilt += result.result.blocksCreated; - } + for (const invocation of invocations) { + const { ranges } = parseCompressArgs(invocation.raw, { + callId: invocation.callId, + }); + if (ranges.length === 0) continue; + const result = core.applyCompression({ + ranges, + messages, + state: working, + config, + }); + working = result.state; + blocksRebuilt += result.result.blocksCreated; + } - return { state: working, blocksRebuilt }; + return { state: working, blocksRebuilt }; } interface CompressInvocation { - callId: string | undefined; - raw: string; + callId: string | undefined; + raw: string; } -function collectCompressInvocations(messages: CoreMessage[]): CompressInvocation[] { - const invocations: CompressInvocation[] = []; - for (const message of messages) { - if (message.toolName !== "compress" || message.contentType !== "tool-call") continue; - invocations.push({ callId: message.toolCallId, raw: message.text ?? "" }); - } - return invocations; +function collectCompressInvocations( + messages: CoreMessage[], +): CompressInvocation[] { + const invocations: CompressInvocation[] = []; + for (const message of messages) { + if (message.toolName !== "compress" || message.contentType !== "tool-call") + continue; + invocations.push({ callId: message.toolCallId, raw: message.text ?? "" }); + } + return invocations; } diff --git a/src/recommend.ts b/src/recommend.ts index e439f32..0ae3934 100644 --- a/src/recommend.ts +++ b/src/recommend.ts @@ -44,9 +44,11 @@ function isSyntheticOrPruned( message: CoreMessage, state: CompressionState, ): boolean { - if (message.text?.startsWith("[Compressed conversation section]")) return true; + if (message.text?.startsWith("[Compressed conversation section]")) + return true; for (const block of state.blocks) { - if (block.active && block.effectiveMessageIds.includes(message.id)) return true; + if (block.active && block.effectiveMessageIds.includes(message.id)) + return true; } return false; } @@ -83,7 +85,14 @@ export function computeProtectedRefs( // counting them toward the last-N window would make them un-compressible // and hide them from recommendations. The message stays fully visible — // this only affects protection scope. - if (isNeverPreserveRecent(msg, config.neverPreserveRecentTools, config.preserveRecentTools)) continue; + if ( + isNeverPreserveRecent( + msg, + config.neverPreserveRecentTools, + config.preserveRecentTools, + ) + ) + continue; const ref = state.messageRefs.byRaw[msg.id]; if (!ref || ref === "BLOCKED") continue; visible.push({ ref, tokens: countMessageTokens(msg, countTokens) }); @@ -99,7 +108,11 @@ export function computeProtectedRefs( // Rule 2: last N tokens (expand backward from tail) if (preserveTokens > 0) { let tokenAccum = 0; - for (let i = visible.length - 1; i >= 0 && tokenAccum < preserveTokens; i--) { + for ( + let i = visible.length - 1; + i >= 0 && tokenAccum < preserveTokens; + i-- + ) { result.add(visible[i]!.ref); tokenAccum += visible[i]!.tokens; } @@ -300,7 +313,9 @@ export function buildCompressibleRanges( cur.chars = (cur.chars ?? 0) + info.chars; if (info.isUser) cur.userMsgs = (cur.userMsgs ?? 0) + 1; if (info.isTool) { - cur.toolPct = Math.round((cur.toolPct * (cur.count - 1) + 100) / cur.count); + cur.toolPct = Math.round( + (cur.toolPct * (cur.count - 1) + 100) / cur.count, + ); } else { cur.toolPct = Math.round((cur.toolPct * (cur.count - 1)) / cur.count); } diff --git a/src/render-refs.ts b/src/render-refs.ts index a7d57ef..7338c46 100644 --- a/src/render-refs.ts +++ b/src/render-refs.ts @@ -43,7 +43,17 @@ const TAG_OPEN = LT + "acp "; const TAG_CLOSE = LT + "/acp" + GT; function acpTag(ref: string, tokens: number, type: string): string { - return TAG_OPEN + 'tokens="' + formatTokens(tokens) + '" type="' + type + '"' + GT + ref + TAG_CLOSE; + return ( + TAG_OPEN + + 'tokens="' + + formatTokens(tokens) + + '" type="' + + type + + '"' + + GT + + ref + + TAG_CLOSE + ); } function renderMessage( @@ -67,7 +77,13 @@ function renderMessage( // Strip own stale tag BEFORE computing tokens (idempotency). // Match the message's own ref only — foreign tags survive (content-corruption fix). const ownTagRe = new RegExp( - "^" + escapeRegex(TAG_OPEN) + "[^>]*" + GT + escapeRegex(ref) + escapeRegex(TAG_CLOSE) + "\\n?", + "^" + + escapeRegex(TAG_OPEN) + + "[^>]*" + + GT + + escapeRegex(ref) + + escapeRegex(TAG_CLOSE) + + "\\n?", ); const cleanText = (message.text || "").replace(ownTagRe, ""); @@ -89,8 +105,7 @@ function renderMessage( export function renderVisibleRefs( messages: CoreMessage[], state: CompressionState, - countTokens: (text: string) => number = (text) => - Math.ceil(text.length / 4), + countTokens: (text: string) => number = (text) => Math.ceil(text.length / 4), strategy: RenderStrategy = "all", ): CoreMessage[] { // Legacy behavior: recompute tokens every render (snapshot = null). diff --git a/src/report.ts b/src/report.ts index 355a62b..99c50f5 100644 --- a/src/report.ts +++ b/src/report.ts @@ -2,422 +2,503 @@ import { clampPrefix } from "./truncate.js"; import { isToolMessage } from "./message-kind.js"; import { refForRaw } from "./refs.js"; import { countMessageTokens } from "./tokenize.js"; -import type { CompressionBlock, CompressionState, CoreMessage } from "./types.js"; +import type { + CompressionBlock, + CompressionState, + CoreMessage, +} from "./types.js"; function formatTokens(n: number): string { - if (!Number.isFinite(n) || n <= 0) return "0"; - return n >= 1000 ? `${(n / 1000).toFixed(1)}K` : String(n); + if (!Number.isFinite(n) || n <= 0) return "0"; + return n >= 1000 ? `${(n / 1000).toFixed(1)}K` : String(n); } function pct(n: number, total: number): number { - if (n <= 0 || total <= 0) return 0; - return Math.round((n / total) * 100); + if (n <= 0 || total <= 0) return 0; + return Math.round((n / total) * 100); } function numericPart(blockId: string): number { - const match = /^b(\d+)$/.exec(blockId); - return match && match[1] !== undefined ? Number(match[1]) : 0; + const match = /^b(\d+)$/.exec(blockId); + return match && match[1] !== undefined ? Number(match[1]) : 0; } -function summaryTokensOf(block: CompressionBlock, countTokens: (t: string) => number): number { - return countTokens(block.summary); +function summaryTokensOf( + block: CompressionBlock, + countTokens: (t: string) => number, +): number { + return countTokens(block.summary); } function effectiveCompressedTokens( - block: CompressionBlock, - _state: CompressionState, - _countTokens: (t: string) => number, + block: CompressionBlock, + _state: CompressionState, + _countTokens: (t: string) => number, ): number { - // block.compressedTokens already records the full input token count of the - // operation that created this block: for a tier-1 block that is the raw - // messages; for a tier-2 block it is the tier-1 summaries + the new - // messages it spans. Recursing into directBlockIds and summing children's - // compressedTokens double-counts the consumed children, so we return the - // block's own value directly. (The previous recursion inflated tier-2+ - // "original" figures and mis-ordered the status report.) - return block.compressedTokens; + // block.compressedTokens already records the full input token count of the + // operation that created this block: for a tier-1 block that is the raw + // messages; for a tier-2 block it is the tier-1 summaries + the new + // messages it spans. Recursing into directBlockIds and summing children's + // compressedTokens double-counts the consumed children, so we return the + // block's own value directly. (The previous recursion inflated tier-2+ + // "original" figures and mis-ordered the status report.) + return block.compressedTokens; } function tierLabel(block: CompressionBlock): string { - return `T${block.tier}`; + return `T${block.tier}`; } function tierBreakdown( - blocks: CompressionBlock[], - countTokens: (t: string) => number, + blocks: CompressionBlock[], + countTokens: (t: string) => number, ): string | null { - const tierTokens: Record = {}; - const tierCounts: Record = {}; - for (const block of blocks) { - tierTokens[block.tier] = (tierTokens[block.tier] ?? 0) + summaryTokensOf(block, countTokens); - tierCounts[block.tier] = (tierCounts[block.tier] ?? 0) + 1; - } - const tiers = Object.keys(tierTokens).map(Number); - if (tiers.length <= 1) return null; - // Counts, not just tokens: the tier-distillation nudge lists every active - // lower-tier block uncapped while the block list below may be truncated — - // the count is what lets the model reconcile the two views (#221). - const parts: string[] = []; - for (const tier of [1, 2, 3]) { - if (tierTokens[tier]) parts.push(`T${tier}: ${formatTokens(tierTokens[tier])} (${tierCounts[tier]} blocks)`); - } - return parts.join(" | "); + const tierTokens: Record = {}; + const tierCounts: Record = {}; + for (const block of blocks) { + tierTokens[block.tier] = + (tierTokens[block.tier] ?? 0) + summaryTokensOf(block, countTokens); + tierCounts[block.tier] = (tierCounts[block.tier] ?? 0) + 1; + } + const tiers = Object.keys(tierTokens).map(Number); + if (tiers.length <= 1) return null; + // Counts, not just tokens: the tier-distillation nudge lists every active + // lower-tier block uncapped while the block list below may be truncated — + // the count is what lets the model reconcile the two views (#221). + const parts: string[] = []; + for (const tier of [1, 2, 3]) { + if (tierTokens[tier]) + parts.push( + `T${tier}: ${formatTokens(tierTokens[tier])} (${tierCounts[tier]} blocks)`, + ); + } + return parts.join(" | "); } interface VisibleMessageInfo { - ref: string; - tokens: number; - tool: string; - isTool: boolean; - index: number; + ref: string; + tokens: number; + tool: string; + isTool: boolean; + index: number; } function collectVisible( - messages: CoreMessage[], - state: CompressionState, - countTokens: (t: string) => number, + messages: CoreMessage[], + state: CompressionState, + countTokens: (t: string) => number, ): { visible: VisibleMessageInfo[]; summaryTokens: number } { - const coveredIds = new Set(); - for (const block of state.blocks) { - if (!block.active) continue; - for (const id of block.effectiveMessageIds) coveredIds.add(id); + const coveredIds = new Set(); + for (const block of state.blocks) { + if (!block.active) continue; + for (const id of block.effectiveMessageIds) coveredIds.add(id); + } + let summaryTokens = 0; + for (const block of state.blocks) { + if (block.active) summaryTokens += summaryTokensOf(block, countTokens); + } + const visible: VisibleMessageInfo[] = []; + // Tool RESULTS carry no toolName of their own — resolve them through + // their call so the tool bucket attributes result payload (#386: with + // `toolName ?? "text"` every tool-result landed in the text bucket and + // the tool bucket showed ~0.6% of the real volume). + const toolCallNames = new Map(); + for (const message of messages) { + if ( + message.contentType === "tool-call" && + message.toolCallId && + message.toolName + ) { + toolCallNames.set(message.toolCallId, message.toolName); } - let summaryTokens = 0; - for (const block of state.blocks) { - if (block.active) summaryTokens += summaryTokensOf(block, countTokens); - } - const visible: VisibleMessageInfo[] = []; - // Tool RESULTS carry no toolName of their own — resolve them through - // their call so the tool bucket attributes result payload (#386: with - // `toolName ?? "text"` every tool-result landed in the text bucket and - // the tool bucket showed ~0.6% of the real volume). - const toolCallNames = new Map(); - for (const message of messages) { - if (message.contentType === "tool-call" && message.toolCallId && message.toolName) { - toolCallNames.set(message.toolCallId, message.toolName); - } - } - messages.forEach((message, index) => { - if (coveredIds.has(message.id)) return; - const ref = refForRaw(state.messageRefs, message.id); - if (!ref) return; - const tokens = countMessageTokens(message, countTokens); - const isTool = isToolMessage(message); - const tool = isTool - ? (message.toolName ?? (message.toolCallId ? toolCallNames.get(message.toolCallId) : undefined) ?? "tool") - : "text"; - if (tokens > 0) visible.push({ ref, tokens, tool, isTool, index }); - }); - return { visible, summaryTokens }; + } + messages.forEach((message, index) => { + if (coveredIds.has(message.id)) return; + const ref = refForRaw(state.messageRefs, message.id); + if (!ref) return; + const tokens = countMessageTokens(message, countTokens); + const isTool = isToolMessage(message); + const tool = isTool + ? (message.toolName ?? + (message.toolCallId + ? toolCallNames.get(message.toolCallId) + : undefined) ?? + "tool") + : "text"; + if (tokens > 0) visible.push({ ref, tokens, tool, isTool, index }); + }); + return { visible, summaryTokens }; } /** Host-declared surface facts for status reports. The kernel never resolves * packs — which pack is active is host policy (billion-context #730); the * kernel only renders what the host declares about the active surface. */ export interface StatusReportMeta { - pack?: string; - packVersion?: string; - host?: string; + pack?: string; + packVersion?: string; + host?: string; } export interface StatusReportOptions { - scope?: "compressed" | "uncompressed"; - view?: "ranges" | "messages"; - tool?: string; - sort?: "size" | "time" | "tool" | "age"; - limit?: number; - meta?: StatusReportMeta; + scope?: "compressed" | "uncompressed"; + view?: "ranges" | "messages"; + tool?: string; + sort?: "size" | "time" | "tool" | "age"; + limit?: number; + meta?: StatusReportMeta; } export function buildStatusReport( - state: CompressionState, - messages: CoreMessage[], - countTokens: (t: string) => number, - options: StatusReportOptions = {}, + state: CompressionState, + messages: CoreMessage[], + countTokens: (t: string) => number, + options: StatusReportOptions = {}, ): string { - const scope = options.scope; - const view = options.view ?? "ranges"; - const toolFilter = options.tool; - const sort = options.sort ?? "size"; - const limit = options.limit ?? 30; - - const activeBlocks = state.blocks - .filter((b) => b.active) - .sort((a, b) => numericPart(a.blockId) - numericPart(b.blockId)); - - if (scope === "compressed") { - return renderCompressedDrilldown(activeBlocks, state, sort, limit, countTokens, options.meta); - } + const scope = options.scope; + const view = options.view ?? "ranges"; + const toolFilter = options.tool; + const sort = options.sort ?? "size"; + const limit = options.limit ?? 30; + + const activeBlocks = state.blocks + .filter((b) => b.active) + .sort((a, b) => numericPart(a.blockId) - numericPart(b.blockId)); + + if (scope === "compressed") { + return renderCompressedDrilldown( + activeBlocks, + state, + sort, + limit, + countTokens, + options.meta, + ); + } - const { visible, summaryTokens } = collectVisible(messages, state, countTokens); + const { visible, summaryTokens } = collectVisible( + messages, + state, + countTokens, + ); - if (scope === "uncompressed") { - if (view === "messages") { - return renderMessageDrilldown(visible, toolFilter, sort, limit); - } - return renderUncompressedRanges(visible); + if (scope === "uncompressed") { + if (view === "messages") { + return renderMessageDrilldown(visible, toolFilter, sort, limit); } - - return renderOverview(visible, summaryTokens, activeBlocks, state, countTokens, limit, options.meta); + return renderUncompressedRanges(visible); + } + + return renderOverview( + visible, + summaryTokens, + activeBlocks, + state, + countTokens, + limit, + options.meta, + ); } function surfaceLine(meta: StatusReportMeta | undefined): string | null { - if (!meta) return null; - const parts: string[] = []; - if (meta.pack) parts.push(`pack=${meta.pack}${meta.packVersion ? ` v${meta.packVersion}` : ""}`); - if (meta.host) parts.push(`host=${meta.host}`); - if (parts.length === 0) return null; - return `ACTIVE SURFACE: ${parts.join(" | ")}`; + if (!meta) return null; + const parts: string[] = []; + if (meta.pack) + parts.push( + `pack=${meta.pack}${meta.packVersion ? ` v${meta.packVersion}` : ""}`, + ); + if (meta.host) parts.push(`host=${meta.host}`); + if (parts.length === 0) return null; + return `ACTIVE SURFACE: ${parts.join(" | ")}`; } function renderOverview( - visible: VisibleMessageInfo[], - summaryTokens: number, - blocks: CompressionBlock[], - state: CompressionState, - countTokens: (t: string) => number, - limit: number, - meta: StatusReportMeta | undefined, + visible: VisibleMessageInfo[], + summaryTokens: number, + blocks: CompressionBlock[], + state: CompressionState, + countTokens: (t: string) => number, + limit: number, + meta: StatusReportMeta | undefined, ): string { - const lines: string[] = []; - const surface = surfaceLine(meta); - if (surface) { - lines.push(surface); - lines.push(""); - } - const toolTypeMap = new Map(); - for (const message of visible) { - toolTypeMap.set(message.tool, (toolTypeMap.get(message.tool) ?? 0) + message.tokens); - } - const topTool = [...toolTypeMap.entries()].sort((a, b) => b[1] - a[1])[0]?.[0]; - - const totalTool = visible - .filter((m) => m.isTool) - .reduce((sum, m) => sum + m.tokens, 0); - const totalText = visible - .filter((m) => !m.isTool) - .reduce((sum, m) => sum + m.tokens, 0); - const total = summaryTokens + totalTool + totalText; - - lines.push("CONTEXT BREAKDOWN"); + const lines: string[] = []; + const surface = surfaceLine(meta); + if (surface) { + lines.push(surface); + lines.push(""); + } + const toolTypeMap = new Map(); + for (const message of visible) { + toolTypeMap.set( + message.tool, + (toolTypeMap.get(message.tool) ?? 0) + message.tokens, + ); + } + const topTool = [...toolTypeMap.entries()].sort( + (a, b) => b[1] - a[1], + )[0]?.[0]; + + const totalTool = visible + .filter((m) => m.isTool) + .reduce((sum, m) => sum + m.tokens, 0); + const totalText = visible + .filter((m) => !m.isTool) + .reduce((sum, m) => sum + m.tokens, 0); + const total = summaryTokens + totalTool + totalText; + + lines.push("CONTEXT BREAKDOWN"); + lines.push( + ` ${formatTokens(totalTool)} tool (${pct(totalTool, total)}%) | ${formatTokens(totalText)} text (${pct(totalText, total)}%) | ${formatTokens(summaryTokens)} summaries (${pct(summaryTokens, total)}%)`, + ); + const topTypes = [...toolTypeMap.entries()] + .sort((a, b) => b[1] - a[1]) + .slice(0, 3); + if (topTypes.length > 0) { lines.push( - ` ${formatTokens(totalTool)} tool (${pct(totalTool, total)}%) | ${formatTokens(totalText)} text (${pct(totalText, total)}%) | ${formatTokens(summaryTokens)} summaries (${pct(summaryTokens, total)}%)`, + ` Top tools: ${topTypes.map(([t, n]) => `${t} (${pct(n, total)}%)`).join(", ")}`, ); - const topTypes = [...toolTypeMap.entries()] - .sort((a, b) => b[1] - a[1]) - .slice(0, 3); - if (topTypes.length > 0) { - lines.push(` Top tools: ${topTypes.map(([t, n]) => `${t} (${pct(n, total)}%)`).join(", ")}`); - } - + } + + lines.push(""); + if (blocks.length === 0) { + lines.push("COMPRESSED BLOCKS"); + lines.push(" No compressed blocks."); + } else { + const totalSummary = blocks.reduce( + (s, b) => s + summaryTokensOf(b, countTokens), + 0, + ); + const totalEffective = blocks.reduce( + (s, b) => s + effectiveCompressedTokens(b, state, countTokens), + 0, + ); + lines.push( + `COMPRESSED BLOCKS — ${blocks.length} active (${formatTokens(totalSummary)} summary, ${formatTokens(totalEffective)} original)`, + ); + const breakdown = tierBreakdown(blocks, countTokens); + if (breakdown) lines.push(` Tier usage: ${breakdown}`); lines.push(""); - if (blocks.length === 0) { - lines.push("COMPRESSED BLOCKS"); - lines.push(" No compressed blocks."); - } else { - const totalSummary = blocks.reduce((s, b) => s + summaryTokensOf(b, countTokens), 0); - const totalEffective = blocks.reduce( - (s, b) => s + effectiveCompressedTokens(b, state, countTokens), - 0, - ); - lines.push( - `COMPRESSED BLOCKS — ${blocks.length} active (${formatTokens(totalSummary)} summary, ${formatTokens(totalEffective)} original)`, - ); - const breakdown = tierBreakdown(blocks, countTokens); - if (breakdown) lines.push(` Tier usage: ${breakdown}`); - lines.push(""); - const sorted = [...blocks].sort( - (a, b) => - effectiveCompressedTokens(b, state, countTokens) - - effectiveCompressedTokens(a, state, countTokens) || - b.createdAt - a.createdAt, - ); - for (const block of sorted.slice(0, limit)) { - const topic = block.topic ?? "(no topic)"; - const eff = effectiveCompressedTokens(block, state, countTokens); - lines.push( - ` ${block.blockId} (${tierLabel(block)}) ${formatTokens(eff)}→${formatTokens(summaryTokensOf(block, countTokens))} ${block.effectiveMessageIds.length} msgs "${topic}"`, - ); - } - if (blocks.length > limit) { - lines.push( - ` ... and ${blocks.length - limit} more blocks not shown (scope:"compressed", limit:${blocks.length} for full list)`, - ); - } + const sorted = [...blocks].sort( + (a, b) => + effectiveCompressedTokens(b, state, countTokens) - + effectiveCompressedTokens(a, state, countTokens) || + b.createdAt - a.createdAt, + ); + for (const block of sorted.slice(0, limit)) { + const topic = block.topic ?? "(no topic)"; + const eff = effectiveCompressedTokens(block, state, countTokens); + lines.push( + ` ${block.blockId} (${tierLabel(block)}) ${formatTokens(eff)}→${formatTokens(summaryTokensOf(block, countTokens))} ${block.effectiveMessageIds.length} msgs "${topic}"`, + ); } + if (blocks.length > limit) { + lines.push( + ` ... and ${blocks.length - limit} more blocks not shown (scope:"compressed", limit:${blocks.length} for full list)`, + ); + } + } - lines.push(""); - lines.push( - `Tip: buildStatusReport({scope:"uncompressed", view:"messages", tool:"${topTool ?? "bash"}"}) for per-message listing`, - ); - return lines.join("\n"); + lines.push(""); + lines.push( + `Tip: buildStatusReport({scope:"uncompressed", view:"messages", tool:"${topTool ?? "bash"}"}) for per-message listing`, + ); + return lines.join("\n"); } function renderUncompressedRanges(visible: VisibleMessageInfo[]): string { - const lines: string[] = []; - const totalTokens = visible.reduce((s, m) => s + m.tokens, 0); - lines.push(`UNCOMPRESSED — ${formatTokens(totalTokens)} | ${visible.length} visible messages`); - lines.push(""); - if (visible.length === 0) { - lines.push(" (no uncompressed messages)"); - return lines.join("\n"); - } - // Merge consecutive messages into ranges (by numeric ref), aggregating - // token counts and dominant tool so the view reads as blocks, not a - // per-message firehose — mirroring the Compressible Ranges output. - interface Merged { startRef: string; endRef: string; startNum: number; count: number; tokens: number; toolTokens: Map; } - const refNum = (ref: string): number => { - const m = ref.match(/\d+/); - return m ? parseInt(m[0], 10) : 0; - }; - const dominantTool = (toolTokens: Map): string => { - let best = "text"; - let bestN = -1; - for (const [tool, n] of toolTokens) { - if (n > bestN) { - best = tool; - bestN = n; - } - } - return best; - }; - const merged: Merged[] = []; - for (const m of visible) { - const num = refNum(m.ref); - const last = merged[merged.length - 1]; - if (last && num === last.startNum + last.count) { - last.endRef = m.ref; - last.count += 1; - last.tokens += m.tokens; - last.toolTokens.set(m.tool, (last.toolTokens.get(m.tool) ?? 0) + m.tokens); - } else { - const toolTokens = new Map([[m.tool, m.tokens]]); - merged.push({ startRef: m.ref, endRef: m.ref, startNum: num, count: 1, tokens: m.tokens, toolTokens }); - } - } - for (const r of merged.slice(0, 30)) { - const range = r.count === 1 ? r.startRef : `${r.startRef}–${r.endRef}`; - lines.push(` ${range} (${r.count} msgs, ${formatTokens(r.tokens)}${r.count > 1 ? ` (${Math.round(r.tokens / r.count)}/msg)` : ""}) ${dominantTool(r.toolTokens)}`); + const lines: string[] = []; + const totalTokens = visible.reduce((s, m) => s + m.tokens, 0); + lines.push( + `UNCOMPRESSED — ${formatTokens(totalTokens)} | ${visible.length} visible messages`, + ); + lines.push(""); + if (visible.length === 0) { + lines.push(" (no uncompressed messages)"); + return lines.join("\n"); + } + // Merge consecutive messages into ranges (by numeric ref), aggregating + // token counts and dominant tool so the view reads as blocks, not a + // per-message firehose — mirroring the Compressible Ranges output. + interface Merged { + startRef: string; + endRef: string; + startNum: number; + count: number; + tokens: number; + toolTokens: Map; + } + const refNum = (ref: string): number => { + const m = ref.match(/\d+/); + return m ? parseInt(m[0], 10) : 0; + }; + const dominantTool = (toolTokens: Map): string => { + let best = "text"; + let bestN = -1; + for (const [tool, n] of toolTokens) { + if (n > bestN) { + best = tool; + bestN = n; + } } - if (merged.length > 30) { - lines.push(` ... and ${merged.length - 30} more ranges`); + return best; + }; + const merged: Merged[] = []; + for (const m of visible) { + const num = refNum(m.ref); + const last = merged[merged.length - 1]; + if (last && num === last.startNum + last.count) { + last.endRef = m.ref; + last.count += 1; + last.tokens += m.tokens; + last.toolTokens.set( + m.tool, + (last.toolTokens.get(m.tool) ?? 0) + m.tokens, + ); + } else { + const toolTokens = new Map([[m.tool, m.tokens]]); + merged.push({ + startRef: m.ref, + endRef: m.ref, + startNum: num, + count: 1, + tokens: m.tokens, + toolTokens, + }); } - return lines.join("\n"); + } + for (const r of merged.slice(0, 30)) { + const range = r.count === 1 ? r.startRef : `${r.startRef}–${r.endRef}`; + lines.push( + ` ${range} (${r.count} msgs, ${formatTokens(r.tokens)}${r.count > 1 ? ` (${Math.round(r.tokens / r.count)}/msg)` : ""}) ${dominantTool(r.toolTokens)}`, + ); + } + if (merged.length > 30) { + lines.push(` ... and ${merged.length - 30} more ranges`); + } + return lines.join("\n"); } function renderMessageDrilldown( - visible: VisibleMessageInfo[], - toolFilter: string | undefined, - sort: string, - limit: number, + visible: VisibleMessageInfo[], + toolFilter: string | undefined, + sort: string, + limit: number, ): string { - let filtered = visible; - if (toolFilter) filtered = filtered.filter((m) => m.tool === toolFilter); - - if (sort === "time") filtered.sort((a, b) => a.index - b.index); - else if (sort === "tool") filtered.sort((a, b) => a.tool.localeCompare(b.tool) || b.tokens - a.tokens); - else filtered.sort((a, b) => b.tokens - a.tokens); - - const totalTokens = filtered.reduce((s, m) => s + m.tokens, 0); - const allTokens = visible.reduce((s, m) => s + m.tokens, 0); - const header = toolFilter - ? `UNCOMPRESSED — ${toolFilter}: ${formatTokens(totalTokens)} | ${filtered.length} msgs | ${pct(totalTokens, allTokens)}% of visible` - : `UNCOMPRESSED — ${formatTokens(totalTokens)} | ${filtered.length} msgs`; - const lines = [header, `Sorted by ${sort}`, ""]; - const shown = filtered.slice(0, limit); - for (const message of shown) { - lines.push(` ${message.ref} (${formatTokens(message.tokens)}) ${message.tool}`); - } - if (filtered.length > shown.length) { - lines.push(""); - lines.push(`${shown.length} of ${filtered.length} shown.`); - } - return lines.join("\n"); + let filtered = visible; + if (toolFilter) filtered = filtered.filter((m) => m.tool === toolFilter); + + if (sort === "time") filtered.sort((a, b) => a.index - b.index); + else if (sort === "tool") + filtered.sort( + (a, b) => a.tool.localeCompare(b.tool) || b.tokens - a.tokens, + ); + else filtered.sort((a, b) => b.tokens - a.tokens); + + const totalTokens = filtered.reduce((s, m) => s + m.tokens, 0); + const allTokens = visible.reduce((s, m) => s + m.tokens, 0); + const header = toolFilter + ? `UNCOMPRESSED — ${toolFilter}: ${formatTokens(totalTokens)} | ${filtered.length} msgs | ${pct(totalTokens, allTokens)}% of visible` + : `UNCOMPRESSED — ${formatTokens(totalTokens)} | ${filtered.length} msgs`; + const lines = [header, `Sorted by ${sort}`, ""]; + const shown = filtered.slice(0, limit); + for (const message of shown) { + lines.push( + ` ${message.ref} (${formatTokens(message.tokens)}) ${message.tool}`, + ); + } + if (filtered.length > shown.length) { + lines.push(""); + lines.push(`${shown.length} of ${filtered.length} shown.`); + } + return lines.join("\n"); } function renderCompressedDrilldown( - blocks: CompressionBlock[], - state: CompressionState, - sort: string, - limit: number, - countTokens: (t: string) => number, - meta: StatusReportMeta | undefined, + blocks: CompressionBlock[], + state: CompressionState, + sort: string, + limit: number, + countTokens: (t: string) => number, + meta: StatusReportMeta | undefined, ): string { - let sorted = [...blocks]; - if (sort === "time") sorted.sort((a, b) => a.createdAt - b.createdAt); - else if (sort === "age") sorted.sort((a, b) => b.survivedCount - a.survivedCount); - else - sorted.sort( - (a, b) => - effectiveCompressedTokens(b, state, countTokens) - - effectiveCompressedTokens(a, state, countTokens) || - b.createdAt - a.createdAt, - ); - - const totalSummary = sorted.reduce((s, b) => s + summaryTokensOf(b, countTokens), 0); - const totalEffective = sorted.reduce( - (s, b) => s + effectiveCompressedTokens(b, state, countTokens), - 0, + let sorted = [...blocks]; + if (sort === "time") sorted.sort((a, b) => a.createdAt - b.createdAt); + else if (sort === "age") + sorted.sort((a, b) => b.survivedCount - a.survivedCount); + else + sorted.sort( + (a, b) => + effectiveCompressedTokens(b, state, countTokens) - + effectiveCompressedTokens(a, state, countTokens) || + b.createdAt - a.createdAt, ); - const lines: string[] = []; - const surface = surfaceLine(meta); - if (surface) { - lines.push(surface); - lines.push(""); - } + + const totalSummary = sorted.reduce( + (s, b) => s + summaryTokensOf(b, countTokens), + 0, + ); + const totalEffective = sorted.reduce( + (s, b) => s + effectiveCompressedTokens(b, state, countTokens), + 0, + ); + const lines: string[] = []; + const surface = surfaceLine(meta); + if (surface) { + lines.push(surface); + lines.push(""); + } + lines.push( + `COMPRESSED — ${sorted.length} blocks | ${formatTokens(totalEffective)} original → ${formatTokens(totalSummary)} summary`, + ); + const breakdown = tierBreakdown(sorted, countTokens); + if (breakdown) lines.push(`Tier usage: ${breakdown}`); + lines.push(""); + const shown = sorted.slice(0, limit); + for (const block of shown) { + const nested = + block.directBlockIds.length > 0 + ? ` nested=[${block.directBlockIds.join(",")}]` + : ""; + const topic = block.topic ?? "(no topic)"; + const eff = effectiveCompressedTokens(block, state, countTokens); lines.push( - `COMPRESSED — ${sorted.length} blocks | ${formatTokens(totalEffective)} original → ${formatTokens(totalSummary)} summary`, + ` ${block.blockId} (${tierLabel(block)}) ${formatTokens(eff)}→${formatTokens(summaryTokensOf(block, countTokens))} ${block.effectiveMessageIds.length} msgs age=${block.survivedCount} ${block.generation}${nested}`, ); - const breakdown = tierBreakdown(sorted, countTokens); - if (breakdown) lines.push(`Tier usage: ${breakdown}`); + lines.push(` "${topic}"`); + } + if (sorted.length > shown.length) { lines.push(""); - const shown = sorted.slice(0, limit); - for (const block of shown) { - const nested = block.directBlockIds.length > 0 ? ` nested=[${block.directBlockIds.join(",")}]` : ""; - const topic = block.topic ?? "(no topic)"; - const eff = effectiveCompressedTokens(block, state, countTokens); - lines.push( - ` ${block.blockId} (${tierLabel(block)}) ${formatTokens(eff)}→${formatTokens(summaryTokensOf(block, countTokens))} ${block.effectiveMessageIds.length} msgs age=${block.survivedCount} ${block.generation}${nested}`, - ); - lines.push(` "${topic}"`); - } - if (sorted.length > shown.length) { - lines.push(""); - lines.push(`${shown.length} of ${sorted.length} shown.`); - } - return lines.join("\n"); + lines.push(`${shown.length} of ${sorted.length} shown.`); + } + return lines.join("\n"); } -export function buildRecap( - state: CompressionState, - blockId?: string, -): string { - const activeBlocks = state.blocks - .filter((b) => b.active) - .sort((a, b) => numericPart(a.blockId) - numericPart(b.blockId)); - - if (blockId !== undefined) { - const block = state.blocks.find((b) => b.blockId === blockId); - if (!block) { - const activeList = activeBlocks.map((b) => b.blockId).join(", "); - return `Block ${blockId} not found. Active blocks: ${activeList}`; - } - if (!block.active) { - return `Block ${blockId} is inactive (deactivated by nested compression).`; - } - const range = `${block.effectiveMessageIds.length} messages`; - return `[Compressed conversation section]\n${block.summary}\n\n[${blockId} | ${range} | topic: "${block.topic ?? "(none)"}"]`; - } +export function buildRecap(state: CompressionState, blockId?: string): string { + const activeBlocks = state.blocks + .filter((b) => b.active) + .sort((a, b) => numericPart(a.blockId) - numericPart(b.blockId)); - if (activeBlocks.length === 0) return "No active compression blocks."; - - const lines = [`Active compression blocks (${activeBlocks.length}):`]; - for (const block of activeBlocks) { - const range = `${block.effectiveMessageIds.length} messages`; - const preview = clampPrefix(block.summary, 200); - lines.push(`\n${block.blockId} | ${range} | "${block.topic ?? "(none)"}"`); - lines.push(` ${preview}${block.summary.length > 200 ? "..." : ""}`); + if (blockId !== undefined) { + const block = state.blocks.find((b) => b.blockId === blockId); + if (!block) { + const activeList = activeBlocks.map((b) => b.blockId).join(", "); + return `Block ${blockId} not found. Active blocks: ${activeList}`; } - lines.push(`\nCall with blockId to get the full summary.`); - return lines.join("\n"); + if (!block.active) { + return `Block ${blockId} is inactive (deactivated by nested compression).`; + } + const range = `${block.effectiveMessageIds.length} messages`; + return `[Compressed conversation section]\n${block.summary}\n\n[${blockId} | ${range} | topic: "${block.topic ?? "(none)"}"]`; + } + + if (activeBlocks.length === 0) return "No active compression blocks."; + + const lines = [`Active compression blocks (${activeBlocks.length}):`]; + for (const block of activeBlocks) { + const range = `${block.effectiveMessageIds.length} messages`; + const preview = clampPrefix(block.summary, 200); + lines.push(`\n${block.blockId} | ${range} | "${block.topic ?? "(none)"}"`); + lines.push(` ${preview}${block.summary.length > 200 ? "..." : ""}`); + } + lines.push(`\nCall with blockId to get the full summary.`); + return lines.join("\n"); } diff --git a/src/search.ts b/src/search.ts index 5cf74bb..c191f04 100644 --- a/src/search.ts +++ b/src/search.ts @@ -1,21 +1,35 @@ /** * Search — re-exports from the modular src/search/ implementation. */ -export { searchBlocks, searchBlocksAsync, blockDocs, messageDocs } from "./search/index.js"; -export { clearDocFeatures, docCacheInfo, docFeatures, setDocCacheCap } from "./search/index.js"; +export { + searchBlocks, + searchBlocksAsync, + blockDocs, + messageDocs, +} from "./search/index.js"; +export { + clearDocFeatures, + docCacheInfo, + docFeatures, + setDocCacheCap, +} from "./search/index.js"; export type { DocFeatures } from "./search/index.js"; export type { - SearchResult, - SearchOptions, - SearchAlgorithm, - AsyncSearchAlgorithm, - AnySearchAlgorithm, - SearchDoc, - SearchDocKind, - ScoredBlock, - MessageRole, - RoleWeights, - MessageInput, + SearchResult, + SearchOptions, + SearchAlgorithm, + AsyncSearchAlgorithm, + AnySearchAlgorithm, + SearchDoc, + SearchDocKind, + ScoredBlock, + MessageRole, + RoleWeights, + MessageInput, } from "./search/types.js"; export { DEFAULT_ALGORITHM, DEFAULT_ROLE_WEIGHTS } from "./search/types.js"; -export { registerSearchAlgorithm, getSearchAlgorithm, listSearchAlgorithms } from "./search/registry.js"; +export { + registerSearchAlgorithm, + getSearchAlgorithm, + listSearchAlgorithms, +} from "./search/registry.js"; diff --git a/src/search/SEARCH.md b/src/search/SEARCH.md index 2122fd2..a640d01 100644 --- a/src/search/SEARCH.md +++ b/src/search/SEARCH.md @@ -18,8 +18,8 @@ import { searchBlocks, blockDocs, messageDocs } from "acp-kernel"; // Search both blocks and historical messages const docs = [ - ...blockDocs(state), // all blocks (active + inactive) - ...messageDocs([...historyMsgs]), // original message text + ...blockDocs(state), // all blocks (active + inactive) + ...messageDocs([...historyMsgs]), // original message text ]; const results = searchBlocks(docs, "auth token", { limit: 5 }); // → [{ kind: "message", ref: "m00350", blockId: "b3", score: 0.92, @@ -34,9 +34,9 @@ locate detail, then pay decompress cost only for what you need. ## Two data sources -| source | text searched | ref | when | -|--------|---------------|-----|------| -| **blocks** | summary (+ topic) | `b3` | active AND inactive blocks (inactive fix) | +| source | text searched | ref | when | +| ------------ | --------------------------------- | -------- | --------------------------------------------- | +| **blocks** | summary (+ topic) | `b3` | active AND inactive blocks (inactive fix) | | **messages** | original content from session log | `m00350` | detail that compression folded into a summary | Messages are host-supplied (pai-acp reads them from pi's append-only session @@ -48,12 +48,12 @@ decompress for surrounding context. Messages carry a role; per-role weights prioritize human intent over noise: -| role | default weight | why | -|------|---------------|-----| -| user | 1.5 | questions, decisions, requirements — high signal | -| assistant | 1.0 | reasoning and findings | -| tool | 0.6 | logs, listings — large volume, lower density | -| block | 1.0 | model-authored summaries | +| role | default weight | why | +| --------- | -------------- | ------------------------------------------------ | +| user | 1.5 | questions, decisions, requirements — high signal | +| assistant | 1.0 | reasoning and findings | +| tool | 0.6 | logs, listings — large volume, lower density | +| block | 1.0 | model-authored summaries | Override via `SearchOptions.roleWeights`. @@ -61,12 +61,13 @@ Override via `SearchOptions.roleWeights`. The default algorithm is `hybrid` (BM25+stem ⊕ fuzzy n-gram). On a 32-block mixed EN/CJK benchmark with 48 queries, against the original substring counter: -| algorithm | MRR | R@1 | R@3 | -|-------------|-------|-------|-------| -| substring | 0.797 | 0.792 | 0.792 | -| bm25 | 0.833 | 0.833 | 0.833 | -| fuzzy | 0.795 | 0.708 | 0.875 | -| **hybrid** | **0.898** | **0.875** | **0.917** | + +| algorithm | MRR | R@1 | R@3 | +| ---------- | --------- | --------- | --------- | +| substring | 0.797 | 0.792 | 0.792 | +| bm25 | 0.833 | 0.833 | 0.833 | +| fuzzy | 0.795 | 0.708 | 0.875 | +| **hybrid** | **0.898** | **0.875** | **0.917** | All rows measured on the rework benchmark (32 blocks / 48 queries) with the final code (segmenter tokenizer + CJK fuzzy gate). @@ -88,27 +89,27 @@ The wins come from three fixes to plain substring matching: starve the recall channel for a whole script. Previews are **match-context snippets**, not arbitrary prefixes — the result -shows the sentence around the hit so the user sees *why* a block matched. +shows the sentence around the hit so the user sees _why_ a block matched. ## Options ```ts interface SearchOptions { - algorithm?: string; // "hybrid" (default) | "bm25" | "fuzzy" | "substring" | - limit?: number; // default 10 + algorithm?: string; // "hybrid" (default) | "bm25" | "fuzzy" | "substring" | + limit?: number; // default 10 previewLength?: number; // default 200 - minScore?: number; // default 0.01 + minScore?: number; // default 0.01 } ``` ## Built-in algorithms -| name | strengths | when to use | -|-------------|---------------------------------------------------|----------------------------------| -| `hybrid` | best all-rounder (default) | always | -| `bm25` | IR-standard, IDF + length norm + stemming | precision on long corpora | -| `fuzzy` | typo-tolerant, cross-script | recall on messy input | -| `substring` | exact, deterministic, occurrence count | backward compat / debugging | +| name | strengths | when to use | +| ----------- | ----------------------------------------- | --------------------------- | +| `hybrid` | best all-rounder (default) | always | +| `bm25` | IR-standard, IDF + length norm + stemming | precision on long corpora | +| `fuzzy` | typo-tolerant, cross-script | recall on messy input | +| `substring` | exact, deterministic, occurrence count | backward compat / debugging | ## Adding a custom algorithm @@ -145,12 +146,14 @@ import { createSemanticAlgorithm } from "acp-kernel/search/algorithms/semantic"; const semantic = createSemanticAlgorithm({ // any backend: @huggingface/transformers (local), OpenAI, Voyage, a local server… - embed: async (texts) => myEmbeddingApi.embed(texts), // → number[][] + embed: async (texts) => myEmbeddingApi.embed(texts), // → number[][] }); registerSearchAlgorithm(semantic); // semantic score() is async → use searchBlocksAsync (searchBlocks throws a clear error) -const results = await searchBlocksAsync(state, "credentials", { algorithm: "semantic" }); +const results = await searchBlocksAsync(state, "credentials", { + algorithm: "semantic", +}); ``` `embeddings` are memoized by content hash — docs only re-embed when their diff --git a/src/search/algorithms/bm25.ts b/src/search/algorithms/bm25.ts index e6d15c1..56fd722 100644 --- a/src/search/algorithms/bm25.ts +++ b/src/search/algorithms/bm25.ts @@ -15,37 +15,40 @@ import { docFeatures } from "../doc-cache.js"; * metric, and the precision component of the hybrid default (see hybrid.ts). */ export const bm25Algorithm: SearchAlgorithm = { - name: "bm25", - description: "BM25 with stemming + CJK bigram tokenization. IR-standard relevance ranking.", - score(docs: SearchDoc[], query: string): ScoredBlock[] { - const N = docs.length; - const k1 = 1.2; - const b = 0.75; - const parsed = docs.map((d) => { - const f = docFeatures(d.text); // memoized: tf + length, cached across calls - return { id: d.ref, tf: f.tf, len: f.len }; - }); - const avgdl = parsed.reduce((s, d) => s + d.len, 0) / (N || 1); + name: "bm25", + description: + "BM25 with stemming + CJK bigram tokenization. IR-standard relevance ranking.", + score(docs: SearchDoc[], query: string): ScoredBlock[] { + const N = docs.length; + const k1 = 1.2; + const b = 0.75; + const parsed = docs.map((d) => { + const f = docFeatures(d.text); // memoized: tf + length, cached across calls + return { id: d.ref, tf: f.tf, len: f.len }; + }); + const avgdl = parsed.reduce((s, d) => s + d.len, 0) / (N || 1); - const qTerms = tokenize(query, { stem: true }); - if (qTerms.length === 0) return docs.map((d) => ({ ref: d.ref, score: 0 })); + const qTerms = tokenize(query, { stem: true }); + if (qTerms.length === 0) return docs.map((d) => ({ ref: d.ref, score: 0 })); - const idf = new Map(); - for (const t of new Set(qTerms)) { - let df = 0; - for (const d of parsed) if (d.tf.has(t)) df++; - idf.set(t, Math.log(1 + (N - df + 0.5) / (df + 0.5))); - } + const idf = new Map(); + for (const t of new Set(qTerms)) { + let df = 0; + for (const d of parsed) if (d.tf.has(t)) df++; + idf.set(t, Math.log(1 + (N - df + 0.5) / (df + 0.5))); + } - return parsed.map((d) => { - let score = 0; - for (const t of qTerms) { - const f = d.tf.get(t) ?? 0; - if (f === 0) continue; - const idfT = idf.get(t) ?? 0; - score += (idfT * (f * (k1 + 1))) / (f + k1 * (1 - b + (b * d.len) / (avgdl || 1))); - } - return { ref: d.id, score }; - }); - }, + return parsed.map((d) => { + let score = 0; + for (const t of qTerms) { + const f = d.tf.get(t) ?? 0; + if (f === 0) continue; + const idfT = idf.get(t) ?? 0; + score += + (idfT * (f * (k1 + 1))) / + (f + k1 * (1 - b + (b * d.len) / (avgdl || 1))); + } + return { ref: d.id, score }; + }); + }, }; diff --git a/src/search/algorithms/fuzzy.ts b/src/search/algorithms/fuzzy.ts index c4bdd98..c6a6d21 100644 --- a/src/search/algorithms/fuzzy.ts +++ b/src/search/algorithms/fuzzy.ts @@ -24,23 +24,28 @@ import { docFeatures } from "../doc-cache.js"; * hybrid default. */ export const fuzzyAlgorithm: SearchAlgorithm = { - name: "fuzzy", - description: "Character bigram overlap. Typo-tolerant, script-agnostic, high recall.", - score(docs: SearchDoc[], query: string): ScoredBlock[] { - // Gate (see header): Latin short tokens are noise, 2-char CJK words - // are real terms — admit the latter so 缓存/登录 reach the scorer. - const qTokens = query.toLowerCase().split(/[\s,]+/).filter((t) => t.length >= 4 || (t.length >= 2 && CJK.test(t))); - if (qTokens.length === 0) return docs.map((d) => ({ ref: d.ref, score: 0 })); + name: "fuzzy", + description: + "Character bigram overlap. Typo-tolerant, script-agnostic, high recall.", + score(docs: SearchDoc[], query: string): ScoredBlock[] { + // Gate (see header): Latin short tokens are noise, 2-char CJK words + // are real terms — admit the latter so 缓存/登录 reach the scorer. + const qTokens = query + .toLowerCase() + .split(/[\s,]+/) + .filter((t) => t.length >= 4 || (t.length >= 2 && CJK.test(t))); + if (qTokens.length === 0) + return docs.map((d) => ({ ref: d.ref, score: 0 })); - const qGrams = new Set(); - for (const t of qTokens) for (const g of charBigrams(t)) qGrams.add(g); - if (qGrams.size === 0) return docs.map((d) => ({ ref: d.ref, score: 0 })); + const qGrams = new Set(); + for (const t of qTokens) for (const g of charBigrams(t)) qGrams.add(g); + if (qGrams.size === 0) return docs.map((d) => ({ ref: d.ref, score: 0 })); - return docs.map((d) => { - const docGrams = docFeatures(d.text).grams; // memoized bigram set - let hits = 0; - for (const g of qGrams) if (docGrams.has(g)) hits++; - return { ref: d.ref, score: hits / qGrams.size }; - }); - }, + return docs.map((d) => { + const docGrams = docFeatures(d.text).grams; // memoized bigram set + let hits = 0; + for (const g of qGrams) if (docGrams.has(g)) hits++; + return { ref: d.ref, score: hits / qGrams.size }; + }); + }, }; diff --git a/src/search/algorithms/hybrid.ts b/src/search/algorithms/hybrid.ts index 3838e6f..dce30c5 100644 --- a/src/search/algorithms/hybrid.ts +++ b/src/search/algorithms/hybrid.ts @@ -23,19 +23,21 @@ const W_BM25 = 0.7; const W_FUZZY = 0.3; export const hybridAlgorithm: SearchAlgorithm = { - name: "hybrid", - description: "Weighted BM25(stem) + fuzzy n-gram. Default — best precision + recall.", - score(docs: SearchDoc[], query: string): ScoredBlock[] { - const bm = bm25Algorithm.score(docs, query); - const fz = fuzzyAlgorithm.score(docs, query); - // reduce, not Math.max(...spread): the spread trips V8's argument limit (~65K–125K docs) → RangeError. - const maxBm = bm.reduce((m, r) => (r.score > m ? r.score : m), 1e-9); - const maxFz = fz.reduce((m, r) => (r.score > m ? r.score : m), 1e-9); - const bmMap = new Map(bm.map((r) => [r.ref, r.score / maxBm])); - const fzMap = new Map(fz.map((r) => [r.ref, r.score / maxFz])); - return docs.map((d) => ({ - ref: d.ref, - score: W_BM25 * (bmMap.get(d.ref) ?? 0) + W_FUZZY * (fzMap.get(d.ref) ?? 0), - })); - }, + name: "hybrid", + description: + "Weighted BM25(stem) + fuzzy n-gram. Default — best precision + recall.", + score(docs: SearchDoc[], query: string): ScoredBlock[] { + const bm = bm25Algorithm.score(docs, query); + const fz = fuzzyAlgorithm.score(docs, query); + // reduce, not Math.max(...spread): the spread trips V8's argument limit (~65K–125K docs) → RangeError. + const maxBm = bm.reduce((m, r) => (r.score > m ? r.score : m), 1e-9); + const maxFz = fz.reduce((m, r) => (r.score > m ? r.score : m), 1e-9); + const bmMap = new Map(bm.map((r) => [r.ref, r.score / maxBm])); + const fzMap = new Map(fz.map((r) => [r.ref, r.score / maxFz])); + return docs.map((d) => ({ + ref: d.ref, + score: + W_BM25 * (bmMap.get(d.ref) ?? 0) + W_FUZZY * (fzMap.get(d.ref) ?? 0), + })); + }, }; diff --git a/src/search/algorithms/semantic.ts b/src/search/algorithms/semantic.ts index a1731c9..c6309e9 100644 --- a/src/search/algorithms/semantic.ts +++ b/src/search/algorithms/semantic.ts @@ -29,78 +29,84 @@ import type { AsyncSearchAlgorithm, SearchDoc, ScoredBlock } from "../types.js"; export interface EmbedFn { - (texts: string[]): Promise; + (texts: string[]): Promise; } export interface SemanticOptions { - embed: EmbedFn; - name?: string; + embed: EmbedFn; + name?: string; } interface CachedEmbedding { - hash: string; - vec: number[]; + hash: string; + vec: number[]; } function hashText(s: string): string { - let h = 0x811c9dc5; - for (let i = 0; i < s.length; i++) { - h ^= s.charCodeAt(i); - h = Math.imul(h, 0x01000193); - } - return (h >>> 0).toString(36); + let h = 0x811c9dc5; + for (let i = 0; i < s.length; i++) { + h ^= s.charCodeAt(i); + h = Math.imul(h, 0x01000193); + } + return (h >>> 0).toString(36); } function cosine(a: number[], b: number[]): number { - let dot = 0, na = 0, nb = 0; - for (let i = 0; i < a.length; i++) { - const av = a[i]!; - const bv = b[i]!; - dot += av * bv; - na += av * av; - nb += bv * bv; - } - if (na === 0 || nb === 0) return 0; - return dot / (Math.sqrt(na) * Math.sqrt(nb)); + let dot = 0, + na = 0, + nb = 0; + for (let i = 0; i < a.length; i++) { + const av = a[i]!; + const bv = b[i]!; + dot += av * bv; + na += av * av; + nb += bv * bv; + } + if (na === 0 || nb === 0) return 0; + return dot / (Math.sqrt(na) * Math.sqrt(nb)); } -export function createSemanticAlgorithm(opts: SemanticOptions): AsyncSearchAlgorithm { - const name = opts.name ?? "semantic"; - const cache = new Map(); +export function createSemanticAlgorithm( + opts: SemanticOptions, +): AsyncSearchAlgorithm { + const name = opts.name ?? "semantic"; + const cache = new Map(); - return { - name, - description: "Embedding cosine similarity (host-supplied embed fn). Catches synonyms/cross-lang lexical algorithms miss.", + return { + name, + description: + "Embedding cosine similarity (host-supplied embed fn). Catches synonyms/cross-lang lexical algorithms miss.", - async score(docs: SearchDoc[], query: string): Promise { - if (docs.length === 0) return []; + async score(docs: SearchDoc[], query: string): Promise { + if (docs.length === 0) return []; - // 1. find docs needing (re)embedding - const stale: Array<{ id: string; text: string; hash: string }> = []; - for (const d of docs) { - const text = d.text; - const hash = hashText(text); - const cached = cache.get(d.ref); - if (!cached || cached.hash !== hash) stale.push({ id: d.ref, text, hash }); - } + // 1. find docs needing (re)embedding + const stale: Array<{ id: string; text: string; hash: string }> = []; + for (const d of docs) { + const text = d.text; + const hash = hashText(text); + const cached = cache.get(d.ref); + if (!cached || cached.hash !== hash) + stale.push({ id: d.ref, text, hash }); + } - // 2. batch-embed stale docs + query in one round-trip - const toEmbed = stale.map((s) => s.text); - toEmbed.push(query); - const vecs = await opts.embed(toEmbed); - if (vecs.length !== toEmbed.length) { - return docs.map((d) => ({ ref: d.ref, score: 0 })); - } - const qVec = vecs[vecs.length - 1]!; - for (let j = 0; j < stale.length; j++) { - cache.set(stale[j]!.id, { hash: stale[j]!.hash, vec: vecs[j]! }); - } + // 2. batch-embed stale docs + query in one round-trip + const toEmbed = stale.map((s) => s.text); + toEmbed.push(query); + const vecs = await opts.embed(toEmbed); + if (vecs.length !== toEmbed.length) { + return docs.map((d) => ({ ref: d.ref, score: 0 })); + } + const qVec = vecs[vecs.length - 1]!; + for (let j = 0; j < stale.length; j++) { + cache.set(stale[j]!.id, { hash: stale[j]!.hash, vec: vecs[j]! }); + } - // 3. cosine similarity - return docs.map((d) => { - const cached = cache.get(d.ref); - return { ref: d.ref, score: cached ? cosine(qVec, cached.vec) : 0 }; - }); - }, - }; + // 3. cosine similarity + return docs.map((d) => { + const cached = cache.get(d.ref); + return { ref: d.ref, score: cached ? cosine(qVec, cached.vec) : 0 }; + }); + }, + }; } diff --git a/src/search/algorithms/substring.ts b/src/search/algorithms/substring.ts index 54db449..5b64c68 100644 --- a/src/search/algorithms/substring.ts +++ b/src/search/algorithms/substring.ts @@ -8,21 +8,26 @@ import { docFeatures } from "../doc-cache.js"; * as a deterministic reference. */ export const substringAlgorithm: SearchAlgorithm = { - name: "substring", - description: "Exact substring counting (original baseline). Predictable, no normalization.", - score(docs: SearchDoc[], query: string): ScoredBlock[] { - const terms = query.toLowerCase().trim().split(/\s+/).filter((t) => t.length > 0); - if (terms.length === 0) return docs.map((d) => ({ ref: d.ref, score: 0 })); - return docs.map((d) => { - const haystack = docFeatures(d.text).lower; // memoized across calls - let score = 0; - for (const term of terms) score += countOccurrences(haystack, term); - return { ref: d.ref, score }; - }); - }, + name: "substring", + description: + "Exact substring counting (original baseline). Predictable, no normalization.", + score(docs: SearchDoc[], query: string): ScoredBlock[] { + const terms = query + .toLowerCase() + .trim() + .split(/\s+/) + .filter((t) => t.length > 0); + if (terms.length === 0) return docs.map((d) => ({ ref: d.ref, score: 0 })); + return docs.map((d) => { + const haystack = docFeatures(d.text).lower; // memoized across calls + let score = 0; + for (const term of terms) score += countOccurrences(haystack, term); + return { ref: d.ref, score }; + }); + }, }; function countOccurrences(haystack: string, needle: string): number { - if (!needle) return 0; - return haystack.split(needle).length - 1; + if (!needle) return 0; + return haystack.split(needle).length - 1; } diff --git a/src/search/doc-cache.ts b/src/search/doc-cache.ts index 620cc7f..63d4600 100644 --- a/src/search/doc-cache.ts +++ b/src/search/doc-cache.ts @@ -41,14 +41,14 @@ import { charBigrams, tfMap } from "./tokenizer.js"; export interface DocFeatures { - /** Stemmed term frequencies (BM25 channel). */ - tf: Map; - /** Total term count (BM25 length normalization). */ - len: number; - /** Lower-cased text (substring + fuzzy channels). */ - lower: string; - /** Unique char bigrams of `lower` (fuzzy channel). */ - grams: Set; + /** Stemmed term frequencies (BM25 channel). */ + tf: Map; + /** Total term count (BM25 length normalization). */ + len: number; + /** Lower-cased text (substring + fuzzy channels). */ + lower: string; + /** Unique char bigrams of `lower` (fuzzy channel). */ + grams: Set; } const DEFAULT_CAP_CHARS = 8 * 1024 * 1024; @@ -57,38 +57,38 @@ const cache = new Map(); let cachedChars = 0; function build(text: string): DocFeatures { - const tf = tfMap(text, true); - let len = 0; - for (const v of tf.values()) len += v; - const lower = text.toLowerCase(); - return { tf, len, lower, grams: new Set(charBigrams(lower)) }; + const tf = tfMap(text, true); + let len = 0; + for (const v of tf.values()) len += v; + const lower = text.toLowerCase(); + return { tf, len, lower, grams: new Set(charBigrams(lower)) }; } export function docFeatures(text: string): DocFeatures { - const hit = cache.get(text); - if (hit) { - // LRU: re-insert at the tail (most-recently-used); eviction takes from the head. - cache.delete(text); - cache.set(text, hit); - return hit; + const hit = cache.get(text); + if (hit) { + // LRU: re-insert at the tail (most-recently-used); eviction takes from the head. + cache.delete(text); + cache.set(text, hit); + return hit; + } + const f = build(text); + if (text.length > 0 && text.length <= capChars) { + while (cachedChars + text.length > capChars && cache.size > 0) { + const k = cache.keys().next().value as string; + cachedChars -= k.length; + cache.delete(k); } - const f = build(text); - if (text.length > 0 && text.length <= capChars) { - while (cachedChars + text.length > capChars && cache.size > 0) { - const k = cache.keys().next().value as string; - cachedChars -= k.length; - cache.delete(k); - } - cache.set(text, f); - cachedChars += text.length; - } - return f; + cache.set(text, f); + cachedChars += text.length; + } + return f; } /** Drop all cached features (e.g. on session shutdown/switch). */ export function clearDocFeatures(): void { - cache.clear(); - cachedChars = 0; + cache.clear(); + cachedChars = 0; } /** @@ -98,15 +98,15 @@ export function clearDocFeatures(): void { * Also used by tests to exercise eviction. */ export function setDocCacheCap(chars: number): void { - capChars = Math.max(1, chars); - while (cachedChars > capChars && cache.size > 0) { - const k = cache.keys().next().value as string; - cachedChars -= k.length; - cache.delete(k); - } + capChars = Math.max(1, chars); + while (cachedChars > capChars && cache.size > 0) { + const k = cache.keys().next().value as string; + cachedChars -= k.length; + cache.delete(k); + } } /** Cache occupancy — for diagnostics. */ export function docCacheInfo(): { entries: number; chars: number } { - return { entries: cache.size, chars: cachedChars }; + return { entries: cache.size, chars: cachedChars }; } diff --git a/src/search/index.ts b/src/search/index.ts index 9dda5a4..ad6c778 100644 --- a/src/search/index.ts +++ b/src/search/index.ts @@ -21,15 +21,15 @@ import { DEFAULT_ALGORITHM, DEFAULT_ROLE_WEIGHTS } from "./types.js"; /** Build SearchDoc[] from all blocks (active AND inactive) of the state. */ export function blockDocs(state: CompressionState): SearchDoc[] { - return state.blocks.map((b: CompressionBlock): SearchDoc => ({ - kind: "block", - ref: b.blockId, - text: `${b.topic ?? ""} ${b.summary ?? ""}`, - title: b.topic ?? b.blockId, - blockId: b.blockId, - tier: b.tier ?? 1, - tokens: b.compressedTokens, - })); + return state.blocks.map((b: CompressionBlock): SearchDoc => ({ + kind: "block", + ref: b.blockId, + text: `${b.topic ?? ""} ${b.summary ?? ""}`, + title: b.topic ?? b.blockId, + blockId: b.blockId, + tier: b.tier ?? 1, + tokens: b.compressedTokens, + })); } /** @@ -41,98 +41,117 @@ export function blockDocs(state: CompressionState): SearchDoc[] { * message hit tells the model exactly which block to decompress for detail. */ export function messageDocs(msgs: MessageInput[]): SearchDoc[] { - return msgs.map((m): SearchDoc => ({ - kind: "message", - ref: m.ref, - text: m.text, - title: `${m.role}: ${clampPrefix(m.text, 60)}`, - role: m.role, - blockId: m.blockId, - tier: m.tier, - tokens: m.tokens, - })); + return msgs.map((m): SearchDoc => ({ + kind: "message", + ref: m.ref, + text: m.text, + title: `${m.role}: ${clampPrefix(m.text, 60)}`, + role: m.role, + blockId: m.blockId, + tier: m.tier, + tokens: m.tokens, + })); } -function applyRoleWeight(scored: ScoredBlock[], docs: SearchDoc[], rw: Required): ScoredBlock[] { - if (docs.length === 0) return scored; - const docByRef = new Map(docs.map((d) => [d.ref, d])); - return scored.map((s) => { - const doc = docByRef.get(s.ref); - if (!doc) return s; - const w = - doc.kind === "message" - ? doc.role === "user" - ? rw.user - : doc.role === "assistant" - ? rw.assistant - : rw.tool - : rw.block; - return { ref: s.ref, score: s.score * w }; - }); +function applyRoleWeight( + scored: ScoredBlock[], + docs: SearchDoc[], + rw: Required, +): ScoredBlock[] { + if (docs.length === 0) return scored; + const docByRef = new Map(docs.map((d) => [d.ref, d])); + return scored.map((s) => { + const doc = docByRef.get(s.ref); + if (!doc) return s; + const w = + doc.kind === "message" + ? doc.role === "user" + ? rw.user + : doc.role === "assistant" + ? rw.assistant + : rw.tool + : rw.block; + return { ref: s.ref, score: s.score * w }; + }); } function runSearch( - docs: SearchDoc[], - query: string, - options: SearchOptions, + docs: SearchDoc[], + query: string, + options: SearchOptions, ): SearchResult[] | Promise { - const limit = options.limit ?? 10; - const previewLength = options.previewLength ?? 200; - const minScore = options.minScore ?? 0.01; - const algoName = options.algorithm ?? DEFAULT_ALGORITHM; - const rw = { ...DEFAULT_ROLE_WEIGHTS, ...options.roleWeights }; - - const algo = getSearchAlgorithm(algoName); - if (!algo) return []; - if (docs.length === 0) return []; - - const scoredOrPromise = algo.score(docs, query); - - const buildResults = (weighted: ScoredBlock[]): SearchResult[] => { - const byRef = new Map(docs.map((d) => [d.ref, d])); - return weighted - .map((s): SearchResult | null => { - const doc = byRef.get(s.ref); - if (!doc) return null; - return { - kind: doc.kind, - ref: doc.ref, - blockId: doc.blockId, - tier: doc.tier ?? 1, - score: s.score, - title: doc.title, - preview: makePreview(doc.text, query, previewLength), - role: doc.role, - tokens: doc.tokens, - }; - }) - .filter((r): r is SearchResult => r !== null && r.score >= minScore) - .sort((a, b) => b.score - a.score) - .slice(0, limit); - }; - - if (scoredOrPromise instanceof Promise) { - return scoredOrPromise.then((raw) => buildResults(applyRoleWeight(raw, docs, rw))); - } - return buildResults(applyRoleWeight(scoredOrPromise, docs, rw)); + const limit = options.limit ?? 10; + const previewLength = options.previewLength ?? 200; + const minScore = options.minScore ?? 0.01; + const algoName = options.algorithm ?? DEFAULT_ALGORITHM; + const rw = { ...DEFAULT_ROLE_WEIGHTS, ...options.roleWeights }; + + const algo = getSearchAlgorithm(algoName); + if (!algo) return []; + if (docs.length === 0) return []; + + const scoredOrPromise = algo.score(docs, query); + + const buildResults = (weighted: ScoredBlock[]): SearchResult[] => { + const byRef = new Map(docs.map((d) => [d.ref, d])); + return weighted + .map((s): SearchResult | null => { + const doc = byRef.get(s.ref); + if (!doc) return null; + return { + kind: doc.kind, + ref: doc.ref, + blockId: doc.blockId, + tier: doc.tier ?? 1, + score: s.score, + title: doc.title, + preview: makePreview(doc.text, query, previewLength), + role: doc.role, + tokens: doc.tokens, + }; + }) + .filter((r): r is SearchResult => r !== null && r.score >= minScore) + .sort((a, b) => b.score - a.score) + .slice(0, limit); + }; + + if (scoredOrPromise instanceof Promise) { + return scoredOrPromise.then((raw) => + buildResults(applyRoleWeight(raw, docs, rw)), + ); + } + return buildResults(applyRoleWeight(scoredOrPromise, docs, rw)); } /** Sync entry — throws for async algorithms. Pass docs from blockDocs() + messageDocs(). */ -export function searchBlocks(docs: SearchDoc[], query: string, options: SearchOptions = {}): SearchResult[] { - const result = runSearch(docs, query, options); - if (result instanceof Promise) { - throw new Error( - `searchBlocks: algorithm "${options.algorithm ?? DEFAULT_ALGORITHM}" is async (e.g. semantic). Use searchBlocksAsync() instead.`, - ); - } - return result; +export function searchBlocks( + docs: SearchDoc[], + query: string, + options: SearchOptions = {}, +): SearchResult[] { + const result = runSearch(docs, query, options); + if (result instanceof Promise) { + throw new Error( + `searchBlocks: algorithm "${options.algorithm ?? DEFAULT_ALGORITHM}" is async (e.g. semantic). Use searchBlocksAsync() instead.`, + ); + } + return result; } -export { clearDocFeatures, docCacheInfo, docFeatures, setDocCacheCap } from "./doc-cache.js"; +export { + clearDocFeatures, + docCacheInfo, + docFeatures, + setDocCacheCap, +} from "./doc-cache.js"; export type { DocFeatures } from "./doc-cache.js"; -export async function searchBlocksAsync(docs: SearchDoc[], query: string, options: SearchOptions = {}): Promise { - return await runSearch(docs, query, options); +export async function searchBlocksAsync( + docs: SearchDoc[], + query: string, + options: SearchOptions = {}, +): Promise { + return await runSearch(docs, query, options); } /** @@ -140,26 +159,30 @@ export async function searchBlocksAsync(docs: SearchDoc[], query: string, option * Falls back to the head when no term hits. */ function makePreview(text: string, query: string, len: number): string { - if (!text) return ""; - const terms = query.toLowerCase().trim().split(/\s+/).filter((t) => t.length > 1); - if (terms.length === 0) return clampPrefix(text, len); - - const lower = text.toLowerCase(); - let hitIdx = -1; - for (const term of terms) { - const idx = lower.indexOf(term); - if (idx >= 0) { - hitIdx = idx; - break; - } + if (!text) return ""; + const terms = query + .toLowerCase() + .trim() + .split(/\s+/) + .filter((t) => t.length > 1); + if (terms.length === 0) return clampPrefix(text, len); + + const lower = text.toLowerCase(); + let hitIdx = -1; + for (const term of terms) { + const idx = lower.indexOf(term); + if (idx >= 0) { + hitIdx = idx; + break; } + } - if (hitIdx < 0) return clampPrefix(text, len); + if (hitIdx < 0) return clampPrefix(text, len); - const half = Math.max(0, Math.floor(len / 2) - 10); - const start = Math.max(0, hitIdx - half); - const end = Math.min(text.length, start + len); - const prefix = start > 0 ? "…" : ""; - const suffix = end < text.length ? "…" : ""; - return prefix + clampWindow(text, start, end).trim() + suffix; + const half = Math.max(0, Math.floor(len / 2) - 10); + const start = Math.max(0, hitIdx - half); + const end = Math.min(text.length, start + len); + const prefix = start > 0 ? "…" : ""; + const suffix = end < text.length ? "…" : ""; + return prefix + clampWindow(text, start, end).trim() + suffix; } diff --git a/src/search/registry.ts b/src/search/registry.ts index 5eeb578..98dd194 100644 --- a/src/search/registry.ts +++ b/src/search/registry.ts @@ -12,15 +12,17 @@ import { hybridAlgorithm } from "./algorithms/hybrid.js"; const registry = new Map(); export function registerSearchAlgorithm(algo: AnySearchAlgorithm): void { - registry.set(algo.name, algo); + registry.set(algo.name, algo); } -export function getSearchAlgorithm(name: string): AnySearchAlgorithm | undefined { - return registry.get(name); +export function getSearchAlgorithm( + name: string, +): AnySearchAlgorithm | undefined { + return registry.get(name); } export function listSearchAlgorithms(): AnySearchAlgorithm[] { - return [...registry.values()]; + return [...registry.values()]; } // Pre-register builtins. Hybrid is the default (see types.ts DEFAULT_ALGORITHM). diff --git a/src/search/stemmer.ts b/src/search/stemmer.ts index 5e288bb..c170781 100644 --- a/src/search/stemmer.ts +++ b/src/search/stemmer.ts @@ -8,19 +8,20 @@ * untouched (handled by bigram tokenization, not stemming). */ export function stem(word: string): string { - let w = word; - if (w.length <= 3) return w; - if (w.endsWith("ies")) w = w.slice(0, -3) + "y"; - else if (w.endsWith("ses") || w.endsWith("xes") || w.endsWith("zes")) w = w.slice(0, -2); - else if (w.endsWith("ches") || w.endsWith("shes")) w = w.slice(0, -2); - else if (w.endsWith("s") && !w.endsWith("ss")) w = w.slice(0, -1); - if (w.endsWith("ing") && w.length > 5) w = w.slice(0, -3); - if (w.endsWith("ed") && w.length > 4) w = w.slice(0, -2); - if (w.endsWith("ation") && w.length > 6) w = w.slice(0, -3); - else if (w.endsWith("tion") && w.length > 5) w = w.slice(0, -4) + "t"; - else if (w.endsWith("ion") && w.length > 4) w = w.slice(0, -3); - if (w.endsWith("ment") && w.length > 6) w = w.slice(0, -4); - if (w.endsWith("ness") && w.length > 6) w = w.slice(0, -4); - if (w.endsWith("ly") && w.length > 4) w = w.slice(0, -2); - return w; + let w = word; + if (w.length <= 3) return w; + if (w.endsWith("ies")) w = w.slice(0, -3) + "y"; + else if (w.endsWith("ses") || w.endsWith("xes") || w.endsWith("zes")) + w = w.slice(0, -2); + else if (w.endsWith("ches") || w.endsWith("shes")) w = w.slice(0, -2); + else if (w.endsWith("s") && !w.endsWith("ss")) w = w.slice(0, -1); + if (w.endsWith("ing") && w.length > 5) w = w.slice(0, -3); + if (w.endsWith("ed") && w.length > 4) w = w.slice(0, -2); + if (w.endsWith("ation") && w.length > 6) w = w.slice(0, -3); + else if (w.endsWith("tion") && w.length > 5) w = w.slice(0, -4) + "t"; + else if (w.endsWith("ion") && w.length > 4) w = w.slice(0, -3); + if (w.endsWith("ment") && w.length > 6) w = w.slice(0, -4); + if (w.endsWith("ness") && w.length > 6) w = w.slice(0, -4); + if (w.endsWith("ly") && w.length > 4) w = w.slice(0, -2); + return w; } diff --git a/src/search/tokenizer.ts b/src/search/tokenizer.ts index 24ce541..34d7708 100644 --- a/src/search/tokenizer.ts +++ b/src/search/tokenizer.ts @@ -43,81 +43,81 @@ const cjkSegmenter = new Intl.Segmenter("zh", { granularity: "word" }); * preserved — this also covers single-char queries like "验". */ function cjkRunTokens(segs: string[]): string[] { - const words = segs.filter((w) => w.length >= 2); - if (words.length > 0) return words; - const run = segs.join(""); - const out: string[] = []; - for (let i = 0; i < run.length - 1; i++) out.push(run.slice(i, i + 2)); - for (const ch of run) out.push(ch); - return out; + const words = segs.filter((w) => w.length >= 2); + if (words.length > 0) return words; + const run = segs.join(""); + const out: string[] = []; + for (let i = 0; i < run.length - 1; i++) out.push(run.slice(i, i + 2)); + for (const ch of run) out.push(ch); + return out; } export interface TokenizeOptions { - stem?: boolean; + stem?: boolean; } export function tokenize(text: string, opts: TokenizeOptions = {}): string[] { - const lower = text.toLowerCase(); - const tokens: string[] = []; + const lower = text.toLowerCase(); + const tokens: string[] = []; - const latin = lower.match(LATIN_WORD) ?? []; - for (let w of latin) { - if (w.length >= 2) { - if (opts.stem) w = stem(w); - tokens.push(w); - } + const latin = lower.match(LATIN_WORD) ?? []; + for (let w of latin) { + if (w.length >= 2) { + if (opts.stem) w = stem(w); + tokens.push(w); } + } - // CJK: one segmenter pass over the whole text instead of one - // segment() call per CJK run. A segment() call has fixed overhead - // (~3µs), and run-heavy text (logs: dozens of short runs per line) made - // per-run calls 10-16× slower than one bulk pass. ICU never merges CJK - // words across non-CJK boundaries, so bulk segmentation yields the same - // words per run (differential-verified against the per-run - // implementation across a mixed-script stress corpus); run boundaries - // are re-derived below to keep the all-OOV bigram fallback. - // - // Guard: skip the segmenter entirely when the text has no CJK at all — - // the old code never called it for pure-Latin text, and a bulk pass - // would pay a full-text scan (12ms → 33ms per MB of English) for nothing. - if (!CJK.test(lower)) return tokens; + // CJK: one segmenter pass over the whole text instead of one + // segment() call per CJK run. A segment() call has fixed overhead + // (~3µs), and run-heavy text (logs: dozens of short runs per line) made + // per-run calls 10-16× slower than one bulk pass. ICU never merges CJK + // words across non-CJK boundaries, so bulk segmentation yields the same + // words per run (differential-verified against the per-run + // implementation across a mixed-script stress corpus); run boundaries + // are re-derived below to keep the all-OOV bigram fallback. + // + // Guard: skip the segmenter entirely when the text has no CJK at all — + // the old code never called it for pure-Latin text, and a bulk pass + // would pay a full-text scan (12ms → 33ms per MB of English) for nothing. + if (!CJK.test(lower)) return tokens; - // Group the bulk segments back into CJK runs: a non-CJK segment is a run - // boundary (the segmenter never puts non-CJK inside a CJK word segment). - const runSegs: string[][] = []; - let cur: string[] | null = null; - for (const s of cjkSegmenter.segment(lower)) { - const t = s.segment; - if (t.length === 0) continue; - if (CJK.test(t)) { - (cur ??= []).push(t); - } else if (cur) { - runSegs.push(cur); - cur = null; - } + // Group the bulk segments back into CJK runs: a non-CJK segment is a run + // boundary (the segmenter never puts non-CJK inside a CJK word segment). + const runSegs: string[][] = []; + let cur: string[] | null = null; + for (const s of cjkSegmenter.segment(lower)) { + const t = s.segment; + if (t.length === 0) continue; + if (CJK.test(t)) { + (cur ??= []).push(t); + } else if (cur) { + runSegs.push(cur); + cur = null; } - if (cur) runSegs.push(cur); + } + if (cur) runSegs.push(cur); - for (const segs of runSegs) { - tokens.push(...cjkRunTokens(segs)); - } + for (const segs of runSegs) { + tokens.push(...cjkRunTokens(segs)); + } - return tokens; + return tokens; } /** Character bigrams over arbitrary text — used by fuzzy matching. */ export function charBigrams(text: string): string[] { - const grams: string[] = []; - for (let i = 0; i < text.length - 1; i++) { - const pair = text.slice(i, i + 2); - if (pair.trim().length === pair.length) grams.push(pair); - } - return grams; + const grams: string[] = []; + for (let i = 0; i < text.length - 1; i++) { + const pair = text.slice(i, i + 2); + if (pair.trim().length === pair.length) grams.push(pair); + } + return grams; } /** Term-frequency map. */ export function tfMap(text: string, stem: boolean): Map { - const m = new Map(); - for (const t of tokenize(text, { stem })) m.set(t, (m.get(t) ?? 0) + 1); - return m; + const m = new Map(); + for (const t of tokenize(text, { stem })) m.set(t, (m.get(t) ?? 0) + 1); + return m; } diff --git a/src/search/types.ts b/src/search/types.ts index 3b1eaae..d91a08a 100644 --- a/src/search/types.ts +++ b/src/search/types.ts @@ -19,91 +19,91 @@ export type MessageRole = "user" | "assistant" | "tool"; /** A unified searchable document — either a block summary or a message. */ export interface SearchDoc { - kind: SearchDocKind; - /** Stable ref for decompress: "b3" for a block, "m00350" for a message. */ - ref: string; - /** Text this doc is scored against (topic+summary for blocks; content for messages). */ - text: string; - /** For preview/title display. */ - title: string; - /** Message role (messages only); undefined for blocks. Drives role weighting. */ - role?: MessageRole; - /** Block owning this doc. For blocks: the block itself. For messages: the block - * that compressed it (so the model knows which block to decompress for detail). */ - blockId?: string; - /** Tier of the owning block (display + grouping). */ - tier?: number; - /** Approx token size (for "how big is this" display). */ - tokens?: number; + kind: SearchDocKind; + /** Stable ref for decompress: "b3" for a block, "m00350" for a message. */ + ref: string; + /** Text this doc is scored against (topic+summary for blocks; content for messages). */ + text: string; + /** For preview/title display. */ + title: string; + /** Message role (messages only); undefined for blocks. Drives role weighting. */ + role?: MessageRole; + /** Block owning this doc. For blocks: the block itself. For messages: the block + * that compressed it (so the model knows which block to decompress for detail). */ + blockId?: string; + /** Tier of the owning block (display + grouping). */ + tier?: number; + /** Approx token size (for "how big is this" display). */ + tokens?: number; } /** Per-role score multipliers. Defaults favor user intent over tool noise. */ export interface RoleWeights { - user?: number; - assistant?: number; - tool?: number; - block?: number; + user?: number; + assistant?: number; + tool?: number; + block?: number; } export const DEFAULT_ROLE_WEIGHTS: Required = { - user: 1.5, - assistant: 1.0, - tool: 0.6, - block: 1.0, + user: 1.5, + assistant: 1.0, + tool: 0.6, + block: 1.0, }; export interface ScoredBlock { - ref: string; - score: number; + ref: string; + score: number; } export interface SearchAlgorithm { - name: string; - description: string; - score(docs: SearchDoc[], query: string): ScoredBlock[]; + name: string; + description: string; + score(docs: SearchDoc[], query: string): ScoredBlock[]; } export interface AsyncSearchAlgorithm { - name: string; - description: string; - score(docs: SearchDoc[], query: string): Promise; + name: string; + description: string; + score(docs: SearchDoc[], query: string): Promise; } export type AnySearchAlgorithm = SearchAlgorithm | AsyncSearchAlgorithm; export interface SearchResult { - /** "block" or "message". */ - kind: SearchDocKind; - /** Ref to pass to decompress: "b3" or "m00350". */ - ref: string; - /** Owning block id (for messages: the block that compressed it). */ - blockId?: string; - tier: number; - score: number; - title: string; - preview: string; - role?: MessageRole; - tokens?: number; + /** "block" or "message". */ + kind: SearchDocKind; + /** Ref to pass to decompress: "b3" or "m00350". */ + ref: string; + /** Owning block id (for messages: the block that compressed it). */ + blockId?: string; + tier: number; + score: number; + title: string; + preview: string; + role?: MessageRole; + tokens?: number; } export interface SearchOptions { - algorithm?: string; - limit?: number; - previewLength?: number; - minScore?: number; - /** Per-role weights (default DEFAULT_ROLE_WEIGHTS). */ - roleWeights?: RoleWeights; + algorithm?: string; + limit?: number; + previewLength?: number; + minScore?: number; + /** Per-role weights (default DEFAULT_ROLE_WEIGHTS). */ + roleWeights?: RoleWeights; } /** Host-supplied historical message, turned into a message SearchDoc. */ export interface MessageInput { - ref: string; - role: MessageRole; - text: string; - tokens?: number; - /** Block id that compressed this message (undefined if still visible). */ - blockId?: string; - tier?: number; + ref: string; + role: MessageRole; + text: string; + tokens?: number; + /** Block id that compressed this message (undefined if still visible). */ + blockId?: string; + tier?: number; } export const DEFAULT_ALGORITHM = "hybrid"; diff --git a/src/surface-config.ts b/src/surface-config.ts index b037af1..bf6eb2f 100644 --- a/src/surface-config.ts +++ b/src/surface-config.ts @@ -83,10 +83,13 @@ export function cloneWithDescriptions( } function cloneNode(node: unknown, overrides: Record): unknown { - if (Array.isArray(node)) return node.map((item) => cloneNode(item, overrides)); + if (Array.isArray(node)) + return node.map((item) => cloneNode(item, overrides)); if (node && typeof node === "object") { const out: Record = {}; - for (const [name, value] of Object.entries(node as Record)) { + for (const [name, value] of Object.entries( + node as Record, + )) { const cloned = cloneNode(value, overrides); if ( Object.hasOwn(overrides, name) && @@ -135,7 +138,9 @@ export function applyAcpToolOverrides( ...tool, function: { ...tool.function, - ...(ov.description !== undefined ? { description: ov.description } : {}), + ...(ov.description !== undefined + ? { description: ov.description } + : {}), ...(ov.paramDescriptions ? { parameters: cloneWithDescriptions( @@ -154,7 +159,9 @@ export function applyAcpToolOverrides( ...(ov.description !== undefined ? { description: ov.description } : {}), ...(ov.paramDescriptions ? hasInputSchema - ? { input_schema: cloneWithDescriptions(schema, ov.paramDescriptions) } + ? { + input_schema: cloneWithDescriptions(schema, ov.paramDescriptions), + } : { parameters: cloneWithDescriptions(schema, ov.paramDescriptions) } : {}), } as T; diff --git a/src/tool-pairs.ts b/src/tool-pairs.ts index 7b53031..0c63a82 100644 --- a/src/tool-pairs.ts +++ b/src/tool-pairs.ts @@ -15,49 +15,53 @@ import type { CoreMessage } from "./types.js"; * @returns Adjusted { startIndex, endIndex } — may be wider than input. */ export function adjustBoundariesForToolPairs( - startIndex: number, - endIndex: number, - messages: CoreMessage[], - maxScan: number = 20, + startIndex: number, + endIndex: number, + messages: CoreMessage[], + maxScan: number = 20, ): { startIndex: number; endIndex: number } { - // Collect all toolCallIds in range (both tool-call and tool-result messages). - // Skip compress tool — it's force-protected and always survives pruning. - const callIdsInRange = new Set(); - for (let i = startIndex; i <= endIndex; i++) { - const msg = messages[i]; - if (!msg || !msg.toolCallId) continue; - if (msg.toolName === "compress") continue; - callIdsInRange.add(msg.toolCallId); - } + // Collect all toolCallIds in range (both tool-call and tool-result messages). + // Skip compress tool — it's force-protected and always survives pruning. + const callIdsInRange = new Set(); + for (let i = startIndex; i <= endIndex; i++) { + const msg = messages[i]; + if (!msg || !msg.toolCallId) continue; + if (msg.toolName === "compress") continue; + callIdsInRange.add(msg.toolCallId); + } - if (callIdsInRange.size === 0) { - return { startIndex, endIndex }; - } + if (callIdsInRange.size === 0) { + return { startIndex, endIndex }; + } - // Extend FORWARD: tool-results typically follow their tool-call. - // Stop at the first gap after finding at least one matching message. - let newEndIndex = endIndex; - for (let i = endIndex + 1; i < messages.length && i <= endIndex + maxScan; i++) { - const msg = messages[i]; - if (!msg) break; - if (msg.toolCallId && callIdsInRange.has(msg.toolCallId)) { - newEndIndex = i; - } else if (newEndIndex > endIndex) { - break; - } + // Extend FORWARD: tool-results typically follow their tool-call. + // Stop at the first gap after finding at least one matching message. + let newEndIndex = endIndex; + for ( + let i = endIndex + 1; + i < messages.length && i <= endIndex + maxScan; + i++ + ) { + const msg = messages[i]; + if (!msg) break; + if (msg.toolCallId && callIdsInRange.has(msg.toolCallId)) { + newEndIndex = i; + } else if (newEndIndex > endIndex) { + break; } + } - // Extend BACKWARD: tool-calls typically precede their tool-result. - let newStartIndex = startIndex; - for (let i = startIndex - 1; i >= 0 && i >= startIndex - maxScan; i--) { - const msg = messages[i]; - if (!msg) break; - if (msg.toolCallId && callIdsInRange.has(msg.toolCallId)) { - newStartIndex = i; - } else if (newStartIndex < startIndex) { - break; - } + // Extend BACKWARD: tool-calls typically precede their tool-result. + let newStartIndex = startIndex; + for (let i = startIndex - 1; i >= 0 && i >= startIndex - maxScan; i--) { + const msg = messages[i]; + if (!msg) break; + if (msg.toolCallId && callIdsInRange.has(msg.toolCallId)) { + newStartIndex = i; + } else if (newStartIndex < startIndex) { + break; } + } - return { startIndex: newStartIndex, endIndex: newEndIndex }; + return { startIndex: newStartIndex, endIndex: newEndIndex }; } diff --git a/src/truncate-tools.ts b/src/truncate-tools.ts index b701d4c..8881a40 100644 --- a/src/truncate-tools.ts +++ b/src/truncate-tools.ts @@ -3,119 +3,123 @@ import type { Config, CoreMessage } from "./types.js"; import { SUMMARY_HEADER, isRenderedSummaryMessage } from "./prune.js"; export interface TruncateOptions { - minOutputTokens?: number; - keepPrefixChars?: number; - keepSuffixChars?: number; - protectRecentMessages?: number; - /** Also consider user/assistant text messages as LAST-RESORT candidates, - * applied only after tool-result truncation fails to reach the target - * (#300). Rendered block summaries are never touched. */ - includeTextMessages?: boolean; + minOutputTokens?: number; + keepPrefixChars?: number; + keepSuffixChars?: number; + protectRecentMessages?: number; + /** Also consider user/assistant text messages as LAST-RESORT candidates, + * applied only after tool-result truncation fails to reach the target + * (#300). Rendered block summaries are never touched. */ + includeTextMessages?: boolean; } export interface TruncateResult { - messages: CoreMessage[]; - truncatedCount: number; - savedTokens: number; - /** Size-qualified candidates found (pre-truncation). 0 explains a silent - * no-op at/above threshold — hosts should warn-log (#300). */ - candidatesFound: number; + messages: CoreMessage[]; + truncatedCount: number; + savedTokens: number; + /** Size-qualified candidates found (pre-truncation). 0 explains a silent + * no-op at/above threshold — hosts should warn-log (#300). */ + candidatesFound: number; } const TRUNCATION_MARKER = "[truncated for context space]"; const DEFAULTS: Required> = { - minOutputTokens: 1000, - keepPrefixChars: 2000, - keepSuffixChars: 2000, - protectRecentMessages: 3, + minOutputTokens: 1000, + keepPrefixChars: 2000, + keepSuffixChars: 2000, + protectRecentMessages: 3, }; export function truncateLargeToolOutputs( - messages: CoreMessage[], - tokenCount: number, - config: Config, - countTokens: (text: string) => number, - options: TruncateOptions = {}, + messages: CoreMessage[], + tokenCount: number, + config: Config, + countTokens: (text: string) => number, + options: TruncateOptions = {}, ): TruncateResult { - const opts = { ...DEFAULTS, ...options }; - const limit = config.modelContextLimit; - if (limit <= 0 || tokenCount < config.truncate.threshold * limit) { - return { messages, truncatedCount: 0, savedTokens: 0, candidatesFound: 0 }; - } + const opts = { ...DEFAULTS, ...options }; + const limit = config.modelContextLimit; + if (limit <= 0 || tokenCount < config.truncate.threshold * limit) { + return { messages, truncatedCount: 0, savedTokens: 0, candidatesFound: 0 }; + } - const protectedIndex = messages.length - opts.protectRecentMessages; + const protectedIndex = messages.length - opts.protectRecentMessages; - const findCandidates = ( - predicate: (message: CoreMessage) => boolean, - ): CoreMessage[] => { - const found: CoreMessage[] = []; - for (let i = 0; i < protectedIndex; i++) { - const message = messages[i]!; - if (!predicate(message)) continue; - const text = message.text ?? ""; - if (text.length === 0 || text.includes(TRUNCATION_MARKER)) continue; - if (countTokens(text) < opts.minOutputTokens) continue; - found.push(message); - } - return found.sort( - (a, b) => countTokens(b.text ?? "") - countTokens(a.text ?? ""), - ); - }; + const findCandidates = ( + predicate: (message: CoreMessage) => boolean, + ): CoreMessage[] => { + const found: CoreMessage[] = []; + for (let i = 0; i < protectedIndex; i++) { + const message = messages[i]!; + if (!predicate(message)) continue; + const text = message.text ?? ""; + if (text.length === 0 || text.includes(TRUNCATION_MARKER)) continue; + if (countTokens(text) < opts.minOutputTokens) continue; + found.push(message); + } + return found.sort( + (a, b) => countTokens(b.text ?? "") - countTokens(a.text ?? ""), + ); + }; - const toolResults = findCandidates((m) => m.contentType === "tool-result"); - // Last-resort stage (#300): oversized visible TEXT (user requests, assistant - // prose) when tool outputs alone cannot close the gap. Role-restricted so - // system prompts / standing prompts stay intact; rendered block summaries - // ("[Compressed conversation section]") are NEVER touched — they are the - // only durable record of compressed content. - const textMessages = options.includeTextMessages - ? findCandidates( - (m) => - m.contentType === "text" && - (m.role === "user" || m.role === "assistant") && - !isRenderedSummaryMessage(m) && - !m.text?.startsWith(SUMMARY_HEADER), - ) - : []; - const candidatesFound = toolResults.length + textMessages.length; + const toolResults = findCandidates((m) => m.contentType === "tool-result"); + // Last-resort stage (#300): oversized visible TEXT (user requests, assistant + // prose) when tool outputs alone cannot close the gap. Role-restricted so + // system prompts / standing prompts stay intact; rendered block summaries + // ("[Compressed conversation section]") are NEVER touched — they are the + // only durable record of compressed content. + const textMessages = options.includeTextMessages + ? findCandidates( + (m) => + m.contentType === "text" && + (m.role === "user" || m.role === "assistant") && + !isRenderedSummaryMessage(m) && + !m.text?.startsWith(SUMMARY_HEADER), + ) + : []; + const candidatesFound = toolResults.length + textMessages.length; - let truncatedCount = 0; - let savedTokens = 0; - let remaining = tokenCount; - const targetTokens = config.truncate.threshold * limit * 0.9; - const replacements = new Map(); + let truncatedCount = 0; + let savedTokens = 0; + let remaining = tokenCount; + const targetTokens = config.truncate.threshold * limit * 0.9; + const replacements = new Map(); - const applyTo = (candidates: CoreMessage[]) => { - for (const candidate of candidates) { - if (remaining <= targetTokens) break; - const original = candidate.text ?? ""; - const tokens = countTokens(original); - if (original.length <= opts.keepPrefixChars + opts.keepSuffixChars) { - continue; - } - const prefix = clampPrefix(original, opts.keepPrefixChars); - const suffix = clampWindow(original, original.length - opts.keepSuffixChars, original.length); - const replacement = `${prefix}\n\n...${TRUNCATION_MARKER} — original ~${tokens} tokens]...\n\n${suffix}`; - replacements.set(candidate.id, replacement); - truncatedCount++; - remaining -= tokens - countTokens(replacement); - savedTokens += tokens - countTokens(replacement); - } - }; + const applyTo = (candidates: CoreMessage[]) => { + for (const candidate of candidates) { + if (remaining <= targetTokens) break; + const original = candidate.text ?? ""; + const tokens = countTokens(original); + if (original.length <= opts.keepPrefixChars + opts.keepSuffixChars) { + continue; + } + const prefix = clampPrefix(original, opts.keepPrefixChars); + const suffix = clampWindow( + original, + original.length - opts.keepSuffixChars, + original.length, + ); + const replacement = `${prefix}\n\n...${TRUNCATION_MARKER} — original ~${tokens} tokens]...\n\n${suffix}`; + replacements.set(candidate.id, replacement); + truncatedCount++; + remaining -= tokens - countTokens(replacement); + savedTokens += tokens - countTokens(replacement); + } + }; - applyTo(toolResults); - applyTo(textMessages); + applyTo(toolResults); + applyTo(textMessages); - if (replacements.size === 0) { - return { messages, truncatedCount: 0, savedTokens: 0, candidatesFound }; - } - return { - messages: messages.map((m) => - replacements.has(m.id) ? { ...m, text: replacements.get(m.id)! } : m, - ), - truncatedCount, - savedTokens, - candidatesFound, - }; + if (replacements.size === 0) { + return { messages, truncatedCount: 0, savedTokens: 0, candidatesFound }; + } + return { + messages: messages.map((m) => + replacements.has(m.id) ? { ...m, text: replacements.get(m.id)! } : m, + ), + truncatedCount, + savedTokens, + candidatesFound, + }; } diff --git a/src/truncate.ts b/src/truncate.ts index 7cf7c3a..12436a9 100644 --- a/src/truncate.ts +++ b/src/truncate.ts @@ -5,25 +5,26 @@ // (ranxianglei/billion-context#816) — a deterministic poison that never heals. function isHighSurrogate(c: number): boolean { - return c >= 0xd800 && c <= 0xdbff; + return c >= 0xd800 && c <= 0xdbff; } function isLowSurrogate(c: number): boolean { - return c >= 0xdc00 && c <= 0xdfff; + return c >= 0xdc00 && c <= 0xdfff; } /** At most maxUnits code units; never ends on a stranded high surrogate. */ export function clampPrefix(text: string, maxUnits: number): string { - const cut = Math.min(maxUnits, text.length); - if (cut > 0 && isHighSurrogate(text.charCodeAt(cut - 1))) return text.slice(0, cut - 1); - return text.slice(0, cut); + const cut = Math.min(maxUnits, text.length); + if (cut > 0 && isHighSurrogate(text.charCodeAt(cut - 1))) + return text.slice(0, cut - 1); + return text.slice(0, cut); } /** A [start, end) window with both edges snapped off surrogate pairs. */ export function clampWindow(text: string, start: number, end: number): string { - let s = Math.max(0, Math.min(start, text.length)); - let e = Math.min(text.length, Math.max(s, end)); - if (s > 0 && isLowSurrogate(text.charCodeAt(s))) s += 1; - if (e > s && isHighSurrogate(text.charCodeAt(e - 1))) e -= 1; - return text.slice(s, Math.max(s, e)); + let s = Math.max(0, Math.min(start, text.length)); + let e = Math.min(text.length, Math.max(s, end)); + if (s > 0 && isLowSurrogate(text.charCodeAt(s))) s += 1; + if (e > s && isHighSurrogate(text.charCodeAt(e - 1))) e -= 1; + return text.slice(s, Math.max(s, e)); } diff --git a/src/turn-integrity.ts b/src/turn-integrity.ts index 88dc14e..9615f0f 100644 --- a/src/turn-integrity.ts +++ b/src/turn-integrity.ts @@ -72,7 +72,10 @@ export function computeTurnGroups(messages: CoreMessage[]): string[][] { s++; return s; })(); - if (burstStart >= messages.length || !isAssistantAct(messages[burstStart]!)) { + if ( + burstStart >= messages.length || + !isAssistantAct(messages[burstStart]!) + ) { // Orphan reasoning run (no companion burst): no pairing constraint. continue; } diff --git a/src/wire/anthropic.ts b/src/wire/anthropic.ts index b571246..4e166bd 100644 --- a/src/wire/anthropic.ts +++ b/src/wire/anthropic.ts @@ -2,212 +2,249 @@ import type { BiliMessage } from "./bili-message.js"; import { hashId } from "./util.js"; import { ClusterCounter, deriveMessageId } from "./message-id.js"; -export type AnthropicTextBlock = { type: "text"; text: string; cache_control?: unknown }; +export type AnthropicTextBlock = { + type: "text"; + text: string; + cache_control?: unknown; +}; export type AnthropicToolUse = { - type: "tool_use"; - id: string; - name: string; - input: unknown; - cache_control?: unknown; + type: "tool_use"; + id: string; + name: string; + input: unknown; + cache_control?: unknown; }; export type AnthropicToolResult = { - type: "tool_result"; - tool_use_id: string; - content: string | (AnthropicTextBlock | AnthropicImage)[]; - is_error?: boolean; - cache_control?: unknown; + type: "tool_result"; + tool_use_id: string; + content: string | (AnthropicTextBlock | AnthropicImage)[]; + is_error?: boolean; + cache_control?: unknown; }; export type AnthropicImage = { type: "image"; source: unknown }; -export type AnthropicThinking = { type: "thinking"; thinking: string; signature?: string }; +export type AnthropicThinking = { + type: "thinking"; + thinking: string; + signature?: string; +}; export type AnthropicBlock = - | AnthropicTextBlock - | AnthropicToolUse - | AnthropicToolResult - | AnthropicImage - | AnthropicThinking; + | AnthropicTextBlock + | AnthropicToolUse + | AnthropicToolResult + | AnthropicImage + | AnthropicThinking; export type AnthropicMessage = { - role: "user" | "assistant"; - content: string | AnthropicBlock[]; + role: "user" | "assistant"; + content: string | AnthropicBlock[]; }; export type AnthropicRequestBody = { - model?: string; - max_tokens?: number; - system?: string | AnthropicTextBlock[]; - messages: AnthropicMessage[]; - tools?: unknown[]; - stream?: boolean; - temperature?: number; - [key: string]: unknown; + model?: string; + max_tokens?: number; + system?: string | AnthropicTextBlock[]; + messages: AnthropicMessage[]; + tools?: unknown[]; + stream?: boolean; + temperature?: number; + [key: string]: unknown; }; - export function extractSystem(system: AnthropicRequestBody["system"]): string { - if (!system) return ""; - if (typeof system === "string") return system; - return system.map((b) => b.text).join("\n\n"); + if (!system) return ""; + if (typeof system === "string") return system; + return system.map((b) => b.text).join("\n\n"); } -export function buildSystem(text: string, original: AnthropicRequestBody["system"]): string | AnthropicTextBlock[] { - if (Array.isArray(original) && original.length > 0) { - const ccBlock = original.find((b) => b.cache_control); - return [{ type: "text", text, ...(ccBlock ? { cache_control: ccBlock.cache_control } : {}) }]; - } - return text; +export function buildSystem( + text: string, + original: AnthropicRequestBody["system"], +): string | AnthropicTextBlock[] { + if (Array.isArray(original) && original.length > 0) { + const ccBlock = original.find((b) => b.cache_control); + return [ + { + type: "text", + text, + ...(ccBlock ? { cache_control: ccBlock.cache_control } : {}), + }, + ]; + } + return text; } type Flat = { msgs: BiliMessage[]; cacheControls: Map }; export function anthropicToCore(body: AnthropicRequestBody): Flat { - const msgs: BiliMessage[] = []; - const cacheControls = new Map(); - const clusters = new ClusterCounter(); - // Backfill toolName onto tool-results from their paired tool_use: the wire - // format only carries the call id on results, but kernel guards - // (excludeTools / protectedTools / isNeverPreserveRecent) are name-based. - // Not part of the result identity seed — ids must stay stable across versions. - const toolNames = new Map(); - for (const m of body.messages) { - const blocks = typeof m.content === "string" ? [{ type: "text" as const, text: m.content }] : m.content; - for (const b of blocks) { - switch (b.type) { - case "text": { - const base = deriveMessageId(m.role, "text", b.text); - const id = clusters.next(base); - msgs.push({ id, role: m.role, contentType: "text", text: b.text }); - if (b.cache_control) cacheControls.set(id, b.cache_control); - break; - } - case "tool_use": { - const base = deriveMessageId("assistant", "tool-call", safeStringify(b.input), { - toolCallId: b.id, - toolName: b.name, - }); - const id = clusters.next(base); - msgs.push({ - id, - role: "assistant", - contentType: "tool-call", - toolName: b.name, - toolCallId: b.id, - text: safeStringify(b.input), - }); - toolNames.set(b.id, b.name); - if (b.cache_control) cacheControls.set(id, b.cache_control); - break; - } - case "tool_result": { - // Non-text blocks (image) carry bytes outside msg.text; ride - // the sidecar so coreToAnthropic re-emits them verbatim (#366). - const hasNonText = Array.isArray(b.content) && b.content.some((c) => c.type !== "text"); - const text = typeof b.content === "string" ? b.content : b.content.map((c) => (c.type === "text" ? c.text : "")).join("\n"); - const base = deriveMessageId("tool", "tool-result", text, { toolCallId: b.tool_use_id }); - const id = clusters.next(base); - const name = toolNames.get(b.tool_use_id); - msgs.push({ - id, - role: "tool", - contentType: "tool-result", - toolCallId: b.tool_use_id, - text, - ...(name ? { toolName: name } : {}), - ...(b.is_error === true ? { toolIsError: true } : {}), - ...(hasNonText ? { rawAnthropicBlock: b } : {}), - }); - if (b.cache_control) cacheControls.set(id, b.cache_control); - break; - } - case "thinking": { - const base = deriveMessageId("assistant", "reasoning", b.thinking); - msgs.push({ - id: clusters.next(base), - role: "assistant", - contentType: "reasoning", - text: b.thinking, - ...(b.signature ? { thinkingSignature: b.signature } : {}), - }); - break; - } - case "image": { - const base = deriveMessageId(m.role, "text", "[image]"); - msgs.push({ - id: clusters.next(base), - role: m.role, - contentType: "text", - text: "[image]", - rawAnthropicBlock: b, - }); - break; - } - } + const msgs: BiliMessage[] = []; + const cacheControls = new Map(); + const clusters = new ClusterCounter(); + // Backfill toolName onto tool-results from their paired tool_use: the wire + // format only carries the call id on results, but kernel guards + // (excludeTools / protectedTools / isNeverPreserveRecent) are name-based. + // Not part of the result identity seed — ids must stay stable across versions. + const toolNames = new Map(); + for (const m of body.messages) { + const blocks = + typeof m.content === "string" + ? [{ type: "text" as const, text: m.content }] + : m.content; + for (const b of blocks) { + switch (b.type) { + case "text": { + const base = deriveMessageId(m.role, "text", b.text); + const id = clusters.next(base); + msgs.push({ id, role: m.role, contentType: "text", text: b.text }); + if (b.cache_control) cacheControls.set(id, b.cache_control); + break; + } + case "tool_use": { + const base = deriveMessageId( + "assistant", + "tool-call", + safeStringify(b.input), + { + toolCallId: b.id, + toolName: b.name, + }, + ); + const id = clusters.next(base); + msgs.push({ + id, + role: "assistant", + contentType: "tool-call", + toolName: b.name, + toolCallId: b.id, + text: safeStringify(b.input), + }); + toolNames.set(b.id, b.name); + if (b.cache_control) cacheControls.set(id, b.cache_control); + break; + } + case "tool_result": { + // Non-text blocks (image) carry bytes outside msg.text; ride + // the sidecar so coreToAnthropic re-emits them verbatim (#366). + const hasNonText = + Array.isArray(b.content) && + b.content.some((c) => c.type !== "text"); + const text = + typeof b.content === "string" + ? b.content + : b.content + .map((c) => (c.type === "text" ? c.text : "")) + .join("\n"); + const base = deriveMessageId("tool", "tool-result", text, { + toolCallId: b.tool_use_id, + }); + const id = clusters.next(base); + const name = toolNames.get(b.tool_use_id); + msgs.push({ + id, + role: "tool", + contentType: "tool-result", + toolCallId: b.tool_use_id, + text, + ...(name ? { toolName: name } : {}), + ...(b.is_error === true ? { toolIsError: true } : {}), + ...(hasNonText ? { rawAnthropicBlock: b } : {}), + }); + if (b.cache_control) cacheControls.set(id, b.cache_control); + break; + } + case "thinking": { + const base = deriveMessageId("assistant", "reasoning", b.thinking); + msgs.push({ + id: clusters.next(base), + role: "assistant", + contentType: "reasoning", + text: b.thinking, + ...(b.signature ? { thinkingSignature: b.signature } : {}), + }); + break; } + case "image": { + const base = deriveMessageId(m.role, "text", "[image]"); + msgs.push({ + id: clusters.next(base), + role: m.role, + contentType: "text", + text: "[image]", + rawAnthropicBlock: b, + }); + break; + } + } } - return { msgs, cacheControls }; + } + return { msgs, cacheControls }; } -export function coreToAnthropic(messages: BiliMessage[], cacheControls?: Map): AnthropicMessage[] { - const out: AnthropicMessage[] = []; - let current: { role: "user" | "assistant"; blocks: AnthropicBlock[] } | null = null; - const flush = () => { - if (current && current.blocks.length > 0) { - out.push({ role: current.role, content: current.blocks }); - } - current = null; - }; - const cc = (id: string): { cache_control?: unknown } => { - const v = cacheControls?.get(id); - return v ? { cache_control: v } : {}; - }; - for (const m of messages) { - const target: "user" | "assistant" = - m.role === "assistant" ? "assistant" : "user"; - if (!current || current.role !== target) { - flush(); - current = { role: target, blocks: [] }; +export function coreToAnthropic( + messages: BiliMessage[], + cacheControls?: Map, +): AnthropicMessage[] { + const out: AnthropicMessage[] = []; + let current: { role: "user" | "assistant"; blocks: AnthropicBlock[] } | null = + null; + const flush = () => { + if (current && current.blocks.length > 0) { + out.push({ role: current.role, content: current.blocks }); + } + current = null; + }; + const cc = (id: string): { cache_control?: unknown } => { + const v = cacheControls?.get(id); + return v ? { cache_control: v } : {}; + }; + for (const m of messages) { + const target: "user" | "assistant" = + m.role === "assistant" ? "assistant" : "user"; + if (!current || current.role !== target) { + flush(); + current = { role: target, blocks: [] }; + } + switch (m.contentType) { + case "text": { + if (m.rawAnthropicBlock) { + current.blocks.push(m.rawAnthropicBlock as AnthropicBlock); + break; } - switch (m.contentType) { - case "text": { - if (m.rawAnthropicBlock) { - current.blocks.push(m.rawAnthropicBlock as AnthropicBlock); - break; - } - current.blocks.push({ type: "text", text: m.text ?? "", ...cc(m.id) }); - break; - } - case "tool-call": - current.blocks.push({ - type: "tool_use", - id: m.toolCallId ?? `call_${m.id}`, - name: m.toolName ?? "unknown", - input: safeParse(m.text), - ...cc(m.id), - }); - break; - case "tool-result": - if (m.rawAnthropicBlock) { - current.blocks.push(m.rawAnthropicBlock as AnthropicBlock); - break; - } - current.blocks.push({ - type: "tool_result", - tool_use_id: m.toolCallId ?? "", - content: m.text ?? "", - ...(m.toolIsError ? { is_error: true } : {}), - ...cc(m.id), - }); - break; - case "reasoning": - current.blocks.push({ - type: "thinking", - thinking: m.text ?? "", - ...(m.thinkingSignature ? { signature: m.thinkingSignature } : {}), - }); - break; + current.blocks.push({ type: "text", text: m.text ?? "", ...cc(m.id) }); + break; + } + case "tool-call": + current.blocks.push({ + type: "tool_use", + id: m.toolCallId ?? `call_${m.id}`, + name: m.toolName ?? "unknown", + input: safeParse(m.text), + ...cc(m.id), + }); + break; + case "tool-result": + if (m.rawAnthropicBlock) { + current.blocks.push(m.rawAnthropicBlock as AnthropicBlock); + break; } + current.blocks.push({ + type: "tool_result", + tool_use_id: m.toolCallId ?? "", + content: m.text ?? "", + ...(m.toolIsError ? { is_error: true } : {}), + ...cc(m.id), + }); + break; + case "reasoning": + current.blocks.push({ + type: "thinking", + thinking: m.text ?? "", + ...(m.thinkingSignature ? { signature: m.thinkingSignature } : {}), + }); + break; } - flush(); - return out; + } + flush(); + return out; } /** Extract the conversation dimension for Anthropic: a client-provided @@ -215,27 +252,30 @@ export function coreToAnthropic(messages: BiliMessage[], cacheControls?: Map m?.role === "user"); - const seed = firstUser ? JSON.stringify(firstUser.content) : "default"; - return hashId(seed); +export function conversationSignalAnthropic( + body: AnthropicRequestBody, + headerValue?: string, +): string { + if (headerValue && headerValue.trim()) return headerValue.trim(); + const msgs = Array.isArray(body?.messages) ? body.messages : []; + const firstUser = msgs.find((m) => m?.role === "user"); + const seed = firstUser ? JSON.stringify(firstUser.content) : "default"; + return hashId(seed); } function safeStringify(v: unknown): string { - try { - return JSON.stringify(v ?? {}); - } catch { - return "{}"; - } + try { + return JSON.stringify(v ?? {}); + } catch { + return "{}"; + } } function safeParse(s: string | undefined): unknown { - if (!s) return {}; - try { - return JSON.parse(s); - } catch { - return {}; - } + if (!s) return {}; + try { + return JSON.parse(s); + } catch { + return {}; + } } diff --git a/src/wire/bili-message.ts b/src/wire/bili-message.ts index dfb96d4..273a300 100644 --- a/src/wire/bili-message.ts +++ b/src/wire/bili-message.ts @@ -24,75 +24,77 @@ import type { CoreMessage } from "acp-kernel"; * reverse conversion can reconstruct losslessly. Every field is optional — * plain text messages (the common case) have none set. */ export interface BiliMessage extends CoreMessage { - /** Anthropic: the original content block for an image or a structured - * tool_result. Restored verbatim by coreToAnthropic. */ - rawAnthropicBlock?: unknown; - /** OpenAI chat: the original content part for an image_url (data: or - * remote URL) or the original message object for a developer-role - * message. */ - rawOpenaiContent?: unknown; - /** Responses API: the original input item (for input_image, or a raw - * function_call / function_call_output we pass through). */ - rawResponsesItem?: unknown; - /** Google/Gemini: the original content parts of this message, in wire - * order (inlineData/fileData, executableCode, functionCall/Response ids, - * thought parts and unknown future part types). coreToGoogle rebuilds - * the content from these when present, so parts the core does not model - * survive a round-trip. */ - rawGoogleParts?: unknown[]; - /** Anthropic thinking signature. Anthropic verifies thinking+signature - * pairs; without it the request is rejected. Stored alongside the - * reasoning text so coreToAnthropic can reattach it. */ - thinkingSignature?: string; - /** Google/Gemini thought signature (thoughtSignature). Gemini 3 validates - * the signature of every replayed thought part, so a reasoning core - * carries it here for coreToGoogle to reattach. */ - googleThoughtSignature?: string; - /** OpenAI reasoning_content (chain-of-thought from DeepSeek-R1, GLM-4.6 - * thinking, Qwen-QwQ). These models require reasoning_content be echoed - * back on subsequent requests or the API returns HTTP 400; stored so - * coreToOpenai can reattach it. */ - reasoningContent?: string; - /** OpenAI: the host's wire carried `reasoning_content` with a BLANK value. - * Strict-echo thinking models (DeepSeek et al.) reject a replayed - * assistant turn whose reasoning_content is MISSING but accept a blank - * one, so "blank" and "absent" are different requests. openaiToCore sets - * this when it found no reasoning text to store; coreToOpenai re-emits - * the key (blank) instead of dropping it. */ - reasoningPresent?: boolean; - /** Anthropic tool_result.is_error. Marks the tool result as an error so - * the model knows the tool failed (not just returned an error string). */ - toolIsError?: boolean; - /** OpenAI: original role was "developer" (reconstructed as "system" by - * openaiToCore for the kernel; coreToOpenai restores "developer"). */ - originalRole?: "system" | "developer"; - /** The original media type for an image (image/png, image/jpeg, image/gif, - * image/webp). Lets coreToOpenai/coreToResponses rebuild image_url / - * input_image with the right data URL. */ - imageMediaType?: string; - /** The base64 data of an image (without the data: prefix). Lets the - * reverse conversion rebuild the full image payload. */ - imageBase64?: string; - /** OpenAI chat: ALL original image_url content parts (data: or remote - * URL) for a user message carrying more than one image, in wire order. - * coreToOpenai re-emits these verbatim (after the text part). The - * singular `rawOpenaiContent` still covers the single-image case and - * legacy persisted state. Typed as unknown[] (not OpenAIContentPart) to - * avoid a circular import with the openai codec. */ - rawOpenaiContentParts?: unknown[]; + /** Anthropic: the original content block for an image or a structured + * tool_result. Restored verbatim by coreToAnthropic. */ + rawAnthropicBlock?: unknown; + /** OpenAI chat: the original content part for an image_url (data: or + * remote URL) or the original message object for a developer-role + * message. */ + rawOpenaiContent?: unknown; + /** Responses API: the original input item (for input_image, or a raw + * function_call / function_call_output we pass through). */ + rawResponsesItem?: unknown; + /** Google/Gemini: the original content parts of this message, in wire + * order (inlineData/fileData, executableCode, functionCall/Response ids, + * thought parts and unknown future part types). coreToGoogle rebuilds + * the content from these when present, so parts the core does not model + * survive a round-trip. */ + rawGoogleParts?: unknown[]; + /** Anthropic thinking signature. Anthropic verifies thinking+signature + * pairs; without it the request is rejected. Stored alongside the + * reasoning text so coreToAnthropic can reattach it. */ + thinkingSignature?: string; + /** Google/Gemini thought signature (thoughtSignature). Gemini 3 validates + * the signature of every replayed thought part, so a reasoning core + * carries it here for coreToGoogle to reattach. */ + googleThoughtSignature?: string; + /** OpenAI reasoning_content (chain-of-thought from DeepSeek-R1, GLM-4.6 + * thinking, Qwen-QwQ). These models require reasoning_content be echoed + * back on subsequent requests or the API returns HTTP 400; stored so + * coreToOpenai can reattach it. */ + reasoningContent?: string; + /** OpenAI: the host's wire carried `reasoning_content` with a BLANK value. + * Strict-echo thinking models (DeepSeek et al.) reject a replayed + * assistant turn whose reasoning_content is MISSING but accept a blank + * one, so "blank" and "absent" are different requests. openaiToCore sets + * this when it found no reasoning text to store; coreToOpenai re-emits + * the key (blank) instead of dropping it. */ + reasoningPresent?: boolean; + /** Anthropic tool_result.is_error. Marks the tool result as an error so + * the model knows the tool failed (not just returned an error string). */ + toolIsError?: boolean; + /** OpenAI: original role was "developer" (reconstructed as "system" by + * openaiToCore for the kernel; coreToOpenai restores "developer"). */ + originalRole?: "system" | "developer"; + /** The original media type for an image (image/png, image/jpeg, image/gif, + * image/webp). Lets coreToOpenai/coreToResponses rebuild image_url / + * input_image with the right data URL. */ + imageMediaType?: string; + /** The base64 data of an image (without the data: prefix). Lets the + * reverse conversion rebuild the full image payload. */ + imageBase64?: string; + /** OpenAI chat: ALL original image_url content parts (data: or remote + * URL) for a user message carrying more than one image, in wire order. + * coreToOpenai re-emits these verbatim (after the text part). The + * singular `rawOpenaiContent` still covers the single-image case and + * legacy persisted state. Typed as unknown[] (not OpenAIContentPart) to + * avoid a circular import with the openai codec. */ + rawOpenaiContentParts?: unknown[]; } /** Narrow a BiliMessage[] to CoreMessage[] for the kernel. The sidecar fields * are transparently carried along — the kernel's `{ ...msg }` copies them. */ export function toCoreMessages(msgs: BiliMessage[]): CoreMessage[] { - return msgs as CoreMessage[]; + return msgs as CoreMessage[]; } /** Re-decode a base64 data URL into media type + data. Returns undefined if * the input is not a recognized data URL. Used by openaiToCore/responsesToCore * to split image_url/input_image into the sidecar fields. */ -export function parseDataUrl(url: string): { mediaType: string; base64: string } | undefined { - const m = /^data:([^;,]+)(?:;base64)?,(.+)$/i.exec(url); - if (!m) return undefined; - return { mediaType: m[1]!, base64: m[2]! }; +export function parseDataUrl( + url: string, +): { mediaType: string; base64: string } | undefined { + const m = /^data:([^;,]+)(?:;base64)?,(.+)$/i.exec(url); + if (!m) return undefined; + return { mediaType: m[1]!, base64: m[2]! }; } diff --git a/src/wire/compress-detect.ts b/src/wire/compress-detect.ts index 7450a3e..52f047e 100644 --- a/src/wire/compress-detect.ts +++ b/src/wire/compress-detect.ts @@ -50,24 +50,45 @@ export function compressToolArgs(call: { name: string; arguments?: unknown }): { } | null { let args = call.arguments; if (typeof args === "string") { - try { args = JSON.parse(args); } catch { return null; } + try { + args = JSON.parse(args); + } catch { + return null; + } } if (!args || typeof args !== "object" || Array.isArray(args)) return null; const a = args as Record; if (call.name === "compress") { - return Array.isArray(a.content) ? { content: a.content, topic: a.topic, summaryMaxChars: a.summaryMaxChars } : null; + return Array.isArray(a.content) + ? { + content: a.content, + topic: a.topic, + summaryMaxChars: a.summaryMaxChars, + } + : null; } if (call.name !== "write") return null; - const path = typeof a.path === "string" ? a.path.split("?")[0]!.replace(/\/+$/, "") : ""; + const path = + typeof a.path === "string" ? a.path.split("?")[0]!.replace(/\/+$/, "") : ""; if (path !== "xd://compress") return null; let inner: unknown = a.content; if (typeof inner === "string") { - try { inner = JSON.parse(inner); } catch { return null; } + try { + inner = JSON.parse(inner); + } catch { + return null; + } } if (!inner || typeof inner !== "object") return null; if (Array.isArray(inner)) return { content: inner }; const ia = inner as Record; - return Array.isArray(ia.content) ? { content: ia.content, topic: ia.topic, summaryMaxChars: ia.summaryMaxChars } : { content: [ia] }; + return Array.isArray(ia.content) + ? { + content: ia.content, + topic: ia.topic, + summaryMaxChars: ia.summaryMaxChars, + } + : { content: [ia] }; } /** toolCallId → toolName for every tool-call piece (protected-piece @@ -75,7 +96,8 @@ export function compressToolArgs(call: { name: string; arguments?: unknown }): { export function toolCallNames(msgs: BiliMessage[]): Map { const names = new Map(); for (const m of msgs) { - if (m.contentType === "tool-call" && m.toolCallId && m.toolName) names.set(m.toolCallId, m.toolName); + if (m.contentType === "tool-call" && m.toolCallId && m.toolName) + names.set(m.toolCallId, m.toolName); } return names; } @@ -103,14 +125,28 @@ export function findCompressCallsCore(msg: BiliMessage): StreamCompressCall[] { const ranges: StreamCompressCall["ranges"] = []; const callTopic = typeof args.topic === "string" ? args.topic : undefined; for (const item of content) { - const r = item as { startId?: unknown; endId?: unknown; summary?: unknown; topic?: unknown }; - if (typeof r.startId !== "string" || typeof r.endId !== "string" || typeof r.summary !== "string" || r.summary.length === 0) continue; + const r = item as { + startId?: unknown; + endId?: unknown; + summary?: unknown; + topic?: unknown; + }; + if ( + typeof r.startId !== "string" || + typeof r.endId !== "string" || + typeof r.summary !== "string" || + r.summary.length === 0 + ) + continue; ranges.push({ startRef: r.startId, endRef: r.endId, summary: r.summary, topic: typeof r.topic === "string" ? r.topic : callTopic, - summaryMaxChars: typeof args.summaryMaxChars === "number" ? args.summaryMaxChars : undefined, + summaryMaxChars: + typeof args.summaryMaxChars === "number" + ? args.summaryMaxChars + : undefined, compressCallId: msg.toolCallId ?? "", }); } @@ -130,12 +166,20 @@ export function corePieceKey(cm: CoreMessage): string { * exact first/last covered pieces. Boundary ids are pre-resolved (byRef / * block lookup) — unlike the pN-space spanFingerprint there is no position * parsing, ids are unique per piece. */ -export function spanFingerprintCore(coreMessages: CoreMessage[], startId: string, endId: string): string { - const find = (id: string): CoreMessage | undefined => coreMessages.find((cm) => cm.id === id); +export function spanFingerprintCore( + coreMessages: CoreMessage[], + startId: string, + endId: string, +): string { + const find = (id: string): CoreMessage | undefined => + coreMessages.find((cm) => cm.id === id); const first = find(startId); const last = find(endId); if (!first || !last) return ""; - return createHash("sha1").update(`${corePieceKey(first)}\u0000${corePieceKey(last)}`).digest("hex").slice(0, 8); + return createHash("sha1") + .update(`${corePieceKey(first)}\u0000${corePieceKey(last)}`) + .digest("hex") + .slice(0, 8); } /** Index-based span fingerprint (issue #91 replay fallback): hash the content @@ -143,11 +187,18 @@ export function spanFingerprintCore(coreMessages: CoreMessage[], startId: string * decides keep/drop — the position is only a recovery hint for a drifted * boundary whose content-hash id no longer matches, so a benign tail drift * (first-4096 intact) is kept while a real rewrite mismatches. */ -export function spanFingerprintCoreIdx(coreMessages: CoreMessage[], startIdx: number, endIdx: number): string { +export function spanFingerprintCoreIdx( + coreMessages: CoreMessage[], + startIdx: number, + endIdx: number, +): string { const first = coreMessages[startIdx]; const last = coreMessages[endIdx]; if (!first || !last) return ""; - return createHash("sha1").update(`${corePieceKey(first)}\u0000${corePieceKey(last)}`).digest("hex").slice(0, 8); + return createHash("sha1") + .update(`${corePieceKey(first)}\u0000${corePieceKey(last)}`) + .digest("hex") + .slice(0, 8); } /** Structural subset the boundary resolvers need from a compression block @@ -175,7 +226,8 @@ export function boundaryRawCore( if (!m) return ""; const block = blocks.find((b) => b.blockId.toLowerCase() === `b${m[1]}`); if (!block) return ""; - const idx = (id: string): number => coreMessages.findIndex((cm) => cm.id === (byRef[id] ?? id)); + const idx = (id: string): number => + coreMessages.findIndex((cm) => cm.id === (byRef[id] ?? id)); let best = -1; for (const id of block.effectiveMessageIds) { const i = idx(id); @@ -203,7 +255,9 @@ export function boundaryIndexCore( const i = coreMessages.findIndex((cm) => cm.id === id); if (i >= 0) return i; } - return fallbackIdx >= 0 && fallbackIdx < coreMessages.length ? fallbackIdx : -1; + return fallbackIdx >= 0 && fallbackIdx < coreMessages.length + ? fallbackIdx + : -1; } /** Structured replay-guard verdict (issue #91, rework): the position @@ -228,10 +282,15 @@ export type ReplayRangeVerdict = { * "" when the piece has no ref (protected) — the replay must fail closed * rather than hand the kernel a ref it does not know. Replay-time only * (replayed compress calls), so the O(refs) scan stays off the hot path. */ -export function refOfPieceCore(coreMessages: BiliMessage[], idx: number, byRef: Record): string { +export function refOfPieceCore( + coreMessages: BiliMessage[], + idx: number, + byRef: Record, +): string { const id = coreMessages[idx]?.id; if (!id) return ""; - for (const [ref, mapped] of Object.entries(byRef)) if (mapped === id) return ref; + for (const [ref, mapped] of Object.entries(byRef)) + if (mapped === id) return ref; return ""; } @@ -249,7 +308,7 @@ export function staleRangeCore( // boundary piece dangles its carried ref — the position recovers it, and // the fingerprint below still decides keep/drop. const pm = resultText.match(/\[pos=([0-9,-]+)\]/); - const pair = pm ? pm[1]!.split(",")[rangeIndex] ?? "-" : "-"; + const pair = pm ? (pm[1]!.split(",")[rangeIndex] ?? "-") : "-"; const hinted = pair !== "-"; const [ps, pe] = pair === "-" ? ["", ""] : pair.split("-"); const fbStart = ps && ps !== "" ? Number.parseInt(ps, 10) : -1; @@ -257,26 +316,57 @@ export function staleRangeCore( // Raw resolution (id → stream index) separately from the fallback, so a // dangling m-ref recovered by position can be flagged for remapping. - const startRaw = boundaryRawCore(r.startRef, byRef, blocks, coreMessages, "min"); + const startRaw = boundaryRawCore( + r.startRef, + byRef, + blocks, + coreMessages, + "min", + ); const endRaw = boundaryRawCore(r.endRef, byRef, blocks, coreMessages, "max"); - const rawStartIdx = startRaw ? coreMessages.findIndex((cm) => cm.id === startRaw) : -1; - const rawEndIdx = endRaw ? coreMessages.findIndex((cm) => cm.id === endRaw) : -1; - const startIdx = rawStartIdx >= 0 ? rawStartIdx : fbStart >= 0 && fbStart < coreMessages.length ? fbStart : -1; - const endIdx = rawEndIdx >= 0 ? rawEndIdx : fbEnd >= 0 && fbEnd < coreMessages.length ? fbEnd : -1; + const rawStartIdx = startRaw + ? coreMessages.findIndex((cm) => cm.id === startRaw) + : -1; + const rawEndIdx = endRaw + ? coreMessages.findIndex((cm) => cm.id === endRaw) + : -1; + const startIdx = + rawStartIdx >= 0 + ? rawStartIdx + : fbStart >= 0 && fbStart < coreMessages.length + ? fbStart + : -1; + const endIdx = + rawEndIdx >= 0 + ? rawEndIdx + : fbEnd >= 0 && fbEnd < coreMessages.length + ? fbEnd + : -1; if (startIdx < 0 || endIdx < 0) { if (!/^b\d+$/i.test(r.startRef.trim()) && !/^b\d+$/i.test(r.endRef.trim())) - return { reject: `unresolved ${r.startRef}..${r.endRef} -> ${startIdx}..${endIdx}`, ...(hinted ? { hint: true } : {}) }; + return { + reject: `unresolved ${r.startRef}..${r.endRef} -> ${startIdx}..${endIdx}`, + ...(hinted ? { hint: true } : {}), + }; return {}; // block ref(s): the kernel resolves them itself (master) } // The end piece must precede the call that issued it — a rewrite moved // the call and the fingerprint check below is meaningless either way. - if (endIdx > callIndex) return { reject: `end idx ${endIdx} > callIndex ${callIndex}`, ...(hinted ? { hint: true } : {}) }; + if (endIdx > callIndex) + return { + reject: `end idx ${endIdx} > callIndex ${callIndex}`, + ...(hinted ? { hint: true } : {}), + }; const m = resultText.match(/\[fp=([0-9a-f,-]+)\]/); if (m) { const want = m[1]!.split(",")[rangeIndex]; if (want !== undefined && want !== "-") { const got = spanFingerprintCoreIdx(coreMessages, startIdx, endIdx); - if (want !== got) return { reject: `fp ${r.startRef}..${r.endRef} want ${want} got ${got} @${startIdx}..${endIdx}`, ...(hinted ? { hint: true } : {}) }; + if (want !== got) + return { + reject: `fp ${r.startRef}..${r.endRef} want ${want} got ${got} @${startIdx}..${endIdx}`, + ...(hinted ? { hint: true } : {}), + }; } } // Remap only the boundaries that actually dangled (m-refs whose recorded id @@ -285,12 +375,20 @@ export function staleRangeCore( const remap: { startRef?: string; endRef?: string } = {}; if (/^m\d+$/i.test(r.startRef.trim()) && rawStartIdx < 0) { const ref = refOfPieceCore(coreMessages, startIdx, byRef); - if (!ref) return { reject: `recovered ${r.startRef} @${startIdx} has no ref (protected piece)`, ...(hinted ? { hint: true } : {}) }; + if (!ref) + return { + reject: `recovered ${r.startRef} @${startIdx} has no ref (protected piece)`, + ...(hinted ? { hint: true } : {}), + }; remap.startRef = ref; } if (/^m\d+$/i.test(r.endRef.trim()) && rawEndIdx < 0) { const ref = refOfPieceCore(coreMessages, endIdx, byRef); - if (!ref) return { reject: `recovered ${r.endRef} @${endIdx} has no ref (protected piece)`, ...(hinted ? { hint: true } : {}) }; + if (!ref) + return { + reject: `recovered ${r.endRef} @${endIdx} has no ref (protected piece)`, + ...(hinted ? { hint: true } : {}), + }; remap.endRef = ref; } if (!remap.startRef && !remap.endRef) return {}; @@ -306,8 +404,16 @@ export function rangeFingerprintsCore( blocks: BlockLike[], ): string[] { return ranges.map((r) => { - const start = boundaryRawCore(r.startRef, byRef, blocks, coreMessages, "min"); - const end = start ? boundaryRawCore(r.endRef, byRef, blocks, coreMessages, "max") : ""; + const start = boundaryRawCore( + r.startRef, + byRef, + blocks, + coreMessages, + "min", + ); + const end = start + ? boundaryRawCore(r.endRef, byRef, blocks, coreMessages, "max") + : ""; if (start && end) { const fp = spanFingerprintCore(coreMessages, start, end); if (fp.length > 0) return fp; @@ -328,7 +434,10 @@ export function rangePositionsCore( ): string[] { return ranges.map((r) => { const s = boundaryIndexCore(r.startRef, byRef, blocks, coreMessages, "min"); - const e = s >= 0 ? boundaryIndexCore(r.endRef, byRef, blocks, coreMessages, "max") : -1; + const e = + s >= 0 + ? boundaryIndexCore(r.endRef, byRef, blocks, coreMessages, "max") + : -1; return s >= 0 && e >= 0 ? `${s}-${e}` : "-"; }); } diff --git a/src/wire/demoted-thinking.ts b/src/wire/demoted-thinking.ts index 1464e12..e81509d 100644 --- a/src/wire/demoted-thinking.ts +++ b/src/wire/demoted-thinking.ts @@ -29,20 +29,20 @@ * uses the same bare rendering), not parsed. */ export type DemotedSplit = { - reasoning: string; - text: string; + reasoning: string; + text: string; }; type DelimitedForm = { - open: string; - close: string; + open: string; + close: string; }; /** The three inline tag forms hosts actually emit (see file comment). */ const FORMS: readonly DelimitedForm[] = [ - { open: "\n", close: "\n" }, - { open: "\n", close: "\n" }, - { open: "```thinking\n", close: "\n```" }, + { open: "\n", close: "\n" }, + { open: "\n", close: "\n" }, + { open: "```thinking\n", close: "\n```" }, ]; /** If `content` starts with one or more inline demoted-thinking blocks, @@ -52,26 +52,26 @@ const FORMS: readonly DelimitedForm[] = [ * content as-is. Malformed (unterminated) or empty blocks do not match; * the content is then treated as plain text, never dropped. */ export function splitDemotedThinking(content: string): DemotedSplit | null { - let rest = content; - const parts: string[] = []; - for (;;) { - let matched = false; - for (const form of FORMS) { - if (!rest.startsWith(form.open)) continue; - const end = rest.indexOf(form.close, form.open.length); - if (end < 0) continue; - const inner = rest.slice(form.open.length, end); - if (inner.length === 0) continue; - parts.push(inner); - rest = rest.slice(end + form.close.length); - // Glue the demotion renderer inserts between a demoted block - // and the block that follows it. - if (rest.startsWith("\n")) rest = rest.slice(1); - matched = true; - break; - } - if (!matched) break; + let rest = content; + const parts: string[] = []; + for (;;) { + let matched = false; + for (const form of FORMS) { + if (!rest.startsWith(form.open)) continue; + const end = rest.indexOf(form.close, form.open.length); + if (end < 0) continue; + const inner = rest.slice(form.open.length, end); + if (inner.length === 0) continue; + parts.push(inner); + rest = rest.slice(end + form.close.length); + // Glue the demotion renderer inserts between a demoted block + // and the block that follows it. + if (rest.startsWith("\n")) rest = rest.slice(1); + matched = true; + break; } - if (parts.length === 0) return null; - return { reasoning: parts.join("\n"), text: rest }; + if (!matched) break; + } + if (parts.length === 0) return null; + return { reasoning: parts.join("\n"), text: rest }; } diff --git a/src/wire/formats.ts b/src/wire/formats.ts index a8f56a0..ab0c90d 100644 --- a/src/wire/formats.ts +++ b/src/wire/formats.ts @@ -1,4 +1,9 @@ -export const WIRE_FORMATS = ["anthropic", "openai", "responses", "google"] as const; +export const WIRE_FORMATS = [ + "anthropic", + "openai", + "responses", + "google", +] as const; export type WireFormat = (typeof WIRE_FORMATS)[number]; export function isWireFormat(value: unknown): value is WireFormat { diff --git a/src/wire/message-id.ts b/src/wire/message-id.ts index 1cb69ae..728e833 100644 --- a/src/wire/message-id.ts +++ b/src/wire/message-id.ts @@ -30,16 +30,16 @@ import { hashId } from "./util.js"; * same cluster numbering (deterministic). */ export function deriveMessageId( - role: string, - contentType: string, - text: string, - options: { - toolCallId?: string; - toolName?: string; - } = {}, + role: string, + contentType: string, + text: string, + options: { + toolCallId?: string; + toolName?: string; + } = {}, ): string { - const seed = `${role}|${contentType}|${options.toolCallId ?? ""}|${options.toolName ?? ""}|${text}`; - return "h_" + hashId(seed); + const seed = `${role}|${contentType}|${options.toolCallId ?? ""}|${options.toolName ?? ""}|${text}`; + return "h_" + hashId(seed); } /** @@ -48,11 +48,11 @@ export function deriveMessageId( * Nth duplicate gets a `_${N}` suffix. */ export class ClusterCounter { - private counts = new Map(); + private counts = new Map(); - next(baseId: string): string { - const n = this.counts.get(baseId) ?? 0; - this.counts.set(baseId, n + 1); - return n === 0 ? baseId : `${baseId}_${n}`; - } + next(baseId: string): string { + const n = this.counts.get(baseId) ?? 0; + this.counts.set(baseId, n + 1); + return n === 0 ? baseId : `${baseId}_${n}`; + } } diff --git a/src/wire/mirror.ts b/src/wire/mirror.ts index d0b2aa3..61299d1 100644 --- a/src/wire/mirror.ts +++ b/src/wire/mirror.ts @@ -31,72 +31,92 @@ import type { BiliMessage } from "./bili-message.js"; */ export type MirrorBlock = - | { type: "text"; text: string } - | { type: "thinking"; thinking: string; signature?: string } - | { type: "toolCall"; id?: string; name?: string; arguments?: unknown }; + | { type: "text"; text: string } + | { type: "thinking"; thinking: string; signature?: string } + | { type: "toolCall"; id?: string; name?: string; arguments?: unknown }; export type MirrorMessage = { - /** `meta` is anything the host sends as out-of-band/system-ish traffic. */ - role: "user" | "assistant" | "toolResult" | "meta"; - blocks?: MirrorBlock[]; - /** toolResult only. */ - toolCallId?: string; - /** meta only: extracted text or summary. */ - text?: string; + /** `meta` is anything the host sends as out-of-band/system-ish traffic. */ + role: "user" | "assistant" | "toolResult" | "meta"; + blocks?: MirrorBlock[]; + /** toolResult only. */ + toolCallId?: string; + /** meta only: extracted text or summary. */ + text?: string; }; function textBlocks(blocks: MirrorBlock[] | undefined): string[] { - const out: string[] = []; - for (const b of blocks ?? []) if (b.type === "text") out.push(b.text); - return out; + const out: string[] = []; + for (const b of blocks ?? []) if (b.type === "text") out.push(b.text); + return out; } /** Openai wire text: text blocks joined with "\n" (whitespace-only kept). */ function joinText(blocks: MirrorBlock[] | undefined): string { - return textBlocks(blocks).join("\n"); + return textBlocks(blocks).join("\n"); } function thinkingText(blocks: MirrorBlock[] | undefined): string { - return blocks - ?.filter((b) => b.type === "thinking" && b.thinking.trim().length > 0) - .map((b) => (b as { thinking: string }).thinking) - .join("\n") ?? ""; + return ( + blocks + ?.filter((b) => b.type === "thinking" && b.thinking.trim().length > 0) + .map((b) => (b as { thinking: string }).thinking) + .join("\n") ?? "" + ); } /** Openai/completions mirror: system message first, then the conversation * with thinking as the `reasoning_content` field (issue #103). */ -export function mirrorOpenaiMessages(view: MirrorMessage[], systemText: string): Array> { - const messages: Array> = [{ role: "system", content: systemText }]; - for (const message of view) { - if (message.role === "user") { - const text = joinText(message.blocks); - if (text) messages.push({ role: "user", content: text }); - } else if (message.role === "assistant") { - const calls = (message.blocks ?? []).filter((b) => b.type === "toolCall") as Array<{ id?: string; name?: string; arguments?: unknown }>; - const reasoning = thinkingText(message.blocks); - const text = joinText(message.blocks); - if (calls.length > 0) { - messages.push({ - role: "assistant", - content: text, - ...(reasoning ? { reasoning_content: reasoning } : {}), - tool_calls: calls.map((c) => ({ - id: c.id, - type: "function", - function: { name: c.name ?? "", arguments: JSON.stringify(c.arguments ?? {}) }, - })), - }); - } else if (text || reasoning) { - messages.push({ role: "assistant", content: text, ...(reasoning ? { reasoning_content: reasoning } : {}) }); - } - } else if (message.role === "toolResult") { - messages.push({ role: "tool", tool_call_id: message.toolCallId ?? "", content: joinText(message.blocks) }); - } else { - const text = message.text ?? ""; - if (text) messages.push({ role: "developer", content: text }); - } +export function mirrorOpenaiMessages( + view: MirrorMessage[], + systemText: string, +): Array> { + const messages: Array> = [ + { role: "system", content: systemText }, + ]; + for (const message of view) { + if (message.role === "user") { + const text = joinText(message.blocks); + if (text) messages.push({ role: "user", content: text }); + } else if (message.role === "assistant") { + const calls = (message.blocks ?? []).filter( + (b) => b.type === "toolCall", + ) as Array<{ id?: string; name?: string; arguments?: unknown }>; + const reasoning = thinkingText(message.blocks); + const text = joinText(message.blocks); + if (calls.length > 0) { + messages.push({ + role: "assistant", + content: text, + ...(reasoning ? { reasoning_content: reasoning } : {}), + tool_calls: calls.map((c) => ({ + id: c.id, + type: "function", + function: { + name: c.name ?? "", + arguments: JSON.stringify(c.arguments ?? {}), + }, + })), + }); + } else if (text || reasoning) { + messages.push({ + role: "assistant", + content: text, + ...(reasoning ? { reasoning_content: reasoning } : {}), + }); + } + } else if (message.role === "toolResult") { + messages.push({ + role: "tool", + tool_call_id: message.toolCallId ?? "", + content: joinText(message.blocks), + }); + } else { + const text = message.text ?? ""; + if (text) messages.push({ role: "developer", content: text }); } - return messages; + } + return messages; } /** Anthropic/messages mirror: no system message (the live request carries it @@ -105,45 +125,65 @@ export function mirrorOpenaiMessages(view: MirrorMessage[], systemText: string): * blocks (issue #103). Unsigned thinking is demoted to text by the live * encoder; sending it as a thinking block diverges, so callers that persist * unsigned thinking should send it as a text block instead. */ -export function mirrorAnthropicMessages(view: MirrorMessage[]): Array> { - const messages: Array> = []; - for (const message of view) { - if (message.role === "user") { - const text = joinText(message.blocks); - if (text) messages.push({ role: "user", content: [{ type: "text", text }] }); - } else if (message.role === "assistant") { - const content: Array> = []; - for (const b of message.blocks ?? []) { - if (b.type === "thinking" && b.thinking.trim().length > 0) { - content.push({ - type: "thinking", - thinking: b.thinking, - ...(typeof b.signature === "string" && b.signature ? { signature: b.signature } : {}), - }); - } else if (b.type === "text" && b.text.trim().length > 0) { - content.push({ type: "text", text: b.text }); - } else if (b.type === "toolCall") { - let input: unknown = {}; - try { - input = b.arguments && typeof b.arguments === "object" ? b.arguments : JSON.parse(JSON.stringify(b.arguments ?? {})); - } catch { - input = {}; - } - content.push({ type: "tool_use", id: b.id, name: b.name ?? "", input }); - } - } - if (content.length > 0) messages.push({ role: "assistant", content }); - } else if (message.role === "toolResult") { - messages.push({ - role: "user", - content: [{ type: "tool_result", tool_use_id: message.toolCallId ?? "", content: joinText(message.blocks) }], - }); - } else { - const text = message.text ?? ""; - if (text) messages.push({ role: "user", content: [{ type: "text", text }] }); +export function mirrorAnthropicMessages( + view: MirrorMessage[], +): Array> { + const messages: Array> = []; + for (const message of view) { + if (message.role === "user") { + const text = joinText(message.blocks); + if (text) + messages.push({ role: "user", content: [{ type: "text", text }] }); + } else if (message.role === "assistant") { + const content: Array> = []; + for (const b of message.blocks ?? []) { + if (b.type === "thinking" && b.thinking.trim().length > 0) { + content.push({ + type: "thinking", + thinking: b.thinking, + ...(typeof b.signature === "string" && b.signature + ? { signature: b.signature } + : {}), + }); + } else if (b.type === "text" && b.text.trim().length > 0) { + content.push({ type: "text", text: b.text }); + } else if (b.type === "toolCall") { + let input: unknown = {}; + try { + input = + b.arguments && typeof b.arguments === "object" + ? b.arguments + : JSON.parse(JSON.stringify(b.arguments ?? {})); + } catch { + input = {}; + } + content.push({ + type: "tool_use", + id: b.id, + name: b.name ?? "", + input, + }); } + } + if (content.length > 0) messages.push({ role: "assistant", content }); + } else if (message.role === "toolResult") { + messages.push({ + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: message.toolCallId ?? "", + content: joinText(message.blocks), + }, + ], + }); + } else { + const text = message.text ?? ""; + if (text) + messages.push({ role: "user", content: [{ type: "text", text }] }); } - return messages; + } + return messages; } /** Responses mirror: the live /v1/responses request carries the system @@ -151,62 +191,102 @@ export function mirrorAnthropicMessages(view: MirrorMessage[]): Array> { - const input: Array> = []; - for (const message of view) { - if (message.role === "user") { - const text = joinText(message.blocks); - if (text) input.push({ type: "message", role: "user", content: [{ type: "input_text", text }] }); - } else if (message.role === "assistant") { - for (const b of message.blocks ?? []) { - if (b.type === "thinking" && b.thinking.trim().length > 0) { - input.push({ type: "reasoning", summary: [{ type: "summary_text", text: b.thinking }] }); - } else if (b.type === "text" && b.text.trim().length > 0) { - input.push({ type: "message", role: "assistant", content: [{ type: "output_text", text: b.text }] }); - } else if (b.type === "toolCall") { - let args = "{}"; - try { - args = JSON.stringify(b.arguments ?? {}); - } catch { - args = "{}"; - } - input.push({ type: "function_call", call_id: b.id ?? "", name: b.name ?? "", arguments: args }); - } - } - } else if (message.role === "toolResult") { - input.push({ type: "function_call_output", call_id: message.toolCallId ?? "", output: joinText(message.blocks) }); - } else { - const text = message.text ?? ""; - if (text) input.push({ type: "message", role: "user", content: [{ type: "input_text", text }] }); +export function mirrorResponsesInput( + view: MirrorMessage[], +): Array> { + const input: Array> = []; + for (const message of view) { + if (message.role === "user") { + const text = joinText(message.blocks); + if (text) + input.push({ + type: "message", + role: "user", + content: [{ type: "input_text", text }], + }); + } else if (message.role === "assistant") { + for (const b of message.blocks ?? []) { + if (b.type === "thinking" && b.thinking.trim().length > 0) { + input.push({ + type: "reasoning", + summary: [{ type: "summary_text", text: b.thinking }], + }); + } else if (b.type === "text" && b.text.trim().length > 0) { + input.push({ + type: "message", + role: "assistant", + content: [{ type: "output_text", text: b.text }], + }); + } else if (b.type === "toolCall") { + let args = "{}"; + try { + args = JSON.stringify(b.arguments ?? {}); + } catch { + args = "{}"; + } + input.push({ + type: "function_call", + call_id: b.id ?? "", + name: b.name ?? "", + arguments: args, + }); } + } + } else if (message.role === "toolResult") { + input.push({ + type: "function_call_output", + call_id: message.toolCallId ?? "", + output: joinText(message.blocks), + }); + } else { + const text = message.text ?? ""; + if (text) + input.push({ + type: "message", + role: "user", + content: [{ type: "input_text", text }], + }); } - return input; + } + return input; } /** Fold the openai mirror through `openaiToCore`. */ -export function mirrorOpenaiToCore(view: MirrorMessage[], systemText: string): BiliMessage[] { - const { msgs } = openaiToCore({ - model: "prime-fold", - messages: mirrorOpenaiMessages(view, systemText) as Parameters[0]["messages"], - }); - return msgs; +export function mirrorOpenaiToCore( + view: MirrorMessage[], + systemText: string, +): BiliMessage[] { + const { msgs } = openaiToCore({ + model: "prime-fold", + messages: mirrorOpenaiMessages(view, systemText) as Parameters< + typeof openaiToCore + >[0]["messages"], + }); + return msgs; } /** Fold the anthropic mirror through `anthropicToCore`. */ export function mirrorAnthropicToCore(view: MirrorMessage[]): BiliMessage[] { - const { msgs } = anthropicToCore({ - model: "prime-fold", - messages: mirrorAnthropicMessages(view) as Parameters[0]["messages"], - }); - return msgs; + const { msgs } = anthropicToCore({ + model: "prime-fold", + messages: mirrorAnthropicMessages(view) as Parameters< + typeof anthropicToCore + >[0]["messages"], + }); + return msgs; } /** Fold the responses mirror through `responsesToCore`. */ -export function mirrorResponsesToCore(view: MirrorMessage[], systemText: string): BiliMessage[] { - const { msgs } = responsesToCore({ - model: "prime-fold", - instructions: systemText, - input: mirrorResponsesInput(view) as Parameters[0]["input"], - }); - return msgs; +export function mirrorResponsesToCore( + view: MirrorMessage[], + systemText: string, +): BiliMessage[] { + const { msgs } = responsesToCore({ + model: "prime-fold", + instructions: systemText, + input: mirrorResponsesInput(view) as Parameters< + typeof responsesToCore + >[0]["input"], + }); + return msgs; } diff --git a/src/wire/openai.ts b/src/wire/openai.ts index 71ea887..d31b471 100644 --- a/src/wire/openai.ts +++ b/src/wire/openai.ts @@ -4,36 +4,36 @@ import { ClusterCounter, deriveMessageId } from "./message-id.js"; import { parseDataUrl, type BiliMessage } from "./bili-message.js"; export type OpenAIContentPart = - | { type: "text"; text: string } - | { type: "image_url"; image_url: { url: string } } - | { type: string; [k: string]: unknown }; + | { type: "text"; text: string } + | { type: "image_url"; image_url: { url: string } } + | { type: string; [k: string]: unknown }; export type OpenAIToolCall = { - id: string; - type: "function"; - function: { name: string; arguments: string }; + id: string; + type: "function"; + function: { name: string; arguments: string }; }; export type OpenAIMessage = { - role: "system" | "developer" | "user" | "assistant" | "tool"; - content?: string | null | OpenAIContentPart[]; - reasoning_content?: string | null; - tool_calls?: OpenAIToolCall[]; - tool_call_id?: string; - name?: string; + role: "system" | "developer" | "user" | "assistant" | "tool"; + content?: string | null | OpenAIContentPart[]; + reasoning_content?: string | null; + tool_calls?: OpenAIToolCall[]; + tool_call_id?: string; + name?: string; }; export type OpenAITool = { - type: "function"; - function: { name: string; description?: string; parameters?: unknown }; + type: "function"; + function: { name: string; description?: string; parameters?: unknown }; }; export type OpenAIRequestBody = { - model?: string; - messages: OpenAIMessage[]; - tools?: OpenAITool[]; - stream?: boolean; - [key: string]: unknown; + model?: string; + messages: OpenAIMessage[]; + tools?: OpenAITool[]; + stream?: boolean; + [key: string]: unknown; }; type Flat = { msgs: BiliMessage[]; systemText: string }; @@ -48,237 +48,315 @@ type Flat = { msgs: BiliMessage[]; systemText: string }; * entirely. Mid-conversation system messages (rare, host-synthetic) stay in * the fold space unchanged. */ export function openaiToCore(body: OpenAIRequestBody): Flat { - const msgs: BiliMessage[] = []; - const systemParts: string[] = []; - const clusters = new ClusterCounter(); - // Backfill toolName onto role:"tool" results from their paired tool_call — - // kernel guards are name-based. Not part of the identity seed (id stability). - const toolNames = new Map(); - for (const m of body.messages) { - switch (m.role) { - case "system": - case "developer": { - if (msgs.length === 0) { - systemParts.push(stringContent(m.content)); - break; - } - const base = deriveMessageId(m.role, "text", stringContent(m.content)); - msgs.push({ id: clusters.next(base), role: "system", contentType: "text", text: stringContent(m.content), originalRole: m.role }); - break; - } - case "user": { - // Image-only messages keep text "" (NO "[image]" placeholder as - // in the responses/anthropic codecs): ids derive from text alone - // and switching the seed would move every pre-existing shape. - const text = stringContent(m.content); - const nonText = allNonTextParts(m.content); - const imgs = allImageParts(m.content); - const firstImg = imgs[0]; - const firstUrl = firstImg ? firstImg.image_url.url : undefined; - const firstParsed = firstUrl ? parseDataUrl(firstUrl) : undefined; - const base = deriveMessageId("user", "text", text); - msgs.push({ - id: clusters.next(base), - role: "user", - contentType: "text", - text, - // A lone string-URL image keeps the legacy singular sidecar - // (byte-stable for pre-existing shapes). Any other mix — - // multiple images, or any unknown part type such as a - // DeepSeek Files API `{type:"file"}` ref — rides the plural - // sidecar verbatim so coreToOpenai can re-emit it - // (billion-context#1205: unknown parts were silently dropped). - ...(imgs.length === 1 && nonText.length === 1 - ? { - rawOpenaiContent: imgs[0], - // data: URLs only — remote URLs ride rawOpenaiContent - // verbatim (#187/#291) - ...(firstParsed ? { imageMediaType: firstParsed.mediaType, imageBase64: firstParsed.base64 } : {}), - } - : nonText.length > 0 - ? { rawOpenaiContentParts: nonText } - : {}), - }); - break; - } - case "assistant": { - const rawReasoning = m.reasoning_content; - const fieldPresent = typeof rawReasoning === "string"; - const fieldReasoning = fieldPresent ? rawReasoning : ""; - let reasoning = fieldReasoning; - let text = stringContent(m.content); - if (!reasoning) { - // Hosts that cannot replay prior-turn reasoning as a - // structured field inline it into content wrapped in a - // dialect tag (, , ```thinking). Split - // it back out BEFORE identity derivation so the inline - // form and the reasoning_content field form of one turn - // land in a single core-id/fingerprint space (issue #64 - // demoted variant). - const split = splitDemotedThinking(text); - if (split) { - reasoning = split.reasoning; - text = split.text; - } - } - // Blank field, nothing inline: the host sent the key with an - // empty value. Record that on the cores so coreToOpenai - // re-emits the key instead of dropping it — strict-echo - // thinking models reject the key's absence, not its emptiness. - const blankEcho = fieldPresent && reasoning.length === 0; - if (reasoning) { - const base = deriveMessageId("assistant", "reasoning", reasoning); - msgs.push({ - id: clusters.next(base), - role: "assistant", - contentType: "reasoning", - text: reasoning, - reasoningContent: reasoning, - }); - } - if (text) { - const base = deriveMessageId("assistant", "text", text); - msgs.push({ id: clusters.next(base), role: "assistant", contentType: "text", text, ...(blankEcho ? { reasoningPresent: true } : {}) }); - } - if (Array.isArray(m.tool_calls)) { - for (const tc of m.tool_calls) { - const base = deriveMessageId("assistant", "tool-call", tc.function.arguments ?? "", { - toolCallId: tc.id, - toolName: tc.function.name, - }); - msgs.push({ - id: clusters.next(base), - role: "assistant", - contentType: "tool-call", - toolName: tc.function.name, - toolCallId: tc.id, - text: tc.function.arguments ?? "", - ...(blankEcho ? { reasoningPresent: true } : {}), - }); - toolNames.set(tc.id, tc.function.name); + const msgs: BiliMessage[] = []; + const systemParts: string[] = []; + const clusters = new ClusterCounter(); + // Backfill toolName onto role:"tool" results from their paired tool_call — + // kernel guards are name-based. Not part of the identity seed (id stability). + const toolNames = new Map(); + for (const m of body.messages) { + switch (m.role) { + case "system": + case "developer": { + if (msgs.length === 0) { + systemParts.push(stringContent(m.content)); + break; + } + const base = deriveMessageId(m.role, "text", stringContent(m.content)); + msgs.push({ + id: clusters.next(base), + role: "system", + contentType: "text", + text: stringContent(m.content), + originalRole: m.role, + }); + break; + } + case "user": { + // Image-only messages keep text "" (NO "[image]" placeholder as + // in the responses/anthropic codecs): ids derive from text alone + // and switching the seed would move every pre-existing shape. + const text = stringContent(m.content); + const nonText = allNonTextParts(m.content); + const imgs = allImageParts(m.content); + const firstImg = imgs[0]; + const firstUrl = firstImg ? firstImg.image_url.url : undefined; + const firstParsed = firstUrl ? parseDataUrl(firstUrl) : undefined; + const base = deriveMessageId("user", "text", text); + msgs.push({ + id: clusters.next(base), + role: "user", + contentType: "text", + text, + // A lone string-URL image keeps the legacy singular sidecar + // (byte-stable for pre-existing shapes). Any other mix — + // multiple images, or any unknown part type such as a + // DeepSeek Files API `{type:"file"}` ref — rides the plural + // sidecar verbatim so coreToOpenai can re-emit it + // (billion-context#1205: unknown parts were silently dropped). + ...(imgs.length === 1 && nonText.length === 1 + ? { + rawOpenaiContent: imgs[0], + // data: URLs only — remote URLs ride rawOpenaiContent + // verbatim (#187/#291) + ...(firstParsed + ? { + imageMediaType: firstParsed.mediaType, + imageBase64: firstParsed.base64, } - } - break; - } - case "tool": { - const base = deriveMessageId("tool", "tool-result", stringContent(m.content), { - toolCallId: m.tool_call_id ?? "", - }); - const name = m.name || toolNames.get(m.tool_call_id ?? ""); - msgs.push({ - id: clusters.next(base), - role: "tool", - contentType: "tool-result", - toolCallId: m.tool_call_id ?? "", - text: stringContent(m.content), - ...(name ? { toolName: name } : {}), - }); - break; - } + : {}), + } + : nonText.length > 0 + ? { rawOpenaiContentParts: nonText } + : {}), + }); + break; + } + case "assistant": { + const rawReasoning = m.reasoning_content; + const fieldPresent = typeof rawReasoning === "string"; + const fieldReasoning = fieldPresent ? rawReasoning : ""; + let reasoning = fieldReasoning; + let text = stringContent(m.content); + if (!reasoning) { + // Hosts that cannot replay prior-turn reasoning as a + // structured field inline it into content wrapped in a + // dialect tag (, , ```thinking). Split + // it back out BEFORE identity derivation so the inline + // form and the reasoning_content field form of one turn + // land in a single core-id/fingerprint space (issue #64 + // demoted variant). + const split = splitDemotedThinking(text); + if (split) { + reasoning = split.reasoning; + text = split.text; + } + } + // Blank field, nothing inline: the host sent the key with an + // empty value. Record that on the cores so coreToOpenai + // re-emits the key instead of dropping it — strict-echo + // thinking models reject the key's absence, not its emptiness. + const blankEcho = fieldPresent && reasoning.length === 0; + if (reasoning) { + const base = deriveMessageId("assistant", "reasoning", reasoning); + msgs.push({ + id: clusters.next(base), + role: "assistant", + contentType: "reasoning", + text: reasoning, + reasoningContent: reasoning, + }); + } + if (text) { + const base = deriveMessageId("assistant", "text", text); + msgs.push({ + id: clusters.next(base), + role: "assistant", + contentType: "text", + text, + ...(blankEcho ? { reasoningPresent: true } : {}), + }); } + if (Array.isArray(m.tool_calls)) { + for (const tc of m.tool_calls) { + const base = deriveMessageId( + "assistant", + "tool-call", + tc.function.arguments ?? "", + { + toolCallId: tc.id, + toolName: tc.function.name, + }, + ); + msgs.push({ + id: clusters.next(base), + role: "assistant", + contentType: "tool-call", + toolName: tc.function.name, + toolCallId: tc.id, + text: tc.function.arguments ?? "", + ...(blankEcho ? { reasoningPresent: true } : {}), + }); + toolNames.set(tc.id, tc.function.name); + } + } + break; + } + case "tool": { + const base = deriveMessageId( + "tool", + "tool-result", + stringContent(m.content), + { + toolCallId: m.tool_call_id ?? "", + }, + ); + const name = m.name || toolNames.get(m.tool_call_id ?? ""); + msgs.push({ + id: clusters.next(base), + role: "tool", + contentType: "tool-result", + toolCallId: m.tool_call_id ?? "", + text: stringContent(m.content), + ...(name ? { toolName: name } : {}), + }); + break; + } } - return { msgs, systemText: systemParts.join("\n\n") }; + } + return { msgs, systemText: systemParts.join("\n\n") }; } export function coreToOpenai(messages: BiliMessage[]): OpenAIMessage[] { - const out: OpenAIMessage[] = []; - let pending: { text: string | null; toolCalls: OpenAIToolCall[]; reasoning: string | null; reasoningPresent: boolean } | null = null; - const flush = () => { - if (!pending) return; - const reasoning = pending.reasoning !== null && pending.reasoning.length > 0 ? pending.reasoning : undefined; - // A blank field means the host sent the key with an empty value and - // expects it back: strict-echo thinking models reject its absence. - const reasoningField = - reasoning !== undefined ? { reasoning_content: reasoning } : pending.reasoningPresent ? { reasoning_content: "" } : {}; - if (pending.toolCalls.length > 0) { - out.push({ - role: "assistant", - content: pending.text ?? null, - tool_calls: pending.toolCalls, - ...reasoningField, + const out: OpenAIMessage[] = []; + let pending: { + text: string | null; + toolCalls: OpenAIToolCall[]; + reasoning: string | null; + reasoningPresent: boolean; + } | null = null; + const flush = () => { + if (!pending) return; + const reasoning = + pending.reasoning !== null && pending.reasoning.length > 0 + ? pending.reasoning + : undefined; + // A blank field means the host sent the key with an empty value and + // expects it back: strict-echo thinking models reject its absence. + const reasoningField = + reasoning !== undefined + ? { reasoning_content: reasoning } + : pending.reasoningPresent + ? { reasoning_content: "" } + : {}; + if (pending.toolCalls.length > 0) { + out.push({ + role: "assistant", + content: pending.text ?? null, + tool_calls: pending.toolCalls, + ...reasoningField, + }); + } else if (pending.text !== null) { + out.push({ role: "assistant", content: pending.text, ...reasoningField }); + } else if (reasoning) { + out.push({ + role: "assistant", + content: null, + reasoning_content: reasoning, + }); + } + pending = null; + }; + for (const m of messages) { + if (m.role === "assistant") { + if (!pending) + pending = { + text: null, + toolCalls: [], + reasoning: null, + reasoningPresent: false, + }; + if (m.reasoningPresent) pending.reasoningPresent = true; + if (m.contentType === "reasoning") { + pending.reasoning = + (pending.reasoning ?? "") + (m.reasoningContent ?? m.text ?? ""); + } else if (m.contentType === "text") { + pending.text = (pending.text ?? "") + (m.text ?? ""); + } else if (m.contentType === "tool-call") { + pending.toolCalls.push({ + id: m.toolCallId ?? `call_${m.id}`, + type: "function", + function: { name: m.toolName ?? "unknown", arguments: m.text ?? "" }, + }); + } + } else { + flush(); + if (m.role === "system") { + out.push({ + role: m.originalRole === "developer" ? "developer" : "system", + content: m.text ?? "", + }); + } else if (m.role === "user") { + if (m.rawOpenaiContent || m.imageBase64 || m.rawOpenaiContentParts) { + const parts: OpenAIContentPart[] = []; + if (m.text) parts.push({ type: "text", text: m.text }); + if (m.rawOpenaiContentParts && m.rawOpenaiContentParts.length > 0) { + for (const part of m.rawOpenaiContentParts) + parts.push(part as OpenAIContentPart); + } else if (m.rawOpenaiContent) { + parts.push(m.rawOpenaiContent as OpenAIContentPart); + } else if (m.imageBase64 && m.imageMediaType) { + parts.push({ + type: "image_url", + image_url: { + url: `data:${m.imageMediaType};base64,${m.imageBase64}`, + }, }); - } else if (pending.text !== null) { - out.push({ role: "assistant", content: pending.text, ...reasoningField }); - } else if (reasoning) { - out.push({ role: "assistant", content: null, reasoning_content: reasoning }); - } - pending = null; - }; - for (const m of messages) { - if (m.role === "assistant") { - if (!pending) pending = { text: null, toolCalls: [], reasoning: null, reasoningPresent: false }; - if (m.reasoningPresent) pending.reasoningPresent = true; - if (m.contentType === "reasoning") { - pending.reasoning = (pending.reasoning ?? "") + (m.reasoningContent ?? m.text ?? ""); - } else if (m.contentType === "text") { - pending.text = (pending.text ?? "") + (m.text ?? ""); - } else if (m.contentType === "tool-call") { - pending.toolCalls.push({ - id: m.toolCallId ?? `call_${m.id}`, - type: "function", - function: { name: m.toolName ?? "unknown", arguments: m.text ?? "" }, - }); - } + } + out.push({ role: "user", content: parts }); } else { - flush(); - if (m.role === "system") { - out.push({ role: m.originalRole === "developer" ? "developer" : "system", content: m.text ?? "" }); - } else if (m.role === "user") { - if (m.rawOpenaiContent || m.imageBase64 || m.rawOpenaiContentParts) { - const parts: OpenAIContentPart[] = []; - if (m.text) parts.push({ type: "text", text: m.text }); - if (m.rawOpenaiContentParts && m.rawOpenaiContentParts.length > 0) { - for (const part of m.rawOpenaiContentParts) parts.push(part as OpenAIContentPart); - } else if (m.rawOpenaiContent) { - parts.push(m.rawOpenaiContent as OpenAIContentPart); - } else if (m.imageBase64 && m.imageMediaType) { - parts.push({ type: "image_url", image_url: { url: `data:${m.imageMediaType};base64,${m.imageBase64}` } }); - } - out.push({ role: "user", content: parts }); - } else { - out.push({ role: "user", content: m.text ?? "" }); - } - } else if (m.role === "tool") { - out.push({ role: "tool", tool_call_id: m.toolCallId ?? "", content: m.text ?? "" }); - } + out.push({ role: "user", content: m.text ?? "" }); } + } else if (m.role === "tool") { + out.push({ + role: "tool", + tool_call_id: m.toolCallId ?? "", + content: m.text ?? "", + }); + } } - flush(); - return out; + } + flush(); + return out; } -export function injectOpenaiSystem(messages: OpenAIMessage[], parts: string[]): OpenAIMessage[] { - if (parts.length === 0) return messages; - const extra = parts.join("\n\n"); - if (messages.length > 0 && (messages[0]?.role === "system" || messages[0]?.role === "developer")) { - const head = messages[0] as OpenAIMessage; - const base = stringContent(head.content); - const merged = base ? `${base}\n\n---\n\n${extra}` : extra; - return [{ ...head, content: merged }, ...messages.slice(1)]; - } - return [{ role: "system", content: extra }, ...messages]; +export function injectOpenaiSystem( + messages: OpenAIMessage[], + parts: string[], +): OpenAIMessage[] { + if (parts.length === 0) return messages; + const extra = parts.join("\n\n"); + if ( + messages.length > 0 && + (messages[0]?.role === "system" || messages[0]?.role === "developer") + ) { + const head = messages[0] as OpenAIMessage; + const base = stringContent(head.content); + const merged = base ? `${base}\n\n---\n\n${extra}` : extra; + return [{ ...head, content: merged }, ...messages.slice(1)]; + } + return [{ role: "system", content: extra }, ...messages]; } /** Extract the conversation dimension for OpenAI Chat: a client-provided * session header if present, else a content fingerprint of the first user * message. See conversationSignalAnthropic for the full rationale. */ -export function conversationSignalOpenai(body: OpenAIRequestBody, headerValue?: string): string { - if (headerValue && headerValue.trim()) return headerValue.trim(); - const msgs = Array.isArray(body?.messages) ? body.messages : []; - const firstUser = msgs.find((m) => m?.role === "user"); - const seed = firstUser ? stringContent(firstUser.content) : "default"; - return hashId(seed); +export function conversationSignalOpenai( + body: OpenAIRequestBody, + headerValue?: string, +): string { + if (headerValue && headerValue.trim()) return headerValue.trim(); + const msgs = Array.isArray(body?.messages) ? body.messages : []; + const firstUser = msgs.find((m) => m?.role === "user"); + const seed = firstUser ? stringContent(firstUser.content) : "default"; + return hashId(seed); } function stringContent(content: OpenAIMessage["content"]): string { - if (content == null) return ""; - if (typeof content === "string") return content; - if (Array.isArray(content)) { - return content - .map((p) => (typeof p === "string" ? p : p.type === "text" ? (p as { text?: string }).text ?? "" : "")) - .join("\n"); - } - return ""; + if (content == null) return ""; + if (typeof content === "string") return content; + if (Array.isArray(content)) { + return content + .map((p) => + typeof p === "string" + ? p + : p.type === "text" + ? ((p as { text?: string }).text ?? "") + : "", + ) + .join("\n"); + } + return ""; } type OpenAIImagePart = { type: "image_url"; image_url: { url: string } }; @@ -289,18 +367,19 @@ type OpenAIImagePart = { type: "image_url"; image_url: { url: string } }; * a later data-URL-only filter then silently dropped every remote-URL * image — issues #187/#291.) */ function allImageParts(content: OpenAIMessage["content"]): OpenAIImagePart[] { - if (!Array.isArray(content)) return []; - const out: OpenAIImagePart[] = []; - for (const p of content) { - if (typeof p !== "object" || p === null) continue; - if (!("type" in p) || p.type !== "image_url" || !("image_url" in p)) continue; - // The union's index-signature member leaves image_url as `unknown`, so - // narrow via a named const before reading the url. - const imagePart = p as { image_url?: { url?: unknown } | null }; - const url = imagePart.image_url?.url; - if (typeof url === "string") out.push(p as OpenAIImagePart); - } - return out; + if (!Array.isArray(content)) return []; + const out: OpenAIImagePart[] = []; + for (const p of content) { + if (typeof p !== "object" || p === null) continue; + if (!("type" in p) || p.type !== "image_url" || !("image_url" in p)) + continue; + // The union's index-signature member leaves image_url as `unknown`, so + // narrow via a named const before reading the url. + const imagePart = p as { image_url?: { url?: unknown } | null }; + const url = imagePart.image_url?.url; + if (typeof url === "string") out.push(p as OpenAIImagePart); + } + return out; } /** Collect ALL non-text object parts of a user content array, in wire order — @@ -308,13 +387,15 @@ function allImageParts(content: OpenAIMessage["content"]): OpenAIImagePart[] { * payloads (e.g. DeepSeek Files API `{type:"file",file_id}` refs) that the * kernel must never drop: they ride rawOpenaiContentParts verbatim and * coreToOpenai re-emits them unchanged (billion-context#1205). */ -function allNonTextParts(content: OpenAIMessage["content"]): OpenAIContentPart[] { - if (!Array.isArray(content)) return []; - const out: OpenAIContentPart[] = []; - for (const p of content) { - if (typeof p !== "object" || p === null) continue; - if (p.type === "text") continue; - out.push(p); - } - return out; +function allNonTextParts( + content: OpenAIMessage["content"], +): OpenAIContentPart[] { + if (!Array.isArray(content)) return []; + const out: OpenAIContentPart[] = []; + for (const p of content) { + if (typeof p !== "object" || p === null) continue; + if (p.type === "text") continue; + out.push(p); + } + return out; } diff --git a/src/wire/responses.ts b/src/wire/responses.ts index 37a6568..23e872d 100644 --- a/src/wire/responses.ts +++ b/src/wire/responses.ts @@ -6,69 +6,69 @@ import { hashId } from "./util.js"; import { parseDataUrl, type BiliMessage } from "./bili-message.js"; export type ResponseContentPart = - | { type: "input_text"; text: string; [key: string]: unknown } - | { type: "output_text"; text: string; [key: string]: unknown } - | { type: "input_image"; image_url: string; [key: string]: unknown } - | { type: string; [key: string]: unknown }; + | { type: "input_text"; text: string; [key: string]: unknown } + | { type: "output_text"; text: string; [key: string]: unknown } + | { type: "input_image"; image_url: string; [key: string]: unknown } + | { type: string; [key: string]: unknown }; export type ResponseInputMessage = { - type: "message"; - role: "system" | "developer" | "user" | "assistant"; - content: string | ResponseContentPart[]; - [key: string]: unknown; + type: "message"; + role: "system" | "developer" | "user" | "assistant"; + content: string | ResponseContentPart[]; + [key: string]: unknown; }; export type ResponseFunctionCall = { - type: "function_call"; - id?: string; - call_id: string; - name: string; - arguments: string; - [key: string]: unknown; + type: "function_call"; + id?: string; + call_id: string; + name: string; + arguments: string; + [key: string]: unknown; }; export type ResponseFunctionCallOutput = { - type: "function_call_output"; - call_id: string; - output: string; - [key: string]: unknown; + type: "function_call_output"; + call_id: string; + output: string; + [key: string]: unknown; }; export type ResponseInputItem = - | ResponseInputMessage - | ResponseFunctionCall - | ResponseFunctionCallOutput - | { type: string; [key: string]: unknown }; + | ResponseInputMessage + | ResponseFunctionCall + | ResponseFunctionCallOutput + | { type: string; [key: string]: unknown }; export type ResponsesRequestBody = { - model?: string; - input: string | ResponseInputItem[]; - instructions?: string; - tools?: unknown[]; - stream?: boolean; - session_id?: string; - previous_response_id?: string; - prompt_cache_key?: string; - metadata?: Record; - [key: string]: unknown; + model?: string; + input: string | ResponseInputItem[]; + instructions?: string; + tools?: unknown[]; + stream?: boolean; + session_id?: string; + previous_response_id?: string; + prompt_cache_key?: string; + metadata?: Record; + [key: string]: unknown; }; type ResponseLayoutSlot = { - original: ResponseInputItem; - coreId?: string; + original: ResponseInputItem; + coreId?: string; }; export type ResponsesProjection = { - msgs: BiliMessage[]; - systemParts: string[]; - preamble: ResponseInputItem[]; - customToolCallIds: Set; - layout: ResponseLayoutSlot[]; - stringInput?: { original: string; coreId: string }; - /** Reasoning items dropped because ACP_REASONING_KEEP=none. 0 by default — - * reasoning is normally routed through the compression pipeline so it is - * hidden automatically once its turn is summarized. */ - droppedReasoning: number; + msgs: BiliMessage[]; + systemParts: string[]; + preamble: ResponseInputItem[]; + customToolCallIds: Set; + layout: ResponseLayoutSlot[]; + stringInput?: { original: string; coreId: string }; + /** Reasoning items dropped because ACP_REASONING_KEEP=none. 0 by default — + * reasoning is normally routed through the compression pipeline so it is + * hidden automatically once its turn is summarized. */ + droppedReasoning: number; }; /** Item types that are host DIRECTIVES (tool/definition listings), not @@ -82,13 +82,10 @@ export type ResponsesProjection = { * hides them once their turn is summarized — preserving them verbatim in the * preamble instead made them accumulate unbounded every turn and broke Codex's * prompt-cache prefix. */ -const OPAQUE_ITEM_TYPES = new Set([ - "additional_tools", - "mcp_list_tools", -]); +const OPAQUE_ITEM_TYPES = new Set(["additional_tools", "mcp_list_tools"]); function isOpaqueItem(item: ResponseInputItem): boolean { - return OPAQUE_ITEM_TYPES.has(item.type); + return OPAQUE_ITEM_TYPES.has(item.type); } const EASY_INPUT_ROLES = new Set(["user", "assistant", "system", "developer"]); @@ -97,26 +94,31 @@ const EASY_INPUT_ROLES = new Set(["user", "assistant", "system", "developer"]); * `{ role: "user", content: "..." }`); without normalization those items fall * through the switch's `default` into the preamble and are never folded. */ function normalizeEasyInputItem(item: ResponseInputItem): ResponseInputItem { - if (typeof item.type === "string") return item; - if (EASY_INPUT_ROLES.has(String((item as { role?: unknown }).role))) { - return { ...(item as { [key: string]: unknown }), type: "message" } as ResponseInputMessage; - } - return item; + if (typeof item.type === "string") return item; + if (EASY_INPUT_ROLES.has(String((item as { role?: unknown }).role))) { + return { + ...(item as { [key: string]: unknown }), + type: "message", + } as ResponseInputMessage; + } + return item; } function shouldDropAllReasoning(): boolean { - return (process.env.ACP_REASONING_KEEP ?? "").trim().toLowerCase() === "none"; + return (process.env.ACP_REASONING_KEEP ?? "").trim().toLowerCase() === "none"; } function partText(part: ResponseContentPart): string { - if (part.type === "input_text" || part.type === "output_text") { - return typeof part.text === "string" ? part.text : ""; - } - return ""; + if (part.type === "input_text" || part.type === "output_text") { + return typeof part.text === "string" ? part.text : ""; + } + return ""; } function messageContent(content: string | ResponseContentPart[]): string { - return typeof content === "string" ? content : content.map(partText).join("\n"); + return typeof content === "string" + ? content + : content.map(partText).join("\n"); } /** Kernel-side text for a user item: its text as given, or the "[image]" @@ -126,11 +128,16 @@ function messageContent(content: string | ResponseContentPart[]): string { * decode-side verbatim condition (coreToResponses): the two sides must agree * byte-for-byte or the raw-item round-trip silently degrades to a text-only * rebuild (#187/#291). */ -function userDisplayText(content: string | ResponseContentPart[], text: string): string { - if (typeof content === "string") return text; - const hasImage = content.some((part) => part.type === "input_image"); - const hasTextPart = content.some((part) => part.type === "input_text" || part.type === "output_text"); - return !text || (hasImage && !hasTextPart) ? "[image]" : text; +function userDisplayText( + content: string | ResponseContentPart[], + text: string, +): string { + if (typeof content === "string") return text; + const hasImage = content.some((part) => part.type === "input_image"); + const hasTextPart = content.some( + (part) => part.type === "input_text" || part.type === "output_text", + ); + return !text || (hasImage && !hasTextPart) ? "[image]" : text; } /** Extract the reasoning text from a responses reasoning item. The host @@ -138,358 +145,561 @@ function userDisplayText(content: string | ResponseContentPart[], text: string): * carries it in `summary` (summary_text parts). Both must yield the same * text so the kernel derives the same core id (issue #64, responses). */ function reasoningText(item: ResponseInputItem): string { - const fromParts = (parts: unknown, type: string): string => { - if (!Array.isArray(parts)) return ""; - const texts: string[] = []; - for (const part of parts) { - if (part && typeof part === "object" && "type" in part && "text" in part) { - const rec = part as { type: unknown; text: unknown }; - if (rec.type === type && typeof rec.text === "string") texts.push(rec.text); - } - } - return texts.join("\n"); - }; - const content = "content" in item ? item.content : undefined; - const summary = "summary" in item ? item.summary : undefined; - return fromParts(content, "reasoning_text") || fromParts(summary, "summary_text"); + const fromParts = (parts: unknown, type: string): string => { + if (!Array.isArray(parts)) return ""; + const texts: string[] = []; + for (const part of parts) { + if ( + part && + typeof part === "object" && + "type" in part && + "text" in part + ) { + const rec = part as { type: unknown; text: unknown }; + if (rec.type === type && typeof rec.text === "string") + texts.push(rec.text); + } + } + return texts.join("\n"); + }; + const content = "content" in item ? item.content : undefined; + const summary = "summary" in item ? item.summary : undefined; + return ( + fromParts(content, "reasoning_text") || fromParts(summary, "summary_text") + ); } -export function responsesToCore(body: ResponsesRequestBody): ResponsesProjection { - const msgs: BiliMessage[] = []; - const systemParts: string[] = []; - const preamble: ResponseInputItem[] = []; - const customToolCallIds = new Set(); - const layout: ResponseLayoutSlot[] = []; - let droppedReasoning = 0; - const clusters = new ClusterCounter(); - // Backfill toolName onto call outputs from their paired calls — kernel - // guards are name-based. Not part of the identity seed (id stability). - const toolNames = new Map(); - let idx = 0; - if (typeof body.instructions === "string" && body.instructions.trim()) systemParts.push(body.instructions); - if (typeof body.input === "string") { - const id = clusters.next(deriveMessageId("user", "text", body.input)); - msgs.push({ id, role: "user", contentType: "text", text: body.input }); - return { msgs, systemParts, preamble, customToolCallIds, layout, droppedReasoning, stringInput: { original: body.input, coreId: id } }; - } - for (const raw of body.input) { - const item = normalizeEasyInputItem(raw); - let coreId: string | undefined; - if (isOpaqueItem(item)) preamble.push(item); - switch (item.type) { - case "reasoning": { - if (shouldDropAllReasoning()) { - droppedReasoning++; - continue; - } - // Key the reasoning piece on its TEXT (deterministic), consistent - // with the anthropic/openai codecs. The host mints a per-request - // item id that the primeFold mirror cannot reproduce; keying on it - // put the mirror in a different ref/fingerprint space, so restart - // replay rejected every in-stream compress call (issue #64, - // responses variant). - const text = reasoningText(item); - const rid = - text.length > 0 - ? text - : "id" in item && typeof item.id === "string" - ? item.id - : hashId(JSON.stringify(item)); - coreId = clusters.next(deriveMessageId("assistant", "reasoning", rid)); - msgs.push({ - id: coreId, - role: "assistant", - contentType: "reasoning", - text: rid, - rawResponsesItem: item, - }); - break; - } - case "message": { - const message = item as ResponseInputMessage; - const text = messageContent(message.content); - if (message.role === "system" || message.role === "developer") { - systemParts.push(text); - idx++; - continue; - } else if (message.role === "user" || (message.role === "assistant" && text)) { - const role = message.role; - let effText = text; - if (role === "assistant") { - // Same normalization as openaiToCore: hosts that demote - // prior-turn reasoning inline it as a dialect tag at the - // head of the message text. Split it out before identity - // derivation so the inline form and the separate - // reasoning-item form of one turn share a single - // core-id/fingerprint space. - const split = splitDemotedThinking(text); - if (split) { - msgs.push({ - id: clusters.next(deriveMessageId("assistant", "reasoning", split.reasoning)), - role: "assistant", - contentType: "reasoning", - text: split.reasoning, - rawResponsesItem: item, - }); - effText = split.text; - } - } - // Image-only user items have no text of their own (≥2 image - // parts join to a truthy "\n" — not real text). Track them - // with the "[image]" placeholder so they enter the - // compression space instead of vanishing from the rebuilt - // input (#187/#291). displayText derives from effText - // (post-split); for user items split never applies, so it - // equals messageContent(content). - const hasImage = Array.isArray(message.content) && message.content.some((part) => part.type === "input_image"); - if (effText || (role === "user" && hasImage)) { - const displayText = role === "user" ? userDisplayText(message.content, effText) : effText; - coreId = clusters.next(deriveMessageId(role, "text", displayText)); - const imageUrl = Array.isArray(message.content) - ? message.content.find((part) => part.type === "input_image" && typeof part.image_url === "string")?.image_url - : undefined; - const image = typeof imageUrl === "string" ? parseDataUrl(imageUrl) : undefined; - msgs.push({ - id: coreId, - role, - contentType: "text", - text: displayText, - rawResponsesItem: item, - ...(image ? { imageMediaType: image.mediaType, imageBase64: image.base64 } : {}), - }); - } - } - break; - } - case "function_call": { - const call = item as ResponseFunctionCall; - coreId = clusters.next(deriveMessageId("assistant", "tool-call", call.arguments ?? "", { - toolCallId: call.call_id, - toolName: call.name, - })); - msgs.push({ - id: coreId, - role: "assistant", - contentType: "tool-call", - toolName: call.name, - toolCallId: call.call_id, - text: call.arguments ?? "", - rawResponsesItem: item, - }); - toolNames.set(call.call_id, call.name); - break; - } - case "function_call_output": { - const output = item as ResponseFunctionCallOutput; - const text = typeof output.output === "string" ? output.output : JSON.stringify(output.output); - coreId = clusters.next(deriveMessageId("tool", "tool-result", text, { toolCallId: output.call_id })); - const name = toolNames.get(output.call_id); - msgs.push({ id: coreId, role: "tool", contentType: "tool-result", toolCallId: output.call_id, text, ...(name ? { toolName: name } : {}), rawResponsesItem: item }); - break; - } - case "computer_call": - case "computer_call_output": - case "file_search_call": - case "web_search_call": - case "image_generation_call": - case "code_interpreter_call": - case "mcp_call": { - const rid = - typeof (item as { id?: unknown }).id === "string" - ? String((item as { id?: string }).id) - : hashId(JSON.stringify(item)); - coreId = clusters.next(deriveMessageId("assistant", "responses-call", rid)); - msgs.push({ id: coreId, role: "assistant", contentType: "reasoning", text: rid, rawResponsesItem: item }); - break; - } - case "custom_tool_call": { - const ctc = item as { call_id?: string; name?: string; input?: string; arguments?: string }; - const callId = ctc.call_id ?? `call_${idx}`; - customToolCallIds.add(callId); - const argText = ctc.input ?? ctc.arguments ?? ""; - coreId = clusters.next(deriveMessageId("assistant", "tool-call", argText, { toolCallId: callId, toolName: ctc.name ?? "custom" })); - msgs.push({ id: coreId, role: "assistant", contentType: "tool-call", toolName: ctc.name ?? "custom", toolCallId: callId, text: argText, rawResponsesItem: item }); - toolNames.set(callId, ctc.name ?? "custom"); - break; - } - case "custom_tool_call_output": { - const ctco = item as { call_id?: string; output?: string }; - const callId = ctco.call_id ?? `call_${idx}`; - customToolCallIds.add(callId); - const outText = typeof ctco.output === "string" ? ctco.output : JSON.stringify(ctco.output ?? ""); - coreId = clusters.next(deriveMessageId("tool", "tool-result", outText, { toolCallId: callId })); - const name = toolNames.get(callId); - msgs.push({ id: coreId, role: "tool", contentType: "tool-result", toolCallId: callId, text: outText, ...(name ? { toolName: name } : {}), rawResponsesItem: item }); - break; +export function responsesToCore( + body: ResponsesRequestBody, +): ResponsesProjection { + const msgs: BiliMessage[] = []; + const systemParts: string[] = []; + const preamble: ResponseInputItem[] = []; + const customToolCallIds = new Set(); + const layout: ResponseLayoutSlot[] = []; + let droppedReasoning = 0; + const clusters = new ClusterCounter(); + // Backfill toolName onto call outputs from their paired calls — kernel + // guards are name-based. Not part of the identity seed (id stability). + const toolNames = new Map(); + let idx = 0; + if (typeof body.instructions === "string" && body.instructions.trim()) + systemParts.push(body.instructions); + if (typeof body.input === "string") { + const id = clusters.next(deriveMessageId("user", "text", body.input)); + msgs.push({ id, role: "user", contentType: "text", text: body.input }); + return { + msgs, + systemParts, + preamble, + customToolCallIds, + layout, + droppedReasoning, + stringInput: { original: body.input, coreId: id }, + }; + } + for (const raw of body.input) { + const item = normalizeEasyInputItem(raw); + let coreId: string | undefined; + if (isOpaqueItem(item)) preamble.push(item); + switch (item.type) { + case "reasoning": { + if (shouldDropAllReasoning()) { + droppedReasoning++; + continue; + } + // Key the reasoning piece on its TEXT (deterministic), consistent + // with the anthropic/openai codecs. The host mints a per-request + // item id that the primeFold mirror cannot reproduce; keying on it + // put the mirror in a different ref/fingerprint space, so restart + // replay rejected every in-stream compress call (issue #64, + // responses variant). + const text = reasoningText(item); + const rid = + text.length > 0 + ? text + : "id" in item && typeof item.id === "string" + ? item.id + : hashId(JSON.stringify(item)); + coreId = clusters.next(deriveMessageId("assistant", "reasoning", rid)); + msgs.push({ + id: coreId, + role: "assistant", + contentType: "reasoning", + text: rid, + rawResponsesItem: item, + }); + break; + } + case "message": { + const message = item as ResponseInputMessage; + const text = messageContent(message.content); + if (message.role === "system" || message.role === "developer") { + systemParts.push(text); + idx++; + continue; + } else if ( + message.role === "user" || + (message.role === "assistant" && text) + ) { + const role = message.role; + let effText = text; + if (role === "assistant") { + // Same normalization as openaiToCore: hosts that demote + // prior-turn reasoning inline it as a dialect tag at the + // head of the message text. Split it out before identity + // derivation so the inline form and the separate + // reasoning-item form of one turn share a single + // core-id/fingerprint space. + const split = splitDemotedThinking(text); + if (split) { + msgs.push({ + id: clusters.next( + deriveMessageId("assistant", "reasoning", split.reasoning), + ), + role: "assistant", + contentType: "reasoning", + text: split.reasoning, + rawResponsesItem: item, + }); + effText = split.text; } - default: - if (!isOpaqueItem(item)) preamble.push(item); - break; + } + // Image-only user items have no text of their own (≥2 image + // parts join to a truthy "\n" — not real text). Track them + // with the "[image]" placeholder so they enter the + // compression space instead of vanishing from the rebuilt + // input (#187/#291). displayText derives from effText + // (post-split); for user items split never applies, so it + // equals messageContent(content). + const hasImage = + Array.isArray(message.content) && + message.content.some((part) => part.type === "input_image"); + if (effText || (role === "user" && hasImage)) { + const displayText = + role === "user" + ? userDisplayText(message.content, effText) + : effText; + coreId = clusters.next(deriveMessageId(role, "text", displayText)); + const imageUrl = Array.isArray(message.content) + ? message.content.find( + (part) => + part.type === "input_image" && + typeof part.image_url === "string", + )?.image_url + : undefined; + const image = + typeof imageUrl === "string" ? parseDataUrl(imageUrl) : undefined; + msgs.push({ + id: coreId, + role, + contentType: "text", + text: displayText, + rawResponsesItem: item, + ...(image + ? { imageMediaType: image.mediaType, imageBase64: image.base64 } + : {}), + }); + } } - layout.push({ original: item, coreId }); - idx++; + break; + } + case "function_call": { + const call = item as ResponseFunctionCall; + coreId = clusters.next( + deriveMessageId("assistant", "tool-call", call.arguments ?? "", { + toolCallId: call.call_id, + toolName: call.name, + }), + ); + msgs.push({ + id: coreId, + role: "assistant", + contentType: "tool-call", + toolName: call.name, + toolCallId: call.call_id, + text: call.arguments ?? "", + rawResponsesItem: item, + }); + toolNames.set(call.call_id, call.name); + break; + } + case "function_call_output": { + const output = item as ResponseFunctionCallOutput; + const text = + typeof output.output === "string" + ? output.output + : JSON.stringify(output.output); + coreId = clusters.next( + deriveMessageId("tool", "tool-result", text, { + toolCallId: output.call_id, + }), + ); + const name = toolNames.get(output.call_id); + msgs.push({ + id: coreId, + role: "tool", + contentType: "tool-result", + toolCallId: output.call_id, + text, + ...(name ? { toolName: name } : {}), + rawResponsesItem: item, + }); + break; + } + case "computer_call": + case "computer_call_output": + case "file_search_call": + case "web_search_call": + case "image_generation_call": + case "code_interpreter_call": + case "mcp_call": { + const rid = + typeof (item as { id?: unknown }).id === "string" + ? String((item as { id?: string }).id) + : hashId(JSON.stringify(item)); + coreId = clusters.next( + deriveMessageId("assistant", "responses-call", rid), + ); + msgs.push({ + id: coreId, + role: "assistant", + contentType: "reasoning", + text: rid, + rawResponsesItem: item, + }); + break; + } + case "custom_tool_call": { + const ctc = item as { + call_id?: string; + name?: string; + input?: string; + arguments?: string; + }; + const callId = ctc.call_id ?? `call_${idx}`; + customToolCallIds.add(callId); + const argText = ctc.input ?? ctc.arguments ?? ""; + coreId = clusters.next( + deriveMessageId("assistant", "tool-call", argText, { + toolCallId: callId, + toolName: ctc.name ?? "custom", + }), + ); + msgs.push({ + id: coreId, + role: "assistant", + contentType: "tool-call", + toolName: ctc.name ?? "custom", + toolCallId: callId, + text: argText, + rawResponsesItem: item, + }); + toolNames.set(callId, ctc.name ?? "custom"); + break; + } + case "custom_tool_call_output": { + const ctco = item as { call_id?: string; output?: string }; + const callId = ctco.call_id ?? `call_${idx}`; + customToolCallIds.add(callId); + const outText = + typeof ctco.output === "string" + ? ctco.output + : JSON.stringify(ctco.output ?? ""); + coreId = clusters.next( + deriveMessageId("tool", "tool-result", outText, { + toolCallId: callId, + }), + ); + const name = toolNames.get(callId); + msgs.push({ + id: coreId, + role: "tool", + contentType: "tool-result", + toolCallId: callId, + text: outText, + ...(name ? { toolName: name } : {}), + rawResponsesItem: item, + }); + break; + } + default: + if (!isOpaqueItem(item)) preamble.push(item); + break; } - return { msgs, systemParts, preamble, customToolCallIds, layout, droppedReasoning }; + layout.push({ original: item, coreId }); + idx++; + } + return { + msgs, + systemParts, + preamble, + customToolCallIds, + layout, + droppedReasoning, + }; } -function patchTextParts(parts: ResponseContentPart[], text: string): ResponseContentPart[] { - const textIndexes = parts.flatMap((part, index) => - part.type === "input_text" || part.type === "output_text" ? [index] : [], - ); - if (textIndexes.length === 0) return [{ type: "input_text", text }, ...parts]; - const first = textIndexes[0]; - const remaining = new Set(textIndexes.slice(1)); - return parts.map((part, index) => { - if (index === first) return { ...part, text }; - if (remaining.has(index)) return { ...part, text: "" }; - return part; - }); +function patchTextParts( + parts: ResponseContentPart[], + text: string, +): ResponseContentPart[] { + const textIndexes = parts.flatMap((part, index) => + part.type === "input_text" || part.type === "output_text" ? [index] : [], + ); + if (textIndexes.length === 0) return [{ type: "input_text", text }, ...parts]; + const first = textIndexes[0]; + const remaining = new Set(textIndexes.slice(1)); + return parts.map((part, index) => { + if (index === first) return { ...part, text }; + if (remaining.has(index)) return { ...part, text: "" }; + return part; + }); } -function patchOriginalItem(original: ResponseInputItem, source: CoreMessage, next: CoreMessage): ResponseInputItem { - if ( - source.text === next.text && - source.toolName === next.toolName && - source.toolCallId === next.toolCallId && - source.role === next.role && - source.contentType === next.contentType - ) return original; - if (original.type === "message") { - const message = original as ResponseInputMessage; - const content = typeof message.content === "string" ? next.text ?? "" : patchTextParts(message.content, next.text ?? ""); - return { ...message, content }; - } - if (original.type === "function_call") { - return { - ...original, - name: next.toolName ?? String(original.name ?? "unknown"), - call_id: next.toolCallId ?? String(original.call_id ?? ""), - arguments: next.text ?? "", - }; - } - if (original.type === "function_call_output") { - return { ...original, call_id: next.toolCallId ?? String(original.call_id ?? ""), output: next.text ?? "" }; - } +function patchOriginalItem( + original: ResponseInputItem, + source: CoreMessage, + next: CoreMessage, +): ResponseInputItem { + if ( + source.text === next.text && + source.toolName === next.toolName && + source.toolCallId === next.toolCallId && + source.role === next.role && + source.contentType === next.contentType + ) return original; + if (original.type === "message") { + const message = original as ResponseInputMessage; + const content = + typeof message.content === "string" + ? (next.text ?? "") + : patchTextParts(message.content, next.text ?? ""); + return { ...message, content }; + } + if (original.type === "function_call") { + return { + ...original, + name: next.toolName ?? String(original.name ?? "unknown"), + call_id: next.toolCallId ?? String(original.call_id ?? ""), + arguments: next.text ?? "", + }; + } + if (original.type === "function_call_output") { + return { + ...original, + call_id: next.toolCallId ?? String(original.call_id ?? ""), + output: next.text ?? "", + }; + } + return original; } -export function patchResponsesInput(projection: ResponsesProjection, messages: CoreMessage[]): string | ResponseInputItem[] { - if (projection.stringInput) { - const original = projection.msgs.find((message) => message.id === projection.stringInput?.coreId); - const next = messages.find((message) => message.id === projection.stringInput?.coreId); - if (original && next && messages.length === 1 && next.role === "user" && next.contentType === "text") { - return next.text === original.text ? projection.stringInput.original : next.text ?? ""; - } - return coreToResponses(messages, projection.customToolCallIds); +export function patchResponsesInput( + projection: ResponsesProjection, + messages: CoreMessage[], +): string | ResponseInputItem[] { + if (projection.stringInput) { + const original = projection.msgs.find( + (message) => message.id === projection.stringInput?.coreId, + ); + const next = messages.find( + (message) => message.id === projection.stringInput?.coreId, + ); + if ( + original && + next && + messages.length === 1 && + next.role === "user" && + next.contentType === "text" + ) { + return next.text === original.text + ? projection.stringInput.original + : (next.text ?? ""); } - const sourceById = new Map(projection.msgs.map((message) => [message.id, message])); - const nextById = new Map(messages.map((message) => [message.id, message])); - const slotById = new Map(); - projection.layout.forEach((slot, index) => { - if (slot.coreId) slotById.set(slot.coreId, index); - }); - const insertions = new Map(); - for (let index = 0; index < messages.length; index++) { - const message = messages[index]!; - if (sourceById.has(message.id)) continue; - let target = projection.layout.length; - for (let nextIndex = index + 1; nextIndex < messages.length; nextIndex++) { - const slot = slotById.get(messages[nextIndex]!.id); - if (slot !== undefined) { - target = slot; - break; - } - } - const generated = coreToResponses([message], projection.customToolCallIds); - if (generated.length > 0) insertions.set(target, [...(insertions.get(target) ?? []), ...generated]); + return coreToResponses(messages, projection.customToolCallIds); + } + const sourceById = new Map( + projection.msgs.map((message) => [message.id, message]), + ); + const nextById = new Map(messages.map((message) => [message.id, message])); + const slotById = new Map(); + projection.layout.forEach((slot, index) => { + if (slot.coreId) slotById.set(slot.coreId, index); + }); + const insertions = new Map(); + for (let index = 0; index < messages.length; index++) { + const message = messages[index]!; + if (sourceById.has(message.id)) continue; + let target = projection.layout.length; + for (let nextIndex = index + 1; nextIndex < messages.length; nextIndex++) { + const slot = slotById.get(messages[nextIndex]!.id); + if (slot !== undefined) { + target = slot; + break; + } } - const out: ResponseInputItem[] = []; - projection.layout.forEach((slot, index) => { - out.push(...(insertions.get(index) ?? [])); - if (!slot.coreId) { - out.push(slot.original); - return; - } - const source = sourceById.get(slot.coreId); - const next = nextById.get(slot.coreId); - if (source && next) out.push(patchOriginalItem(slot.original, source, next)); - }); - out.push(...(insertions.get(projection.layout.length) ?? [])); - return out; + const generated = coreToResponses([message], projection.customToolCallIds); + if (generated.length > 0) + insertions.set(target, [...(insertions.get(target) ?? []), ...generated]); + } + const out: ResponseInputItem[] = []; + projection.layout.forEach((slot, index) => { + out.push(...(insertions.get(index) ?? [])); + if (!slot.coreId) { + out.push(slot.original); + return; + } + const source = sourceById.get(slot.coreId); + const next = nextById.get(slot.coreId); + if (source && next) + out.push(patchOriginalItem(slot.original, source, next)); + }); + out.push(...(insertions.get(projection.layout.length) ?? [])); + return out; } export function coreToResponses( - messages: CoreMessage[], - customToolCallIds: Set = new Set(), + messages: CoreMessage[], + customToolCallIds: Set = new Set(), ): ResponseInputItem[] { - const out: ResponseInputItem[] = []; - for (const message of messages) { - const biliMessage = message as BiliMessage; - const raw = biliMessage.rawResponsesItem as ResponseInputItem | undefined; - if (message.role === "system") { - out.push({ type: "message", role: "developer", content: message.text ?? "" }); - } else if (message.role === "user") { - const rawMessage = raw?.type === "message" ? (raw as ResponseInputMessage) : undefined; - if (rawMessage && userDisplayText(rawMessage.content, messageContent(rawMessage.content)) === (message.text ?? "")) out.push(rawMessage); - else out.push({ type: "message", role: "user", content: message.text ?? "" }); - } else if (message.role === "assistant") { - if (message.contentType === "text") { - out.push({ type: "message", role: "assistant", content: message.text ?? "" }); - } else if (message.contentType === "tool-call") { - const callId = message.toolCallId ?? `call_${message.id}`; - if (customToolCallIds.has(callId)) { - out.push({ type: "custom_tool_call", call_id: callId, name: message.toolName ?? "unknown", input: message.text ?? "", status: "completed" } as ResponseInputItem); - } else { - out.push({ type: "function_call", call_id: callId, name: message.toolName ?? "unknown", arguments: message.text ?? "" }); - } - } else if (message.contentType === "reasoning") { - if (raw) out.push(raw); - } - } else if (message.role === "tool") { - const callId = message.toolCallId ?? ""; - if (customToolCallIds.has(callId)) { - out.push({ type: "custom_tool_call_output", call_id: callId, output: message.text ?? "" } as ResponseInputItem); - } else { - out.push({ type: "function_call_output", call_id: callId, output: message.text ?? "" }); - } + const out: ResponseInputItem[] = []; + for (const message of messages) { + const biliMessage = message as BiliMessage; + const raw = biliMessage.rawResponsesItem as ResponseInputItem | undefined; + if (message.role === "system") { + out.push({ + type: "message", + role: "developer", + content: message.text ?? "", + }); + } else if (message.role === "user") { + const rawMessage = + raw?.type === "message" ? (raw as ResponseInputMessage) : undefined; + if ( + rawMessage && + userDisplayText( + rawMessage.content, + messageContent(rawMessage.content), + ) === (message.text ?? "") + ) + out.push(rawMessage); + else + out.push({ + type: "message", + role: "user", + content: message.text ?? "", + }); + } else if (message.role === "assistant") { + if (message.contentType === "text") { + out.push({ + type: "message", + role: "assistant", + content: message.text ?? "", + }); + } else if (message.contentType === "tool-call") { + const callId = message.toolCallId ?? `call_${message.id}`; + if (customToolCallIds.has(callId)) { + out.push({ + type: "custom_tool_call", + call_id: callId, + name: message.toolName ?? "unknown", + input: message.text ?? "", + status: "completed", + } as ResponseInputItem); + } else { + out.push({ + type: "function_call", + call_id: callId, + name: message.toolName ?? "unknown", + arguments: message.text ?? "", + }); } + } else if (message.contentType === "reasoning") { + if (raw) out.push(raw); + } + } else if (message.role === "tool") { + const callId = message.toolCallId ?? ""; + if (customToolCallIds.has(callId)) { + out.push({ + type: "custom_tool_call_output", + call_id: callId, + output: message.text ?? "", + } as ResponseInputItem); + } else { + out.push({ + type: "function_call_output", + call_id: callId, + output: message.text ?? "", + }); + } } - return out; + } + return out; } export function injectResponsesDeveloperMessage( - input: string | ResponseInputItem[], - content: string, + input: string | ResponseInputItem[], + content: string, ): ResponseInputItem[] { - const items: ResponseInputItem[] = typeof input === "string" - ? [{ type: "message", role: "user", content: input }] - : [...input]; - let index = 0; - while (items[index]?.type === "additional_tools") index++; - items.splice(index, 0, { type: "message", role: "developer", content }); - return items; + const items: ResponseInputItem[] = + typeof input === "string" + ? [{ type: "message", role: "user", content: input }] + : [...input]; + let index = 0; + while (items[index]?.type === "additional_tools") index++; + items.splice(index, 0, { type: "message", role: "developer", content }); + return items; } export function conversationIdentityResponses( - body: ResponsesRequestBody, - headerValue?: string, + body: ResponsesRequestBody, + headerValue?: string, ): ConversationIdentity { - if (headerValue?.trim()) return { value: headerValue.trim(), source: "header", clientProvided: true }; - if (typeof body.session_id === "string" && body.session_id.trim()) { - return { value: body.session_id.trim(), source: "body-session", clientProvided: true }; - } - const metadataSession = body.metadata?.session_id; - if (typeof metadataSession === "string" && metadataSession.trim()) { - return { value: metadataSession.trim(), source: "metadata-session", clientProvided: true }; - } - if (typeof body.previous_response_id === "string" && body.previous_response_id.trim()) { - return { value: body.previous_response_id.trim(), source: "previous-response", clientProvided: false }; - } - return { value: hashId(JSON.stringify(body.input ?? [])), source: "content-fingerprint", clientProvided: false }; + if (headerValue?.trim()) + return { + value: headerValue.trim(), + source: "header", + clientProvided: true, + }; + if (typeof body.session_id === "string" && body.session_id.trim()) { + return { + value: body.session_id.trim(), + source: "body-session", + clientProvided: true, + }; + } + const metadataSession = body.metadata?.session_id; + if (typeof metadataSession === "string" && metadataSession.trim()) { + return { + value: metadataSession.trim(), + source: "metadata-session", + clientProvided: true, + }; + } + if ( + typeof body.previous_response_id === "string" && + body.previous_response_id.trim() + ) { + return { + value: body.previous_response_id.trim(), + source: "previous-response", + clientProvided: false, + }; + } + return { + value: hashId(JSON.stringify(body.input ?? [])), + source: "content-fingerprint", + clientProvided: false, + }; } -export function conversationSignalResponses(body: ResponsesRequestBody, headerValue?: string): string { - return conversationIdentityResponses(body, headerValue).value; +export function conversationSignalResponses( + body: ResponsesRequestBody, + headerValue?: string, +): string { + return conversationIdentityResponses(body, headerValue).value; } // Codex subagents (guardian approval reviewer, etc.) reuse the main @@ -503,10 +713,10 @@ export function conversationSignalResponses(body: ResponsesRequestBody, headerVa // compression state. Subagent requests are self-contained replays, so the // fresh namespace is lossless. export interface SubagentNamespaces { - /** Resolve the compression-state namespace for a request: the identity - * itself for the anchored (main) instructions, `identity|sub:` for - * any other instructions value. First-seen instructions anchor. */ - namespaceFor(identityValue: string, instructions: unknown): string; + /** Resolve the compression-state namespace for a request: the identity + * itself for the anchored (main) instructions, `identity|sub:` for + * any other instructions value. First-seen instructions anchor. */ + namespaceFor(identityValue: string, instructions: unknown): string; } /** Host-owned subagent-namespace store. @@ -518,19 +728,20 @@ export interface SubagentNamespaces { * first request it sees, which after a restart may be a subagent request, * orphaning the main conversation's stored compression state). */ export function createSubagentNamespaces(): SubagentNamespaces { - const anchors = new Map(); - return { - namespaceFor(identityValue: string, instructions: unknown): string { - if (typeof instructions !== "string" || instructions.trim().length === 0) return identityValue; - const fp = hashId(instructions); - const anchor = anchors.get(identityValue); - if (anchor === undefined) { - anchors.set(identityValue, fp); - return identityValue; - } - return anchor === fp ? identityValue : `${identityValue}|sub:${fp}`; - }, - }; + const anchors = new Map(); + return { + namespaceFor(identityValue: string, instructions: unknown): string { + if (typeof instructions !== "string" || instructions.trim().length === 0) + return identityValue; + const fp = hashId(instructions); + const anchor = anchors.get(identityValue); + if (anchor === undefined) { + anchors.set(identityValue, fp); + return identityValue; + } + return anchor === fp ? identityValue : `${identityValue}|sub:${fp}`; + }, + }; } // Convenience singleton for single-process proxies that don't manage @@ -540,6 +751,9 @@ export function createSubagentNamespaces(): SubagentNamespaces { // createSubagentNamespaces() and own its lifecycle. const defaultNamespaces = createSubagentNamespaces(); -export function subagentNamespace(identityValue: string, instructions: unknown): string { - return defaultNamespaces.namespaceFor(identityValue, instructions); +export function subagentNamespace( + identityValue: string, + instructions: unknown, +): string { + return defaultNamespaces.namespaceFor(identityValue, instructions); } diff --git a/src/wire/strip-images.ts b/src/wire/strip-images.ts index 15ad4d1..b391ca2 100644 --- a/src/wire/strip-images.ts +++ b/src/wire/strip-images.ts @@ -17,7 +17,8 @@ // never sent unpaired). Shared across hosts // (proxy + in-process adapters) so "which field carries an image" has one home. -export type StripProtocol = "anthropic" | "openai" | "responses" | "google" | null; +export type StripProtocol = + "anthropic" | "openai" | "responses" | "google" | null; export interface StripResult { body: unknown; diff --git a/src/wire/util.ts b/src/wire/util.ts index af01fca..d00b3b1 100644 --- a/src/wire/util.ts +++ b/src/wire/util.ts @@ -4,14 +4,20 @@ import { createHash } from "node:crypto"; * stay byte-identical across the extraction (re-keying would orphan every * downstream map keyed on these ids). */ export function hashId(s: string): string { - return createHash("sha256").update(s, "utf8").digest("hex").slice(0, 16); + return createHash("sha256").update(s, "utf8").digest("hex").slice(0, 16); } /** How a conversation's identity was derived (kept verbatim from the proxy's * session-id.ts — only the TYPE moves; session-key derivation stays in the * proxy, which owns multi-tenant state). */ export type ConversationIdentity = { - value: string; - source: "header" | "body-session" | "metadata-session" | "previous-response" | "content-fingerprint" | "generated"; - clientProvided: boolean; + value: string; + source: + | "header" + | "body-session" + | "metadata-session" + | "previous-response" + | "content-fingerprint" + | "generated"; + clientProvided: boolean; }; diff --git a/tests/block-map.test.ts b/tests/block-map.test.ts index 7390c8e..c7c46bc 100644 --- a/tests/block-map.test.ts +++ b/tests/block-map.test.ts @@ -43,7 +43,11 @@ function config(): Config { }; } -function msg(id: string, text: string, role: CoreMessage["role"] = "user"): CoreMessage { +function msg( + id: string, + text: string, + role: CoreMessage["role"] = "user", +): CoreMessage { return { id, role, contentType: "text", text }; } @@ -56,7 +60,9 @@ function makeState(ids: string[]): ReturnType { return state; } -function makeBlock(overrides: Partial = {}): CompressionBlock { +function makeBlock( + overrides: Partial = {}, +): CompressionBlock { return { blockId: "b1", runId: "r1", @@ -166,11 +172,15 @@ test("tier-distilled block created via bN..bM boundaries reports an m-ref span", { startRef: "m00001", endRef: "m00010", summary: "first ten" }, { startRef: "m00011", endRef: "m00020", summary: "next ten" }, ], - messages, state, config: config(), + messages, + state, + config: config(), }).state; state = core.applyCompression({ ranges: [{ startRef: "b1", endRef: "b2", summary: "distilled" }], - messages, state, config: config(), + messages, + state, + config: config(), }).state; const t2 = state.blocks[state.blocks.length - 1]!; assert.equal(t2.tier, 2); @@ -187,8 +197,18 @@ test("tier-distilled block created via bN..bM boundaries reports an m-ref span", test("activeBlockSpans skips inactive and unresolvable blocks, keeps order", () => { const state = makeState(["a", "b"]); state.blocks = [ - makeBlock({ blockId: "b1", effectiveMessageIds: ["a"], startRef: "m00001", endRef: "m00001" }), - makeBlock({ blockId: "b2", active: false, startRef: "m00002", endRef: "m00002" }), + makeBlock({ + blockId: "b1", + effectiveMessageIds: ["a"], + startRef: "m00001", + endRef: "m00001", + }), + makeBlock({ + blockId: "b2", + active: false, + startRef: "m00002", + endRef: "m00002", + }), makeBlock({ blockId: "b3", effectiveMessageIds: ["vanished"], tier: 2 }), ]; const spans = activeBlockSpans(state); @@ -233,7 +253,12 @@ test("buildCompressibleRanges counts user messages per range", () => { ]; const state = makeState(messages.map((m) => m.id)); const cfg = config(); - const protectedRefs = computeProtectedRefs(messages, state, cfg, defaultCountTokens); + const protectedRefs = computeProtectedRefs( + messages, + state, + cfg, + defaultCountTokens, + ); const ranges = buildCompressibleRanges( messages, state, diff --git a/tests/cache-report.test.ts b/tests/cache-report.test.ts index 9d78308..cfe8233 100644 --- a/tests/cache-report.test.ts +++ b/tests/cache-report.test.ts @@ -294,7 +294,11 @@ test("miss bucket label is honest: upstream-ttl-or-client-rewrite, never ttl/oth /upstream-ttl-or-client-rewrite \(unattributed\)/, `${detail} mode must use the honest bucket label`, ); - assert.match(text, /kernel cannot distinguish/, `${detail} mode keeps the uncertainty wording`); + assert.match( + text, + /kernel cannot distinguish/, + `${detail} mode keeps the uncertainty wording`, + ); assert.ok( !text.includes("ttl/other"), `${detail} mode must not claim ttl/other (#392)`, @@ -410,7 +414,12 @@ test("formatCacheReport detail:'full' keeps the legacy every-line listing", () = test("formatCacheReport summary lists every fold when few, with TTL spike idle gap", () => { const samples = [sample(0, 100_000, 99_000), sample(1, 100_000, 99_000)]; const folds: FoldEvent[] = [ - { at: T0 + 0.5 * min, tokensCompressed: 20_000, summaryTokens: 500, firstFoldStartTokens: 3_000 }, + { + at: T0 + 0.5 * min, + tokensCompressed: 20_000, + summaryTokens: 500, + firstFoldStartTokens: 3_000, + }, ]; samples.push(sample(10, 100_000, 10_000)); const text = formatCacheReport(buildCacheReport(samples, folds), "s4"); diff --git a/tests/checkpoint-carrier.test.ts b/tests/checkpoint-carrier.test.ts index 1ff3a50..1f67001 100644 --- a/tests/checkpoint-carrier.test.ts +++ b/tests/checkpoint-carrier.test.ts @@ -60,7 +60,13 @@ function buildCarrierScenario() { nextIndex: 1, }).map; const first = core.applyCompression({ - ranges: [{ startRef: "m00002", endRef: "m00005", summary: "block A distills the refactor" }], + ranges: [ + { + startRef: "m00002", + endRef: "m00005", + summary: "block A distills the refactor", + }, + ], messages, state, config: config(), @@ -82,7 +88,13 @@ function buildCarrierScenario() { test("plain message-ref range skips a live host checkpoint carrier and keeps it visible (#335)", () => { const { core, state, messages, blockA } = buildCarrierScenario(); const second = core.applyCompression({ - ranges: [{ startRef: "m00001", endRef: "m00008", summary: "plain range across the checkpoint" }], + ranges: [ + { + startRef: "m00001", + endRef: "m00008", + summary: "plain range across the checkpoint", + }, + ], messages, state, config: config(), @@ -90,10 +102,17 @@ test("plain message-ref range skips a live host checkpoint carrier and keeps it assert.equal(second.result.blocksCreated, 1); assert.equal(second.result.errors.length, 0); const blockB = second.state.blocks[second.state.blocks.length - 1]!; - assert.ok(!blockB.effectiveMessageIds.includes("7"), "carrier must not be folded into the plain range block"); + assert.ok( + !blockB.effectiveMessageIds.includes("7"), + "carrier must not be folded into the plain range block", + ); assert.ok(!blockB.directMessageIds.includes("7")); assert.ok(blockB.effectiveMessageIds.includes("8")); - assert.ok(blockB.effectiveMessageIds.includes("2") && blockB.effectiveMessageIds.includes("5"), "the block's own range is still re-covered"); + assert.ok( + blockB.effectiveMessageIds.includes("2") && + blockB.effectiveMessageIds.includes("5"), + "the block's own range is still re-covered", + ); assert.ok( (second.result.warnings ?? []).some((w) => w.includes("checkpoint")), "the skip must be reported, not silent", @@ -137,7 +156,13 @@ test("block-ref boundary (bN..bM) folds host checkpoint carriers — distillatio config: config(), }).state; const distilled = core.applyCompression({ - ranges: [{ startRef: "b1", endRef: "b2", summary: "tier two across both blocks and the carrier" }], + ranges: [ + { + startRef: "b1", + endRef: "b2", + summary: "tier two across both blocks and the carrier", + }, + ], messages, state, config: config(), @@ -146,7 +171,10 @@ test("block-ref boundary (bN..bM) folds host checkpoint carriers — distillatio assert.equal(distilled.result.errors.length, 0); const t2 = distilled.state.blocks[distilled.state.blocks.length - 1]!; assert.equal(t2.tier, 2); - assert.ok(t2.effectiveMessageIds.includes("6"), "block-boundary distillation folds the carrier"); + assert.ok( + t2.effectiveMessageIds.includes("6"), + "block-boundary distillation folds the carrier", + ); assert.ok( !(distilled.result.warnings ?? []).some((w) => w.includes("checkpoint")), "no checkpoint warning on the distillation path", @@ -172,16 +200,31 @@ test("stale carriers (block inactive or unknown) fold like ordinary messages", ( nextIndex: 9, }).map; const result = core.applyCompression({ - ranges: [{ startRef: "m00001", endRef: "m00011", summary: "plain range over stale carriers" }], + ranges: [ + { + startRef: "m00001", + endRef: "m00011", + summary: "plain range over stale carriers", + }, + ], messages, state, config: config(), }); assert.equal(result.result.blocksCreated, 1); const block = result.state.blocks[result.state.blocks.length - 1]!; - assert.ok(block.effectiveMessageIds.includes("9"), "inactive-block carrier folds"); - assert.ok(block.effectiveMessageIds.includes("10"), "unknown-block carrier folds"); - assert.ok(block.effectiveMessageIds.includes("7"), "carrier of the now-dead block folds too"); + assert.ok( + block.effectiveMessageIds.includes("9"), + "inactive-block carrier folds", + ); + assert.ok( + block.effectiveMessageIds.includes("10"), + "unknown-block carrier folds", + ); + assert.ok( + block.effectiveMessageIds.includes("7"), + "carrier of the now-dead block folds too", + ); }); test("dsh shape: plain range with tool-pair growth consumes the old block yet skips its carrier (#335 exact scenario)", () => { @@ -189,13 +232,55 @@ test("dsh shape: plain range with tool-pair growth consumes the old block yet sk let state = createInitialState(); const messages: CoreMessage[] = [ msg("1", "user one"), - { id: "2", role: "assistant", contentType: "tool-call", toolName: "bash", toolCallId: "c1", text: "{}" }, - { id: "2#c1", role: "assistant", contentType: "tool-call", toolName: "bash", toolCallId: "c2", text: "{}" }, - { id: "3", role: "user", contentType: "tool-result", toolName: "bash", toolCallId: "c1", text: "result c1" }, - { id: "4", role: "user", contentType: "tool-result", toolName: "bash", toolCallId: "c2", text: "" }, + { + id: "2", + role: "assistant", + contentType: "tool-call", + toolName: "bash", + toolCallId: "c1", + text: "{}", + }, + { + id: "2#c1", + role: "assistant", + contentType: "tool-call", + toolName: "bash", + toolCallId: "c2", + text: "{}", + }, + { + id: "3", + role: "user", + contentType: "tool-result", + toolName: "bash", + toolCallId: "c1", + text: "result c1", + }, + { + id: "4", + role: "user", + contentType: "tool-result", + toolName: "bash", + toolCallId: "c2", + text: "", + }, msg("5", "user two"), - { id: "6", role: "assistant", contentType: "tool-call", toolName: "bash", toolCallId: "c3", text: "{}" }, - { id: "7", role: "user", contentType: "tool-result", toolName: "bash", toolCallId: "c3", text: "" }, + { + id: "6", + role: "assistant", + contentType: "tool-call", + toolName: "bash", + toolCallId: "c3", + text: "{}", + }, + { + id: "7", + role: "user", + contentType: "tool-result", + toolName: "bash", + toolCallId: "c3", + text: "", + }, msg("8", "user three"), ]; state.messageRefs = assignRefs(messages, { @@ -203,7 +288,13 @@ test("dsh shape: plain range with tool-pair growth consumes the old block yet sk nextIndex: 1, }).map; state = core.applyCompression({ - ranges: [{ startRef: "m00002", endRef: "m00006", summary: "block A distills the tool work" }], + ranges: [ + { + startRef: "m00002", + endRef: "m00006", + summary: "block A distills the tool work", + }, + ], messages, state, config: config(), @@ -224,7 +315,13 @@ test("dsh shape: plain range with tool-pair growth consumes the old block yet sk }).map; const lastRef = `m${String(messages.length).padStart(5, "0")}`; const second = core.applyCompression({ - ranges: [{ startRef: "m00001", endRef: lastRef, summary: "plain range over everything" }], + ranges: [ + { + startRef: "m00001", + endRef: lastRef, + summary: "plain range over everything", + }, + ], messages, state, config: config(), @@ -232,15 +329,30 @@ test("dsh shape: plain range with tool-pair growth consumes the old block yet sk assert.equal(second.result.blocksCreated, 1); assert.equal(second.result.errors.length, 0); const blockB = second.state.blocks[second.state.blocks.length - 1]!; - assert.ok(!blockB.effectiveMessageIds.includes("9"), "carrier of the consumed block must still be skipped"); + assert.ok( + !blockB.effectiveMessageIds.includes("9"), + "carrier of the consumed block must still be skipped", + ); assert.ok(!blockB.directMessageIds.includes("9")); assert.ok( (second.result.warnings ?? []).some((w) => w.includes("checkpoint")), "skip is reported", ); - const blockACopy = second.state.blocks.find((b) => b.blockId === blockA.blockId)!; - assert.equal(blockACopy.active, false, "tool-pair growth re-scan consumes the nested block (dsh ledger shape)"); - assert.ok(blockB.directBlockIds.includes(blockA.blockId), "parents link recorded"); + const blockACopy = second.state.blocks.find( + (b) => b.blockId === blockA.blockId, + )!; + assert.equal( + blockACopy.active, + false, + "tool-pair growth re-scan consumes the nested block (dsh ledger shape)", + ); + assert.ok( + blockB.directBlockIds.includes(blockA.blockId), + "parents link recorded", + ); const visible = prune(messages, second.state); - assert.ok(visible.some((m) => m.id === "9"), "carrier stays visible"); + assert.ok( + visible.some((m) => m.id === "9"), + "carrier stays visible", + ); }); diff --git a/tests/compress-detect.test.ts b/tests/compress-detect.test.ts index 8af79dd..b9d521e 100644 --- a/tests/compress-detect.test.ts +++ b/tests/compress-detect.test.ts @@ -16,7 +16,9 @@ import { } from "../src/wire/compress-detect.js"; import type { BiliMessage } from "../src/wire/bili-message.js"; -function msg(partial: Partial & { id: string; role: BiliMessage["role"] }): BiliMessage { +function msg( + partial: Partial & { id: string; role: BiliMessage["role"] }, +): BiliMessage { return { contentType: "text", text: "", @@ -32,12 +34,21 @@ const stream: BiliMessage[] = [ msg({ id: "h5", role: "assistant", text: "final answer" }), ]; -const byRef: Record = { m00001: "h1", m00002: "h2", m00003: "h3", m00004: "h4", m00005: "h5" }; +const byRef: Record = { + m00001: "h1", + m00002: "h2", + m00003: "h3", + m00004: "h4", + m00005: "h5", +}; test("compressToolArgs accepts direct compress and legacy xd://compress write shapes", () => { const direct = compressToolArgs({ name: "compress", - arguments: JSON.stringify({ content: [{ startId: "m00001", endId: "m00002", summary: "s" }], topic: "T" }), + arguments: JSON.stringify({ + content: [{ startId: "m00001", endId: "m00002", summary: "s" }], + topic: "T", + }), }); assert.ok(direct); assert.equal(direct.content.length, 1); @@ -47,7 +58,9 @@ test("compressToolArgs accepts direct compress and legacy xd://compress write sh name: "write", arguments: { path: "xd://compress", - content: JSON.stringify({ content: [{ startId: "m00001", endId: "m00003", summary: "s" }] }), + content: JSON.stringify({ + content: [{ startId: "m00001", endId: "m00003", summary: "s" }], + }), }, }); assert.ok(legacy); @@ -55,12 +68,25 @@ test("compressToolArgs accepts direct compress and legacy xd://compress write sh const legacySingle = compressToolArgs({ name: "write", - arguments: { path: "xd://compress", content: JSON.stringify({ startId: "m00001", endRef: "m00002", summary: "s" }) }, + arguments: { + path: "xd://compress", + content: JSON.stringify({ + startId: "m00001", + endRef: "m00002", + summary: "s", + }), + }, }); assert.ok(legacySingle); assert.equal(legacySingle.content.length, 1); - assert.equal(compressToolArgs({ name: "write", arguments: { path: "xd://other", content: "x" } }), null); + assert.equal( + compressToolArgs({ + name: "write", + arguments: { path: "xd://other", content: "x" }, + }), + null, + ); assert.equal(compressToolArgs({ name: "bogus", arguments: {} }), null); }); @@ -71,11 +97,13 @@ test("findCompressCallsCore validates ranges and stamps compressCallId", () => { contentType: "tool-call", toolCallId: "call-7", toolName: "compress", - text: JSON.stringify({ content: [ - { startId: "m00001", endId: "m00003", summary: "first" }, - { startId: "m00001", endId: "m00002" }, // no summary -> skipped - { summary: "no ids" }, // skipped - ] }), + text: JSON.stringify({ + content: [ + { startId: "m00001", endId: "m00003", summary: "first" }, + { startId: "m00001", endId: "m00002" }, // no summary -> skipped + { summary: "no ids" }, // skipped + ], + }), }); const calls = findCompressCallsCore(call); assert.equal(calls.length, 1); @@ -83,13 +111,29 @@ test("findCompressCallsCore validates ranges and stamps compressCallId", () => { assert.equal(calls[0]!.ranges.length, 1); assert.equal(calls[0]!.ranges[0]!.compressCallId, "call-7"); assert.equal(calls[0]!.ranges[0]!.summary, "first"); - assert.deepEqual(findCompressCallsCore(msg({ id: "h1", role: "user", text: "x" })), []); + assert.deepEqual( + findCompressCallsCore(msg({ id: "h1", role: "user", text: "x" })), + [], + ); }); test("toolCallNames and toolResultTextsCore map by toolCallId", () => { const s: BiliMessage[] = [ - msg({ id: "a", role: "assistant", contentType: "tool-call", toolCallId: "t1", toolName: "bash", text: "{}" }), - msg({ id: "b", role: "tool", contentType: "tool-result", toolCallId: "t1", text: "out" }), + msg({ + id: "a", + role: "assistant", + contentType: "tool-call", + toolCallId: "t1", + toolName: "bash", + text: "{}", + }), + msg({ + id: "b", + role: "tool", + contentType: "tool-result", + toolCallId: "t1", + text: "out", + }), ]; assert.deepEqual([...toolCallNames(s)], [["t1", "bash"]]); assert.deepEqual([...toolResultTextsCore(s)], [["t1", "out"]]); @@ -101,18 +145,30 @@ test("span fingerprints hash first/last piece content keys (index and id forms a assert.equal(fp, fpIdx); assert.match(fp, /^[0-9a-f]{8}$/); // same boundaries, different content -> different fp - assert.notEqual(spanFingerprintCoreIdx(stream, 0, 2), spanFingerprintCoreIdx(stream, 0, 4)); + assert.notEqual( + spanFingerprintCoreIdx(stream, 0, 2), + spanFingerprintCoreIdx(stream, 0, 4), + ); // drift BEYOND the first 4096 chars keeps the key; drift inside it does not - const longBase = ["x".repeat(5000), "x".repeat(4096) + "y".repeat(1000)].map((text, i) => - msg({ id: `t${i}`, role: "assistant", text }), + const longBase = ["x".repeat(5000), "x".repeat(4096) + "y".repeat(1000)].map( + (text, i) => msg({ id: `t${i}`, role: "assistant", text }), ); assert.equal(corePieceKey(longBase[0]!), corePieceKey(longBase[1]!)); - const shortDrift = msg({ id: "t2", role: "assistant", text: "x".repeat(4095) + "y" }); + const shortDrift = msg({ + id: "t2", + role: "assistant", + text: "x".repeat(4095) + "y", + }); assert.notEqual(corePieceKey(longBase[0]!), corePieceKey(shortDrift)); // role/contentType/toolName participate in the key assert.notEqual( corePieceKey({ id: "h1", role: "user", text: "a", contentType: "text" }), - corePieceKey({ id: "h1", role: "assistant", text: "a", contentType: "text" }), + corePieceKey({ + id: "h1", + role: "assistant", + text: "a", + contentType: "text", + }), ); }); @@ -134,30 +190,82 @@ function resultTextFor(fp: string, pos: string): string { test("staleRangeCore: matching fp passes, mismatch rejects with diagnostics", () => { const fp = spanFingerprintCore(stream, "h1", "h3"); - const ok = staleRangeCore({ startRef: "m00001", endRef: "m00003" }, 0, resultTextFor(fp, "0-2"), stream, 4, byRef, blocks); + const ok = staleRangeCore( + { startRef: "m00001", endRef: "m00003" }, + 0, + resultTextFor(fp, "0-2"), + stream, + 4, + byRef, + blocks, + ); assert.deepEqual(ok, {}); - const bad = staleRangeCore({ startRef: "m00001", endRef: "m00003" }, 0, resultTextFor("deadbeef", "0-2"), stream, 4, byRef, blocks); + const bad = staleRangeCore( + { startRef: "m00001", endRef: "m00003" }, + 0, + resultTextFor("deadbeef", "0-2"), + stream, + 4, + byRef, + blocks, + ); assert.match(bad.reject ?? "", /fp m00001\.\.m00003 want deadbeef got/); assert.equal(bad.hint, true); }); test("staleRangeCore: unresolved refs and end-after-call reject; block refs defer to kernel", () => { - const unresolved = staleRangeCore({ startRef: "m00042", endRef: "m00043" }, 0, "x", stream, 4, byRef, blocks); + const unresolved = staleRangeCore( + { startRef: "m00042", endRef: "m00043" }, + 0, + "x", + stream, + 4, + byRef, + blocks, + ); assert.match(unresolved.reject ?? "", /unresolved m00042\.\.m00043/); const fp = spanFingerprintCore(stream, "h1", "h3"); - const afterCall = staleRangeCore({ startRef: "m00001", endRef: "m00005" }, 0, resultTextFor(fp, "0-4"), stream, 2, byRef, blocks); + const afterCall = staleRangeCore( + { startRef: "m00001", endRef: "m00005" }, + 0, + resultTextFor(fp, "0-4"), + stream, + 2, + byRef, + blocks, + ); assert.match(afterCall.reject ?? "", /end idx 4 > callIndex 2/); - const blockRef = staleRangeCore({ startRef: "b1", endRef: "b1" }, 0, "x", stream, 4, byRef, blocks); + const blockRef = staleRangeCore( + { startRef: "b1", endRef: "b1" }, + 0, + "x", + stream, + 4, + byRef, + blocks, + ); assert.deepEqual(blockRef, {}); // unresolved block refs: kernel resolves itself }); test("staleRangeCore: benign tail drift recovers by position and remaps dangling m-refs", () => { // h3 drifted (re-hashed): its recorded ref dangles, but [pos=] recovers the index. const drifted = [...stream]; - drifted[2] = msg({ id: "h3x", role: "assistant", text: "the answer is 42 (edited)" }); + drifted[2] = msg({ + id: "h3x", + role: "assistant", + text: "the answer is 42 (edited)", + }); const driftedByRef = { ...byRef, m00003: "h3x" }; const fpWant = spanFingerprintCoreIdx(stream, 0, 2); - const v = staleRangeCore({ startRef: "m00001", endRef: "m00003" }, 0, resultTextFor(fpWant, "0-2"), drifted, 4, driftedByRef, blocks); + const v = staleRangeCore( + { startRef: "m00001", endRef: "m00003" }, + 0, + resultTextFor(fpWant, "0-2"), + drifted, + 4, + driftedByRef, + blocks, + ); // content changed -> fp mismatch rejects (edit beyond first-4096 changes the key) assert.match(v.reject ?? "", /want/); assert.equal(v.hint, true); @@ -167,16 +275,40 @@ test("staleRangeCore: protected recovered piece without a ref fails closed", () const fpWant = spanFingerprintCoreIdx(stream, 0, 2); // start ref dangles and the recovered piece has no ref in byRef const emptyByRef: Record = {}; - const v = staleRangeCore({ startRef: "m00001", endRef: "m00003" }, 0, resultTextFor(fpWant, "0-2"), stream, 4, emptyByRef, blocks); + const v = staleRangeCore( + { startRef: "m00001", endRef: "m00003" }, + 0, + resultTextFor(fpWant, "0-2"), + stream, + 4, + emptyByRef, + blocks, + ); // fp still computed from positions 0..2 and matches; m00001 remap finds no ref assert.match(v.reject ?? "", /no ref \(protected piece\)/); }); test("rangeFingerprintsCore and rangePositionsCore emit per-range pairs", () => { - const fps = rangeFingerprintsCore([{ startRef: "m00001", endRef: "m00003" }, { startRef: "m00009", endRef: "m00010" }], stream, byRef, blocks); + const fps = rangeFingerprintsCore( + [ + { startRef: "m00001", endRef: "m00003" }, + { startRef: "m00009", endRef: "m00010" }, + ], + stream, + byRef, + blocks, + ); assert.match(fps[0]!, /^[0-9a-f]{8}$/); assert.equal(fps[1], "-"); // unresolvable range - const pos = rangePositionsCore([{ startRef: "m00001", endRef: "m00003" }, { startRef: "b1", endRef: "b1" }], stream, byRef, blocks); + const pos = rangePositionsCore( + [ + { startRef: "m00001", endRef: "m00003" }, + { startRef: "b1", endRef: "b1" }, + ], + stream, + byRef, + blocks, + ); assert.equal(pos[0], "0-2"); assert.equal(pos[1], "0-2"); // block resolves to h1..h3 = 0..2 }); diff --git a/tests/compress-schema-contract.test.ts b/tests/compress-schema-contract.test.ts index 452bde1..a36afbc 100644 --- a/tests/compress-schema-contract.test.ts +++ b/tests/compress-schema-contract.test.ts @@ -15,7 +15,8 @@ import { parseCompressArgs } from "../src/parse-compress-input.js"; * kernel; whatever the kernel accepts must not be killed here. */ function typeOk(t: string, v: unknown): boolean { if (t === "array") return Array.isArray(v); - if (t === "object") return v !== null && typeof v === "object" && !Array.isArray(v); + if (t === "object") + return v !== null && typeof v === "object" && !Array.isArray(v); return typeof v === t; } @@ -25,14 +26,23 @@ function validate(schema: Record, value: unknown): boolean { if (!types.some((t) => typeOk(String(t), value))) return false; } if (Array.isArray(value) && schema.items !== undefined) { - if (!value.every((item) => validate(schema.items as Record, item))) return false; + if ( + !value.every((item) => + validate(schema.items as Record, item), + ) + ) + return false; } if (value !== null && typeof value === "object" && !Array.isArray(value)) { const obj = value as Record; - if (Array.isArray(schema.required) && !(schema.required as string[]).every((k) => obj[k] !== undefined)) { + if ( + Array.isArray(schema.required) && + !(schema.required as string[]).every((k) => obj[k] !== undefined) + ) { return false; } - const props = schema.properties as Record> | undefined; + const props = schema.properties as + Record> | undefined; if (props) { for (const [k, sub] of Object.entries(props)) { if (obj[k] !== undefined && !validate(sub, obj[k])) return false; @@ -40,7 +50,12 @@ function validate(schema: Record, value: unknown): boolean { } } if (Array.isArray(schema.anyOf)) { - if (!(schema.anyOf as Record[]).some((sub) => validate(sub, value))) return false; + if ( + !(schema.anyOf as Record[]).some((sub) => + validate(sub, value), + ) + ) + return false; } return true; } @@ -52,7 +67,11 @@ function acceptedByBoth(input: unknown, expectedRanges: number): void { `schema rejected a shape the kernel accepts: ${JSON.stringify(input)}`, ); const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(ranges.length, expectedRanges, `parser recovered ${ranges.length}, kind=${diagnostics.kind}`); + assert.equal( + ranges.length, + expectedRanges, + `parser recovered ${ranges.length}, kind=${diagnostics.kind}`, + ); assert.ok(diagnostics.ok); } @@ -74,22 +93,46 @@ test("all three wire shapes carry the identical shared parameter schema", () => test("issue #374 repro shapes: flat form and string content are accepted by both layers", () => { // Repro 1: flat {topic, startId, endId, summary} at top level (no content). - acceptedByBoth({ topic: "T", startId: "m00150", endId: "m00220", summary: "S" }, 1); + acceptedByBoth( + { topic: "T", startId: "m00150", endId: "m00220", summary: "S" }, + 1, + ); // Repro 2: content as a bare line-form string. acceptedByBoth({ content: "m00150–m00220 T\nS body" }, 1); }); test("every shape parseCompressArgs accepts passes the declared schema", () => { acceptedByBoth({ content: ["m00150-m00220\nS body"] }, 1); - acceptedByBoth({ content: [{ startId: "m00150", endId: "m00220", summary: "S" }] }, 1); acceptedByBoth( - { content: ["m00150-m00160 A\nfirst", { startId: "m00170", endId: "m00220", summary: "second", topic: "B" }] }, + { content: [{ startId: "m00150", endId: "m00220", summary: "S" }] }, + 1, + ); + acceptedByBoth( + { + content: [ + "m00150-m00160 A\nfirst", + { startId: "m00170", endId: "m00220", summary: "second", topic: "B" }, + ], + }, 2, ); - acceptedByBoth({ content: [{ startRef: "m00150", endRef: "m00220", summary: "S" }] }, 1); + acceptedByBoth( + { content: [{ startRef: "m00150", endRef: "m00220", summary: "S" }] }, + 1, + ); acceptedByBoth({ startRef: "m00150", endRef: "m00220", summary: "S" }, 1); - acceptedByBoth({ content: JSON.stringify([{ startId: "m00001", endId: "m00002", summary: "S" }]) }, 1); - acceptedByBoth({ content: "m00150–m00160 A\nfirst\nm00170–m00220 B\nsecond" }, 2); + acceptedByBoth( + { + content: JSON.stringify([ + { startId: "m00001", endId: "m00002", summary: "S" }, + ]), + }, + 1, + ); + acceptedByBoth( + { content: "m00150–m00160 A\nfirst\nm00170–m00220 B\nsecond" }, + 2, + ); }); test("structural violations are rejected by both layers", () => { @@ -110,7 +153,11 @@ test("empty-ish calls are the parser's job, not the schema's (bili #1299)", () = validate(COMPRESS_PARAMETERS as Record, input), `schema must stay permissive at top level: ${JSON.stringify(input)}`, ); - assert.equal(parseCompressArgs(input).ranges.length, 0, `parser rejected: ${JSON.stringify(input)}`); + assert.equal( + parseCompressArgs(input).ranges.length, + 0, + `parser rejected: ${JSON.stringify(input)}`, + ); } }); @@ -120,20 +167,29 @@ test("an object under content stays rejected (parser yields zero ranges for it)" // content as "content-not-array" and recovers nothing — so the schema must // not widen to accept it either (that would trade a loud rejection for a // silent no-op). - rejectedByBoth({ content: { startId: "m00150", endId: "m00220", summary: "S" } }); + rejectedByBoth({ + content: { startId: "m00150", endId: "m00220", summary: "S" }, + }); }); test("empty content array passes the schema; the parser degrades gracefully", () => { // Deliberate leniency pin: no minItems. The kernel handles an empty batch // with a warning and zero blocks instead of failing the turn. - assert.ok(validate(COMPRESS_PARAMETERS as Record, { content: [] })); + assert.ok( + validate(COMPRESS_PARAMETERS as Record, { content: [] }), + ); assert.equal(parseCompressArgs({ content: [] }).ranges.length, 0); }); test("the content description still teaches the line form and the string form", () => { - const desc = (COMPRESS_PARAMETERS.properties as Record).content.description ?? ""; + const desc = + (COMPRESS_PARAMETERS.properties as Record) + .content.description ?? ""; assert.ok(desc.includes("m00150–m00220"), "line-form header example kept"); - assert.ok(desc.includes("JSON-encoded array"), "stringified-array form documented"); + assert.ok( + desc.includes("JSON-encoded array"), + "stringified-array form documented", + ); }); test("typed alternatives preserve the former schema's acceptance set", () => { @@ -146,20 +202,47 @@ test("typed alternatives preserve the former schema's acceptance set", () => { ...COMPRESS_PARAMETERS.properties, content: { type: ["array", "string"], - items: { anyOf: [entries[0], { ...object, anyOf: entries.slice(1).map(({ required }) => ({ required })) }] }, + items: { + anyOf: [ + entries[0], + { + ...object, + anyOf: entries.slice(1).map(({ required }) => ({ required })), + }, + ], + }, }, }, }; - const values: unknown[] = [undefined, null, false, 42, "s", [], {}, ["line"], [42]]; + const values: unknown[] = [ + undefined, + null, + false, + 42, + "s", + [], + {}, + ["line"], + [42], + ]; const keys = ["startId", "endId", "startRef", "endRef", "summary", "topic"]; for (let mask = 0; mask < 1 << keys.length; mask++) { - const range = Object.fromEntries(keys.filter((_, i) => mask & (1 << i)).map((key) => [key, "s"])); + const range = Object.fromEntries( + keys.filter((_, i) => mask & (1 << i)).map((key) => [key, "s"]), + ); values.push([range], ["line", range], [{ ...range, extra: true }]); for (const key of keys) values.push([{ ...range, [key]: 42 }]); } for (const content of values) { - for (const input of [{ content }, { content, startId: "m1", endId: "m2", summary: "s" }]) { - assert.equal(validate(COMPRESS_PARAMETERS, input), validate(former, input), JSON.stringify(input)); + for (const input of [ + { content }, + { content, startId: "m1", endId: "m2", summary: "s" }, + ]) { + assert.equal( + validate(COMPRESS_PARAMETERS, input), + validate(former, input), + JSON.stringify(input), + ); } } }); diff --git a/tests/compress-tools.test.ts b/tests/compress-tools.test.ts index fa18a19..17e1750 100644 --- a/tests/compress-tools.test.ts +++ b/tests/compress-tools.test.ts @@ -26,20 +26,36 @@ import { } from "../src/compress-tools.js"; test("ACP tool sets classify every ACP tool by mutation semantics", () => { - assert.deepEqual([...ACP_TOOL_NAMES].sort(), ["acp_cache", "acp_status", "compress", "decompress", "search_context"]); + assert.deepEqual([...ACP_TOOL_NAMES].sort(), [ + "acp_cache", + "acp_status", + "compress", + "decompress", + "search_context", + ]); assert.deepEqual([...ACP_MUTATING_TOOLS].sort(), ["compress", "decompress"]); - assert.deepEqual([...ACP_READONLY_TOOLS].sort(), ["acp_cache", "acp_status", "search_context"]); + assert.deepEqual([...ACP_READONLY_TOOLS].sort(), [ + "acp_cache", + "acp_status", + "search_context", + ]); }); test("tool arrays expose compress in every wire format", () => { - const names = (arr: Array<{ name?: string; function?: { name?: string } }>) => arr.map((t) => t.name ?? t.function?.name); + const names = (arr: Array<{ name?: string; function?: { name?: string } }>) => + arr.map((t) => t.name ?? t.function?.name); assert.ok(names(ACP_TOOLS_ANTHROPIC).includes("compress")); assert.ok(names(ACP_TOOLS_OPENAI).includes("compress")); assert.ok(names(ACP_TOOLS_RESPONSES).includes("compress")); assert.equal(names(ACP_READONLY_TOOLS_RESPONSES).includes("compress"), false); assert.equal(COMPRESS_TOOL.name, "compress"); assert.equal(COMPRESS_TOOL_OPENAI.function?.name, "compress"); - assert.deepEqual(COMPRESS_TOOL_OPENAI.function.parameters.properties.content.anyOf.map((entry) => entry.type), ["array", "string"]); + assert.deepEqual( + COMPRESS_TOOL_OPENAI.function.parameters.properties.content.anyOf.map( + (entry) => entry.type, + ), + ["array", "string"], + ); }); test("text tags are paired delimiters", () => { @@ -54,7 +70,11 @@ test("text tags are paired delimiters", () => { }); test("prompt builders produce non-empty guidance mentioning the compress tool", () => { - for (const p of [buildCompressSystemPrompt(), buildCompressTextSystemPrompt(), buildCompressHybridSystemPrompt()]) { + for (const p of [ + buildCompressSystemPrompt(), + buildCompressTextSystemPrompt(), + buildCompressHybridSystemPrompt(), + ]) { assert.ok(p.includes("compress")); assert.ok(p.length > 1000); } @@ -65,21 +85,41 @@ test("prompt builders produce non-empty guidance mentioning the compress tool", test("parseCompressInput accepts single object, array content, and JSON-string content", () => { const warns: string[] = []; const onWarn = (m: string) => warns.push(m); - const range = { startId: "m00001", endId: "m00005", summary: "s", topic: "t" }; + const range = { + startId: "m00001", + endId: "m00005", + summary: "s", + topic: "t", + }; const single = parseCompressInput(range, "call-1", onWarn); assert.equal(single.length, 1); assert.equal(single[0]?.compressCallId, "call-1"); assert.equal(single[0]?.startRef, "m00001"); assert.equal(single[0]?.endRef, "m00005"); - const arr = parseCompressInput({ content: [range, { startId: "m00006", endId: "m00009", summary: "s2" }] }, "call-2", onWarn); + const arr = parseCompressInput( + { content: [range, { startId: "m00006", endId: "m00009", summary: "s2" }] }, + "call-2", + onWarn, + ); assert.equal(arr.length, 2); assert.equal(arr[1]?.topic, undefined); // JSON-string content field (non-strict providers stringify array args) - const json = parseCompressInput({ content: JSON.stringify([range]) }, "call-3", onWarn); + const json = parseCompressInput( + { content: JSON.stringify([range]) }, + "call-3", + onWarn, + ); assert.equal(json.length, 1); assert.equal(json[0]?.topic, "t"); // endId/endRef spellings both accepted - assert.equal(parseCompressInput({ startRef: "m1", endRef: "m2", summary: "s" }, "c", onWarn).length, 1); + assert.equal( + parseCompressInput( + { startRef: "m1", endRef: "m2", summary: "s" }, + "c", + onWarn, + ).length, + 1, + ); assert.deepEqual(warns, []); }); @@ -88,6 +128,9 @@ test("parseCompressInput rejects malformed payloads and warns", () => { const onWarn = (m: string) => warns.push(m); assert.deepEqual(parseCompressInput(null, "c", onWarn), []); assert.deepEqual(parseCompressInput({ content: "nope" }, "c", onWarn), []); - assert.deepEqual(parseCompressInput({ content: [{ startId: "m1" }] }, "c", onWarn), []); + assert.deepEqual( + parseCompressInput({ content: [{ startId: "m1" }] }, "c", onWarn), + [], + ); assert.ok(warns.length > 0); }); diff --git a/tests/compress.test.ts b/tests/compress.test.ts index 6000910..6da4b73 100644 --- a/tests/compress.test.ts +++ b/tests/compress.test.ts @@ -703,11 +703,7 @@ test("consumed plus unknown ranges keep the already-compressed message", () => { test("first compress of a fresh session with foreign refs reports a new generation, not a prior compress (billion-context#387)", () => { const core = createCore(); const state = createInitialState(); - const messages = [ - msg("u", "the task"), - msg("a", "alpha"), - msg("b", "beta"), - ]; + const messages = [msg("u", "the task"), msg("a", "alpha"), msg("b", "beta")]; state.messageRefs = assignRefs(messages, { existing: state.messageRefs, nextIndex: 1, diff --git a/tests/decompress-report.test.ts b/tests/decompress-report.test.ts index b2ab0fc..c3df8cb 100644 --- a/tests/decompress-report.test.ts +++ b/tests/decompress-report.test.ts @@ -1,271 +1,358 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { - parseBlockIdArg, - findBlocksOverlappingMessages, - findActiveAncestor, - deactivateBlock, - buildRestoredContentPreview, - collectBlockContent, + parseBlockIdArg, + findBlocksOverlappingMessages, + findActiveAncestor, + deactivateBlock, + buildRestoredContentPreview, + collectBlockContent, } from "../src/decompress.js"; import { buildStatusReport, buildRecap } from "../src/report.js"; import { createInitialState } from "../src/state.js"; import { defaultCountTokens } from "../src/tokenize.js"; -import type { CompressionBlock, CompressionState, CoreMessage } from "../src/types.js"; +import type { + CompressionBlock, + CompressionState, + CoreMessage, +} from "../src/types.js"; function block(overrides: Partial): CompressionBlock { - return { - blockId: "b0", - runId: "r0", - tier: 1, - summary: "summary", - directMessageIds: [], - effectiveMessageIds: [], - directBlockIds: [], - createdAt: 1000, - survivedCount: 0, - generation: "young", - active: true, - ...overrides, - }; + return { + blockId: "b0", + runId: "r0", + tier: 1, + summary: "summary", + directMessageIds: [], + effectiveMessageIds: [], + directBlockIds: [], + createdAt: 1000, + survivedCount: 0, + generation: "young", + active: true, + ...overrides, + }; } test("parseBlockIdArg accepts bN and plain N", () => { - assert.equal(parseBlockIdArg("b3"), "b3"); - assert.equal(parseBlockIdArg("3"), "b3"); - assert.equal(parseBlockIdArg("b005"), "b5"); - assert.equal(parseBlockIdArg("b5"), "b5"); - assert.equal(parseBlockIdArg("garbage"), null); + assert.equal(parseBlockIdArg("b3"), "b3"); + assert.equal(parseBlockIdArg("3"), "b3"); + assert.equal(parseBlockIdArg("b005"), "b5"); + assert.equal(parseBlockIdArg("b5"), "b5"); + assert.equal(parseBlockIdArg("garbage"), null); }); test("findBlocksOverlappingMessages returns active blocks covering given messages", () => { - const state: CompressionState = { - ...createInitialState(), - blocks: [ - block({ blockId: "b1", effectiveMessageIds: ["m1", "m2"], active: true }), - block({ blockId: "b2", effectiveMessageIds: ["m3"], active: true }), - block({ blockId: "b3", effectiveMessageIds: ["m2"], active: false }), - ], - }; - const result = findBlocksOverlappingMessages(state, new Set(["m2"])); - assert.equal(result.length, 1); - assert.equal(result[0]!.blockId, "b1"); + const state: CompressionState = { + ...createInitialState(), + blocks: [ + block({ blockId: "b1", effectiveMessageIds: ["m1", "m2"], active: true }), + block({ blockId: "b2", effectiveMessageIds: ["m3"], active: true }), + block({ blockId: "b3", effectiveMessageIds: ["m2"], active: false }), + ], + }; + const result = findBlocksOverlappingMessages(state, new Set(["m2"])); + assert.equal(result.length, 1); + assert.equal(result[0]!.blockId, "b1"); }); test("findActiveAncestor walks nested lineage to nearest active parent", () => { - const state: CompressionState = { - ...createInitialState(), - blocks: [ - block({ blockId: "b1", active: true }), - block({ blockId: "b2", directBlockIds: ["b1"], active: false }), - block({ blockId: "b3", directBlockIds: ["b2"], active: true }), - ], - }; - assert.equal(findActiveAncestor(state, "b3"), "b1"); - assert.equal(findActiveAncestor(state, "b2"), "b1"); + const state: CompressionState = { + ...createInitialState(), + blocks: [ + block({ blockId: "b1", active: true }), + block({ blockId: "b2", directBlockIds: ["b1"], active: false }), + block({ blockId: "b3", directBlockIds: ["b2"], active: true }), + ], + }; + assert.equal(findActiveAncestor(state, "b3"), "b1"); + assert.equal(findActiveAncestor(state, "b2"), "b1"); }); test("deactivateBlock returns new state with target inactive (immutable)", () => { - const original: CompressionState = { - ...createInitialState(), - blocks: [block({ blockId: "b1", active: true }), block({ blockId: "b2", active: true })], - }; - const updated = deactivateBlock(original, ["b1"]); - assert.equal(updated.blocks[0]!.active, false); - assert.equal(updated.blocks[1]!.active, true); - assert.equal(original.blocks[0]!.active, true); + const original: CompressionState = { + ...createInitialState(), + blocks: [ + block({ blockId: "b1", active: true }), + block({ blockId: "b2", active: true }), + ], + }; + const updated = deactivateBlock(original, ["b1"]); + assert.equal(updated.blocks[0]!.active, false); + assert.equal(updated.blocks[1]!.active, true); + assert.equal(original.blocks[0]!.active, true); }); test("deactivateBlock deep cascades through nested lineage", () => { - const state: CompressionState = { - ...createInitialState(), - blocks: [ - block({ blockId: "b1", active: true }), - block({ blockId: "b2", directBlockIds: ["b1"], active: true }), - ], - }; - const updated = deactivateBlock(state, ["b2"], { deep: true }); - assert.equal(updated.blocks[0]!.active, false); - assert.equal(updated.blocks[1]!.active, false); + const state: CompressionState = { + ...createInitialState(), + blocks: [ + block({ blockId: "b1", active: true }), + block({ blockId: "b2", directBlockIds: ["b1"], active: true }), + ], + }; + const updated = deactivateBlock(state, ["b2"], { deep: true }); + assert.equal(updated.blocks[0]!.active, false); + assert.equal(updated.blocks[1]!.active, false); }); test("buildRestoredContentPreview lists messages no longer covered", () => { - const messages: CoreMessage[] = [ - { id: "m1", role: "user", contentType: "text", text: "hello world" }, - { id: "m2", role: "assistant", contentType: "tool-result", toolName: "bash", text: "output" }, - ]; - const before = new Set(["m1", "m2"]); - const state: CompressionState = { - ...createInitialState(), - blocks: [], - }; - const result = buildRestoredContentPreview(messages, before, state); - assert.equal(result.restoredCount, 2); - assert.ok(result.preview.includes("hello world")); + const messages: CoreMessage[] = [ + { id: "m1", role: "user", contentType: "text", text: "hello world" }, + { + id: "m2", + role: "assistant", + contentType: "tool-result", + toolName: "bash", + text: "output", + }, + ]; + const before = new Set(["m1", "m2"]); + const state: CompressionState = { + ...createInitialState(), + blocks: [], + }; + const result = buildRestoredContentPreview(messages, before, state); + assert.equal(result.restoredCount, 2); + assert.ok(result.preview.includes("hello world")); }); test("buildStatusReport overview shows breakdown and active blocks", () => { - const state: CompressionState = { - ...createInitialState(), - blocks: [ - block({ blockId: "b1", summary: "compressed summary", effectiveMessageIds: ["old1"], active: true, tier: 1 }), - ], - }; - const messages: CoreMessage[] = [ - { id: "live1", role: "user", contentType: "text", text: "visible text message" }, - ]; - const report = buildStatusReport(state, messages, defaultCountTokens); - assert.ok(report.includes("CONTEXT BREAKDOWN")); - assert.ok(report.includes("COMPRESSED BLOCKS")); - assert.ok(report.includes("b1")); + const state: CompressionState = { + ...createInitialState(), + blocks: [ + block({ + blockId: "b1", + summary: "compressed summary", + effectiveMessageIds: ["old1"], + active: true, + tier: 1, + }), + ], + }; + const messages: CoreMessage[] = [ + { + id: "live1", + role: "user", + contentType: "text", + text: "visible text message", + }, + ]; + const report = buildStatusReport(state, messages, defaultCountTokens); + assert.ok(report.includes("CONTEXT BREAKDOWN")); + assert.ok(report.includes("COMPRESSED BLOCKS")); + assert.ok(report.includes("b1")); }); test("buildStatusReport compressed scope lists blocks with details", () => { - const state: CompressionState = { - ...createInitialState(), - blocks: [ - block({ blockId: "b1", summary: "first block", topic: "auth", active: true, survivedCount: 3 }), - block({ blockId: "b2", summary: "second block", topic: "deploy", active: true }), - ], - }; - const report = buildStatusReport(state, [], defaultCountTokens, { scope: "compressed" }); - assert.ok(report.includes("COMPRESSED — 2 blocks")); - assert.ok(report.includes('"auth"')); - assert.ok(report.includes('"deploy"')); + const state: CompressionState = { + ...createInitialState(), + blocks: [ + block({ + blockId: "b1", + summary: "first block", + topic: "auth", + active: true, + survivedCount: 3, + }), + block({ + blockId: "b2", + summary: "second block", + topic: "deploy", + active: true, + }), + ], + }; + const report = buildStatusReport(state, [], defaultCountTokens, { + scope: "compressed", + }); + assert.ok(report.includes("COMPRESSED — 2 blocks")); + assert.ok(report.includes('"auth"')); + assert.ok(report.includes('"deploy"')); }); test("buildRecap lists all active blocks when no blockId", () => { - const state: CompressionState = { - ...createInitialState(), - blocks: [ - block({ blockId: "b1", summary: "first summary content", active: true }), - block({ blockId: "b2", summary: "second", active: true }), - ], - }; - const recap = buildRecap(state); - assert.ok(recap.includes("Active compression blocks (2)")); - assert.ok(recap.includes("b1")); - assert.ok(recap.includes("b2")); + const state: CompressionState = { + ...createInitialState(), + blocks: [ + block({ blockId: "b1", summary: "first summary content", active: true }), + block({ blockId: "b2", summary: "second", active: true }), + ], + }; + const recap = buildRecap(state); + assert.ok(recap.includes("Active compression blocks (2)")); + assert.ok(recap.includes("b1")); + assert.ok(recap.includes("b2")); }); test("buildRecap returns full summary for a specific block", () => { - const state: CompressionState = { - ...createInitialState(), - blocks: [ - block({ blockId: "b1", summary: "the full detailed summary text", topic: "research", active: true }), - ], - }; - const recap = buildRecap(state, "b1"); - assert.ok(recap.includes("the full detailed summary text")); - assert.ok(recap.includes("research")); + const state: CompressionState = { + ...createInitialState(), + blocks: [ + block({ + blockId: "b1", + summary: "the full detailed summary text", + topic: "research", + active: true, + }), + ], + }; + const recap = buildRecap(state, "b1"); + assert.ok(recap.includes("the full detailed summary text")); + assert.ok(recap.includes("research")); }); test("buildRecap reports missing and inactive blocks", () => { - const state: CompressionState = { - ...createInitialState(), - blocks: [block({ blockId: "b1", active: false })], - }; - const missing = buildRecap(state, "b9"); - assert.ok(missing.includes("not found")); - const inactive = buildRecap(state, "b1"); - assert.ok(inactive.includes("inactive")); + const state: CompressionState = { + ...createInitialState(), + blocks: [block({ blockId: "b1", active: false })], + }; + const missing = buildRecap(state, "b9"); + assert.ok(missing.includes("not found")); + const inactive = buildRecap(state, "b1"); + assert.ok(inactive.includes("inactive")); }); test("collectBlockContent: full=false shows nested active child summary + direct messages", () => { - const messages: CoreMessage[] = [ - { id: "m1", role: "user", contentType: "text", text: "hello" }, - { id: "m2", role: "assistant", contentType: "tool-result", toolName: "read", text: "file a" }, - { id: "m3", role: "assistant", contentType: "text", text: "found bug" }, - { id: "m4", role: "user", contentType: "text", text: "direct message" }, - ]; - const state: CompressionState = { - ...createInitialState(), - blocks: [ - // T1 child covering m1,m2 — stays folded, its summary shown - block({ blockId: "b1", effectiveMessageIds: ["m1", "m2"], active: true, summary: "child summary", topic: "intro" }), - // T2 parent covering m1..m4, consumes b1 - block({ blockId: "b2", effectiveMessageIds: ["m1", "m2", "m3", "m4"], directBlockIds: ["b1"], active: true }), - ], - }; - const parent = state.blocks.find((b) => b.blockId === "b2")!; - const result = collectBlockContent(state, parent, messages); - assert.ok(result.count > 0); - // child summary rendered with topic label - assert.match(result.text, /\[Compressed conversation section\] — b1: intro/); - assert.ok(result.text.includes("child summary")); - // m1,m2 NOT shown (covered by active child); m3,m4 shown in full - assert.ok(!result.text.includes("hello")); - assert.ok(!result.text.includes("file a")); - assert.ok(result.text.includes("found bug")); - assert.ok(result.text.includes("direct message")); + const messages: CoreMessage[] = [ + { id: "m1", role: "user", contentType: "text", text: "hello" }, + { + id: "m2", + role: "assistant", + contentType: "tool-result", + toolName: "read", + text: "file a", + }, + { id: "m3", role: "assistant", contentType: "text", text: "found bug" }, + { id: "m4", role: "user", contentType: "text", text: "direct message" }, + ]; + const state: CompressionState = { + ...createInitialState(), + blocks: [ + // T1 child covering m1,m2 — stays folded, its summary shown + block({ + blockId: "b1", + effectiveMessageIds: ["m1", "m2"], + active: true, + summary: "child summary", + topic: "intro", + }), + // T2 parent covering m1..m4, consumes b1 + block({ + blockId: "b2", + effectiveMessageIds: ["m1", "m2", "m3", "m4"], + directBlockIds: ["b1"], + active: true, + }), + ], + }; + const parent = state.blocks.find((b) => b.blockId === "b2")!; + const result = collectBlockContent(state, parent, messages); + assert.ok(result.count > 0); + // child summary rendered with topic label + assert.match(result.text, /\[Compressed conversation section\] — b1: intro/); + assert.ok(result.text.includes("child summary")); + // m1,m2 NOT shown (covered by active child); m3,m4 shown in full + assert.ok(!result.text.includes("hello")); + assert.ok(!result.text.includes("file a")); + assert.ok(result.text.includes("found bug")); + assert.ok(result.text.includes("direct message")); }); test("collectBlockContent: full=true recurses to all original messages", () => { - const messages: CoreMessage[] = [ - { id: "m1", role: "user", contentType: "text", text: "hello" }, - { id: "m2", role: "assistant", contentType: "tool-result", toolName: "read", text: "file a" }, - { id: "m3", role: "assistant", contentType: "text", text: "found bug" }, - ]; - const state: CompressionState = { - ...createInitialState(), - blocks: [ - block({ blockId: "b1", effectiveMessageIds: ["m1", "m2"], active: true, summary: "child summary" }), - block({ blockId: "b2", effectiveMessageIds: ["m1", "m2", "m3"], directBlockIds: ["b1"], active: true }), - ], - }; - const parent = state.blocks.find((b) => b.blockId === "b2")!; - const result = collectBlockContent(state, parent, messages, { full: true }); - assert.equal(result.count, 3); - // ALL original messages shown, no summary - assert.ok(result.text.includes("hello")); - assert.ok(result.text.includes("file a")); - assert.ok(result.text.includes("found bug")); - assert.ok(!result.text.includes("child summary")); - assert.ok(!result.text.includes("[Compressed")); + const messages: CoreMessage[] = [ + { id: "m1", role: "user", contentType: "text", text: "hello" }, + { + id: "m2", + role: "assistant", + contentType: "tool-result", + toolName: "read", + text: "file a", + }, + { id: "m3", role: "assistant", contentType: "text", text: "found bug" }, + ]; + const state: CompressionState = { + ...createInitialState(), + blocks: [ + block({ + blockId: "b1", + effectiveMessageIds: ["m1", "m2"], + active: true, + summary: "child summary", + }), + block({ + blockId: "b2", + effectiveMessageIds: ["m1", "m2", "m3"], + directBlockIds: ["b1"], + active: true, + }), + ], + }; + const parent = state.blocks.find((b) => b.blockId === "b2")!; + const result = collectBlockContent(state, parent, messages, { full: true }); + assert.equal(result.count, 3); + // ALL original messages shown, no summary + assert.ok(result.text.includes("hello")); + assert.ok(result.text.includes("file a")); + assert.ok(result.text.includes("found bug")); + assert.ok(!result.text.includes("child summary")); + assert.ok(!result.text.includes("[Compressed")); }); test("collectBlockContent: full=false with inactive nested child shows child's messages directly", () => { - // If the nested child is inactive (e.g. already decompressed), its messages - // are NOT considered covered and are shown in full — only ACTIVE children fold. - const messages: CoreMessage[] = [ - { id: "m1", role: "user", contentType: "text", text: "hello" }, - { id: "m2", role: "assistant", contentType: "text", text: "world" }, - ]; - const state: CompressionState = { - ...createInitialState(), - blocks: [ - block({ blockId: "b1", effectiveMessageIds: ["m1"], active: false, summary: "inactive child" }), - block({ blockId: "b2", effectiveMessageIds: ["m1", "m2"], directBlockIds: ["b1"], active: true }), - ], - }; - const parent = state.blocks.find((b) => b.blockId === "b2")!; - const result = collectBlockContent(state, parent, messages); - assert.equal(result.count, 2); - assert.ok(result.text.includes("hello")); - assert.ok(result.text.includes("world")); - assert.ok(!result.text.includes("inactive child")); + // If the nested child is inactive (e.g. already decompressed), its messages + // are NOT considered covered and are shown in full — only ACTIVE children fold. + const messages: CoreMessage[] = [ + { id: "m1", role: "user", contentType: "text", text: "hello" }, + { id: "m2", role: "assistant", contentType: "text", text: "world" }, + ]; + const state: CompressionState = { + ...createInitialState(), + blocks: [ + block({ + blockId: "b1", + effectiveMessageIds: ["m1"], + active: false, + summary: "inactive child", + }), + block({ + blockId: "b2", + effectiveMessageIds: ["m1", "m2"], + directBlockIds: ["b1"], + active: true, + }), + ], + }; + const parent = state.blocks.find((b) => b.blockId === "b2")!; + const result = collectBlockContent(state, parent, messages); + assert.equal(result.count, 2); + assert.ok(result.text.includes("hello")); + assert.ok(result.text.includes("world")); + assert.ok(!result.text.includes("inactive child")); }); test("collectBlockContent: does not mutate state", () => { - const messages: CoreMessage[] = [ - { id: "m1", role: "user", contentType: "text", text: "hello" }, - ]; - const state: CompressionState = { - ...createInitialState(), - blocks: [block({ blockId: "b1", effectiveMessageIds: ["m1"], active: true })], - }; - const before = JSON.stringify(state); - collectBlockContent(state, state.blocks[0]!, messages, { full: true }); - const after = JSON.stringify(state); - assert.equal(after, before, "state must not be mutated"); + const messages: CoreMessage[] = [ + { id: "m1", role: "user", contentType: "text", text: "hello" }, + ]; + const state: CompressionState = { + ...createInitialState(), + blocks: [ + block({ blockId: "b1", effectiveMessageIds: ["m1"], active: true }), + ], + }; + const before = JSON.stringify(state); + collectBlockContent(state, state.blocks[0]!, messages, { full: true }); + const after = JSON.stringify(state); + assert.equal(after, before, "state must not be mutated"); }); test("collectBlockContent: empty block returns empty", () => { - const state: CompressionState = { - ...createInitialState(), - blocks: [block({ blockId: "b1", effectiveMessageIds: [], active: true })], - }; - const result = collectBlockContent(state, state.blocks[0]!, []); - assert.equal(result.count, 0); - assert.equal(result.text, ""); + const state: CompressionState = { + ...createInitialState(), + blocks: [block({ blockId: "b1", effectiveMessageIds: [], active: true })], + }; + const result = collectBlockContent(state, state.blocks[0]!, []); + assert.equal(result.count, 0); + assert.equal(result.text, ""); }); diff --git a/tests/doc-cache.test.ts b/tests/doc-cache.test.ts index 47091e2..4456398 100644 --- a/tests/doc-cache.test.ts +++ b/tests/doc-cache.test.ts @@ -43,7 +43,11 @@ test("docFeatures: derived fields are correct", () => { clearDocFeatures(); const f = docFeatures("The quick Cache cache CACHE, 缓存 缓存 hit"); assert.ok(f.lower.startsWith("the quick cache"), "lower must be lowercased"); - assert.equal(f.tf.get("cache"), 3, "stemmed tf counts case-insensitive repeats"); + assert.equal( + f.tf.get("cache"), + 3, + "stemmed tf counts case-insensitive repeats", + ); assert.equal(f.tf.get("缓存"), 2, "CJK word tokens counted per occurrence"); assert.ok(f.len >= f.tf.size, "len = sum of tf values"); assert.ok(f.grams.has("qu"), "bigrams of the lower-cased text"); @@ -81,11 +85,15 @@ test("LRU: re-accessing a doc keeps it alive (FIFO would evict it)", () => { setDocCacheCap(25); try { const a = docFeatures("a".repeat(10)); // cache [a] - docFeatures("b".repeat(10)); // cache [a, b] - docFeatures("a".repeat(10)); // re-access a → LRU order [b, a] - docFeatures("c".repeat(15)); // 20+15>25 → evict LRU head (b); a survives + docFeatures("b".repeat(10)); // cache [a, b] + docFeatures("a".repeat(10)); // re-access a → LRU order [b, a] + docFeatures("c".repeat(15)); // 20+15>25 → evict LRU head (b); a survives const a2 = docFeatures("a".repeat(10)); - assert.equal(a2, a, "LRU must keep the re-accessed doc; FIFO would have evicted it"); + assert.equal( + a2, + a, + "LRU must keep the re-accessed doc; FIFO would have evicted it", + ); } finally { setDocCacheCap(DEFAULT_CAP); clearDocFeatures(); @@ -99,7 +107,11 @@ test("setDocCacheCap(Infinity): whole corpus cached, no eviction", () => { docFeatures("a".repeat(100)); docFeatures("b".repeat(100)); docFeatures("c".repeat(100)); - assert.equal(docCacheInfo().entries, 3, "no eviction under an Infinity cap"); + assert.equal( + docCacheInfo().entries, + 3, + "no eviction under an Infinity cap", + ); assert.equal(docCacheInfo().chars, 300); } finally { setDocCacheCap(DEFAULT_CAP); @@ -114,7 +126,11 @@ test("setDocCacheCap: lowering the cap evicts immediately", () => { docFeatures("x".repeat(100)); assert.equal(docCacheInfo().entries, 1); setDocCacheCap(50); - assert.equal(docCacheInfo().entries, 0, "oversized-after-lowering doc must be evicted"); + assert.equal( + docCacheInfo().entries, + 0, + "oversized-after-lowering doc must be evicted", + ); assert.equal(docCacheInfo().chars, 0); } finally { setDocCacheCap(DEFAULT_CAP); @@ -128,7 +144,11 @@ test("docs larger than the cap are never cached", () => { setDocCacheCap(100); const f = docFeatures("alpha ".repeat(100)); // 600 chars > 100 cap assert.ok(f.tf.has("alpha"), "features are still returned"); - assert.equal(docCacheInfo().entries, 0, "oversized doc must not enter the cache"); + assert.equal( + docCacheInfo().entries, + 0, + "oversized doc must not enter the cache", + ); } finally { setDocCacheCap(DEFAULT_CAP); clearDocFeatures(); @@ -138,14 +158,42 @@ test("docs larger than the cap are never cached", () => { test("searchBlocks: results identical on warm cache (no score drift)", () => { clearDocFeatures(); const docs: SearchDoc[] = [ - { kind: "block", ref: "b1", text: "缓存服务 命中率 0.87 身份验证失败 请求超时", title: "b1", blockId: "b1", tier: 1, tokens: 100 }, - { kind: "message", ref: "m1", text: "the auth flow failed twice, then the cache warm-up finished", title: "m1", role: "tool", blockId: "b2", tier: 1, tokens: 50 }, - { kind: "block", ref: "b2", text: "auth failure 身份验证 cache warmup 缓存预热", title: "b2", blockId: "b2", tier: 1, tokens: 80 }, + { + kind: "block", + ref: "b1", + text: "缓存服务 命中率 0.87 身份验证失败 请求超时", + title: "b1", + blockId: "b1", + tier: 1, + tokens: 100, + }, + { + kind: "message", + ref: "m1", + text: "the auth flow failed twice, then the cache warm-up finished", + title: "m1", + role: "tool", + blockId: "b2", + tier: 1, + tokens: 50, + }, + { + kind: "block", + ref: "b2", + text: "auth failure 身份验证 cache warmup 缓存预热", + title: "b2", + blockId: "b2", + tier: 1, + tokens: 80, + }, ]; const cold = searchBlocks(docs, "缓存 身份验证"); const warm = searchBlocks(docs, "缓存 身份验证"); - assert.deepEqual(warm.map((r) => [r.ref, r.score]), cold.map((r) => [r.ref, r.score]), - "cached features must not change scores"); + assert.deepEqual( + warm.map((r) => [r.ref, r.score]), + cold.map((r) => [r.ref, r.score]), + "cached features must not change scores", + ); assert.ok(cold.length >= 1); clearDocFeatures(); }); @@ -165,6 +213,12 @@ test("single-pass CJK segmentation: run grouping edge cases", () => { // mixed: latin + several CJK runs, latin first, runs in text order const mixed = tokenize("cache 缓存 pool 连接池", {}); assert.ok(mixed.includes("cache") && mixed.includes("pool")); - assert.ok(mixed.indexOf("缓存") !== -1, `缓存 (OOV bigram) missing in ${JSON.stringify(mixed)}`); - assert.ok(mixed.indexOf("缓存") < mixed.indexOf("连接"), "run order must follow text order"); + assert.ok( + mixed.indexOf("缓存") !== -1, + `缓存 (OOV bigram) missing in ${JSON.stringify(mixed)}`, + ); + assert.ok( + mixed.indexOf("缓存") < mixed.indexOf("连接"), + "run order must follow text order", + ); }); diff --git a/tests/effective-gate-chars.test.ts b/tests/effective-gate-chars.test.ts index eddcc21..a671bb1 100644 --- a/tests/effective-gate-chars.test.ts +++ b/tests/effective-gate-chars.test.ts @@ -35,7 +35,11 @@ function buildConfig(overrides: Partial = {}): Config { promotionThreshold: 5, truncate: { threshold: 1 }, merge: { maxSummaryLength: 3000, minOldGenBlocks: 3 }, - compress: { minCompressRange: 5000, maxSummaryLength: 3000, minSummaryLength: 100 }, + compress: { + minCompressRange: 5000, + maxSummaryLength: 3000, + minSummaryLength: 100, + }, protectedTools: [], preserveRecentMessages: 0, preserveRecentTokens: 0, @@ -61,20 +65,38 @@ test("mergeRangesToThreshold: batches by real chars, not tokens*4 (CJK tokenizer // Two hand-built ranges WITHOUT chars exercise the legacy fallback; // ranges WITH chars exercise the real accounting. const legacyA: CompressibleRange = { - startRef: "m00001", endRef: "m00002", count: 2, tokens: 1250, toolPct: 0, textPct: 100, + startRef: "m00001", + endRef: "m00002", + count: 2, + tokens: 1250, + toolPct: 0, + textPct: 100, }; const out = mergeRangesToThreshold([legacyA], 5000); - assert.equal(out.length, 1, "fallback tokens*4=5000 still clears the 5000 threshold"); + assert.equal( + out.length, + 1, + "fallback tokens*4=5000 still clears the 5000 threshold", + ); // 3000 chars of CJK = 3000 tokens under a CJK tokenizer. tokens*4 = 12000 // used to "clear" 5000; real chars (3000) do not — tail stays sub-threshold // and pendingByTier must not count it as effective. const cjk: CompressibleRange = { - startRef: "m00001", endRef: "m00006", count: 6, tokens: 3000, chars: 3000, - toolPct: 0, textPct: 100, + startRef: "m00001", + endRef: "m00006", + count: 6, + tokens: 3000, + chars: 3000, + toolPct: 0, + textPct: 100, }; assert.equal(cjk.chars < 5000, true); - assert.equal(cjk.tokens * 4 >= 5000, true, "pre-fix this range looked effective"); + assert.equal( + cjk.tokens * 4 >= 5000, + true, + "pre-fix this range looked effective", + ); }); test("nudge: CJK session below minCompressRange chars is NOT offered (apply would reject)", () => { @@ -83,11 +105,20 @@ test("nudge: CJK session below minCompressRange chars is NOT offered (apply woul const messages = cjkMessages(500, 6); // 3000 chars total < 5000 min let state = createInitialState(); - state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; + state = core.processTurn({ + messages, + state, + config, + tokenCount: 10000, + }).state; // usage 95% >= maxContextLimitPct 0.9 → pressure path const turn = core.processTurn({ messages, state, config, tokenCount: 95000 }); - assert.equal(turn.nudge.shouldInject, false, "3000 chars < minCompressRange 5000 — nudge must not offer it"); + assert.equal( + turn.nudge.shouldInject, + false, + "3000 chars < minCompressRange 5000 — nudge must not offer it", + ); assert.match( turn.nudge.reason, /no tier has effective compressible content/, @@ -96,7 +127,9 @@ test("nudge: CJK session below minCompressRange chars is NOT offered (apply woul // The apply side agrees: the same range is atomically rejected. const applied = core.applyCompression({ - ranges: [{ startRef: "m00001", endRef: "m00006", summary: "s", topic: "t" }], + ranges: [ + { startRef: "m00001", endRef: "m00006", summary: "s", topic: "t" }, + ], messages, state: turn.state, config, @@ -114,19 +147,39 @@ test("nudge: CJK session above minCompressRange chars IS offered (control)", () const messages = cjkMessages(1000, 6); // 6000 chars total >= 5000 min let state = createInitialState(); - state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; + state = core.processTurn({ + messages, + state, + config, + tokenCount: 10000, + }).state; const turn = core.processTurn({ messages, state, config, tokenCount: 95000 }); - assert.equal(turn.nudge.shouldInject, true, "6000 chars >= 5000 — effective T1 pending exists"); + assert.equal( + turn.nudge.shouldInject, + true, + "6000 chars >= 5000 — effective T1 pending exists", + ); assert.match(turn.nudge.reason, /T1/); // And the apply side accepts the same range — both gates agree on chars. const applied = core.applyCompression({ - ranges: [{ startRef: "m00001", endRef: "m00006", summary: "总结".repeat(60), topic: "t" }], + ranges: [ + { + startRef: "m00001", + endRef: "m00006", + summary: "总结".repeat(60), + topic: "t", + }, + ], messages, state: turn.state, config, }); - assert.equal(applied.result.blocksCreated, 1, "apply accepts: 6000 real chars >= 5000"); + assert.equal( + applied.result.blocksCreated, + 1, + "apply accepts: 6000 real chars >= 5000", + ); assert.deepEqual(applied.result.errors, []); }); diff --git a/tests/handoff.test.ts b/tests/handoff.test.ts index d5ba1e5..944a09e 100644 --- a/tests/handoff.test.ts +++ b/tests/handoff.test.ts @@ -4,11 +4,17 @@ import { createInitialState } from "../src/state.js"; import { renderHandoff, renderMessage, matchSession } from "../src/handoff.js"; import type { CompressionBlock, CoreMessage } from "../src/types.js"; -function msg(id: string, role: CoreMessage["role"] = "user", text: string = id): CoreMessage { +function msg( + id: string, + role: CoreMessage["role"] = "user", + text: string = id, +): CoreMessage { return { id, role, contentType: "text", text }; } -function makeBlock(overrides: Partial & { blockId: string }): CompressionBlock { +function makeBlock( + overrides: Partial & { blockId: string }, +): CompressionBlock { return { runId: "r1", tier: 1, @@ -26,28 +32,77 @@ function makeBlock(overrides: Partial & { blockId: string }): } test("renderMessage renders each content type", () => { - assert.equal(renderMessage({ id: "a", role: "user", contentType: "text", text: "hello" }), "hello\n"); assert.equal( - renderMessage({ id: "b", role: "assistant", contentType: "tool-call", toolName: "bash", toolCallId: "t1", text: "ls" }), + renderMessage({ + id: "a", + role: "user", + contentType: "text", + text: "hello", + }), + "hello\n", + ); + assert.equal( + renderMessage({ + id: "b", + role: "assistant", + contentType: "tool-call", + toolName: "bash", + toolCallId: "t1", + text: "ls", + }), "`bash(t1)` args: ls\n", ); assert.equal( - renderMessage({ id: "c", role: "tool", contentType: "tool-result", toolName: "bash", toolCallId: "t1", text: "ok" }), + renderMessage({ + id: "c", + role: "tool", + contentType: "tool-result", + toolName: "bash", + toolCallId: "t1", + text: "ok", + }), "`bash(t1)` → ok\n", ); assert.equal( - renderMessage({ id: "d", role: "assistant", contentType: "reasoning", text: "thinking" }), + renderMessage({ + id: "d", + role: "assistant", + contentType: "reasoning", + text: "thinking", + }), "_reasoning_: thinking\n", ); - assert.equal(renderMessage({ id: "e", role: "user", contentType: "text", text: "" }), "_(empty)_"); - assert.equal(renderMessage({ id: "f", role: "assistant", contentType: "tool-call" }), "`?()` args:\n"); - assert.equal(renderMessage({ id: "g", role: "tool", contentType: "tool-result", toolName: "bash" }), "`bash()` →\n"); + assert.equal( + renderMessage({ id: "e", role: "user", contentType: "text", text: "" }), + "_(empty)_", + ); + assert.equal( + renderMessage({ id: "f", role: "assistant", contentType: "tool-call" }), + "`?()` args:\n", + ); + assert.equal( + renderMessage({ + id: "g", + role: "tool", + contentType: "tool-result", + toolName: "bash", + }), + "`bash()` →\n", + ); }); test("renderHandoff full view shows every original message", () => { - const messages = [msg("m1", "user", "hello"), msg("m2", "assistant", "world")]; + const messages = [ + msg("m1", "user", "hello"), + msg("m2", "assistant", "world"), + ]; const state = createInitialState(); - const out = renderHandoff({ coreMessages: messages, state, full: true, meta: { sessionId: "s1", title: "T" } }); + const out = renderHandoff({ + coreMessages: messages, + state, + full: true, + meta: { sessionId: "s1", title: "T" }, + }); assert.match(out, /## Full conversation \(2 messages\)/); assert.match(out, /### user/); assert.match(out, /hello/); @@ -56,29 +111,67 @@ test("renderHandoff full view shows every original message", () => { }); test("renderHandoff folded view injects the block summary and drops the covered message", () => { - const messages = [msg("m1", "user", "hello"), msg("m2", "user", "secret-covered"), msg("m3", "assistant", "hi")]; + const messages = [ + msg("m1", "user", "hello"), + msg("m2", "user", "secret-covered"), + msg("m3", "assistant", "hi"), + ]; const state = createInitialState(); - state.blocks.push(makeBlock({ blockId: "b1", summary: "COVERED SUMMARY", effectiveMessageIds: ["m2"] })); - const folded = renderHandoff({ coreMessages: messages, state, full: false, meta: { sessionId: "s1" } }); - assert.match(folded, /## Conversation \(folded view as the model saw it, 3 client messages\)/); + state.blocks.push( + makeBlock({ + blockId: "b1", + summary: "COVERED SUMMARY", + effectiveMessageIds: ["m2"], + }), + ); + const folded = renderHandoff({ + coreMessages: messages, + state, + full: false, + meta: { sessionId: "s1" }, + }); + assert.match( + folded, + /## Conversation \(folded view as the model saw it, 3 client messages\)/, + ); assert.match(folded, /COVERED SUMMARY/); assert.doesNotMatch(folded, /secret-covered/); assert.match(folded, /hello/); assert.match(folded, /hi/); - const full = renderHandoff({ coreMessages: messages, state, full: true, meta: { sessionId: "s1" } }); + const full = renderHandoff({ + coreMessages: messages, + state, + full: true, + meta: { sessionId: "s1" }, + }); assert.match(full, /secret-covered/); }); test("renderHandoff renders metadata bullets in order and omits optionals", () => { const messages = [msg("m1", "user", "hi")]; const state = createInitialState(); - state.blocks.push(makeBlock({ blockId: "b1", summary: "s", effectiveMessageIds: [] })); - state.blocks.push(makeBlock({ blockId: "b2", summary: "s", effectiveMessageIds: [], active: false })); + state.blocks.push( + makeBlock({ blockId: "b1", summary: "s", effectiveMessageIds: [] }), + ); + state.blocks.push( + makeBlock({ + blockId: "b2", + summary: "s", + effectiveMessageIds: [], + active: false, + }), + ); const out = renderHandoff({ coreMessages: messages, state, full: true, - meta: { sessionId: "abc", title: "My Title", label: "my-label", contextTokens: 1234, extraBullets: ["- requests: 7"] }, + meta: { + sessionId: "abc", + title: "My Title", + label: "my-label", + contextTokens: 1234, + extraBullets: ["- requests: 7"], + }, }); assert.match(out, /- title: My Title/); assert.match(out, /- label: my-label/); @@ -92,17 +185,38 @@ test("renderHandoff renders metadata bullets in order and omits optionals", () = const idxReq = out.indexOf("- requests:"); const idxTok = out.indexOf("- last context tokens:"); const idxBlocks = out.indexOf("- compression blocks:"); - assert.ok(idxTitle < idxLabel && idxLabel < idxId && idxId < idxReq && idxReq < idxTok && idxTok < idxBlocks); - const noOptional = renderHandoff({ coreMessages: messages, state, full: true, meta: { sessionId: "abc" } }); + assert.ok( + idxTitle < idxLabel && + idxLabel < idxId && + idxId < idxReq && + idxReq < idxTok && + idxTok < idxBlocks, + ); + const noOptional = renderHandoff({ + coreMessages: messages, + state, + full: true, + meta: { sessionId: "abc" }, + }); assert.doesNotMatch(noOptional, /- label:/); assert.doesNotMatch(noOptional, /- last context tokens:/); assert.match(noOptional, /- title: \(untitled\)/); - const zeroTokens = renderHandoff({ coreMessages: messages, state, full: true, meta: { sessionId: "abc", contextTokens: 0 } }); + const zeroTokens = renderHandoff({ + coreMessages: messages, + state, + full: true, + meta: { sessionId: "abc", contextTokens: 0 }, + }); assert.doesNotMatch(zeroTokens, /- last context tokens:/); }); test("renderHandoff empty conversation shows a placeholder", () => { - const out = renderHandoff({ coreMessages: [], state: createInitialState(), full: true, meta: { sessionId: "s" } }); + const out = renderHandoff({ + coreMessages: [], + state: createInitialState(), + full: true, + meta: { sessionId: "s" }, + }); assert.match(out, /No conversation messages to export\./); }); @@ -112,43 +226,99 @@ test("matchSession matches by exact id, label, and prefix", () => { { id: "sess-bbb-222", label: "beta" }, ]; const labelOf = (s: { id: string; label?: string }) => s.label; - assert.deepEqual(matchSession(sessions, "sess-aaa-111", labelOf).map((s) => s.id), ["sess-aaa-111"]); - assert.deepEqual(matchSession(sessions, "beta", labelOf).map((s) => s.id), ["sess-bbb-222"]); - assert.deepEqual(matchSession(sessions, "sess-aaa", labelOf).map((s) => s.id), ["sess-aaa-111"]); - assert.deepEqual(matchSession(sessions, "alpha", labelOf).map((s) => s.id), ["sess-aaa-111"]); - assert.deepEqual(matchSession(sessions, "alp", labelOf).map((s) => s.id), ["sess-aaa-111"]); - assert.deepEqual(matchSession(sessions, "sess", labelOf).map((s) => s.id), ["sess-aaa-111", "sess-bbb-222"]); + assert.deepEqual( + matchSession(sessions, "sess-aaa-111", labelOf).map((s) => s.id), + ["sess-aaa-111"], + ); + assert.deepEqual( + matchSession(sessions, "beta", labelOf).map((s) => s.id), + ["sess-bbb-222"], + ); + assert.deepEqual( + matchSession(sessions, "sess-aaa", labelOf).map((s) => s.id), + ["sess-aaa-111"], + ); + assert.deepEqual( + matchSession(sessions, "alpha", labelOf).map((s) => s.id), + ["sess-aaa-111"], + ); + assert.deepEqual( + matchSession(sessions, "alp", labelOf).map((s) => s.id), + ["sess-aaa-111"], + ); + assert.deepEqual( + matchSession(sessions, "sess", labelOf).map((s) => s.id), + ["sess-aaa-111", "sess-bbb-222"], + ); assert.deepEqual(matchSession(sessions, "nope", labelOf), []); }); test("renderHandoff folded snapshot renders as-is without re-pruning", () => { const state = createInitialState(); - state.blocks.push(makeBlock({ blockId: "b0", effectiveMessageIds: ["m1", "m2"], directMessageIds: ["m1", "m2"] })); + state.blocks.push( + makeBlock({ + blockId: "b0", + effectiveMessageIds: ["m1", "m2"], + directMessageIds: ["m1", "m2"], + }), + ); // Folded persisted snapshot: covered ids already replaced by a summary // anchor; ids m1/m2 gone. Re-pruning must not resurrect them. - const snapshot = [msg("m3", "assistant", "hi"), msg("m4", "user", "final question")]; - const out = renderHandoff({ coreMessages: snapshot, state, full: false, folded: true, meta: { sessionId: "s1" } }); - assert.match(out, /## Conversation \(persisted folded snapshot, 2 messages\)/); + const snapshot = [ + msg("m3", "assistant", "hi"), + msg("m4", "user", "final question"), + ]; + const out = renderHandoff({ + coreMessages: snapshot, + state, + full: false, + folded: true, + meta: { sessionId: "s1" }, + }); + assert.match( + out, + /## Conversation \(persisted folded snapshot, 2 messages\)/, + ); assert.match(out, /final question/); assert.doesNotMatch(out, /summary/); }); test("re-pruning the same snapshot without folded resurrects the summary at index 0", () => { const state = createInitialState(); - state.blocks.push(makeBlock({ blockId: "b0", effectiveMessageIds: ["m1", "m2"], directMessageIds: ["m1", "m2"] })); - const snapshot = [msg("m3", "assistant", "hi"), msg("m4", "user", "final question")]; - const out = renderHandoff({ coreMessages: snapshot, state, full: false, meta: { sessionId: "s1" } }); - assert.match(out, /## Conversation \(folded view as the model saw it, 2 client messages\)/); + state.blocks.push( + makeBlock({ + blockId: "b0", + effectiveMessageIds: ["m1", "m2"], + directMessageIds: ["m1", "m2"], + }), + ); + const snapshot = [ + msg("m3", "assistant", "hi"), + msg("m4", "user", "final question"), + ]; + const out = renderHandoff({ + coreMessages: snapshot, + state, + full: false, + meta: { sessionId: "s1" }, + }); + assert.match( + out, + /## Conversation \(folded view as the model saw it, 2 client messages\)/, + ); assert.match(out, /\[Compressed conversation section\]/); assert.ok( - out.indexOf("[Compressed conversation section]") < out.indexOf("### assistant"), + out.indexOf("[Compressed conversation section]") < + out.indexOf("### assistant"), "resurrected summary must land before the first surviving message", ); }); test("renderHandoff full+folded appends block originals after the tail", () => { const state = createInitialState(); - state.blocks.push(makeBlock({ blockId: "b0", topic: "investigation", summary: "summary" })); + state.blocks.push( + makeBlock({ blockId: "b0", topic: "investigation", summary: "summary" }), + ); const snapshot = [msg("m3", "assistant", "hi")]; const out = renderHandoff({ coreMessages: snapshot, @@ -156,12 +326,20 @@ test("renderHandoff full+folded appends block originals after the tail", () => { full: true, folded: true, blocksFull: [ - { blockId: "b0", topic: "investigation", count: 7, fullText: "hello proxy\n778899" }, + { + blockId: "b0", + topic: "investigation", + count: 7, + fullText: "hello proxy\n778899", + }, { blockId: "b1", count: 2, fullText: "tail content\n\n" }, ], meta: { sessionId: "s1" }, }); - assert.match(out, /## Conversation \(persisted folded snapshot, 1 messages\)/); + assert.match( + out, + /## Conversation \(persisted folded snapshot, 1 messages\)/, + ); assert.match(out, /## Block b0 — investigation/); assert.match(out, /### Original messages \(7\)/); assert.match(out, /778899/); @@ -169,7 +347,10 @@ test("renderHandoff full+folded appends block originals after the tail", () => { assert.doesNotMatch(out, /## Block b1 —/); assert.match(out, /### Original messages \(2\)/); assert.doesNotMatch(out, /tail content\n\n/); - assert.ok(out.indexOf("## Block b0") > out.indexOf("hi"), "blocks must follow the conversation"); + assert.ok( + out.indexOf("## Block b0") > out.indexOf("hi"), + "blocks must follow the conversation", + ); }); test("renderHandoff folded flag is ignored for block append when not full", () => { diff --git a/tests/hide-consumed-stub.test.ts b/tests/hide-consumed-stub.test.ts index 2b233e2..bc6bc26 100644 --- a/tests/hide-consumed-stub.test.ts +++ b/tests/hide-consumed-stub.test.ts @@ -3,165 +3,228 @@ import assert from "node:assert/strict"; import { hideConsumedCompressCalls } from "../src/hide-consumed.js"; import { renderVisibleRefs } from "../src/render-refs.js"; import { createInitialState } from "../src/state.js"; -import type { CompressionBlock, CompressionState, CoreMessage } from "../src/types.js"; +import type { + CompressionBlock, + CompressionState, + CoreMessage, +} from "../src/types.js"; function block(overrides: Partial): CompressionBlock { - return { - blockId: "b0", - runId: "r0", - tier: 1, - summary: "summary", - directMessageIds: [], - effectiveMessageIds: [], - directBlockIds: [], - createdAt: 1000, - survivedCount: 0, - generation: "young", - active: true, - ...overrides, - }; + return { + blockId: "b0", + runId: "r0", + tier: 1, + summary: "summary", + directMessageIds: [], + effectiveMessageIds: [], + directBlockIds: [], + createdAt: 1000, + survivedCount: 0, + generation: "young", + active: true, + ...overrides, + }; } function callWith(text: string, toolCallId = "call1"): CoreMessage { - return { id: "mc1", role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId, text }; + return { + id: "mc1", + role: "assistant", + contentType: "tool-call", + toolName: "compress", + toolCallId, + text, + }; } -function liveState(summary: string, startRef: string, endRef: string): CompressionState { - const state = createInitialState(); - state.blocks.push(block({ blockId: "b1", compressCallId: "call1", summary, startRef, endRef })); - return state; +function liveState( + summary: string, + startRef: string, + endRef: string, +): CompressionState { + const state = createInitialState(); + state.blocks.push( + block({ + blockId: "b1", + compressCallId: "call1", + summary, + startRef, + endRef, + }), + ); + return state; } interface StubEntry { - startId?: string; - summary?: string; - topic?: string; + startId?: string; + summary?: string; + topic?: string; } function parsedContent(text: string | undefined): StubEntry[] { - return (JSON.parse(text ?? "") as { content: StubEntry[] }).content; + return (JSON.parse(text ?? "") as { content: StubEntry[] }).content; } test("fully-live compress call args get their long summaries stubbed", () => { - const longSummary = "x".repeat(5000); - const callText = JSON.stringify({ - content: [{ startId: "m00001", endId: "m00050", topic: "T", summary: longSummary }], - }); - const state = liveState(longSummary, "m00001", "m00050"); - - const { messages } = hideConsumedCompressCalls(state, [callWith(callText)]); - - assert.equal(messages.length, 1); - const content = parsedContent(messages[0]!.text); - assert.equal(content.length, 1); - assert.equal(content[0]!.summary!.length, 200); - assert.ok(content[0]!.summary!.endsWith("…")); - assert.ok(content[0]!.summary!.startsWith("xxxx")); - assert.equal(content[0]!.topic, "T"); - assert.equal(content[0]!.startId, "m00001"); + const longSummary = "x".repeat(5000); + const callText = JSON.stringify({ + content: [ + { startId: "m00001", endId: "m00050", topic: "T", summary: longSummary }, + ], + }); + const state = liveState(longSummary, "m00001", "m00050"); + + const { messages } = hideConsumedCompressCalls(state, [callWith(callText)]); + + assert.equal(messages.length, 1); + const content = parsedContent(messages[0]!.text); + assert.equal(content.length, 1); + assert.equal(content[0]!.summary!.length, 200); + assert.ok(content[0]!.summary!.endsWith("…")); + assert.ok(content[0]!.summary!.startsWith("xxxx")); + assert.equal(content[0]!.topic, "T"); + assert.equal(content[0]!.startId, "m00001"); }); test("short summaries are left byte-identical; kept orphans with long summaries are stubbed", () => { - const shortText = JSON.stringify({ - content: [{ startId: "m00001", endId: "m00050", summary: "short" }], - }); - const state = liveState("short", "m00001", "m00050"); - - const { messages } = hideConsumedCompressCalls(state, [callWith(shortText)]); - assert.equal(messages[0]!.text, shortText); - - const orphanText = JSON.stringify({ - content: [{ startId: "m00001", endId: "m00050", summary: "y".repeat(4000) }], - }); - const { messages: orphanOut } = hideConsumedCompressCalls(createInitialState(), [callWith(orphanText, "orphan")]); - assert.equal(orphanOut.length, 1); - assert.equal(parsedContent(orphanOut[0]!.text)[0]!.summary!.length, 200); + const shortText = JSON.stringify({ + content: [{ startId: "m00001", endId: "m00050", summary: "short" }], + }); + const state = liveState("short", "m00001", "m00050"); + + const { messages } = hideConsumedCompressCalls(state, [callWith(shortText)]); + assert.equal(messages[0]!.text, shortText); + + const orphanText = JSON.stringify({ + content: [ + { startId: "m00001", endId: "m00050", summary: "y".repeat(4000) }, + ], + }); + const { messages: orphanOut } = hideConsumedCompressCalls( + createInitialState(), + [callWith(orphanText, "orphan")], + ); + assert.equal(orphanOut.length, 1); + assert.equal(parsedContent(orphanOut[0]!.text)[0]!.summary!.length, 200); }); test("tag-prefixed compress call text is still parsed and stubbed (ref tag precedes the JSON)", () => { - const longSummary = "z".repeat(3000); - const state = liveState(longSummary, "m00001", "m00050"); - state.messageRefs.byRaw["mc1"] = "m00010"; - state.messageRefs.byRef["m00010"] = "mc1"; - const callText = JSON.stringify({ - content: [{ startId: "m00001", endId: "m00050", summary: longSummary }], - }); - const tagged = renderVisibleRefs([callWith(callText)], state)[0]!; - assert.ok(tagged.text!.startsWith(" { - const state: CompressionState = { - ...createInitialState(), - blocks: [ - block({ blockId: "b5", compressCallId: "call-batch", active: true, startRef: "m5", endRef: "m6" }), - block({ blockId: "b8", compressCallId: "call-batch", active: false, startRef: "m8", endRef: "m9" }), - ], - }; - state.messageRefs.byRaw["mc"] = "m00010"; - state.messageRefs.byRef["m00010"] = "mc"; - const call: CoreMessage = { - id: "mc", - role: "assistant", - contentType: "tool-call", - toolName: "compress", - toolCallId: "call-batch", - text: JSON.stringify({ - content: [ - { startId: "m5", endId: "m6", summary: "live entry summary" }, - { startId: "m8", endId: "m9", summary: "c".repeat(300) }, - ], - }), - }; - const tagged = renderVisibleRefs([call], state)[0]!; - - const { messages } = hideConsumedCompressCalls(state, [tagged]); - const kept = messages.find((m) => m.toolCallId === "call-batch")!; - const text = kept.text!; - assert.ok(text.startsWith(" m.toolCallId === "call-batch")!; + const text = kept.text!; + assert.ok(text.startsWith(" { - const state: CompressionState = { - ...createInitialState(), - blocks: [ - block({ blockId: "b1", compressCallId: "call-str", active: true, startRef: "m1", endRef: "m2" }), - block({ blockId: "b2", compressCallId: "call-str", active: false, startRef: "m3", endRef: "m4" }), - ], - }; - const inner = JSON.stringify([ - { startId: "m1", endId: "m2", summary: "x".repeat(300) }, - { startId: "m3", endId: "m4", summary: "y".repeat(300) }, - ]); - const messages: CoreMessage[] = [ - { - id: "m5", - role: "assistant", - contentType: "tool-call", - toolName: "compress", - toolCallId: "call-str", - text: JSON.stringify({ content: inner }), - }, - ]; - const result = hideConsumedCompressCalls(state, messages); - const kept = result.messages.find((m) => m.toolCallId === "call-str"); - assert.ok(kept, "live call kept"); - const parsed = JSON.parse(kept!.text!.replace(/^[^{]*/, "")) as { content: unknown }; - assert.equal(typeof parsed.content, "string", "string shape preserved"); - const entries = JSON.parse(parsed.content as string) as { startId: string; summary: string }[]; - assert.equal(entries.length, 1); - assert.equal(entries[0]!.startId, "m1"); - assert.ok(entries[0]!.summary.length <= 200, "summary stubbed"); - assert.ok(entries[0]!.summary.endsWith("…")); + const state: CompressionState = { + ...createInitialState(), + blocks: [ + block({ + blockId: "b1", + compressCallId: "call-str", + active: true, + startRef: "m1", + endRef: "m2", + }), + block({ + blockId: "b2", + compressCallId: "call-str", + active: false, + startRef: "m3", + endRef: "m4", + }), + ], + }; + const inner = JSON.stringify([ + { startId: "m1", endId: "m2", summary: "x".repeat(300) }, + { startId: "m3", endId: "m4", summary: "y".repeat(300) }, + ]); + const messages: CoreMessage[] = [ + { + id: "m5", + role: "assistant", + contentType: "tool-call", + toolName: "compress", + toolCallId: "call-str", + text: JSON.stringify({ content: inner }), + }, + ]; + const result = hideConsumedCompressCalls(state, messages); + const kept = result.messages.find((m) => m.toolCallId === "call-str"); + assert.ok(kept, "live call kept"); + const parsed = JSON.parse(kept!.text!.replace(/^[^{]*/, "")) as { + content: unknown; + }; + assert.equal(typeof parsed.content, "string", "string shape preserved"); + const entries = JSON.parse(parsed.content as string) as { + startId: string; + summary: string; + }[]; + assert.equal(entries.length, 1); + assert.equal(entries[0]!.startId, "m1"); + assert.ok(entries[0]!.summary.length <= 200, "summary stubbed"); + assert.ok(entries[0]!.summary.endsWith("…")); }); diff --git a/tests/hide-merge-rebuild.test.ts b/tests/hide-merge-rebuild.test.ts index b9567bd..96d5e9b 100644 --- a/tests/hide-merge-rebuild.test.ts +++ b/tests/hide-merge-rebuild.test.ts @@ -4,296 +4,578 @@ import { hideConsumedCompressCalls } from "../src/hide-consumed.js"; import { rebuildCompressionState } from "../src/rebuild.js"; import { createInitialState } from "../src/state.js"; import { defaultConfig } from "../src/config.js"; -import type { CompressionBlock, CompressionState, CoreMessage } from "../src/types.js"; +import type { + CompressionBlock, + CompressionState, + CoreMessage, +} from "../src/types.js"; function block(overrides: Partial): CompressionBlock { - return { - blockId: "b0", - runId: "r0", - tier: 1, - summary: "summary", - directMessageIds: [], - effectiveMessageIds: [], - directBlockIds: [], - createdAt: 1000, - survivedCount: 0, - generation: "young", - active: true, - ...overrides, - }; + return { + blockId: "b0", + runId: "r0", + tier: 1, + summary: "summary", + directMessageIds: [], + effectiveMessageIds: [], + directBlockIds: [], + createdAt: 1000, + survivedCount: 0, + generation: "young", + active: true, + ...overrides, + }; } test("hideConsumedCompressCalls keeps active-block calls, hides consumed, keeps newest two orphans", () => { - const state: CompressionState = { - ...createInitialState(), - blocks: [block({ blockId: "b1", compressCallId: "call-active", active: true })], - }; - const messages: CoreMessage[] = [ - { id: "m1", role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId: "call-active", text: "{}" }, - { id: "m2", role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId: "call-consumed", text: "{}" }, - { id: "m3", role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId: "call-orphan1", text: "{}" }, - { id: "m4", role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId: "call-orphan2", text: "{}" }, - { id: "m5", role: "user", contentType: "text", text: "hello" }, - ]; - const result = hideConsumedCompressCalls(state, messages); - // active-block call kept; consumed hidden; newest KEEP_LAST_ORPHANED=2 - // orphans kept visible (failure observability — billion-context-pi #9). - assert.equal(result.hidden, 1); - const remainingCallIds = result.messages.filter((m) => m.toolName === "compress").map((m) => m.toolCallId); - assert.ok(remainingCallIds.includes("call-active")); - assert.ok(!remainingCallIds.includes("call-consumed")); - assert.ok(remainingCallIds.includes("call-orphan1")); - assert.ok(remainingCallIds.includes("call-orphan2")); - assert.ok(result.messages.some((m) => m.text === "hello")); + const state: CompressionState = { + ...createInitialState(), + blocks: [ + block({ blockId: "b1", compressCallId: "call-active", active: true }), + ], + }; + const messages: CoreMessage[] = [ + { + id: "m1", + role: "assistant", + contentType: "tool-call", + toolName: "compress", + toolCallId: "call-active", + text: "{}", + }, + { + id: "m2", + role: "assistant", + contentType: "tool-call", + toolName: "compress", + toolCallId: "call-consumed", + text: "{}", + }, + { + id: "m3", + role: "assistant", + contentType: "tool-call", + toolName: "compress", + toolCallId: "call-orphan1", + text: "{}", + }, + { + id: "m4", + role: "assistant", + contentType: "tool-call", + toolName: "compress", + toolCallId: "call-orphan2", + text: "{}", + }, + { id: "m5", role: "user", contentType: "text", text: "hello" }, + ]; + const result = hideConsumedCompressCalls(state, messages); + // active-block call kept; consumed hidden; newest KEEP_LAST_ORPHANED=2 + // orphans kept visible (failure observability — billion-context-pi #9). + assert.equal(result.hidden, 1); + const remainingCallIds = result.messages + .filter((m) => m.toolName === "compress") + .map((m) => m.toolCallId); + assert.ok(remainingCallIds.includes("call-active")); + assert.ok(!remainingCallIds.includes("call-consumed")); + assert.ok(remainingCallIds.includes("call-orphan1")); + assert.ok(remainingCallIds.includes("call-orphan2")); + assert.ok(result.messages.some((m) => m.text === "hello")); }); test("orphaned compress residue is bounded: only the newest two pairs survive any session length", () => { - const state: CompressionState = { - ...createInitialState(), - blocks: [block({ blockId: "b1", compressCallId: "call-active", active: true })], - }; - const messages: CoreMessage[] = [ - { id: "m1", role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId: "call-active", text: "{}" }, - ]; - for (let i = 1; i <= 12; i++) { - messages.push({ id: `c${i}`, role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId: `call-fail${i}`, text: "{}" }); - messages.push({ id: `r${i}`, role: "user", contentType: "tool-result", toolName: "compress", toolCallId: `call-fail${i}`, text: "nothing to compress" }); - } - messages.push({ id: "mend", role: "user", contentType: "text", text: "tail" }); + const state: CompressionState = { + ...createInitialState(), + blocks: [ + block({ blockId: "b1", compressCallId: "call-active", active: true }), + ], + }; + const messages: CoreMessage[] = [ + { + id: "m1", + role: "assistant", + contentType: "tool-call", + toolName: "compress", + toolCallId: "call-active", + text: "{}", + }, + ]; + for (let i = 1; i <= 12; i++) { + messages.push({ + id: `c${i}`, + role: "assistant", + contentType: "tool-call", + toolName: "compress", + toolCallId: `call-fail${i}`, + text: "{}", + }); + messages.push({ + id: `r${i}`, + role: "user", + contentType: "tool-result", + toolName: "compress", + toolCallId: `call-fail${i}`, + text: "nothing to compress", + }); + } + messages.push({ + id: "mend", + role: "user", + contentType: "text", + text: "tail", + }); - const result = hideConsumedCompressCalls(state, messages); - // 12 failed pairs in → only the newest 2 pairs + the active call remain: - // PR #18's unbounded accumulation cannot recur under KEEP_LAST_ORPHANED=2. - assert.equal(result.hidden, 20); - const remainingCallIds = result.messages.filter((m) => m.toolName === "compress" && m.contentType === "tool-call").map((m) => m.toolCallId); - assert.deepEqual(remainingCallIds, ["call-active", "call-fail11", "call-fail12"]); - const remainingResultIds = result.messages.filter((m) => m.contentType === "tool-result").map((m) => m.toolCallId); - assert.deepEqual(remainingResultIds, ["call-fail11", "call-fail12"]); + const result = hideConsumedCompressCalls(state, messages); + // 12 failed pairs in → only the newest 2 pairs + the active call remain: + // PR #18's unbounded accumulation cannot recur under KEEP_LAST_ORPHANED=2. + assert.equal(result.hidden, 20); + const remainingCallIds = result.messages + .filter((m) => m.toolName === "compress" && m.contentType === "tool-call") + .map((m) => m.toolCallId); + assert.deepEqual(remainingCallIds, [ + "call-active", + "call-fail11", + "call-fail12", + ]); + const remainingResultIds = result.messages + .filter((m) => m.contentType === "tool-result") + .map((m) => m.toolCallId); + assert.deepEqual(remainingResultIds, ["call-fail11", "call-fail12"]); }); test("a failed compress call and its result stay visible (issue #9 fixed-point precondition removed)", () => { - const state: CompressionState = { ...createInitialState(), blocks: [] }; - const messages: CoreMessage[] = [ - { id: "m1", role: "user", contentType: "text", text: "q" }, - { id: "m2", role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId: "call-fail", text: "{}" }, - { id: "m3", role: "user", contentType: "tool-result", toolName: "compress", toolCallId: "call-fail", text: "Requested range(s) already compressed; nothing to do" }, - ]; - const result = hideConsumedCompressCalls(state, messages); - assert.equal(result.hidden, 0); - assert.ok(result.messages.some((m) => m.toolCallId === "call-fail" && m.contentType === "tool-call")); - assert.ok(result.messages.some((m) => m.toolCallId === "call-fail" && m.contentType === "tool-result")); + const state: CompressionState = { ...createInitialState(), blocks: [] }; + const messages: CoreMessage[] = [ + { id: "m1", role: "user", contentType: "text", text: "q" }, + { + id: "m2", + role: "assistant", + contentType: "tool-call", + toolName: "compress", + toolCallId: "call-fail", + text: "{}", + }, + { + id: "m3", + role: "user", + contentType: "tool-result", + toolName: "compress", + toolCallId: "call-fail", + text: "Requested range(s) already compressed; nothing to do", + }, + ]; + const result = hideConsumedCompressCalls(state, messages); + assert.equal(result.hidden, 0); + assert.ok( + result.messages.some( + (m) => m.toolCallId === "call-fail" && m.contentType === "tool-call", + ), + ); + assert.ok( + result.messages.some( + (m) => m.toolCallId === "call-fail" && m.contentType === "tool-result", + ), + ); }); test("rebuildCompressionState replays historical compress tool calls", () => { - const compressInput = JSON.stringify({ - content: [ - { startId: "m00001", endId: "m00002", summary: "rebuilt summary of early talk", topic: "intro" }, - ], - }); - const messages: CoreMessage[] = [ - { id: "raw1", role: "user", contentType: "text", text: "first message content" }, - { id: "raw2", role: "assistant", contentType: "text", text: "second message content" }, - { id: "raw3", role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId: "call1", text: compressInput }, - { id: "raw4", role: "user", contentType: "text", text: "later message" }, - ]; - const config = defaultConfig(200000, { compress: { minCompressRange: 0, maxSummaryLength: 0, minSummaryLength: 0 }, preserveRecentMessages: 0, preserveRecentTokens: 0 }); - const result = rebuildCompressionState(createInitialState(), messages, config); - assert.ok(result.blocksRebuilt >= 1); - const rebuiltBlock = result.state.blocks.find((b) => b.summary.includes("rebuilt summary")); - assert.ok(rebuiltBlock); - assert.equal(rebuiltBlock!.compressCallId, "call1"); - assert.ok(result.state.messageRefs.byRaw["raw1"]); + const compressInput = JSON.stringify({ + content: [ + { + startId: "m00001", + endId: "m00002", + summary: "rebuilt summary of early talk", + topic: "intro", + }, + ], + }); + const messages: CoreMessage[] = [ + { + id: "raw1", + role: "user", + contentType: "text", + text: "first message content", + }, + { + id: "raw2", + role: "assistant", + contentType: "text", + text: "second message content", + }, + { + id: "raw3", + role: "assistant", + contentType: "tool-call", + toolName: "compress", + toolCallId: "call1", + text: compressInput, + }, + { id: "raw4", role: "user", contentType: "text", text: "later message" }, + ]; + const config = defaultConfig(200000, { + compress: { minCompressRange: 0, maxSummaryLength: 0, minSummaryLength: 0 }, + preserveRecentMessages: 0, + preserveRecentTokens: 0, + }); + const result = rebuildCompressionState( + createInitialState(), + messages, + config, + ); + assert.ok(result.blocksRebuilt >= 1); + const rebuiltBlock = result.state.blocks.find((b) => + b.summary.includes("rebuilt summary"), + ); + assert.ok(rebuiltBlock); + assert.equal(rebuiltBlock!.compressCallId, "call1"); + assert.ok(result.state.messageRefs.byRaw["raw1"]); }); test("rebuildCompressionState returns zero blocks when no compress calls exist", () => { - const messages: CoreMessage[] = [ - { id: "raw1", role: "user", contentType: "text", text: "plain" }, - ]; - const result = rebuildCompressionState(createInitialState(), messages, defaultConfig(200000, { preserveRecentMessages: 0, preserveRecentTokens: 0 })); - assert.equal(result.blocksRebuilt, 0); + const messages: CoreMessage[] = [ + { id: "raw1", role: "user", contentType: "text", text: "plain" }, + ]; + const result = rebuildCompressionState( + createInitialState(), + messages, + defaultConfig(200000, { + preserveRecentMessages: 0, + preserveRecentTokens: 0, + }), + ); + assert.equal(result.blocksRebuilt, 0); }); -function compressInput(entries: Array<{ startId: string; endId: string; summary: string }>): string { - return JSON.stringify({ content: entries }); +function compressInput( + entries: Array<{ startId: string; endId: string; summary: string }>, +): string { + return JSON.stringify({ content: entries }); } -function parseContent(text: string | undefined): Array<{ startId?: string; endId?: string; summary?: string }> { - return (JSON.parse(text ?? "{}") as { content?: Array<{ startId?: string; endId?: string; summary?: string }> }).content ?? []; +function parseContent( + text: string | undefined, +): Array<{ startId?: string; endId?: string; summary?: string }> { + return ( + ( + JSON.parse(text ?? "{}") as { + content?: Array<{ startId?: string; endId?: string; summary?: string }>; + } + ).content ?? [] + ); } test("batched compress: rewrites kept call to drop consumed sibling entries (issue #288)", () => { - const state: CompressionState = { - ...createInitialState(), - blocks: [ - block({ blockId: "b5", compressCallId: "call-batch", active: true, startRef: "m5", endRef: "m6" }), - block({ blockId: "b8", compressCallId: "call-batch", active: false, startRef: "m8", endRef: "m9" }), - ], - }; - const messages: CoreMessage[] = [ - { - id: "mc", role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId: "call-batch", - text: compressInput([ - { startId: "m5", endId: "m6", summary: "live entry summary" }, - { startId: "m8", endId: "m9", summary: "consumed entry summary" }, - ]), - }, - ]; - const result = hideConsumedCompressCalls(state, messages); - assert.equal(result.hidden, 0, "kept batch is not removed"); - const kept = result.messages.find((m) => m.toolCallId === "call-batch"); - assert.ok(kept, "batch call survives (one live sibling)"); - const content = parseContent(kept!.text); - assert.equal(content.length, 1, "consumed entry dropped, live entry retained"); - assert.equal(content[0]!.startId, "m5"); - assert.equal(content[0]!.summary, "live entry summary"); + const state: CompressionState = { + ...createInitialState(), + blocks: [ + block({ + blockId: "b5", + compressCallId: "call-batch", + active: true, + startRef: "m5", + endRef: "m6", + }), + block({ + blockId: "b8", + compressCallId: "call-batch", + active: false, + startRef: "m8", + endRef: "m9", + }), + ], + }; + const messages: CoreMessage[] = [ + { + id: "mc", + role: "assistant", + contentType: "tool-call", + toolName: "compress", + toolCallId: "call-batch", + text: compressInput([ + { startId: "m5", endId: "m6", summary: "live entry summary" }, + { startId: "m8", endId: "m9", summary: "consumed entry summary" }, + ]), + }, + ]; + const result = hideConsumedCompressCalls(state, messages); + assert.equal(result.hidden, 0, "kept batch is not removed"); + const kept = result.messages.find((m) => m.toolCallId === "call-batch"); + assert.ok(kept, "batch call survives (one live sibling)"); + const content = parseContent(kept!.text); + assert.equal( + content.length, + 1, + "consumed entry dropped, live entry retained", + ); + assert.equal(content[0]!.startId, "m5"); + assert.equal(content[0]!.summary, "live entry summary"); }); test("batched compress: no rewrite when all sibling blocks are live", () => { - const state: CompressionState = { - ...createInitialState(), - blocks: [ - block({ blockId: "b5", compressCallId: "call-batch", active: true, startRef: "m5", endRef: "m6" }), - block({ blockId: "b8", compressCallId: "call-batch", active: true, startRef: "m8", endRef: "m9" }), - ], - }; - const originalText = compressInput([ - { startId: "m5", endId: "m6", summary: "S1" }, - { startId: "m8", endId: "m9", summary: "S2" }, - ]); - const messages: CoreMessage[] = [ - { - id: "mc", role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId: "call-batch", - text: originalText, - }, - ]; - const result = hideConsumedCompressCalls(state, messages); - assert.equal(result.hidden, 0); - const kept = result.messages.find((m) => m.toolCallId === "call-batch")!; - assert.equal(kept.text, originalText, "byte-identical: rewrite path did not fire/re-serialize"); - assert.equal(parseContent(kept.text).length, 2, "both live entries retained"); + const state: CompressionState = { + ...createInitialState(), + blocks: [ + block({ + blockId: "b5", + compressCallId: "call-batch", + active: true, + startRef: "m5", + endRef: "m6", + }), + block({ + blockId: "b8", + compressCallId: "call-batch", + active: true, + startRef: "m8", + endRef: "m9", + }), + ], + }; + const originalText = compressInput([ + { startId: "m5", endId: "m6", summary: "S1" }, + { startId: "m8", endId: "m9", summary: "S2" }, + ]); + const messages: CoreMessage[] = [ + { + id: "mc", + role: "assistant", + contentType: "tool-call", + toolName: "compress", + toolCallId: "call-batch", + text: originalText, + }, + ]; + const result = hideConsumedCompressCalls(state, messages); + assert.equal(result.hidden, 0); + const kept = result.messages.find((m) => m.toolCallId === "call-batch")!; + assert.equal( + kept.text, + originalText, + "byte-identical: rewrite path did not fire/re-serialize", + ); + assert.equal(parseContent(kept.text).length, 2, "both live entries retained"); }); test("batched compress: fully removed when all sibling blocks consumed", () => { - const state: CompressionState = { - ...createInitialState(), - blocks: [ - block({ blockId: "b5", compressCallId: "call-batch", active: false, startRef: "m5", endRef: "m6" }), - block({ blockId: "b8", compressCallId: "call-batch", active: false, startRef: "m8", endRef: "m9" }), - ], - }; - const messages: CoreMessage[] = [ - { - id: "mc", role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId: "call-batch", - text: compressInput([ - { startId: "m5", endId: "m6", summary: "S1" }, - { startId: "m8", endId: "m9", summary: "S2" }, - ]), - }, - ]; - const result = hideConsumedCompressCalls(state, messages); - assert.equal(result.hidden, 1, "fully-consumed batch removed"); - assert.equal(result.messages.find((m) => m.toolCallId === "call-batch"), undefined); + const state: CompressionState = { + ...createInitialState(), + blocks: [ + block({ + blockId: "b5", + compressCallId: "call-batch", + active: false, + startRef: "m5", + endRef: "m6", + }), + block({ + blockId: "b8", + compressCallId: "call-batch", + active: false, + startRef: "m8", + endRef: "m9", + }), + ], + }; + const messages: CoreMessage[] = [ + { + id: "mc", + role: "assistant", + contentType: "tool-call", + toolName: "compress", + toolCallId: "call-batch", + text: compressInput([ + { startId: "m5", endId: "m6", summary: "S1" }, + { startId: "m8", endId: "m9", summary: "S2" }, + ]), + }, + ]; + const result = hideConsumedCompressCalls(state, messages); + assert.equal(result.hidden, 1, "fully-consumed batch removed"); + assert.equal( + result.messages.find((m) => m.toolCallId === "call-batch"), + undefined, + ); }); test("batched compress: legacy live block (no startRef/endRef) keeps whole part intact", () => { - const state: CompressionState = { - ...createInitialState(), - blocks: [ - block({ blockId: "b5", compressCallId: "call-batch", active: true, startRef: "m5", endRef: "m6" }), - block({ blockId: "b8", compressCallId: "call-batch", active: true }), - ], - }; - const messages: CoreMessage[] = [ - { - id: "mc", role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId: "call-batch", - text: compressInput([ - { startId: "m5", endId: "m6", summary: "S1" }, - { startId: "m8", endId: "m9", summary: "S2" }, - ]), - }, - ]; - const result = hideConsumedCompressCalls(state, messages); - assert.equal(result.hidden, 0); - const kept = result.messages.find((m) => m.toolCallId === "call-batch")!; - assert.equal(parseContent(kept.text).length, 2, "legacy live block → no rewrite, all entries kept"); + const state: CompressionState = { + ...createInitialState(), + blocks: [ + block({ + blockId: "b5", + compressCallId: "call-batch", + active: true, + startRef: "m5", + endRef: "m6", + }), + block({ blockId: "b8", compressCallId: "call-batch", active: true }), + ], + }; + const messages: CoreMessage[] = [ + { + id: "mc", + role: "assistant", + contentType: "tool-call", + toolName: "compress", + toolCallId: "call-batch", + text: compressInput([ + { startId: "m5", endId: "m6", summary: "S1" }, + { startId: "m8", endId: "m9", summary: "S2" }, + ]), + }, + ]; + const result = hideConsumedCompressCalls(state, messages); + assert.equal(result.hidden, 0); + const kept = result.messages.find((m) => m.toolCallId === "call-batch")!; + assert.equal( + parseContent(kept.text).length, + 2, + "legacy live block → no rewrite, all entries kept", + ); }); test("batched compress: message-mode entry matched via messageId fallback", () => { - const state: CompressionState = { - ...createInitialState(), - blocks: [ - block({ blockId: "b1", compressCallId: "call-batch", active: true, startRef: "msg-3", endRef: "msg-3" }), - block({ blockId: "b2", compressCallId: "call-batch", active: false, startRef: "msg-7", endRef: "msg-7" }), + const state: CompressionState = { + ...createInitialState(), + blocks: [ + block({ + blockId: "b1", + compressCallId: "call-batch", + active: true, + startRef: "msg-3", + endRef: "msg-3", + }), + block({ + blockId: "b2", + compressCallId: "call-batch", + active: false, + startRef: "msg-7", + endRef: "msg-7", + }), + ], + }; + const messages: CoreMessage[] = [ + { + id: "mc", + role: "assistant", + contentType: "tool-call", + toolName: "compress", + toolCallId: "call-batch", + text: JSON.stringify({ + content: [ + { messageId: "msg-3", summary: "live message-mode summary" }, + { messageId: "msg-7", summary: "consumed message-mode summary" }, ], - }; - const messages: CoreMessage[] = [ - { - id: "mc", role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId: "call-batch", - text: JSON.stringify({ - content: [ - { messageId: "msg-3", summary: "live message-mode summary" }, - { messageId: "msg-7", summary: "consumed message-mode summary" }, - ], - }), - }, - ]; - const result = hideConsumedCompressCalls(state, messages); - assert.equal(result.hidden, 0); - const kept = result.messages.find((m) => m.toolCallId === "call-batch")!; - const content = parseContent(kept.text) as Array<{ messageId?: string; summary?: string }>; - assert.equal(content.length, 1, "consumed message-mode entry dropped"); - assert.equal(content[0]!.messageId, "msg-3"); - assert.equal(content[0]!.summary, "live message-mode summary"); + }), + }, + ]; + const result = hideConsumedCompressCalls(state, messages); + assert.equal(result.hidden, 0); + const kept = result.messages.find((m) => m.toolCallId === "call-batch")!; + const content = parseContent(kept.text) as Array<{ + messageId?: string; + summary?: string; + }>; + assert.equal(content.length, 1, "consumed message-mode entry dropped"); + assert.equal(content[0]!.messageId, "msg-3"); + assert.equal(content[0]!.summary, "live message-mode summary"); }); test("batched compress: matching-miss keeps original message intact (no data loss)", () => { - const state: CompressionState = { - ...createInitialState(), - blocks: [ - block({ blockId: "b1", compressCallId: "call-batch", active: true, startRef: "m5", endRef: "m6" }), - ], - }; - const originalText = compressInput([ - { startId: "mx", endId: "my", summary: "unmatched entry" }, - { startId: "mz", endId: "mw", summary: "another unmatched" }, - ]); - const messages: CoreMessage[] = [ - { - id: "mc", role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId: "call-batch", - text: originalText, - }, - ]; - const result = hideConsumedCompressCalls(state, messages); - assert.equal(result.hidden, 0); - const kept = result.messages.find((m) => m.toolCallId === "call-batch")!; - assert.equal(kept.text, originalText, "no entries matched live keys → message preserved verbatim"); + const state: CompressionState = { + ...createInitialState(), + blocks: [ + block({ + blockId: "b1", + compressCallId: "call-batch", + active: true, + startRef: "m5", + endRef: "m6", + }), + ], + }; + const originalText = compressInput([ + { startId: "mx", endId: "my", summary: "unmatched entry" }, + { startId: "mz", endId: "mw", summary: "another unmatched" }, + ]); + const messages: CoreMessage[] = [ + { + id: "mc", + role: "assistant", + contentType: "tool-call", + toolName: "compress", + toolCallId: "call-batch", + text: originalText, + }, + ]; + const result = hideConsumedCompressCalls(state, messages); + assert.equal(result.hidden, 0); + const kept = result.messages.find((m) => m.toolCallId === "call-batch")!; + assert.equal( + kept.text, + originalText, + "no entries matched live keys → message preserved verbatim", + ); }); test("batched compress: rewritten batch keeps its tool-result message", () => { - const state: CompressionState = { - ...createInitialState(), - blocks: [ - block({ blockId: "b5", compressCallId: "call-batch", active: true, startRef: "m5", endRef: "m6" }), - block({ blockId: "b8", compressCallId: "call-batch", active: false, startRef: "m8", endRef: "m9" }), - ], - }; - const messages: CoreMessage[] = [ - { - id: "mc", role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId: "call-batch", - text: compressInput([ - { startId: "m5", endId: "m6", summary: "S1" }, - { startId: "m8", endId: "m9", summary: "S2" }, - ]), - }, - { - id: "mc-result", role: "tool", contentType: "tool-result", toolCallId: "call-batch", - text: '{"ok":true}', - }, - ]; - const result = hideConsumedCompressCalls(state, messages); - assert.equal(result.hidden, 0, "neither tool-call nor tool-result removed on a kept/rewritten batch"); - assert.ok(result.messages.find((m) => m.id === "mc"), "rewritten tool-call survives"); - assert.ok(result.messages.find((m) => m.id === "mc-result"), "tool-result for kept callId survives"); - const kept = result.messages.find((m) => m.id === "mc")!; - assert.equal(parseContent(kept.text).length, 1, "tool-call text rewritten to drop consumed entry"); + const state: CompressionState = { + ...createInitialState(), + blocks: [ + block({ + blockId: "b5", + compressCallId: "call-batch", + active: true, + startRef: "m5", + endRef: "m6", + }), + block({ + blockId: "b8", + compressCallId: "call-batch", + active: false, + startRef: "m8", + endRef: "m9", + }), + ], + }; + const messages: CoreMessage[] = [ + { + id: "mc", + role: "assistant", + contentType: "tool-call", + toolName: "compress", + toolCallId: "call-batch", + text: compressInput([ + { startId: "m5", endId: "m6", summary: "S1" }, + { startId: "m8", endId: "m9", summary: "S2" }, + ]), + }, + { + id: "mc-result", + role: "tool", + contentType: "tool-result", + toolCallId: "call-batch", + text: '{"ok":true}', + }, + ]; + const result = hideConsumedCompressCalls(state, messages); + assert.equal( + result.hidden, + 0, + "neither tool-call nor tool-result removed on a kept/rewritten batch", + ); + assert.ok( + result.messages.find((m) => m.id === "mc"), + "rewritten tool-call survives", + ); + assert.ok( + result.messages.find((m) => m.id === "mc-result"), + "tool-result for kept callId survives", + ); + const kept = result.messages.find((m) => m.id === "mc")!; + assert.equal( + parseContent(kept.text).length, + 1, + "tool-call text rewritten to drop consumed entry", + ); }); diff --git a/tests/image-compress.test.ts b/tests/image-compress.test.ts index 1785881..e919b7b 100644 --- a/tests/image-compress.test.ts +++ b/tests/image-compress.test.ts @@ -155,7 +155,8 @@ function jpegBytes(w: number, h: number, app1Pad = 0): Uint8Array { } const OFF_CFG = () => defaultConfig(200_000); -const ON_CFG = () => defaultConfig(200_000, { imageCompression: { enabled: true } }); +const ON_CFG = () => + defaultConfig(200_000, { imageCompression: { enabled: true } }); const PHONE: ImageMeta = { mediaType: "image/png", @@ -215,7 +216,10 @@ function stateWithRefOnly(): CompressionState { function stateWithShrink(): CompressionState { const state = stateWithRefOnly(); - return recordImageShrink(state, shrinkRecord("m00001", 2_000_000, 180_000, 2125, 765)); + return recordImageShrink( + state, + shrinkRecord("m00001", 2_000_000, 180_000, 2125, 765), + ); } test("pixelTileEstimate: known values and bounds", () => { @@ -230,9 +234,16 @@ test("pixelTileEstimate: known values and bounds", () => { }); test("estimateImageTokens: bytes mode counts ceil(base64len/4)", () => { - assert.equal(estimateImageTokens({ mediaType: "image/png", base64Length: 1003 }), 251); assert.equal( - estimateImageTokens({ mediaType: "image/png", base64Length: 1003, billing: "bytes" }), + estimateImageTokens({ mediaType: "image/png", base64Length: 1003 }), + 251, + ); + assert.equal( + estimateImageTokens({ + mediaType: "image/png", + base64Length: 1003, + billing: "bytes", + }), 251, ); }); @@ -277,31 +288,70 @@ test("estimateImageTokens: pixels mode falls back to 16384 for unknown dims", () }); test("parseImageDimensions: PNG / GIF / BMP / WebP variants", () => { - assert.deepEqual(parseImageDimensions(pngBytes(1280, 960)), { width: 1280, height: 960 }); - assert.deepEqual(parseImageDimensions(gifBytes(320, 200)), { width: 320, height: 200 }); - assert.deepEqual(parseImageDimensions(bmpBytes(1024, 768)), { width: 1024, height: 768 }); - assert.deepEqual(parseImageDimensions(bmpBytes(1024, -768)), { width: 1024, height: 768 }); - assert.deepEqual(parseImageDimensions(webpVp8xBytes(640, 480)), { width: 640, height: 480 }); - assert.deepEqual(parseImageDimensions(webpVp8lBytes(100, 200)), { width: 100, height: 200 }); - assert.deepEqual(parseImageDimensions(webpVp8LossyBytes(800, 600)), { width: 800, height: 600 }); + assert.deepEqual(parseImageDimensions(pngBytes(1280, 960)), { + width: 1280, + height: 960, + }); + assert.deepEqual(parseImageDimensions(gifBytes(320, 200)), { + width: 320, + height: 200, + }); + assert.deepEqual(parseImageDimensions(bmpBytes(1024, 768)), { + width: 1024, + height: 768, + }); + assert.deepEqual(parseImageDimensions(bmpBytes(1024, -768)), { + width: 1024, + height: 768, + }); + assert.deepEqual(parseImageDimensions(webpVp8xBytes(640, 480)), { + width: 640, + height: 480, + }); + assert.deepEqual(parseImageDimensions(webpVp8lBytes(100, 200)), { + width: 100, + height: 200, + }); + assert.deepEqual(parseImageDimensions(webpVp8LossyBytes(800, 600)), { + width: 800, + height: 600, + }); }); test("parseImageDimensions: JPEG SOF scan incl. APP1 skip", () => { - assert.deepEqual(parseImageDimensions(jpegBytes(1280, 960)), { width: 1280, height: 960 }); - assert.deepEqual(parseImageDimensions(jpegBytes(1920, 1080, 60)), { width: 1920, height: 1080 }); + assert.deepEqual(parseImageDimensions(jpegBytes(1280, 960)), { + width: 1280, + height: 960, + }); + assert.deepEqual(parseImageDimensions(jpegBytes(1920, 1080, 60)), { + width: 1920, + height: 1080, + }); }); test("parseImageDimensions: unknown or truncated inputs → undefined", () => { - assert.equal(parseImageDimensions(new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])), undefined); - assert.equal(parseImageDimensions(new Uint8Array([0x89, 0x50, 0x4e, 0x47])), undefined); + assert.equal( + parseImageDimensions(new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])), + undefined, + ); + assert.equal( + parseImageDimensions(new Uint8Array([0x89, 0x50, 0x4e, 0x47])), + undefined, + ); assert.equal(parseImageDimensions(new Uint8Array([])), undefined); }); test("parseImageDimensionsFromBase64: round-trips headers; JPEG needs the extended scan", () => { const png = Buffer.from(pngBytes(1568, 2248)).toString("base64"); - assert.deepEqual(parseImageDimensionsFromBase64(png), { width: 1568, height: 2248 }); + assert.deepEqual(parseImageDimensionsFromBase64(png), { + width: 1568, + height: 2248, + }); const jpeg = Buffer.from(jpegBytes(1920, 1080, 60)).toString("base64"); - assert.deepEqual(parseImageDimensionsFromBase64(jpeg), { width: 1920, height: 1080 }); + assert.deepEqual(parseImageDimensionsFromBase64(jpeg), { + width: 1920, + height: 1080, + }); assert.equal(parseImageDimensionsFromBase64(""), undefined); assert.equal(parseImageDimensionsFromBase64("bm90YW4gaW1hZ2U="), undefined); }); @@ -313,18 +363,38 @@ test("heuristic classifier: aspect band + short-side floor", () => { assert.equal(c.isScreenshotLike(DESKTOP), true); assert.equal(c.isScreenshotLike(SQUARE_PHOTO), false); assert.equal(c.isScreenshotLike(BANNER), false); - assert.equal(c.isScreenshotLike({ ...PHONE, width: 640, height: 960 }), false); - assert.equal(c.isScreenshotLike({ mediaType: "image/png", base64Length: 1000 }), false); + assert.equal( + c.isScreenshotLike({ ...PHONE, width: 640, height: 960 }), + false, + ); + assert.equal( + c.isScreenshotLike({ mediaType: "image/png", base64Length: 1000 }), + false, + ); }); test("heuristic classifier: custom options", () => { - const c = createHeuristicClassifier({ aspectRatioMin: 2.0, aspectRatioMax: 3.0, minShortSide: 100 }); + const c = createHeuristicClassifier({ + aspectRatioMin: 2.0, + aspectRatioMax: 3.0, + minShortSide: 100, + }); assert.equal( - c.isScreenshotLike({ mediaType: "image/png", base64Length: 0, width: 100, height: 300 }), + c.isScreenshotLike({ + mediaType: "image/png", + base64Length: 0, + width: 100, + height: 300, + }), true, ); assert.equal( - c.isScreenshotLike({ mediaType: "image/png", base64Length: 0, width: 800, height: 1200 }), + c.isScreenshotLike({ + mediaType: "image/png", + base64Length: 0, + width: 800, + height: 1200, + }), false, ); }); @@ -351,7 +421,11 @@ test("decideImageRoute: screenshot-like ≥ minTokens ⇒ downsample with recipe assert.equal(d.action, "downsample"); assert.equal(d.reason, "downsample"); assert.equal(d.estimatedTokens, 2125); - assert.deepEqual(d.recipe, { maxDimension: 1280, quality: 80, format: "webp" }); + assert.deepEqual(d.recipe, { + maxDimension: 1280, + quality: 80, + format: "webp", + }); }); test("decideImageRoute: below-min-tokens passes untouched", () => { @@ -369,10 +443,19 @@ test("decideImageRoute: below-min-tokens passes untouched", () => { }); test("decideImageRoute: non-screenshot-like and non-image pass untouched", () => { - assert.equal(decideImageRoute(SQUARE_PHOTO, ON_CFG()).reason, "not-screenshot-like"); - assert.equal(decideImageRoute(BANNER, ON_CFG()).reason, "not-screenshot-like"); assert.equal( - decideImageRoute({ mediaType: "application/pdf", base64Length: 5_000_000 }, ON_CFG()).reason, + decideImageRoute(SQUARE_PHOTO, ON_CFG()).reason, + "not-screenshot-like", + ); + assert.equal( + decideImageRoute(BANNER, ON_CFG()).reason, + "not-screenshot-like", + ); + assert.equal( + decideImageRoute( + { mediaType: "application/pdf", base64Length: 5_000_000 }, + ON_CFG(), + ).reason, "not-an-image", ); assert.equal( @@ -386,18 +469,34 @@ test("decideImageRoute: non-screenshot-like and non-image pass untouched", () => test("decideImageRoute: custom config values flow into the recipe", () => { const cfg = defaultConfig(200_000, { - imageCompression: { enabled: true, minTokens: 100, maxDimension: 1000, quality: 60, format: "jpeg" }, + imageCompression: { + enabled: true, + minTokens: 100, + maxDimension: 1000, + quality: 60, + format: "jpeg", + }, }); const d = decideImageRoute(PHONE, cfg); - assert.deepEqual(d.recipe, { maxDimension: 1000, quality: 60, format: "jpeg" }); + assert.deepEqual(d.recipe, { + maxDimension: 1000, + quality: 60, + format: "jpeg", + }); }); test("decideImageRoute: pluggable classifier injection", () => { const alwaysTrue = { name: "always", isScreenshotLike: () => true }; const alwaysFalse = { name: "never", isScreenshotLike: () => false }; - assert.equal(decideImageRoute(SQUARE_PHOTO, ON_CFG(), alwaysTrue).action, "downsample"); + assert.equal( + decideImageRoute(SQUARE_PHOTO, ON_CFG(), alwaysTrue).action, + "downsample", + ); assert.equal(decideImageRoute(PHONE, ON_CFG(), alwaysFalse).action, "pass"); - assert.equal(decideImageRoute(PHONE, ON_CFG(), alwaysFalse).reason, "not-screenshot-like"); + assert.equal( + decideImageRoute(PHONE, ON_CFG(), alwaysFalse).reason, + "not-screenshot-like", + ); }); test("decideImageRoute: deterministic across repeated calls", () => { @@ -418,12 +517,18 @@ test("decideImageRoute: deterministic across repeated calls", () => { test("recordImageShrink: appends record and bumps cumulative stats", () => { let state = createInitialState(); - state = recordImageShrink(state, shrinkRecord("m00001", 2_000_000, 180_000, 2125, 765)); + state = recordImageShrink( + state, + shrinkRecord("m00001", 2_000_000, 180_000, 2125, 765), + ); assert.equal(state.stats.imagesShrunk, 1); assert.equal(state.stats.imageBytesSaved, 1_820_000); assert.equal(state.stats.imageTokensSaved, 1360); assert.equal(state.imageShrinks?.length, 1); - state = recordImageShrink(state, shrinkRecord("m00002", 1_000_000, 900_000, 2125, 765)); + state = recordImageShrink( + state, + shrinkRecord("m00002", 1_000_000, 900_000, 2125, 765), + ); assert.equal(state.stats.imagesShrunk, 2); assert.equal(state.stats.imageBytesSaved, 1_920_000); }); @@ -433,7 +538,10 @@ test("recordImageShrink: negative deltas clamp to zero; works on pre-feature sta delete legacy.imageShrinks; delete legacy.imageFullRestored; delete legacy.stats.imagesShrunk; - const grown = recordImageShrink(legacy, shrinkRecord("m00001", 1000, 1200, 700, 900)); + const grown = recordImageShrink( + legacy, + shrinkRecord("m00001", 1000, 1200, 700, 900), + ); assert.equal(grown.stats.imageBytesSaved, 0); assert.equal(grown.stats.imageTokensSaved, 0); assert.equal(grown.imageShrinks?.length, 1); @@ -451,14 +559,25 @@ test("applyImageFull: happy path marks sticky restore", () => { const state = stateWithShrink(); const out = applyImageFull({ ref: "m00001", state, config: ON_CFG() }); assert.equal(out.ok, true); - assert.match(out.resultText, /^restored original-resolution image\(s\) for m00001 \(1 image\)/); + assert.match( + out.resultText, + /^restored original-resolution image\(s\) for m00001 \(1 image\)/, + ); assert.deepEqual(out.state.imageFullRestored, ["m00001"]); assert.equal(isImageFullRestored(out.state, "m00001"), true); }); test("applyImageFull: idempotent second call", () => { - const first = applyImageFull({ ref: "m00001", state: stateWithShrink(), config: ON_CFG() }); - const second = applyImageFull({ ref: "m00001", state: first.state, config: ON_CFG() }); + const first = applyImageFull({ + ref: "m00001", + state: stateWithShrink(), + config: ON_CFG(), + }); + const second = applyImageFull({ + ref: "m00001", + state: first.state, + config: ON_CFG(), + }); assert.equal(second.ok, true); assert.match(second.resultText, /^already restored \(m00001\)/); assert.deepEqual(second.state.imageFullRestored, ["m00001"]); @@ -472,7 +591,11 @@ test("applyImageFull: failure modes use the FAILED marker", () => { assert.ok(unknown.resultText.startsWith(IMAGE_FULL_FAILURE_MARKER)); assert.match(unknown.resultText, /ref m00099 does not exist/); - const noShrink = applyImageFull({ ref: "m00001", state: stateWithRefOnly(), config: ON_CFG() }); + const noShrink = applyImageFull({ + ref: "m00001", + state: stateWithRefOnly(), + config: ON_CFG(), + }); assert.equal(noShrink.ok, false); assert.ok(noShrink.resultText.startsWith(IMAGE_FULL_FAILURE_MARKER)); assert.match(noShrink.resultText, /no downscaled image/); @@ -486,7 +609,11 @@ test("applyImageFull: failure modes use the FAILED marker", () => { test("resetImageFullState: clears ref-keyed entries, keeps stats and blocks", () => { const state = stateWithShrink(); - const restored = applyImageFull({ ref: "m00001", state, config: ON_CFG() }).state; + const restored = applyImageFull({ + ref: "m00001", + state, + config: ON_CFG(), + }).state; const reset = resetImageFullState(restored); assert.deepEqual(reset.imageFullRestored, []); assert.deepEqual(reset.imageShrinks, []); @@ -507,9 +634,13 @@ test("buildImageFullSystemNote: byte-stable, count-aware, mentions the contract" test("parseImageFullInput: object, JSON string, aliases, trim", () => { assert.deepEqual(parseImageFullInput({ ref: "m00001" }), { ref: "m00001" }); assert.deepEqual(parseImageFullInput('{"ref":"m00002"}'), { ref: "m00002" }); - assert.deepEqual(parseImageFullInput({ messageId: "m00003" }), { ref: "m00003" }); + assert.deepEqual(parseImageFullInput({ messageId: "m00003" }), { + ref: "m00003", + }); assert.deepEqual(parseImageFullInput({ of: "m00004" }), { ref: "m00004" }); - assert.deepEqual(parseImageFullInput({ ref: " m00005 " }), { ref: "m00005" }); + assert.deepEqual(parseImageFullInput({ ref: " m00005 " }), { + ref: "m00005", + }); assert.deepEqual(parseImageFullInput({ ref: "m00001" }, "call-9"), { ref: "m00001", callId: "call-9", @@ -535,7 +666,9 @@ test("image_full tool schemas: three wire shapes, required ref", () => { assert.equal(IMAGE_FULL_TOOL.input_schema.properties.ref.type, "string"); assert.equal(IMAGE_FULL_TOOL_OPENAI.type, "function"); assert.equal(IMAGE_FULL_TOOL_OPENAI.function.name, "image_full"); - assert.deepEqual(IMAGE_FULL_TOOL_OPENAI.function.parameters.required, ["ref"]); + assert.deepEqual(IMAGE_FULL_TOOL_OPENAI.function.parameters.required, [ + "ref", + ]); assert.equal(IMAGE_FULL_TOOL_RESPONSES.type, "function"); assert.deepEqual(IMAGE_FULL_TOOL_RESPONSES.parameters.required, ["ref"]); }); @@ -548,7 +681,10 @@ test("defaultConfig: imageCompression defaults are off with issue spec values", quality: 80, format: "webp", }); - assert.deepEqual(DEFAULT_IMAGE_COMPRESSION_CONFIG, defaultConfig(200_000).imageCompression); + assert.deepEqual( + DEFAULT_IMAGE_COMPRESSION_CONFIG, + defaultConfig(200_000).imageCompression, + ); }); test("defaultConfig: sub-field merge preserves other defaults (absorb-style)", () => { @@ -566,13 +702,28 @@ test("defaultConfig: sub-field merge preserves other defaults (absorb-style)", ( test("validateConfig: rejects invalid imageCompression fields", () => { const good = validateConfig( defaultConfig(200_000, { - imageCompression: { enabled: true, minTokens: 0, maxDimension: 16, quality: 1, format: "png" }, + imageCompression: { + enabled: true, + minTokens: 0, + maxDimension: 16, + quality: 1, + format: "png", + }, }), ); - assert.deepEqual(good.filter((e) => e.startsWith("imageCompression")), []); + assert.deepEqual( + good.filter((e) => e.startsWith("imageCompression")), + [], + ); const bad = validateConfig( defaultConfig(200_000, { - imageCompression: { enabled: true, minTokens: -1, maxDimension: 8, quality: 0, format: "gif" }, + imageCompression: { + enabled: true, + minTokens: -1, + maxDimension: 8, + quality: 0, + format: "gif", + }, }), ); assert.deepEqual(bad, [ @@ -601,7 +752,11 @@ test("createInitialState seeds image counters and empty sticky arrays", () => { test("mergeCompressionState carries image fields through persist/load", () => { const state = stateWithShrink(); - const restored = applyImageFull({ ref: "m00001", state, config: ON_CFG() }).state; + const restored = applyImageFull({ + ref: "m00001", + state, + config: ON_CFG(), + }).state; const merged = mergeCompressionState(restored); assert.deepEqual(merged.imageFullRestored, ["m00001"]); assert.equal(merged.imageShrinks?.length, 1); diff --git a/tests/issue684-turn-integrity.test.ts b/tests/issue684-turn-integrity.test.ts index a2f8a84..af7f366 100644 --- a/tests/issue684-turn-integrity.test.ts +++ b/tests/issue684-turn-integrity.test.ts @@ -8,139 +8,226 @@ import { adjustBoundariesForReasoningPairs } from "../src/reasoning-pairs.js"; import type { CoreMessage } from "../src/types.js"; function user(id: string, text = "u"): CoreMessage { - return { id, role: "user", contentType: "text", text: `${text}-${id} ` + "x".repeat(200) }; + return { + id, + role: "user", + contentType: "text", + text: `${text}-${id} ` + "x".repeat(200), + }; } function reasoning(id: string): CoreMessage { - return { id, role: "assistant", contentType: "reasoning", text: `think-${id} ` + "t".repeat(400) }; + return { + id, + role: "assistant", + contentType: "reasoning", + text: `think-${id} ` + "t".repeat(400), + }; } function toolCall(id: string, callId: string, toolName = "pwsh"): CoreMessage { - return { id, role: "assistant", contentType: "tool-call", toolName, toolCallId: callId, text: `call-${callId}` }; + return { + id, + role: "assistant", + contentType: "tool-call", + toolName, + toolCallId: callId, + text: `call-${callId}`, + }; } -function toolResult(id: string, callId: string, toolName = "pwsh"): CoreMessage { - return { id, role: "user", contentType: "tool-result", toolName, toolCallId: callId, text: `result-${callId} ` + "r".repeat(200) }; +function toolResult( + id: string, + callId: string, + toolName = "pwsh", +): CoreMessage { + return { + id, + role: "user", + contentType: "tool-result", + toolName, + toolCallId: callId, + text: `result-${callId} ` + "r".repeat(200), + }; } function buildMessages(): CoreMessage[] { - return [ - user("m00001"), - user("m00002"), - reasoning("m00003"), - toolCall("m00004", "call_00_x"), - toolCall("m00005", "call_01_x"), - toolResult("m00006", "call_00_x"), - toolResult("m00007", "call_01_x"), - user("m00008"), - user("m00009"), - ]; + return [ + user("m00001"), + user("m00002"), + reasoning("m00003"), + toolCall("m00004", "call_00_x"), + toolCall("m00005", "call_01_x"), + toolResult("m00006", "call_00_x"), + toolResult("m00007", "call_01_x"), + user("m00008"), + user("m00009"), + ]; } const config = (over: Record = {}) => ({ - ...defaultConfig(150000), - compress: { ...defaultConfig(150000).compress, minCompressRange: 1, minSummaryLength: 1, ...over }, + ...defaultConfig(150000), + compress: { + ...defaultConfig(150000).compress, + minCompressRange: 1, + minSummaryLength: 1, + ...over, + }, }); describe("computeTurnGroups", () => { - it("groups reasoning + multi-call burst + all results as one atomic turn", () => { - const groups = computeTurnGroups(buildMessages()); - const big = groups.find((g) => g.includes("m00003"))!; - assert.deepEqual(new Set(big), new Set(["m00003", "m00004", "m00005", "m00006", "m00007"])); - }); + it("groups reasoning + multi-call burst + all results as one atomic turn", () => { + const groups = computeTurnGroups(buildMessages()); + const big = groups.find((g) => g.includes("m00003"))!; + assert.deepEqual( + new Set(big), + new Set(["m00003", "m00004", "m00005", "m00006", "m00007"]), + ); + }); - it("groups a no-reasoning burst with its results", () => { - const msgs = [user("m1"), toolCall("m2", "c1"), toolResult("m3", "c1"), user("m4")]; - const groups = computeTurnGroups(msgs); - assert.deepEqual(groups, [["m2", "m3"]]); - }); + it("groups a no-reasoning burst with its results", () => { + const msgs = [ + user("m1"), + toolCall("m2", "c1"), + toolResult("m3", "c1"), + user("m4"), + ]; + const groups = computeTurnGroups(msgs); + assert.deepEqual(groups, [["m2", "m3"]]); + }); - it("leaves orphan reasoning ungrouped", () => { - const msgs = [user("m1"), reasoning("m2"), user("m3")]; - assert.deepEqual(computeTurnGroups(msgs), []); - }); + it("leaves orphan reasoning ungrouped", () => { + const msgs = [user("m1"), reasoning("m2"), user("m3")]; + assert.deepEqual(computeTurnGroups(msgs), []); + }); }); describe("adjustBoundariesForReasoningPairs (#684 whole-burst pull)", () => { - it("forward pull covers the entire multi-call burst, not just the first companion", () => { - const msgs = buildMessages(); - const r = adjustBoundariesForReasoningPairs(0, 2, msgs); - assert.equal(r.endIndex, 4); - }); + it("forward pull covers the entire multi-call burst, not just the first companion", () => { + const msgs = buildMessages(); + const r = adjustBoundariesForReasoningPairs(0, 2, msgs); + assert.equal(r.endIndex, 4); + }); }); describe("#684 regression: protected carve must not split a turn", () => { - it("withdraws the whole turn when the carve keeps one call alive", () => { - const core = createCore(); - let state = createInitialState(); - const messages = buildMessages(); - const out = core.processTurn({ messages, state, config: config(), tokenCount: (t) => Math.ceil(t.length / 4) }); - state = out.state; - const refs = state.messageRefs.byRaw; - // recent-zone protection covers call_01 + its result (as in the snapshot) - const protectedRefs = new Set([refs["m00005"]!, refs["m00007"]!]); - const res = core.applyCompression({ - state, - messages, - config: config(), - protectedMessageIds: protectedRefs, - ranges: [{ startRef: "m00001", endRef: "m00007", summary: "s".repeat(400) }], - }); - const block = res.state.blocks.find((b) => b.active); - assert.ok(block, "block created for the non-split part"); - for (const id of ["m00003", "m00004", "m00005", "m00006", "m00007"]) { - assert.ok(!block.effectiveMessageIds.includes(id), `${id} must not fold (turn split by protected zone)`); - } - assert.ok(res.result.warnings.some((w) => /turn/.test(w)), JSON.stringify(res.result.warnings)); - // the surviving view keeps the WHOLE turn visible: reasoning + both calls + results - const view = core.processTurn({ messages, state: res.state, config: config(), tokenCount: (t) => Math.ceil(t.length / 4) }); - const visibleIds = new Set(view.messages.map((m) => (m as { id?: string }).id).filter(Boolean)); - for (const id of ["m00003", "m00004", "m00005", "m00006", "m00007"]) { - assert.ok(visibleIds.has(id), `${id} must stay visible`); - } + it("withdraws the whole turn when the carve keeps one call alive", () => { + const core = createCore(); + let state = createInitialState(); + const messages = buildMessages(); + const out = core.processTurn({ + messages, + state, + config: config(), + tokenCount: (t) => Math.ceil(t.length / 4), }); + state = out.state; + const refs = state.messageRefs.byRaw; + // recent-zone protection covers call_01 + its result (as in the snapshot) + const protectedRefs = new Set([refs["m00005"]!, refs["m00007"]!]); + const res = core.applyCompression({ + state, + messages, + config: config(), + protectedMessageIds: protectedRefs, + ranges: [ + { startRef: "m00001", endRef: "m00007", summary: "s".repeat(400) }, + ], + }); + const block = res.state.blocks.find((b) => b.active); + assert.ok(block, "block created for the non-split part"); + for (const id of ["m00003", "m00004", "m00005", "m00006", "m00007"]) { + assert.ok( + !block.effectiveMessageIds.includes(id), + `${id} must not fold (turn split by protected zone)`, + ); + } + assert.ok( + res.result.warnings.some((w) => /turn/.test(w)), + JSON.stringify(res.result.warnings), + ); + // the surviving view keeps the WHOLE turn visible: reasoning + both calls + results + const view = core.processTurn({ + messages, + state: res.state, + config: config(), + tokenCount: (t) => Math.ceil(t.length / 4), + }); + const visibleIds = new Set( + view.messages.map((m) => (m as { id?: string }).id).filter(Boolean), + ); + for (const id of ["m00003", "m00004", "m00005", "m00006", "m00007"]) { + assert.ok(visibleIds.has(id), `${id} must stay visible`); + } + }); - it("fails with a clear error when the range is exactly the split turn", () => { - const core = createCore(); - let state = createInitialState(); - const messages = buildMessages(); - const out = core.processTurn({ messages, state, config: config(), tokenCount: (t) => Math.ceil(t.length / 4) }); - state = out.state; - const refs = state.messageRefs.byRaw; - const res = core.applyCompression({ - state, - messages, - config: config(), - protectedMessageIds: new Set([refs["m00005"]!, refs["m00007"]!]), - ranges: [{ startRef: "m00003", endRef: "m00007", summary: "s".repeat(400) }], - }); - assert.equal(res.result.blocksCreated, 0); - assert.equal(res.result.errors.length, 1); - assert.match(res.result.errors[0]!, /split 1 turn/); + it("fails with a clear error when the range is exactly the split turn", () => { + const core = createCore(); + let state = createInitialState(); + const messages = buildMessages(); + const out = core.processTurn({ + messages, + state, + config: config(), + tokenCount: (t) => Math.ceil(t.length / 4), + }); + state = out.state; + const refs = state.messageRefs.byRaw; + const res = core.applyCompression({ + state, + messages, + config: config(), + protectedMessageIds: new Set([refs["m00005"]!, refs["m00007"]!]), + ranges: [ + { startRef: "m00003", endRef: "m00007", summary: "s".repeat(400) }, + ], }); + assert.equal(res.result.blocksCreated, 0); + assert.equal(res.result.errors.length, 1); + assert.match(res.result.errors[0]!, /split 1 turn/); + }); - it("#564 shape stays allowed: fold call+result, keep reasoning+text (directional)", () => { - const core = createCore(); - const state = createInitialState(); - const messages: CoreMessage[] = [ - user("m1"), - reasoning("m2"), - { id: "m3", role: "assistant", contentType: "text", text: "TEXT-A " + "x".repeat(200) }, - toolCall("m4", "ca"), - toolResult("m5", "ca"), - user("m6"), - user("m7"), - ]; - const out = core.processTurn({ messages, state, config: config(), tokenCount: (t) => Math.ceil(t.length / 4) }); - const res = core.applyCompression({ - state: out.state, - messages, - config: config(), - protectedMessageIds: new Set(), - ranges: [{ startRef: "m00004", endRef: "m00005", summary: "s".repeat(400) }], - }); - assert.equal(res.result.blocksCreated, 1, `call+result must fold; errors=${JSON.stringify(res.result.errors)}`); - const block = res.state.blocks.find((b) => b.active); - assert.ok(block); - assert.ok(block.effectiveMessageIds.includes("m4")); - assert.ok(block.effectiveMessageIds.includes("m5")); - assert.ok(!res.result.warnings.some((w) => /turn/.test(w)), JSON.stringify(res.result.warnings)); + it("#564 shape stays allowed: fold call+result, keep reasoning+text (directional)", () => { + const core = createCore(); + const state = createInitialState(); + const messages: CoreMessage[] = [ + user("m1"), + reasoning("m2"), + { + id: "m3", + role: "assistant", + contentType: "text", + text: "TEXT-A " + "x".repeat(200), + }, + toolCall("m4", "ca"), + toolResult("m5", "ca"), + user("m6"), + user("m7"), + ]; + const out = core.processTurn({ + messages, + state, + config: config(), + tokenCount: (t) => Math.ceil(t.length / 4), + }); + const res = core.applyCompression({ + state: out.state, + messages, + config: config(), + protectedMessageIds: new Set(), + ranges: [ + { startRef: "m00004", endRef: "m00005", summary: "s".repeat(400) }, + ], }); + assert.equal( + res.result.blocksCreated, + 1, + `call+result must fold; errors=${JSON.stringify(res.result.errors)}`, + ); + const block = res.state.blocks.find((b) => b.active); + assert.ok(block); + assert.ok(block.effectiveMessageIds.includes("m4")); + assert.ok(block.effectiveMessageIds.includes("m5")); + assert.ok( + !res.result.warnings.some((w) => /turn/.test(w)), + JSON.stringify(res.result.warnings), + ); + }); }); diff --git a/tests/nudge-text.test.ts b/tests/nudge-text.test.ts index ea6caab..4936776 100644 --- a/tests/nudge-text.test.ts +++ b/tests/nudge-text.test.ts @@ -1,7 +1,11 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { renderNudgeText } from "../src/nudge-text.js"; -import type { NudgeDecision, CompressibleRange, BlockSpan } from "../src/types.js"; +import type { + NudgeDecision, + CompressibleRange, + BlockSpan, +} from "../src/types.js"; function makeRanges(count: number): CompressibleRange[] { return Array.from({ length: count }, (_, i) => ({ @@ -29,8 +33,14 @@ function makeDecision(overrides: Partial = {}): NudgeDecision { test("gentle mode: voice and header text", () => { const result = renderNudgeText(makeDecision({ contextUsage: 0.5 })); assert.equal(result.voice, "gentle"); - assert.ok(result.text.includes("efficiency nudge"), "should contain gentle header"); - assert.ok(result.text.includes("not an overflow warning"), "should reassure it's not overflow"); + assert.ok( + result.text.includes("efficiency nudge"), + "should contain gentle header", + ); + assert.ok( + result.text.includes("not an overflow warning"), + "should reassure it's not overflow", + ); }); test("gentle mode: tail tip is conditional and licenses deferral (#1198)", () => { @@ -41,12 +51,29 @@ test("gentle mode: tail tip is conditional and licenses deferral (#1198)", () => // Emergency mode keeps its unconditional directive and must not inherit // this line. const gentle = renderNudgeText(makeDecision({ contextUsage: 0.5 })); - assert.ok(gentle.text.includes("If you compress"), "tip must be conditional on the model choosing to compress"); - assert.ok(!gentle.text.includes("Compress all ranges in one call"), "unconditional compress-all directive must be gone"); - assert.ok(gentle.text.includes("they reappear in later nudges"), "skipped ranges must be framed as deferrable, not lost"); - const emergency = renderNudgeText(makeDecision({ contextUsage: 0.99, breakdown: { emergencyOverride: 1 } })); - assert.ok(!emergency.text.includes("If you compress"), "emergency stays unconditional"); - assert.ok(emergency.text.includes("compress now"), "emergency keeps its mandatory directive"); + assert.ok( + gentle.text.includes("If you compress"), + "tip must be conditional on the model choosing to compress", + ); + assert.ok( + !gentle.text.includes("Compress all ranges in one call"), + "unconditional compress-all directive must be gone", + ); + assert.ok( + gentle.text.includes("they reappear in later nudges"), + "skipped ranges must be framed as deferrable, not lost", + ); + const emergency = renderNudgeText( + makeDecision({ contextUsage: 0.99, breakdown: { emergencyOverride: 1 } }), + ); + assert.ok( + !emergency.text.includes("If you compress"), + "emergency stays unconditional", + ); + assert.ok( + emergency.text.includes("compress now"), + "emergency keeps its mandatory directive", + ); }); test("emergency mode: voice and header text", () => { @@ -57,22 +84,34 @@ test("emergency mode: voice and header text", () => { }), ); assert.equal(result.voice, "emergency"); - assert.ok(result.text.includes("Context limit reached"), "should contain emergency header"); + assert.ok( + result.text.includes("Context limit reached"), + "should contain emergency header", + ); assert.ok(result.text.includes("compress now"), "should demand compression"); }); test("gentle mode does NOT contain emergency language", () => { const result = renderNudgeText(makeDecision({ contextUsage: 0.5 })); assert.ok(!result.text.includes("MUST compress"), "gentle should not demand"); - assert.ok(!result.text.includes("⚠️"), "gentle should not have warning symbol"); + assert.ok( + !result.text.includes("⚠️"), + "gentle should not have warning symbol", + ); }); test("emergency mode does NOT contain gentle language", () => { const result = renderNudgeText( makeDecision({ contextUsage: 0.99, breakdown: { emergencyOverride: 1 } }), ); - assert.ok(!result.text.includes("EFFICIENCY NUDGE"), "emergency should not be gentle"); - assert.ok(!result.text.includes("not an overflow warning"), "emergency should not reassure"); + assert.ok( + !result.text.includes("EFFICIENCY NUDGE"), + "emergency should not be gentle", + ); + assert.ok( + !result.text.includes("not an overflow warning"), + "emergency should not reassure", + ); }); test("tier-2 distillation: text contains tier header", () => { @@ -90,15 +129,27 @@ test("tier-3 distillation: text contains tier header", () => { test("tier-2 distillation: guidance warns raw messages in span are absorbed", () => { const result = renderNudgeText(makeDecision({ tier: 2 })); assert.ok(result.text.includes("raw"), "should mention raw messages in span"); - assert.ok(result.text.includes("absorbed"), "should state raw messages are absorbed into the tier-2 block"); - assert.ok(result.text.includes("HOW TO COMPRESS"), "should direct raw messages to HOW TO COMPRESS rules"); + assert.ok( + result.text.includes("absorbed"), + "should state raw messages are absorbed into the tier-2 block", + ); + assert.ok( + result.text.includes("HOW TO COMPRESS"), + "should direct raw messages to HOW TO COMPRESS rules", + ); }); test("tier-3 condensation: guidance warns raw messages in span are absorbed", () => { const result = renderNudgeText(makeDecision({ tier: 3 })); assert.ok(result.text.includes("raw"), "should mention raw messages in span"); - assert.ok(result.text.includes("absorbed"), "should state raw messages are absorbed into the tier-3 block"); - assert.ok(result.text.includes("HOW TO COMPRESS"), "should direct raw messages to HOW TO COMPRESS rules"); + assert.ok( + result.text.includes("absorbed"), + "should state raw messages are absorbed into the tier-3 block", + ); + assert.ok( + result.text.includes("HOW TO COMPRESS"), + "should direct raw messages to HOW TO COMPRESS rules", + ); }); test("emergency + tier 2: emergency voice with distillation guidance", () => { @@ -131,10 +182,22 @@ test("both modes include compressible ranges", () => { makeDecision({ contextUsage: 0.99, breakdown: { emergencyOverride: 1 } }), ); - for (const [label, result] of [["gentle", gentle], ["emergency", emergency]] as const) { - assert.ok(result.text.includes("Compressible ranges"), `${label} should list ranges`); - assert.ok(result.text.includes("m00001"), `${label} should contain range ref`); - assert.ok(result.text.includes("1.0K") || result.text.includes("6.0K"), `${label} should show token estimate`); + for (const [label, result] of [ + ["gentle", gentle], + ["emergency", emergency], + ] as const) { + assert.ok( + result.text.includes("Compressible ranges"), + `${label} should list ranges`, + ); + assert.ok( + result.text.includes("m00001"), + `${label} should contain range ref`, + ); + assert.ok( + result.text.includes("1.0K") || result.text.includes("6.0K"), + `${label} should show token estimate`, + ); } }); @@ -144,11 +207,26 @@ test("both modes include compression guide (KEEP/DROP)", () => { makeDecision({ contextUsage: 0.99, breakdown: { emergencyOverride: 1 } }), ); - for (const [label, result] of [["gentle", gentle], ["emergency", emergency]] as const) { - assert.ok(result.text.includes("KEEP VERBATIM"), `${label} should include KEEP rules`); - assert.ok(result.text.includes("DROP"), `${label} should include DROP rules`); - assert.ok(result.text.includes("PRIORITY"), `${label} should include priority ordering`); - assert.ok(result.text.includes("file paths"), `${label} should mention file paths rule`); + for (const [label, result] of [ + ["gentle", gentle], + ["emergency", emergency], + ] as const) { + assert.ok( + result.text.includes("KEEP VERBATIM"), + `${label} should include KEEP rules`, + ); + assert.ok( + result.text.includes("DROP"), + `${label} should include DROP rules`, + ); + assert.ok( + result.text.includes("PRIORITY"), + `${label} should include priority ordering`, + ); + assert.ok( + result.text.includes("file paths"), + `${label} should mention file paths rule`, + ); } }); @@ -156,7 +234,10 @@ test("empty compressible ranges handled gracefully", () => { const result = renderNudgeText( makeDecision({ compressibleRanges: [], contextUsage: 0.5 }), ); - assert.ok(result.text.includes("No specific ranges"), "should handle empty ranges"); + assert.ok( + result.text.includes("No specific ranges"), + "should handle empty ranges", + ); }); test("percentage is NOT shown (only token amounts)", () => { @@ -187,8 +268,15 @@ test("over-limit renders with emergency voice (MAJOR-2 fix)", () => { breakdown: { overLimit: 1 }, }), ); - assert.equal(result.voice, "emergency", "over-limit should use emergency voice, not gentle"); - assert.ok(!result.text.includes("not an overflow warning"), "should NOT contain gentle reassurance"); + assert.equal( + result.voice, + "emergency", + "over-limit should use emergency voice, not gentle", + ); + assert.ok( + !result.text.includes("not an overflow warning"), + "should NOT contain gentle reassurance", + ); }); function makeSpans(n: number): BlockSpan[] { @@ -201,9 +289,13 @@ function makeSpans(n: number): BlockSpan[] { } test("gentle nudge renders active block map", () => { - const result = renderNudgeText(makeDecision({ activeBlockSpans: makeSpans(2) })); + const result = renderNudgeText( + makeDecision({ activeBlockSpans: makeSpans(2) }), + ); assert.ok( - result.text.includes("Active blocks (2): b1=m00001–m00009 · b2=m00011–m00019"), + result.text.includes( + "Active blocks (2): b1=m00001–m00009 · b2=m00011–m00019", + ), "should list each active block with its ref span", ); }); @@ -215,18 +307,32 @@ test("block map marks non-tier-1 blocks with tier suffix", () => { }); test("block map truncates beyond 8 blocks, keeping newest", () => { - const result = renderNudgeText(makeDecision({ activeBlockSpans: makeSpans(10) })); - assert.ok(result.text.includes("Active blocks (10): …+2 older · "), "should show hidden count"); - assert.ok(!result.text.includes("b1="), "oldest hidden blocks must not be listed"); + const result = renderNudgeText( + makeDecision({ activeBlockSpans: makeSpans(10) }), + ); + assert.ok( + result.text.includes("Active blocks (10): …+2 older · "), + "should show hidden count", + ); + assert.ok( + !result.text.includes("b1="), + "oldest hidden blocks must not be listed", + ); assert.ok(result.text.includes("b8=")); assert.ok(result.text.includes("b10=")); }); test("no block map line when absent or empty", () => { const r1 = renderNudgeText(makeDecision()); - assert.ok(!r1.text.includes("Active blocks"), "absent activeBlockSpans → no line"); + assert.ok( + !r1.text.includes("Active blocks"), + "absent activeBlockSpans → no line", + ); const r2 = renderNudgeText(makeDecision({ activeBlockSpans: [] })); - assert.ok(!r2.text.includes("Active blocks"), "empty activeBlockSpans → no line"); + assert.ok( + !r2.text.includes("Active blocks"), + "empty activeBlockSpans → no line", + ); }); test("emergency nudge also renders block map", () => { @@ -243,8 +349,24 @@ test("emergency nudge also renders block map", () => { test("range lines annotate user message count", () => { const ranges: CompressibleRange[] = [ - { startRef: "m00001", endRef: "m00005", count: 5, tokens: 2000, toolPct: 0.5, textPct: 0.5, userMsgs: 3 }, - { startRef: "m00010", endRef: "m00011", count: 2, tokens: 500, toolPct: 1, textPct: 0, userMsgs: 1 }, + { + startRef: "m00001", + endRef: "m00005", + count: 5, + tokens: 2000, + toolPct: 0.5, + textPct: 0.5, + userMsgs: 3, + }, + { + startRef: "m00010", + endRef: "m00011", + count: 2, + tokens: 500, + toolPct: 1, + textPct: 0, + userMsgs: 1, + }, ]; const result = renderNudgeText(makeDecision({ compressibleRanges: ranges })); assert.ok(result.text.includes("· 3 user msgs")); @@ -253,5 +375,8 @@ test("range lines annotate user message count", () => { test("no user-msg annotation when count is zero or absent", () => { const result = renderNudgeText(makeDecision()); - assert.ok(!result.text.includes("user msg"), "makeRanges fixtures carry no userMsgs"); + assert.ok( + !result.text.includes("user msg"), + "makeRanges fixtures carry no userMsgs", + ); }); diff --git a/tests/nudge.test.ts b/tests/nudge.test.ts index ed74ace..6a109c7 100644 --- a/tests/nudge.test.ts +++ b/tests/nudge.test.ts @@ -32,13 +32,21 @@ function buildConfig(overrides: Partial = {}): Config { }; } -function textMessage(role: CoreMessage["role"], id: string, text: string): CoreMessage { +function textMessage( + role: CoreMessage["role"], + id: string, + text: string, +): CoreMessage { return { id, role, contentType: "text", text }; } function makeMessages(count: number): CoreMessage[] { return Array.from({ length: count }, (_, i) => - textMessage(i % 2 === 0 ? "user" : "assistant", `m${i}`, `message ${i} `.repeat(2000)), + textMessage( + i % 2 === 0 ? "user" : "assistant", + `m${i}`, + `message ${i} `.repeat(2000), + ), ); } @@ -53,9 +61,18 @@ test("nudge: baseline stamped on first turn, no nudge below threshold", () => { const messages = makeMessages(10); const state = createInitialState(); - const turn1 = core.processTurn({ messages, state, config, tokenCount: 10000 }); + const turn1 = core.processTurn({ + messages, + state, + config, + tokenCount: 10000, + }); assert.equal(turn1.nudge.shouldInject, false); - assert.equal(turn1.state.nudge.lastPerMessageNudgeTokens, 10000, "baseline stamped"); + assert.equal( + turn1.state.nudge.lastPerMessageNudgeTokens, + 10000, + "baseline stamped", + ); assert.equal(turn1.state.nudge.lastNudgeShownTokens, 0, "no nudge shown"); }); @@ -65,10 +82,28 @@ test("nudge: fires when growth exceeds threshold AND usage over min limit", () = const messages = makeMessages(10); let state = createInitialState(); - state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; - const turn2 = core.processTurn({ messages, state, config, tokenCount: 55000 }); - assert.equal(turn2.nudge.shouldInject, true, "growth 45000 >= 6000, usage 55% >= 45%"); - assert.equal(turn2.state.nudge.lastNudgeShownTokens, 55000, "shown tokens stamped"); + state = core.processTurn({ + messages, + state, + config, + tokenCount: 10000, + }).state; + const turn2 = core.processTurn({ + messages, + state, + config, + tokenCount: 55000, + }); + assert.equal( + turn2.nudge.shouldInject, + true, + "growth 45000 >= 6000, usage 55% >= 45%", + ); + assert.equal( + turn2.state.nudge.lastNudgeShownTokens, + 55000, + "shown tokens stamped", + ); }); test("nudge: growth-gating suppresses repeat nudge without sufficient growth", () => { @@ -77,11 +112,30 @@ test("nudge: growth-gating suppresses repeat nudge without sufficient growth", ( const messages = makeMessages(10); let state = createInitialState(); - state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; - state = core.processTurn({ messages, state, config, tokenCount: 55000 }).state; + state = core.processTurn({ + messages, + state, + config, + tokenCount: 10000, + }).state; + state = core.processTurn({ + messages, + state, + config, + tokenCount: 55000, + }).state; - const turn3 = core.processTurn({ messages, state, config, tokenCount: 58000 }); - assert.equal(turn3.nudge.shouldInject, false, "growth 3000 < 6000 effectiveThreshold"); + const turn3 = core.processTurn({ + messages, + state, + config, + tokenCount: 58000, + }); + assert.equal( + turn3.nudge.shouldInject, + false, + "growth 3000 < 6000 effectiveThreshold", + ); }); test("nudge: pending nudge halves threshold for faster re-nudge", () => { @@ -90,13 +144,28 @@ test("nudge: pending nudge halves threshold for faster re-nudge", () => { const messages = makeMessages(10); let state = createInitialState(); - state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; - state = core.processTurn({ messages, state, config, tokenCount: 55000 }).state; + state = core.processTurn({ + messages, + state, + config, + tokenCount: 10000, + }).state; + state = core.processTurn({ + messages, + state, + config, + tokenCount: 55000, + }).state; assert.equal(state.nudge.lastNudgeShownTokens, 55000, "pending nudge set"); // Halved threshold = floor(6000/2) = 3000. Growth must also pass growthFloor=5000. // Growth of 5500 >= 5000 → fires despite being below the unhalved 6000. - const turn3 = core.processTurn({ messages, state, config, tokenCount: 60500 }); + const turn3 = core.processTurn({ + messages, + state, + config, + tokenCount: 60500, + }); assert.equal( turn3.nudge.shouldInject, true, @@ -117,11 +186,25 @@ test("nudge: growth floor gate suppresses when growth below floor", () => { let state = createInitialState(); // nudgeGrowthTokens = 6000, growthFloor = max(10000, 0.9*6000) = max(10000, 5400) = 10000 - state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; + state = core.processTurn({ + messages, + state, + config, + tokenCount: 10000, + }).state; // Growth 45000 but growthFloor=10000 → 45000 >= 10000 → passes. Let's test suppression: // Growth 7000 >= effectiveThreshold 6000 but < growthFloor 10000 → suppressed - const turn2 = core.processTurn({ messages, state, config, tokenCount: 17000 }); - assert.equal(turn2.nudge.shouldInject, false, "growth 7000 >= 6000 threshold but < 10000 floor"); + const turn2 = core.processTurn({ + messages, + state, + config, + tokenCount: 17000, + }); + assert.equal( + turn2.nudge.shouldInject, + false, + "growth 7000 >= 6000 threshold but < 10000 floor", + ); }); test("nudge: emergency override fires at 98% regardless of growth", () => { @@ -130,10 +213,24 @@ test("nudge: emergency override fires at 98% regardless of growth", () => { const messages = makeMessages(10); let state = createInitialState(); - state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; + state = core.processTurn({ + messages, + state, + config, + tokenCount: 10000, + }).state; - const turn2 = core.processTurn({ messages, state, config, tokenCount: 99000 }); - assert.equal(turn2.nudge.shouldInject, true, "emergency: 99% >= 98% override"); + const turn2 = core.processTurn({ + messages, + state, + config, + tokenCount: 99000, + }); + assert.equal( + turn2.nudge.shouldInject, + true, + "emergency: 99% >= 98% override", + ); assert.ok( turn2.nudge.reason.includes("EMERGENCY"), `reason should mention EMERGENCY, got: ${turn2.nudge.reason}`, @@ -146,9 +243,23 @@ test("nudge: baseline correction when tokens drop significantly", () => { const messages = makeMessages(10); let state = createInitialState(); - state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; - state = core.processTurn({ messages, state, config, tokenCount: 55000 }).state; - assert.equal(state.nudge.lastPerMessageNudgeTokens, 10000, "baseline at 10000"); + state = core.processTurn({ + messages, + state, + config, + tokenCount: 10000, + }).state; + state = core.processTurn({ + messages, + state, + config, + tokenCount: 55000, + }).state; + assert.equal( + state.nudge.lastPerMessageNudgeTokens, + 10000, + "baseline at 10000", + ); // Token drops below baseline - nudgeGrowthTokens (10000 - 6000 = 4000) // 3000 < 4000 → baseline corrected to 3000 @@ -158,7 +269,11 @@ test("nudge: baseline correction when tokens drop significantly", () => { 3000, "baseline corrected on significant drop", ); - assert.equal(turn3.state.nudge.lastNudgeShownTokens, 0, "pending nudge cleared"); + assert.equal( + turn3.state.nudge.lastNudgeShownTokens, + 0, + "pending nudge cleared", + ); }); test("nudge: compress resets baseline, preventing feedback loop", () => { @@ -167,22 +282,54 @@ test("nudge: compress resets baseline, preventing feedback loop", () => { const messages = makeMessages(10); let state = createInitialState(); - state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; - state = core.processTurn({ messages, state, config, tokenCount: 55000 }).state; + state = core.processTurn({ + messages, + state, + config, + tokenCount: 10000, + }).state; + state = core.processTurn({ + messages, + state, + config, + tokenCount: 55000, + }).state; state = core.applyCompression({ - ranges: [{ startRef: "m00001", endRef: "m00003", summary: "compressed early msgs" }], + ranges: [ + { + startRef: "m00001", + endRef: "m00003", + summary: "compressed early msgs", + }, + ], messages, state, config, }).state; - assert.equal(state.nudge.lastPerMessageNudgeTokens, 0, "baseline cleared post-compress"); - assert.equal(state.nudge.lastNudgeShownTokens, 0, "shown cleared post-compress"); + assert.equal( + state.nudge.lastPerMessageNudgeTokens, + 0, + "baseline cleared post-compress", + ); + assert.equal( + state.nudge.lastNudgeShownTokens, + 0, + "shown cleared post-compress", + ); const turn = core.processTurn({ messages, state, config, tokenCount: 30000 }); - assert.equal(turn.nudge.shouldInject, false, "30% < 45% threshold → no nudge"); - assert.equal(turn.state.nudge.lastPerMessageNudgeTokens, 30000, "baseline re-established"); + assert.equal( + turn.nudge.shouldInject, + false, + "30% < 45% threshold → no nudge", + ); + assert.equal( + turn.state.nudge.lastPerMessageNudgeTokens, + 30000, + "baseline re-established", + ); }); test("nudge: full growth cycle — baseline → growth → nudge → compress → new baseline → growth → nudge", () => { @@ -191,7 +338,12 @@ test("nudge: full growth cycle — baseline → growth → nudge → compress const messages = makeMessages(20); let state = createInitialState(); - state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; + state = core.processTurn({ + messages, + state, + config, + tokenCount: 10000, + }).state; assert.equal(state.nudge.lastPerMessageNudgeTokens, 10000); let turn = core.processTurn({ messages, state, config, tokenCount: 55000 }); @@ -199,7 +351,9 @@ test("nudge: full growth cycle — baseline → growth → nudge → compress state = turn.state; state = core.applyCompression({ - ranges: [{ startRef: "m00001", endRef: "m00005", summary: "compressed batch" }], + ranges: [ + { startRef: "m00001", endRef: "m00005", summary: "compressed batch" }, + ], messages, state, config, @@ -208,22 +362,38 @@ test("nudge: full growth cycle — baseline → growth → nudge → compress turn = core.processTurn({ messages, state, config, tokenCount: 30000 }); assert.equal(turn.nudge.shouldInject, false, "30% < 45%"); - assert.equal(turn.state.nudge.lastPerMessageNudgeTokens, 30000, "new baseline at 30000"); + assert.equal( + turn.state.nudge.lastPerMessageNudgeTokens, + 30000, + "new baseline at 30000", + ); state = turn.state; turn = core.processTurn({ messages, state, config, tokenCount: 55000 }); - assert.equal(turn.nudge.shouldInject, true, "growth 25000 >= 6000, usage 55% >= 45%"); + assert.equal( + turn.nudge.shouldInject, + true, + "growth 25000 >= 6000, usage 55% >= 45%", + ); }); test("nudge: tier distillation fires when lower-tier blocks accumulate enough", () => { const core = createCore(); // Preserve recent messages so pendingT1 (raw compressible) stays below // threshold — we want to isolate the T2 distillation path, not T1. - const config = buildConfig({ tiers: { enabled: true, tier2Trigger: 3, tier3Trigger: 10 }, preserveRecentMessages: 30 }); + const config = buildConfig({ + tiers: { enabled: true, tier2Trigger: 3, tier3Trigger: 10 }, + preserveRecentMessages: 30, + }); const messages = makeMessages(30); let state = createInitialState(); // First turn establishes a baseline at a high token count. - state = core.processTurn({ messages, state, config, tokenCount: 50000 }).state; + state = core.processTurn({ + messages, + state, + config, + tokenCount: 50000, + }).state; // Inject three active T1 blocks with large summaries so pendingT2 (sum of // their summary tokens) exceeds the nudge threshold (buildConfig → 6000). @@ -233,7 +403,9 @@ test("nudge: tier distillation fires when lower-tier blocks accumulate enough", blockId: `b${i + 1}`, runId: "r1", tier: 1 as const, - summary: `tier-1 summary block ${i} with substantial content `.repeat(1200), + summary: `tier-1 summary block ${i} with substantial content `.repeat( + 1200, + ), directMessageIds: [`m${i}`], effectiveMessageIds: [`m${i}`], directBlockIds: [], @@ -244,18 +416,33 @@ test("nudge: tier distillation fires when lower-tier blocks accumulate enough", active: true, })), }; - assert.equal(state.blocks.filter((b) => b.active && b.tier === 1).length, 3, "three T1 blocks present"); + assert.equal( + state.blocks.filter((b) => b.active && b.tier === 1).length, + 3, + "three T1 blocks present", + ); // growth is zero — must NOT inject even though T2 blocks are ready. let turn = core.processTurn({ messages, state, config, tokenCount: 50000 }); - assert.equal(turn.nudge.shouldInject, false, "no growth → no injection even with T2 ready"); + assert.equal( + turn.nudge.shouldInject, + false, + "no growth → no injection even with T2 ready", + ); // Once growth passes the floor (buildConfig floor 5000), the nudge fires // WITH tier-2 distillation info (T1 blocked by preserve → T2 wins). turn = core.processTurn({ messages, state, config, tokenCount: 60000 }); assert.equal(turn.nudge.shouldInject, true, "growth past floor → injects"); - assert.equal(turn.nudge.tier, 2, "nudge carries tier-2 distillation guidance (T1 blocked by preserve)"); - assert.ok((turn.nudge.tierTargetBlocks?.length ?? 0) >= 1, "target T1 blocks listed"); + assert.equal( + turn.nudge.tier, + 2, + "nudge carries tier-2 distillation guidance (T1 blocked by preserve)", + ); + assert.ok( + (turn.nudge.tierTargetBlocks?.length ?? 0) >= 1, + "target T1 blocks listed", + ); }); test("nudge: production config (preserveRecentMessages > 0) computes compressible ranges", () => { @@ -263,11 +450,19 @@ test("nudge: production config (preserveRecentMessages > 0) computes compressibl const config = buildConfig({ preserveRecentMessages: 5 }); const messages = makeMessages(15); let state = createInitialState(); - state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; + state = core.processTurn({ + messages, + state, + config, + tokenCount: 10000, + }).state; const turn = core.processTurn({ messages, state, config, tokenCount: 55000 }); assert.equal(turn.nudge.shouldInject, true); - assert.ok(turn.nudge.compressibleRanges.length > 0, "compressible ranges reported"); + assert.ok( + turn.nudge.compressibleRanges.length > 0, + "compressible ranges reported", + ); for (const range of turn.nudge.compressibleRanges) { assert.ok(!range.startRef.startsWith("m0001"), "preserved tail excluded"); } @@ -278,7 +473,12 @@ test("nudge: compressible ranges exclude messages covered by active blocks", () const config = buildConfig({ preserveRecentMessages: 2 }); const messages = makeMessages(12); let state = createInitialState(); - state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; + state = core.processTurn({ + messages, + state, + config, + tokenCount: 10000, + }).state; state = core.applyCompression({ ranges: [{ startRef: "m00001", endRef: "m00005", summary: "covered" }], @@ -341,25 +541,49 @@ test("arbitration: non-emergency T1 effective >= threshold → tier 1", () => { // minCompressRange > 0 so merge + effective filter actually engage — without // this, pendingByTier bypasses the filter (minCompressRange > 0 ? filter : merged) // and the test exercises the OLD T1 definition. - const config = buildConfig({ compress: { minCompressRange: 5000, maxSummaryLength: 0, minSummaryLength: 0 } }); + const config = buildConfig({ + compress: { + minCompressRange: 5000, + maxSummaryLength: 0, + minSummaryLength: 0, + }, + }); const messages = makeMessages(10); let state = createInitialState(); - state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; + state = core.processTurn({ + messages, + state, + config, + tokenCount: 10000, + }).state; const turn = core.processTurn({ messages, state, config, tokenCount: 55000 }); assert.equal(turn.nudge.shouldInject, true); - assert.equal(turn.nudge.tier, 1, "T1 effective (~50K) >= 6000 → tier 1, no T2 blocks present"); + assert.equal( + turn.nudge.tier, + 1, + "T1 effective (~50K) >= 6000 → tier 1, no T2 blocks present", + ); }); test("arbitration: non-emergency T2 >= 1.5x threshold AND > T1 effective → tier 2", () => { const core = createCore(); // minCompressRange > 0 so the effective filter engages (see test above). const config = buildConfig({ - compress: { minCompressRange: 5000, maxSummaryLength: 0, minSummaryLength: 0 }, + compress: { + minCompressRange: 5000, + maxSummaryLength: 0, + minSummaryLength: 0, + }, preserveRecentMessages: 9, }); const messages = makeMessages(10); let state = createInitialState(); - state = core.processTurn({ messages, state, config, tokenCount: 50000 }).state; + state = core.processTurn({ + messages, + state, + config, + tokenCount: 50000, + }).state; // Two T1 blocks anchored to real messages (m1, m2) with ~6K-token summaries // each → T2 ~12K >= 9000 (1.5x); T1 effective is ~0 (preserve + blocks cover // all visible messages). @@ -376,15 +600,30 @@ test("arbitration: non-emergency T2 >= 1.5x threshold AND > T1 effective → tie test("arbitration: non-emergency T2 large but T1 effective >= threshold → tier 1 wins", () => { const core = createCore(); // minCompressRange > 0 so the effective filter engages. - const config = buildConfig({ compress: { minCompressRange: 5000, maxSummaryLength: 0, minSummaryLength: 0 } }); + const config = buildConfig({ + compress: { + minCompressRange: 5000, + maxSummaryLength: 0, + minSummaryLength: 0, + }, + }); const messages = makeMessages(10); let state = createInitialState(); - state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; + state = core.processTurn({ + messages, + state, + config, + tokenCount: 10000, + }).state; // Large T2 pending (blocks anchored to m0, m1), but T1 effective (~40K from // m2..m9) still dominates the threshold. state = { ...state, blocks: t1Blocks([["m0"], ["m1"]], 40000) }; const turn = core.processTurn({ messages, state, config, tokenCount: 55000 }); - assert.equal(turn.nudge.tier, 1, "T1 effective >= threshold wins even when T2 is large"); + assert.equal( + turn.nudge.tier, + 1, + "T1 effective >= threshold wins even when T2 is large", + ); }); test("arbitration: emergency T2 > T1 effective → tier 2 with emergencyOverride", () => { @@ -392,7 +631,12 @@ test("arbitration: emergency T2 > T1 effective → tier 2 with emergencyOverride const config = buildConfig(); const messages = makeMessages(10); let state = createInitialState(); - state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; + state = core.processTurn({ + messages, + state, + config, + tokenCount: 10000, + }).state; // One T1 block covering ALL messages → compressible empty (T1 effective 0), // with a large summary → T2 pending ~10K dominates. state = { @@ -416,7 +660,11 @@ test("arbitration: emergency T2 > T1 effective → tier 2 with emergencyOverride }; const turn = core.processTurn({ messages, state, config, tokenCount: 99000 }); assert.equal(turn.nudge.shouldInject, true); - assert.equal(turn.nudge.tier, 2, "emergency picks T2 (max pending) when T2 > T1 effective"); + assert.equal( + turn.nudge.tier, + 2, + "emergency picks T2 (max pending) when T2 > T1 effective", + ); assert.equal(turn.nudge.breakdown.emergencyOverride, 1); }); @@ -425,11 +673,20 @@ test("arbitration: emergency T1 effective largest → tier 1", () => { const config = buildConfig(); const messages = makeMessages(10); let state = createInitialState(); - state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; + state = core.processTurn({ + messages, + state, + config, + tokenCount: 10000, + }).state; // No blocks → T2/T3 = 0; T1 effective ~50K is the max. const turn = core.processTurn({ messages, state, config, tokenCount: 99000 }); assert.equal(turn.nudge.shouldInject, true); - assert.equal(turn.nudge.tier, 1, "emergency picks T1 when its effective pending is max"); + assert.equal( + turn.nudge.tier, + 1, + "emergency picks T1 when its effective pending is max", + ); assert.equal(turn.nudge.breakdown.emergencyOverride, 1); }); @@ -439,12 +696,21 @@ test("arbitration: T2 boundary — must NOT fire when pending ∈ [nudgeGrowthTo // range (T2 pending ~7500 ∈ [6000, 9000)) proves the boundary. Diverges from // master, which injected T2 here. const config = buildConfig({ - compress: { minCompressRange: 5000, maxSummaryLength: 0, minSummaryLength: 0 }, + compress: { + minCompressRange: 5000, + maxSummaryLength: 0, + minSummaryLength: 0, + }, preserveRecentMessages: 10, }); const messages = makeMessages(10); let state = createInitialState(); - state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; + state = core.processTurn({ + messages, + state, + config, + tokenCount: 10000, + }).state; // preserveRecentMessages:10 → no compressible msgs → T1 effective = 0. // 1 T1 block, summary 'x'.repeat(30000) → 7500 tokens ∈ [6000, 9000). state = { ...state, blocks: t1Blocks([["m0"]], 30000) }; @@ -461,12 +727,21 @@ test("arbitration: non-emergency T3 >= 1.5× threshold AND > T2 AND > T1 effecti // Validates FIX 1: master had no T3 non-emergency branch → tier would be // null here. NEW code injects T3. const config = buildConfig({ - compress: { minCompressRange: 5000, maxSummaryLength: 0, minSummaryLength: 0 }, + compress: { + minCompressRange: 5000, + maxSummaryLength: 0, + minSummaryLength: 0, + }, preserveRecentMessages: 10, }); const messages = makeMessages(10); let state = createInitialState(); - state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; + state = core.processTurn({ + messages, + state, + config, + tokenCount: 10000, + }).state; // T1 effective = 0 (preserveRecentMessages:10). // T2 pending: 1 small T1 block, summary 4000 chars → 1000 tokens (< 9000). // T3 pending: 2 T2 blocks × 20000-char summaries → 5000 tokens each = 10000 (>= 9000). @@ -487,11 +762,20 @@ test("arbitration: non-emergency T3 >= 1.5× threshold AND > T2 AND > T1 effecti test("arbitration: emergency argmax picks T3 when T3 > T2 > T1 effective", () => { const core = createCore(); const config = buildConfig({ - compress: { minCompressRange: 5000, maxSummaryLength: 0, minSummaryLength: 0 }, + compress: { + minCompressRange: 5000, + maxSummaryLength: 0, + minSummaryLength: 0, + }, }); const messages = makeMessages(10); let state = createInitialState(); - state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; + state = core.processTurn({ + messages, + state, + config, + tokenCount: 10000, + }).state; // T1 block covers ALL messages → compressible empty → T1 effective = 0. // Its summary 'x'.repeat(20000) → 5000 tokens = T2 pending. // 3 T2 blocks × 20000-char summaries → 5000 each = T3 pending 15000 (largest). @@ -512,7 +796,11 @@ test("arbitration: emergency argmax picks T3 when T3 > T2 > T1 effective", () => generation: "young" as const, active: true, }, - ...t2Blocks(3, 20000, messages.map((m) => m.id)), + ...t2Blocks( + 3, + 20000, + messages.map((m) => m.id), + ), ], }; const turn = core.processTurn({ messages, state, config, tokenCount: 99000 }); @@ -534,13 +822,22 @@ test("arbitration: sub-threshold tail folds into preceding batch (pendingT1 span // into it → ONE range m0..m9 (2000 tokens, 8000 chars) that clears 5000 on // its own → pendingT1 = 2000 and nudge lists exactly that one range. const config = buildConfig({ - compress: { minCompressRange: 5000, maxSummaryLength: 0, minSummaryLength: 0 }, + compress: { + minCompressRange: 5000, + maxSummaryLength: 0, + minSummaryLength: 0, + }, }); const messages = Array.from({ length: 10 }, (_, i) => textMessage(i % 2 === 0 ? "user" : "assistant", `m${i}`, "x".repeat(800)), ); let state = createInitialState(); - state = core.processTurn({ messages, state, config, tokenCount: 10000 }).state; + state = core.processTurn({ + messages, + state, + config, + tokenCount: 10000, + }).state; const turn = core.processTurn({ messages, state, config, tokenCount: 35000 }); assert.equal( turn.nudge.breakdown.pendingT1, @@ -559,64 +856,153 @@ test("over-limit fires force-nudge when compressible content exists", () => { const core = createCore(); const config = buildConfig({ nudge: { ...buildConfig().nudge, maxContextLimitPct: 0.75 }, - compress: { minCompressRange: 5000, maxSummaryLength: 0, minSummaryLength: 0 }, + compress: { + minCompressRange: 5000, + maxSummaryLength: 0, + minSummaryLength: 0, + }, preserveRecentMessages: 0, }); const messages = makeMessages(10); - const turn = core.processTurn({ messages, state: createInitialState(), config, tokenCount: 80000 }); - assert.equal(turn.nudge.shouldInject, true, "force-nudge should fire with compressible content"); + const turn = core.processTurn({ + messages, + state: createInitialState(), + config, + tokenCount: 80000, + }); + assert.equal( + turn.nudge.shouldInject, + true, + "force-nudge should fire with compressible content", + ); assert.equal(turn.nudge.breakdown!.overLimit, 1, "overLimit flag set"); - assert.ok(turn.nudge.reason.includes("OVER-LIMIT"), `reason: ${turn.nudge.reason}`); + assert.ok( + turn.nudge.reason.includes("OVER-LIMIT"), + `reason: ${turn.nudge.reason}`, + ); }); test("over-limit does NOT inject when nothing is compressible (MAJOR-1 fix)", () => { const core = createCore(); const config = buildConfig({ nudge: { ...buildConfig().nudge, maxContextLimitPct: 0.75 }, - compress: { minCompressRange: 5000, maxSummaryLength: 0, minSummaryLength: 0 }, + compress: { + minCompressRange: 5000, + maxSummaryLength: 0, + minSummaryLength: 0, + }, preserveRecentMessages: 5, }); const messages = [ textMessage("user", "a", "hello"), textMessage("assistant", "b", "world"), ]; - const turn = core.processTurn({ messages, state: createInitialState(), config, tokenCount: 80000 }); - assert.equal(turn.nudge.shouldInject, false, "no spam when nothing to compress"); - assert.equal(turn.nudge.breakdown!.overLimit, 1, "overLimit flag still set for reporting"); - assert.ok(turn.nudge.reason.includes("OVER-LIMIT"), `reason: ${turn.nudge.reason}`); + const turn = core.processTurn({ + messages, + state: createInitialState(), + config, + tokenCount: 80000, + }); + assert.equal( + turn.nudge.shouldInject, + false, + "no spam when nothing to compress", + ); + assert.equal( + turn.nudge.breakdown!.overLimit, + 1, + "overLimit flag still set for reporting", + ); + assert.ok( + turn.nudge.reason.includes("OVER-LIMIT"), + `reason: ${turn.nudge.reason}`, + ); }); test("emergency suppresses nudge when only sub-minCompressRange tail ranges remain", () => { const core = createCore(); const config = buildConfig({ - nudge: { ...buildConfig().nudge, emergencyThresholdPct: 0.95, maxContextLimitPct: 0.75 }, - compress: { minCompressRange: 5000, maxSummaryLength: 0, minSummaryLength: 0 }, + nudge: { + ...buildConfig().nudge, + emergencyThresholdPct: 0.95, + maxContextLimitPct: 0.75, + }, + compress: { + minCompressRange: 5000, + maxSummaryLength: 0, + minSummaryLength: 0, + }, preserveRecentMessages: 0, }); // Compressible content exists but every range is far below minCompressRange // (min 5000 chars = 1250 tokens; these sum to ~300). Recommending them // would guarantee atomic rejection — the exact failure mode the endurance // report showed at 60% failure rate. - const messages = ["a", "b", "c"].map((t, i) => textMessage(i % 2 === 0 ? "user" : "assistant", `m${i}`, `tiny ${t} `.repeat(10))); - const turn = core.processTurn({ messages, state: createInitialState(), config, tokenCount: 96000 }); - assert.equal(turn.nudge.shouldInject, false, "no emergency spam when nothing is effectively compressible"); - assert.equal(turn.nudge.breakdown!.emergencyOverride, 1, "emergency flag still set for reporting"); - assert.ok(turn.nudge.reason.includes("EMERGENCY"), `reason: ${turn.nudge.reason}`); - assert.ok(turn.nudge.reason.includes("minCompressRange"), `reason explains the suppression: ${turn.nudge.reason}`); + const messages = ["a", "b", "c"].map((t, i) => + textMessage( + i % 2 === 0 ? "user" : "assistant", + `m${i}`, + `tiny ${t} `.repeat(10), + ), + ); + const turn = core.processTurn({ + messages, + state: createInitialState(), + config, + tokenCount: 96000, + }); + assert.equal( + turn.nudge.shouldInject, + false, + "no emergency spam when nothing is effectively compressible", + ); + assert.equal( + turn.nudge.breakdown!.emergencyOverride, + 1, + "emergency flag still set for reporting", + ); + assert.ok( + turn.nudge.reason.includes("EMERGENCY"), + `reason: ${turn.nudge.reason}`, + ); + assert.ok( + turn.nudge.reason.includes("minCompressRange"), + `reason explains the suppression: ${turn.nudge.reason}`, + ); }); test("emergency with effective pending routes to the max-pending tier (post-fix regression)", () => { const core = createCore(); const config = buildConfig({ - nudge: { ...buildConfig().nudge, emergencyThresholdPct: 0.95, maxContextLimitPct: 0.75 }, - compress: { minCompressRange: 5000, maxSummaryLength: 0, minSummaryLength: 0 }, + nudge: { + ...buildConfig().nudge, + emergencyThresholdPct: 0.95, + maxContextLimitPct: 0.75, + }, + compress: { + minCompressRange: 5000, + maxSummaryLength: 0, + minSummaryLength: 0, + }, preserveRecentMessages: 0, }); const messages = makeMessages(10); - const turn = core.processTurn({ messages, state: createInitialState(), config, tokenCount: 96000 }); - assert.equal(turn.nudge.shouldInject, true, "emergency fires when effective content exists"); + const turn = core.processTurn({ + messages, + state: createInitialState(), + config, + tokenCount: 96000, + }); + assert.equal( + turn.nudge.shouldInject, + true, + "emergency fires when effective content exists", + ); assert.equal(turn.nudge.tier, 1, "T1 has the max pending here"); - assert.ok(turn.nudge.reason.includes("EMERGENCY"), `reason: ${turn.nudge.reason}`); + assert.ok( + turn.nudge.reason.includes("EMERGENCY"), + `reason: ${turn.nudge.reason}`, + ); }); test("re-baseline after a tokenCount scale drop also resets per-tier cadence stamps", () => { @@ -635,54 +1021,117 @@ test("re-baseline after a tokenCount scale drop also resets per-tier cadence sta state.nudge.lastShownByTier = { 1: 366_000, 2: 366_000 }; const messages = makeMessages(10); - const turn = core.processTurn({ messages, state, config, tokenCount: 40_000 }); + const turn = core.processTurn({ + messages, + state, + config, + tokenCount: 40_000, + }); const stamped = turn.state.nudge; - assert.equal(stamped.lastPerMessageNudgeTokens, 40_000, "baseline re-anchored at the new scale"); - assert.equal(stamped.lastNudgeShownTokens, 0, "shared cadence baseline cleared"); - assert.deepEqual(stamped.lastShownByTier, {}, "per-tier cadence stamps must not survive a scale drop"); + assert.equal( + stamped.lastPerMessageNudgeTokens, + 40_000, + "baseline re-anchored at the new scale", + ); + assert.equal( + stamped.lastNudgeShownTokens, + 0, + "shared cadence baseline cleared", + ); + assert.deepEqual( + stamped.lastShownByTier, + {}, + "per-tier cadence stamps must not survive a scale drop", + ); }); test("arbitration: 5 summary blocks stay silent with default triggers (#379 — count path default-off)", () => { const core = createCore(); - const config = buildConfig({ preserveRecentMessages: 30, tiers: { enabled: true, tier2Trigger: 1000, tier3Trigger: 2000 } }); + const config = buildConfig({ + preserveRecentMessages: 30, + tiers: { enabled: true, tier2Trigger: 1000, tier3Trigger: 2000 }, + }); const messages = makeMessages(30); - let state = core.processTurn({ messages, state: createInitialState(), config, tokenCount: 50_000 }).state; - state = { ...state, blocks: t1Blocks([["m1"], ["m2"], ["m3"], ["m4"], ["m5"]], 400) }; + let state = core.processTurn({ + messages, + state: createInitialState(), + config, + tokenCount: 50_000, + }).state; + state = { + ...state, + blocks: t1Blocks([["m1"], ["m2"], ["m3"], ["m4"], ["m5"]], 400), + }; // The billion-context#1249 repro shape: five condensed-summary blocks at 40% // usage. Under default triggers the count path is OFF — no nudge, no hint. - const turn = core.processTurn({ messages, state, config, tokenCount: 40_000 }); + const turn = core.processTurn({ + messages, + state, + config, + tokenCount: 40_000, + }); assert.equal(turn.nudge.shouldInject, false, `reason: ${turn.nudge.reason}`); assert.doesNotMatch(turn.nudge.reason ?? "", /T2 distill ready/); - assert.doesNotMatch(turn.nudge.reason ?? "", /T2 \d+ blocks \(count\)/, "5 blocks must not read as count-ready under default triggers"); + assert.doesNotMatch( + turn.nudge.reason ?? "", + /T2 \d+ blocks \(count\)/, + "5 blocks must not read as count-ready under default triggers", + ); }); test("arbitration: count-triggered T3 fires below the old usage band when explicitly configured (#379 — #238 gate deleted)", () => { const core = createCore(); - const config = buildConfig({ tiers: { enabled: true, tier2Trigger: 2, tier3Trigger: 3 }, preserveRecentMessages: 30 }); + const config = buildConfig({ + tiers: { enabled: true, tier2Trigger: 2, tier3Trigger: 3 }, + preserveRecentMessages: 30, + }); const messages = makeMessages(30); - let state = core.processTurn({ messages, state: createInitialState(), config, tokenCount: 25_000 }).state; + let state = core.processTurn({ + messages, + state: createInitialState(), + config, + tokenCount: 25_000, + }).state; state = { ...state, - blocks: t1Blocks([["m1"], ["m2"], ["m3"], ["m4"], ["m5"]], 400).map((b) => ({ ...b, tier: 2 })), + blocks: t1Blocks([["m1"], ["m2"], ["m3"], ["m4"], ["m5"]], 400).map( + (b) => ({ ...b, tier: 2 }), + ), }; // 40k / 100k = 40% < the old minContextLimitPct 45% band; growth 15k >= // floor 6000. With the #238 usage gate deleted, explicit count triggers // fire on pure count — percentages no longer gate compression. - const turn = core.processTurn({ messages, state, config, tokenCount: 40_000 }); + const turn = core.processTurn({ + messages, + state, + config, + tokenCount: 40_000, + }); assert.equal(turn.nudge.shouldInject, true, `reason: ${turn.nudge.reason}`); assert.equal(turn.nudge.tier, 3); - assert.match(turn.nudge.reason ?? "", /T3 condense ready: 5 tier-2 blocks >= tier3Trigger 3/); + assert.match( + turn.nudge.reason ?? "", + /T3 condense ready: 5 tier-2 blocks >= tier3Trigger 3/, + ); }); test("arbitration: T2 fires on tier-1 block COUNT (tier2Trigger) even when summary tokens are small", () => { const core = createCore(); const config = buildConfig({ preserveRecentMessages: 30 }); const messages = makeMessages(30); - let state = core.processTurn({ messages, state: createInitialState(), config, tokenCount: 50000 }).state; + let state = core.processTurn({ + messages, + state: createInitialState(), + config, + tokenCount: 50000, + }).state; // 5 blocks x ~100-token summaries = ~500 pending tokens, far below the // 9000 token gate — only the count path (5 >= tier2Trigger 5) can fire, // and raw T1 is fully preserved so t1Eff cannot dominate. - state = { ...state, blocks: t1Blocks([["m1"], ["m2"], ["m3"], ["m4"], ["m5"]], 400) }; + state = { + ...state, + blocks: t1Blocks([["m1"], ["m2"], ["m3"], ["m4"], ["m5"]], 400), + }; const turn = core.processTurn({ messages, state, config, tokenCount: 60000 }); assert.equal(turn.nudge.shouldInject, true, `reason: ${turn.nudge.reason}`); assert.equal(turn.nudge.tier, 2); @@ -694,26 +1143,50 @@ test("arbitration: below tier2Trigger count and below token gate -> no T2 nudge" const core = createCore(); const config = buildConfig({ preserveRecentMessages: 30 }); const messages = makeMessages(30); - let state = core.processTurn({ messages, state: createInitialState(), config, tokenCount: 50000 }).state; + let state = core.processTurn({ + messages, + state: createInitialState(), + config, + tokenCount: 50000, + }).state; state = { ...state, blocks: t1Blocks([["m1"], ["m2"], ["m3"], ["m4"]], 400) }; const turn = core.processTurn({ messages, state, config, tokenCount: 60000 }); assert.equal(turn.nudge.shouldInject, false, `reason: ${turn.nudge.reason}`); - assert.match(turn.nudge.reason ?? "", /max compressible 400 < threshold 6000/); + assert.match( + turn.nudge.reason ?? "", + /max compressible 400 < threshold 6000/, + ); assert.doesNotMatch(turn.nudge.reason ?? "", /tier2Trigger/); }); test("arbitration: count-triggered T2 does not preempt ready T1 raw compression", () => { const core = createCore(); const config = buildConfig({ - compress: { minCompressRange: 5000, maxSummaryLength: 0, minSummaryLength: 0 }, + compress: { + minCompressRange: 5000, + maxSummaryLength: 0, + minSummaryLength: 0, + }, preserveRecentMessages: 0, }); const messages = makeMessages(10); - let state = core.processTurn({ messages, state: createInitialState(), config, tokenCount: 50000 }).state; - state = { ...state, blocks: t1Blocks([["m1"], ["m2"], ["m3"], ["m4"], ["m5"]], 400) }; + let state = core.processTurn({ + messages, + state: createInitialState(), + config, + tokenCount: 50000, + }).state; + state = { + ...state, + blocks: t1Blocks([["m1"], ["m2"], ["m3"], ["m4"], ["m5"]], 400), + }; const turn = core.processTurn({ messages, state, config, tokenCount: 60000 }); assert.equal(turn.nudge.shouldInject, true, `reason: ${turn.nudge.reason}`); - assert.equal(turn.nudge.tier, 1, "T1 raw pending still has priority over count-ready T2"); + assert.equal( + turn.nudge.tier, + 1, + "T1 raw pending still has priority over count-ready T2", + ); assert.match(turn.nudge.reason ?? "", /T1 effective/); }); @@ -721,12 +1194,34 @@ test("arbitration: count-triggered T2 still respects the growthFloor cadence", ( const core = createCore(); const config = buildConfig({ preserveRecentMessages: 30 }); const messages = makeMessages(30); - let state = core.processTurn({ messages, state: createInitialState(), config, tokenCount: 50000 }).state; - state = { ...state, blocks: t1Blocks([["m1"], ["m2"], ["m3"], ["m4"], ["m5"]], 400) }; - const first = core.processTurn({ messages, state, config, tokenCount: 60000 }); + let state = core.processTurn({ + messages, + state: createInitialState(), + config, + tokenCount: 50000, + }).state; + state = { + ...state, + blocks: t1Blocks([["m1"], ["m2"], ["m3"], ["m4"], ["m5"]], 400), + }; + const first = core.processTurn({ + messages, + state, + config, + tokenCount: 60000, + }); assert.equal(first.nudge.tier, 2, `reason: ${first.nudge.reason}`); - const second = core.processTurn({ messages, state: first.state, config, tokenCount: 61000 }); - assert.equal(second.nudge.shouldInject, false, `reason: ${second.nudge.reason}`); + const second = core.processTurn({ + messages, + state: first.state, + config, + tokenCount: 61000, + }); + assert.equal( + second.nudge.shouldInject, + false, + `reason: ${second.nudge.reason}`, + ); assert.match(second.nudge.reason ?? "", /T2 \(cadence\)/); }); @@ -734,7 +1229,12 @@ test("arbitration: T3 fires on tier-2 block COUNT (tier3Trigger) even when summa const core = createCore(); const config = buildConfig({ preserveRecentMessages: 30 }); const messages = makeMessages(30); - let state = core.processTurn({ messages, state: createInitialState(), config, tokenCount: 50000 }).state; + let state = core.processTurn({ + messages, + state: createInitialState(), + config, + tokenCount: 50000, + }).state; // 2 tier-1 blocks (below tier2Trigger) + 10 tier-2 blocks with ~100-token // summaries: t3Pen ~1000 << 9000 token gate — only the count path fires. state = { @@ -750,10 +1250,24 @@ test("arbitration: T3 fires on tier-2 block COUNT (tier3Trigger) even when summa test("arbitration: tiers disabled -> count trigger cannot fire T2", () => { const core = createCore(); - const config = buildConfig({ tiers: { enabled: false, tier2Trigger: 5, tier3Trigger: 10 }, preserveRecentMessages: 30 }); + const config = buildConfig({ + tiers: { enabled: false, tier2Trigger: 5, tier3Trigger: 10 }, + preserveRecentMessages: 30, + }); const messages = makeMessages(30); - let state = core.processTurn({ messages, state: createInitialState(), config, tokenCount: 50000 }).state; - state = { ...state, blocks: t1Blocks([["m1"], ["m2"], ["m3"], ["m4"], ["m5"], ["m6"], ["m7"]], 400) }; + let state = core.processTurn({ + messages, + state: createInitialState(), + config, + tokenCount: 50000, + }).state; + state = { + ...state, + blocks: t1Blocks( + [["m1"], ["m2"], ["m3"], ["m4"], ["m5"], ["m6"], ["m7"]], + 400, + ), + }; const turn = core.processTurn({ messages, state, config, tokenCount: 60000 }); assert.equal(turn.nudge.shouldInject, false, `reason: ${turn.nudge.reason}`); assert.equal(turn.nudge.tier, null); @@ -772,7 +1286,12 @@ test("nudge: first-sight mass bypass fires on big ingest despite growth 0 (#194) // One-shot ingest: usage 47% (inside the 45% band), growth 0 from the // tokenCount-seeded reference, T1 effective ~55K >= 6000 ready. - const turn1 = core.processTurn({ messages, state, config, tokenCount: 47000 }); + const turn1 = core.processTurn({ + messages, + state, + config, + tokenCount: 47000, + }); assert.equal( turn1.nudge.shouldInject, true, @@ -789,7 +1308,12 @@ test("nudge: first-sight bypass stays off below the min usage band (#194)", () = // 44% < 45% minContextLimitPct: a fresh session below the band still waits // (the bypass must not turn first-turn compression on for small sessions). - const turn1 = core.processTurn({ messages, state, config, tokenCount: 44000 }); + const turn1 = core.processTurn({ + messages, + state, + config, + tokenCount: 44000, + }); assert.equal(turn1.nudge.shouldInject, false, "below min band → no bypass"); }); @@ -806,12 +1330,26 @@ test("nudge: first-sight bypass paces normally after the first nudge (#194)", () const messages = makeMessages(10); let state = createInitialState(); - const turn1 = core.processTurn({ messages, state, config, tokenCount: 46000 }); - assert.equal(turn1.nudge.shouldInject, true, "bypass fires in band (growth 0)"); + const turn1 = core.processTurn({ + messages, + state, + config, + tokenCount: 46000, + }); + assert.equal( + turn1.nudge.shouldInject, + true, + "bypass fires in band (growth 0)", + ); assert.match(turn1.nudge.reason, /\[first-sight mass\]/); state = turn1.state; - const turn2 = core.processTurn({ messages, state, config, tokenCount: 50000 }); + const turn2 = core.processTurn({ + messages, + state, + config, + tokenCount: 50000, + }); assert.equal( turn2.nudge.shouldInject, false, @@ -837,22 +1375,42 @@ test("nudge: first-sight bypass re-arms after a successful compression while sti const messages = makeMessages(20); let state = createInitialState(); - const turn1 = core.processTurn({ messages, state, config, tokenCount: 47000 }); + const turn1 = core.processTurn({ + messages, + state, + config, + tokenCount: 47000, + }); assert.equal(turn1.nudge.shouldInject, true, "big ingest in band fires"); assert.match(turn1.nudge.reason, /\[first-sight mass\]/); // Model executes the compress → success clears baseline + shown stamps. state = core.applyCompression({ - ranges: [{ startRef: "m00001", endRef: "m00005", summary: "drained a slice of the backlog" }], + ranges: [ + { + startRef: "m00001", + endRef: "m00005", + summary: "drained a slice of the backlog", + }, + ], messages, state: turn1.state, config, }).state; - assert.equal(state.nudge.lastNudgeShownTokens, 0, "shown cleared by the compression"); + assert.equal( + state.nudge.lastNudgeShownTokens, + 0, + "shown cleared by the compression", + ); // Still in band (48%) with growth 0 post-compress: the backlog is not yet // drained, so the bypass must fire again — NOT wait for 10000 new tokens. - const turn2 = core.processTurn({ messages, state, config, tokenCount: 48000 }); + const turn2 = core.processTurn({ + messages, + state, + config, + tokenCount: 48000, + }); assert.equal( turn2.nudge.shouldInject, true, @@ -869,16 +1427,36 @@ test("nudge: first-sight bypass falls silent once usage drops below the band (#1 const messages = makeMessages(20); let state = createInitialState(); - const turn1 = core.processTurn({ messages, state, config, tokenCount: 55000 }); + const turn1 = core.processTurn({ + messages, + state, + config, + tokenCount: 55000, + }); assert.equal(turn1.nudge.shouldInject, true); state = core.applyCompression({ - ranges: [{ startRef: "m00001", endRef: "m00005", summary: "drained most of the backlog" }], + ranges: [ + { + startRef: "m00001", + endRef: "m00005", + summary: "drained most of the backlog", + }, + ], messages, state: turn1.state, config, }).state; - const turn2 = core.processTurn({ messages, state, config, tokenCount: 20000 }); - assert.equal(turn2.nudge.shouldInject, false, "20% < 45% band → drain complete, pacing resumes"); + const turn2 = core.processTurn({ + messages, + state, + config, + tokenCount: 20000, + }); + assert.equal( + turn2.nudge.shouldInject, + false, + "20% < 45% band → drain complete, pacing resumes", + ); }); diff --git a/tests/openai-system-hoist.test.ts b/tests/openai-system-hoist.test.ts index 0c678f3..2828a8d 100644 --- a/tests/openai-system-hoist.test.ts +++ b/tests/openai-system-hoist.test.ts @@ -67,11 +67,17 @@ test("system content changes never shift conversation ids (restart regression)", // span fingerprint covering pos 0 breaks and restart replay is rejected. const a = openaiToCore({ model: "m", - messages: [{ role: "system", content: "LIVE system, 45k chars of host state" }, ...conv], + messages: [ + { role: "system", content: "LIVE system, 45k chars of host state" }, + ...conv, + ], } as never); const b = openaiToCore({ model: "m", - messages: [{ role: "system", content: "RECONSTRUCTED system, shorter" }, ...conv], + messages: [ + { role: "system", content: "RECONSTRUCTED system, shorter" }, + ...conv, + ], } as never); assert.deepEqual( a.msgs.map((m) => m.id), @@ -80,8 +86,14 @@ test("system content changes never shift conversation ids (restart regression)", }); test("mirrorOpenaiToCore converges with the live space regardless of systemText", () => { - const t = (text: string): MirrorMessage => ({ role: "user", blocks: [{ type: "text", text }] }); - const a = (text: string): MirrorMessage => ({ role: "assistant", blocks: [{ type: "text", text }] }); + const t = (text: string): MirrorMessage => ({ + role: "user", + blocks: [{ type: "text", text }], + }); + const a = (text: string): MirrorMessage => ({ + role: "assistant", + blocks: [{ type: "text", text }], + }); const view: MirrorMessage[] = [ t("hello"), a("hi there"), @@ -108,7 +120,10 @@ test("coreToOpenai output has no head system; injectOpenaiSystem re-injects it", } as never); const rebuilt = coreToOpenai(msgs); assert.notEqual(rebuilt[0]?.role, "system"); - const withSystem = injectOpenaiSystem(rebuilt, [systemText, "compress prompt"]); + const withSystem = injectOpenaiSystem(rebuilt, [ + systemText, + "compress prompt", + ]); assert.equal(withSystem[0]?.role, "system"); assert.match(String(withSystem[0]?.content), /you are terse/); assert.match(String(withSystem[0]?.content), /compress prompt/); diff --git a/tests/orphan-fixes.test.ts b/tests/orphan-fixes.test.ts index 09f69e2..129011e 100644 --- a/tests/orphan-fixes.test.ts +++ b/tests/orphan-fixes.test.ts @@ -39,7 +39,9 @@ function buildConfig(overrides: Partial = {}): Config { }; } -function makeBlock(overrides: Partial = {}): CompressionBlock { +function makeBlock( + overrides: Partial = {}, +): CompressionBlock { return { blockId: "b0", runId: "r0", @@ -107,11 +109,7 @@ test("hideConsumedCompressCalls: hides BOTH tool-call AND tool-result for consum const result = hideConsumedCompressCalls(state, messages); assert.equal(result.hidden, 2, "should hide both tool-call and tool-result"); - assert.equal( - result.messages.length, - 2, - "only user text messages remain", - ); + assert.equal(result.messages.length, 2, "only user text messages remain"); const ids = result.messages.map((m) => m.id); assert.ok(ids.includes("u1"), "first user message kept"); assert.ok(ids.includes("u2"), "second user message kept"); @@ -197,10 +195,7 @@ test("prune: strips orphaned tool-result when tool-call is in compression range" const result = prune(messages, state); const resultIds = result.map((m) => m.id); - assert.ok( - !resultIds.includes("a1"), - "tool-call in range should be pruned", - ); + assert.ok(!resultIds.includes("a1"), "tool-call in range should be pruned"); assert.ok( !resultIds.includes("a1r"), "orphaned tool-result should be stripped", @@ -255,7 +250,12 @@ test("computeProtectedRefs: last visible user message is protected when preserve const messages: CoreMessage[] = [ { id: "u0", role: "user", contentType: "text", text: "first user" }, ...makeMessages(20), - { id: "uLast", role: "user", contentType: "text", text: "most recent user" }, + { + id: "uLast", + role: "user", + contentType: "text", + text: "most recent user", + }, { id: "a1", role: "assistant", contentType: "text", text: "reply" }, { id: "a2", role: "assistant", contentType: "text", text: "reply2" }, ]; @@ -316,7 +316,12 @@ test("computeProtectedRefs: first user message also gets protected by token zone test("prune: first user message pruned when covered (no duplication)", () => { const messages: CoreMessage[] = [ - { id: "u0", role: "user", contentType: "text", text: "critical first prompt" }, + { + id: "u0", + role: "user", + contentType: "text", + text: "critical first prompt", + }, { id: "a0", role: "assistant", contentType: "text", text: "reply" }, { id: "u1", role: "user", contentType: "text", text: "second" }, { id: "a1", role: "assistant", contentType: "text", text: "reply2" }, @@ -340,14 +345,8 @@ test("prune: first user message pruned when covered (no duplication)", () => { resultIds.includes("u0"), "first user message always survives (even when covered — some providers reject 0-user)", ); - assert.ok( - !resultIds.includes("a0"), - "covered assistant message is pruned", - ); - assert.ok( - !resultIds.includes("u1"), - "covered second user message is pruned", - ); + assert.ok(!resultIds.includes("a0"), "covered assistant message is pruned"); + assert.ok(!resultIds.includes("u1"), "covered second user message is pruned"); assert.ok( resultIds.some((id) => id.startsWith("acp_summary_")), "summary is inserted", diff --git a/tests/packs.test.ts b/tests/packs.test.ts index a56e9f5..5bd7345 100644 --- a/tests/packs.test.ts +++ b/tests/packs.test.ts @@ -34,7 +34,10 @@ test("sanitizePackSurface keeps known keys, drops malformed", () => { promptSections: { acpTags: "tags", tools: null, summariesInContext: 7 }, nudgeSections: { efficiencyNote: "e", t2Guidance: null, bogus: "x" }, toolPrompts: { - compress: { description: "d", paramDescriptions: { content: "c", startId: 3 } }, + compress: { + description: "d", + paramDescriptions: { content: "c", startId: 3 }, + }, search_context: { description: null }, }, adapters: { pi: { any: "thing" } }, @@ -42,20 +45,30 @@ test("sanitizePackSurface keeps known keys, drops malformed", () => { assert.deepEqual(s.prompts, { compressPhilosophy: "p" }); assert.deepEqual(s.promptSections, { acpTags: "tags", tools: null }); assert.deepEqual(s.nudgeSections, { efficiencyNote: "e", t2Guidance: null }); - assert.deepEqual(s.toolPrompts, { compress: { description: "d", paramDescriptions: { content: "c" } } }); + assert.deepEqual(s.toolPrompts, { + compress: { description: "d", paramDescriptions: { content: "c" } }, + }); assert.deepEqual(s.adapters, { pi: { any: "thing" } }); }); test("sanitizePackSurface on null/garbage returns empty surface", () => { assert.deepEqual(sanitizePackSurface(null), {}); - assert.deepEqual(sanitizePackSurface({ prompts: "nope", adapters: [1] }), { prompts: {}, promptSections: {}, nudgeSections: {}, toolPrompts: {} }); + assert.deepEqual(sanitizePackSurface({ prompts: "nope", adapters: [1] }), { + prompts: {}, + promptSections: {}, + nudgeSections: {}, + toolPrompts: {}, + }); }); test("builtin registry resolves default and lean; unknown misses", () => { assert.equal(builtinSource.resolve("default"), defaultPack); assert.equal(builtinSource.resolve("lean"), leanPack); assert.equal(builtinSource.resolve("nope"), null); - const names = builtinSource.list().map((p) => p.name).sort(); + const names = builtinSource + .list() + .map((p) => p.name) + .sort(); assert.deepEqual(names, ["default", "lean"]); }); @@ -65,26 +78,58 @@ test("default pack has empty surface", () => { test("lean pack keeps rules default, one-line tool descriptions, adapters namespace", () => { assert.deepEqual(leanPack.surface.prompts ?? {}, {}); - assert.equal(leanPack.surface.toolPrompts?.compress?.description, "Replace consumed conversation ranges with self-contained summaries using mNNNNN or bN refs."); - assert.ok(leanPack.surface.adapters && typeof leanPack.surface.adapters === "object"); + assert.equal( + leanPack.surface.toolPrompts?.compress?.description, + "Replace consumed conversation ranges with self-contained summaries using mNNNNN or bN refs.", + ); + assert.ok( + leanPack.surface.adapters && typeof leanPack.surface.adapters === "object", + ); assert.equal(Object.keys(leanPack.surface.adapters).length, 1); }); test("lean carries a condensed how-to-compress style contract in the pi slot", () => { - const pi = leanPack.surface.adapters?.pi as { promptSections: Record }; + const pi = leanPack.surface.adapters?.pi as { + promptSections: Record; + }; const howTo = pi.promptSections.howToCompress ?? ""; - assert.ok(howTo.length > 800 && howTo.length < 2600, `condensed, not full (len=${howTo.length})`); - for (const marker of ["TASK AS OF THIS BLOCK", "PENDING", "no Q&A lists", "KEEP VERBATIM", "chose X over Y because Z", "PRIORITY", "Do not mimic"]) { + assert.ok( + howTo.length > 800 && howTo.length < 2600, + `condensed, not full (len=${howTo.length})`, + ); + for (const marker of [ + "TASK AS OF THIS BLOCK", + "PENDING", + "no Q&A lists", + "KEEP VERBATIM", + "chose X over Y because Z", + "PRIORITY", + "Do not mimic", + ]) { assert.ok(howTo.includes(marker), `missing: ${marker}`); } // #265: summariesInContext is a compact trust guardrail now, not null - assert.equal(typeof pi.promptSections.summariesInContext, "string", "compact guardrail retained"); - assert.equal(pi.promptSections.philosophy, null, "philosophy stays dropped (howToCompress is the operative contract)"); - assert.equal(pi.promptSections.howToCompress, LEAN_HOW_TO_COMPRESS, "pi slot is the exported LEAN_HOW_TO_COMPRESS constant"); + assert.equal( + typeof pi.promptSections.summariesInContext, + "string", + "compact guardrail retained", + ); + assert.equal( + pi.promptSections.philosophy, + null, + "philosophy stays dropped (howToCompress is the operative contract)", + ); + assert.equal( + pi.promptSections.howToCompress, + LEAN_HOW_TO_COMPRESS, + "pi slot is the exported LEAN_HOW_TO_COMPRESS constant", + ); }); test("lean acpTags carries BOTH the summary-trust guardrail and the post-compress verification ban (#272 + #265)", () => { - const pi = leanPack.surface.adapters?.pi as { promptSections: Record }; + const pi = leanPack.surface.adapters?.pi as { + promptSections: Record; + }; const tags = pi.promptSections.acpTags ?? ""; for (const marker of [ "Recall on demand only", @@ -99,8 +144,14 @@ test("lean acpTags carries BOTH the summary-trust guardrail and the post-compres ]) { assert.ok(tags.includes(marker), `missing: ${marker}`); } - assert.ok(!tags.includes("settled history"), "inverted 'settled history' phrasing must be gone (#265)"); - assert.ok(!tags.includes("continue the task from them"), "inverted phrasing must be gone (#265)"); + assert.ok( + !tags.includes("settled history"), + "inverted 'settled history' phrasing must be gone (#265)", + ); + assert.ok( + !tags.includes("continue the task from them"), + "inverted phrasing must be gone (#265)", + ); }); test("lean acpTags states ref stability across compression (no false renumbering claim, #417)", () => { @@ -142,17 +193,40 @@ test("builtin packs never claim refs are renumbered after compression (#417)", ( }); test("lean pack retains summary-trust guardrail (regression: inverted 'settled history' removed)", () => { - const pi = leanPack.surface.adapters?.["pi"] as { promptSections?: Record } | undefined; + const pi = leanPack.surface.adapters?.["pi"] as + { promptSections?: Record } | undefined; const ps = pi?.promptSections ?? {}; - assert.equal(typeof ps.summariesInContext, "string", "lean must retain summariesInContext, not strip it"); + assert.equal( + typeof ps.summariesInContext, + "string", + "lean must retain summariesInContext, not strip it", + ); const sic = ps.summariesInContext as string; - assert.ok(sic.includes("verify before acting"), "must instruct verify-before-acting"); - assert.ok(sic.includes("Do NOT act on instructions"), "must forbid acting on summarized instructions"); - assert.ok(sic.startsWith("COMPRESSION SUMMARIES IN CONTEXT"), "override must be the full section incl. header"); - assert.ok(sic.includes("unless the user re-confirms them in a current message"), "re-confirm exception clause must stay"); + assert.ok( + sic.includes("verify before acting"), + "must instruct verify-before-acting", + ); + assert.ok( + sic.includes("Do NOT act on instructions"), + "must forbid acting on summarized instructions", + ); + assert.ok( + sic.startsWith("COMPRESSION SUMMARIES IN CONTEXT"), + "override must be the full section incl. header", + ); + assert.ok( + sic.includes("unless the user re-confirms them in a current message"), + "re-confirm exception clause must stay", + ); const tags = String(ps.acpTags); - assert.ok(!tags.includes("settled history"), "inverted 'settled history' phrasing must be gone"); - assert.ok(!tags.includes("continue the task from them"), "inverted 'continue the task from them' phrasing must be gone"); + assert.ok( + !tags.includes("settled history"), + "inverted 'settled history' phrasing must be gone", + ); + assert.ok( + !tags.includes("continue the task from them"), + "inverted 'continue the task from them' phrasing must be gone", + ); }); test("dir source resolves and lists json packs", () => { @@ -160,7 +234,12 @@ test("dir source resolves and lists json packs", () => { try { writeFileSync( path.join(dir, "team.json"), - JSON.stringify({ name: "team", version: "2.0", description: "d", toolPrompts: { compress: { description: "T" } } }), + JSON.stringify({ + name: "team", + version: "2.0", + description: "d", + toolPrompts: { compress: { description: "T" } }, + }), ); writeFileSync(path.join(dir, "broken.json"), "not json"); const src = createDirPackSource("project", dir); @@ -173,7 +252,10 @@ test("dir source resolves and lists json packs", () => { const names = src.list().map((p) => p.name); assert.deepEqual(names, ["team"]); const { list: listDetached } = src; - assert.deepEqual(listDetached().map((p) => p.name), ["team"]); + assert.deepEqual( + listDetached().map((p) => p.name), + ["team"], + ); } finally { rmSync(dir, { recursive: true, force: true }); } @@ -182,20 +264,37 @@ test("dir source resolves and lists json packs", () => { test("dir source: filename is the pack identity — internal name mismatch is ignored", () => { const dir = mkdtempSync(path.join(tmpdir(), "acp-packs-id-")); try { - writeFileSync(path.join(dir, "team.json"), JSON.stringify({ name: "other", toolPrompts: { compress: { description: "T" } } })); + writeFileSync( + path.join(dir, "team.json"), + JSON.stringify({ + name: "other", + toolPrompts: { compress: { description: "T" } }, + }), + ); const src = createDirPackSource("project", dir); const pack = src.resolve("team"); assert.equal(pack?.name, "team"); assert.equal(pack?.surface.toolPrompts?.compress?.description, "T"); - assert.equal(src.resolve("other"), null, "content-side name is not a resolution key"); - assert.deepEqual(src.list().map((p) => p.name), ["team"], "list() reports the filename too"); + assert.equal( + src.resolve("other"), + null, + "content-side name is not a resolution key", + ); + assert.deepEqual( + src.list().map((p) => p.name), + ["team"], + "list() reports the filename too", + ); } finally { rmSync(dir, { recursive: true, force: true }); } }); test("dir source on missing directory yields empty list and no resolve", () => { - const src = createDirPackSource("user", path.join(tmpdir(), "acp-no-such-dir")); + const src = createDirPackSource( + "user", + path.join(tmpdir(), "acp-no-such-dir"), + ); assert.equal(src.resolve("lean"), null); assert.deepEqual(src.list(), []); }); @@ -203,51 +302,110 @@ test("dir source on missing directory yields empty list and no resolve", () => { test("resolver consults sources in order; first hit wins", () => { const a: PackSource = { id: "a", - resolve: (name) => (name === "x" ? { name: "x", surface: {}, source: "a" } : null), + resolve: (name) => + name === "x" ? { name: "x", surface: {}, source: "a" } : null, }; const b: PackSource = { id: "b", - resolve: (name) => (name === "x" || name === "y" ? { name, surface: {}, source: "b" } : null), - list: () => [{ name: "x", surface: {}, source: "b" }, { name: "z", surface: {}, source: "b" }], + resolve: (name) => + name === "x" || name === "y" ? { name, surface: {}, source: "b" } : null, + list: () => [ + { name: "x", surface: {}, source: "b" }, + { name: "z", surface: {}, source: "b" }, + ], }; const r = createPackResolver([a, b, builtinSource]); assert.equal(r.resolve("x")?.source, "a"); assert.equal(r.resolve("y")?.source, "b"); assert.equal(r.resolve("lean")?.name, "lean"); assert.equal(r.resolve("../bad"), null); - assert.deepEqual(r.listPacks().map((p) => `${p.name}:${p.source}`), ["x:b", "z:b", "default:builtin:default", "lean:builtin:lean"]); + assert.deepEqual( + r.listPacks().map((p) => `${p.name}:${p.source}`), + ["x:b", "z:b", "default:builtin:default", "lean:builtin:lean"], + ); }); test("custom source can shadow builtin lean", () => { const custom: PackSource = { id: "managed", - resolve: (name) => (name === "lean" ? { name: "lean", surface: { toolPrompts: { acp_status: { description: "custom" } } }, source: "managed:lean" } : null), + resolve: (name) => + name === "lean" + ? { + name: "lean", + surface: { toolPrompts: { acp_status: { description: "custom" } } }, + source: "managed:lean", + } + : null, }; const r = createPackResolver([custom, builtinSource]); - assert.equal(r.resolve("lean")?.surface.toolPrompts?.acp_status?.description, "custom"); + assert.equal( + r.resolve("lean")?.surface.toolPrompts?.acp_status?.description, + "custom", + ); }); test("defaultPackSources orders project, user, builtin", () => { - const sources = defaultPackSources({ projectDir: "/p", userDirs: ["/u1", "/u2"] }); - assert.deepEqual(sources.map((s) => s.id), ["project", "user", "user", "builtin"]); + const sources = defaultPackSources({ + projectDir: "/p", + userDirs: ["/u1", "/u2"], + }); + assert.deepEqual( + sources.map((s) => s.id), + ["project", "user", "user", "builtin"], + ); }); test("lean surface applies to wire tools via applyAcpToolOverrides", () => { - const tools = applyAcpToolOverrides(ACP_TOOLS_OPENAI, leanPack.surface.toolPrompts); + const tools = applyAcpToolOverrides( + ACP_TOOLS_OPENAI, + leanPack.surface.toolPrompts, + ); const compress = tools.find((t) => t.function.name === "compress"); assert.ok(compress); - assert.equal(compress.function.description, leanPack.surface.toolPrompts?.compress?.description); - const params = compress.function.parameters as { properties: { content: { anyOf: { items?: { anyOf: { type: string; properties?: Record }[] } }[] } } }; - const objectForm = params.properties.content.anyOf[0].items!.anyOf.find((v) => v.type === "object"); + assert.equal( + compress.function.description, + leanPack.surface.toolPrompts?.compress?.description, + ); + const params = compress.function.parameters as { + properties: { + content: { + anyOf: { + items?: { + anyOf: { + type: string; + properties?: Record; + }[]; + }; + }[]; + }; + }; + }; + const objectForm = params.properties.content.anyOf[0].items!.anyOf.find( + (v) => v.type === "object", + ); assert.ok(objectForm?.properties?.startId); - assert.equal(objectForm!.properties!.startId!.description, "Inclusive first mNNNNN or bN ref."); - const lineForm = params.properties.content.anyOf[0].items!.anyOf.find((v) => v.type === "string"); + assert.equal( + objectForm!.properties!.startId!.description, + "Inclusive first mNNNNN or bN ref.", + ); + const lineForm = params.properties.content.anyOf[0].items!.anyOf.find( + (v) => v.type === "string", + ); assert.ok(lineForm, "line-form string variant is taught in the schema"); }); test("pack promptSections flow into buildCompressSystemPrompt", () => { - const pack: Pack = { name: "quiet", surface: { promptSections: { summariesInContext: null, tools: "TOOLS-OVERRIDE" } }, source: "test" }; - const text = buildCompressSystemPrompt(undefined, pack.surface.promptSections); + const pack: Pack = { + name: "quiet", + surface: { + promptSections: { summariesInContext: null, tools: "TOOLS-OVERRIDE" }, + }, + source: "test", + }; + const text = buildCompressSystemPrompt( + undefined, + pack.surface.promptSections, + ); assert.ok(text.includes("TOOLS-OVERRIDE")); assert.ok(!text.includes("COMPRESSION SUMMARIES IN CONTEXT")); }); @@ -259,7 +417,10 @@ test("file pack round-trips through dir source into resolver", () => { try { writeFileSync( path.join(sub, "rt.json"), - JSON.stringify({ promptSections: { acpTags: "RT" }, adapters: { pi: { delegatePrompt: "D" } } }), + JSON.stringify({ + promptSections: { acpTags: "RT" }, + adapters: { pi: { delegatePrompt: "D" } }, + }), ); const r = createPackResolver(defaultPackSources({ projectDir: sub })); const pack = r.resolve("rt"); diff --git a/tests/panel.test.ts b/tests/panel.test.ts index 1967523..ee09220 100644 --- a/tests/panel.test.ts +++ b/tests/panel.test.ts @@ -1,6 +1,12 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { buildStatusPanel, topicFallback, formatCompactTokens, cacheHitStats, formatHitRate } from "../src/panel/index.js"; +import { + buildStatusPanel, + topicFallback, + formatCompactTokens, + cacheHitStats, + formatHitRate, +} from "../src/panel/index.js"; import { defaultCountTokens } from "../src/tokenize.js"; import { VIABLE_RANGE_MIN_TOKENS } from "../src/viable.js"; @@ -15,7 +21,15 @@ test("panel separates session accounting from sent view", () => { contextUsage: 0.43, tier: null, breakdown: { emergencyOverride: 0 }, - contextBreakdown: { system: 0, tool: 20_000, text: 4_000, code: 0, summaries: 0, total: 24_000, growth: 6_100 }, + contextBreakdown: { + system: 0, + tool: 20_000, + text: 4_000, + code: 0, + summaries: 0, + total: 24_000, + growth: 6_100, + }, }; const state = { blocks: [], @@ -34,20 +48,47 @@ test("panel separates session accounting from sent view", () => { modelContextLimit: 1_000_000, }); - assert.match(text, /Context \(session accounting, host footer scale\): 43% \(430k \/ 1\.0M\) — includes compressed originals; shrinks slower than the sent view/); - assert.match(text, /Sent to LLM \(after compression, est\.\): 24k \(2% of limit\)/); - assert.doesNotMatch(text, /Session-only/, "omitted without unprunedTokens — no cross-scale subtraction"); + assert.match( + text, + /Context \(session accounting, host footer scale\): 43% \(430k \/ 1\.0M\) — includes compressed originals; shrinks slower than the sent view/, + ); + assert.match( + text, + /Sent to LLM \(after compression, est\.\): 24k \(2% of limit\)/, + ); + assert.doesNotMatch( + text, + /Session-only/, + "omitted without unprunedTokens — no cross-scale subtraction", + ); assert.match(text, /Token Breakdown \(sent view\):/); assert.doesNotMatch(text, /Framework/, "no fake Framework bucket"); const toolLine = text.split("\n").find((l) => l.trim().startsWith("Tool"))!; - assert.match(toolLine, / 83%/, `bar percentages must use the sent view: ${toolLine}`); - assert.doesNotMatch(text, /m00002\.\.m00005/, "sub-viability ranges must not be listed"); + assert.match( + toolLine, + / 83%/, + `bar percentages must use the sent view: ${toolLine}`, + ); + assert.doesNotMatch( + text, + /m00002\.\.m00005/, + "sub-viability ranges must not be listed", + ); }); test("panel renders blocks with topic fallback and version line", () => { const state = { blocks: [ - { blockId: "b1", tier: 1, active: true, summary: "Plugin discovery and registration walkthrough.", compressedTokens: 25_000, effectiveMessageIds: [], coveredRawIds: [], createdAt: 1 }, + { + blockId: "b1", + tier: 1, + active: true, + summary: "Plugin discovery and registration walkthrough.", + compressedTokens: 25_000, + effectiveMessageIds: [], + coveredRawIds: [], + createdAt: 1, + }, ], messageRefs: { byRaw: {}, byRef: {} }, nudge: {}, @@ -55,8 +96,17 @@ test("panel renders blocks with topic fallback and version line", () => { nextBlockId: 2, nextRunId: 1, }; - const text = buildStatusPanel({ tokenCount: 1_000, systemPromptTokens: 0, state: state as never, nudge: undefined, modelContextLimit: 200_000 }); - assert.match(text, /Blocks: 1 active \/ 1 total \(25k tokens compressed, cumulative\)/); + const text = buildStatusPanel({ + tokenCount: 1_000, + systemPromptTokens: 0, + state: state as never, + nudge: undefined, + modelContextLimit: 200_000, + }); + assert.match( + text, + /Blocks: 1 active \/ 1 total \(25k tokens compressed, cumulative\)/, + ); assert.match(text, /\[b1\] T1 25k→\d+.*Plugin discovery and registrat…/); assert.doesNotMatch(text, /acp-kernel@/, "no version line when omitted"); }); @@ -72,9 +122,24 @@ test("session-only derives on the estimation scale, never cross-scale", () => { const nudge = { shouldInject: false, reason: "idle", - contextBreakdown: { system: 0, tool: 20_000, text: 4_000, code: 0, summaries: 0, total: 24_000, growth: 0 }, + contextBreakdown: { + system: 0, + tool: 20_000, + text: 4_000, + code: 0, + summaries: 0, + total: 24_000, + growth: 0, + }, + }; + const state = { + blocks: [], + messageRefs: { byRaw: {}, byRef: {} }, + nudge: {}, + stats: { tokensCompressed: 0 }, + nextBlockId: 1, + nextRunId: 1, }; - const state = { blocks: [], messageRefs: { byRaw: {}, byRef: {} }, nudge: {}, stats: { tokensCompressed: 0 }, nextBlockId: 1, nextRunId: 1 }; const text = buildStatusPanel({ tokenCount: 430_000, systemPromptTokens: 0, @@ -83,7 +148,10 @@ test("session-only derives on the estimation scale, never cross-scale", () => { modelContextLimit: 1_000_000, unprunedTokens: 134_000, }); - assert.match(text, /Session-only \(compressed originals, est\.\): 110k — pruned from every request/); + assert.match( + text, + /Session-only \(compressed originals, est\.\): 110k — pruned from every request/, + ); }); test("panel session-only stays on the core's estimator scale for CJK (issue #390)", () => { @@ -95,9 +163,24 @@ test("panel session-only stays on the core's estimator scale for CJK (issue #390 const nudge = { shouldInject: false, reason: "idle", - contextBreakdown: { system: 0, tool: 20_000, text: 4_000, code: 0, summaries: 0, total: 24_000, growth: 0 }, + contextBreakdown: { + system: 0, + tool: 20_000, + text: 4_000, + code: 0, + summaries: 0, + total: 24_000, + growth: 0, + }, + }; + const state = { + blocks: [], + messageRefs: { byRaw: {}, byRef: {} }, + nudge: {}, + stats: { tokensCompressed: 0 }, + nextBlockId: 1, + nextRunId: 1, }; - const state = { blocks: [], messageRefs: { byRaw: {}, byRef: {} }, nudge: {}, stats: { tokensCompressed: 0 }, nextBlockId: 1, nextRunId: 1 }; const text = buildStatusPanel({ tokenCount: 430_000, systemPromptTokens: 0, @@ -106,11 +189,17 @@ test("panel session-only stays on the core's estimator scale for CJK (issue #390 modelContextLimit: 1_000_000, unprunedTokens: unpruned, }); - assert.match(text, /Session-only \(compressed originals, est\.\): 1\.0k — pruned from every request/); + assert.match( + text, + /Session-only \(compressed originals, est\.\): 1\.0k — pruned from every request/, + ); }); test("topicFallback takes the first sentence segment, ≤30 chars", () => { - assert.equal(topicFallback("Database migration steps failed twice."), "Database migration steps faile…"); + assert.equal( + topicFallback("Database migration steps failed twice."), + "Database migration steps faile…", + ); assert.equal(topicFallback('He said "hello". More text.'), 'He said "hello"'); assert.equal(topicFallback("Short."), "Short"); }); @@ -162,8 +251,21 @@ test("formatHitRate clamps and formats one decimal", () => { }); test("panel renders prompt cache line and omits it without cache signal", () => { - const state = { blocks: [], messageRefs: { byRaw: {}, byRef: {} }, nudge: {}, stats: { tokensCompressed: 0 }, nextBlockId: 1, nextRunId: 1 }; - const base = { tokenCount: 430_000, systemPromptTokens: 0, state: state as never, nudge: undefined, modelContextLimit: 1_000_000 }; + const state = { + blocks: [], + messageRefs: { byRaw: {}, byRef: {} }, + nudge: {}, + stats: { tokensCompressed: 0 }, + nextBlockId: 1, + nextRunId: 1, + }; + const base = { + tokenCount: 430_000, + systemPromptTokens: 0, + state: state as never, + nudge: undefined, + modelContextLimit: 1_000_000, + }; const withCache = buildStatusPanel({ ...base, cacheUsages: [ @@ -172,9 +274,23 @@ test("panel renders prompt cache line and omits it without cache signal", () => ], }); // session = (99k + 180k) / (100k + 200k) = 93.0%; last = 180k/200k = 90.0% - assert.match(withCache, /Prompt cache \(provider-reported\): 90\.0% last · 93\.0% session avg — 279k of 300k billed prompt tokens served from cache \(2 req\)/); - const noSignal = buildStatusPanel({ ...base, cacheUsages: [{ input: 5_000, cacheRead: 0, cacheWrite: 0 }] }); - assert.doesNotMatch(noSignal, /Prompt cache/, "omitted when no request reported cache activity"); + assert.match( + withCache, + /Prompt cache \(provider-reported\): 90\.0% last · 93\.0% session avg — 279k of 300k billed prompt tokens served from cache \(2 req\)/, + ); + const noSignal = buildStatusPanel({ + ...base, + cacheUsages: [{ input: 5_000, cacheRead: 0, cacheWrite: 0 }], + }); + assert.doesNotMatch( + noSignal, + /Prompt cache/, + "omitted when no request reported cache activity", + ); const omitted = buildStatusPanel(base); - assert.doesNotMatch(omitted, /Prompt cache/, "omitted without cacheUsages field"); + assert.doesNotMatch( + omitted, + /Prompt cache/, + "omitted without cacheUsages field", + ); }); diff --git a/tests/parse-compress-input.test.ts b/tests/parse-compress-input.test.ts index 3a8f263..1a02bd4 100644 --- a/tests/parse-compress-input.test.ts +++ b/tests/parse-compress-input.test.ts @@ -11,114 +11,137 @@ import type { CompressRangeSpec, CoreMessage } from "../src/types.js"; // --------------------------------------------------------------------------- test("parseCompressArgs parses a valid object with all fields", () => { - const input = { - content: [ - { startRef: "m00001", endRef: "m00002", summary: "intro", topic: "setup", summaryMaxChars: 5000 }, - ], - }; - const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "ok"); - assert.equal(diagnostics.ok, true); - assert.equal(diagnostics.invalidItems, 0); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]?.startRef, "m00001"); - assert.equal(ranges[0]?.endRef, "m00002"); - assert.equal(ranges[0]?.summary, "intro"); - assert.equal(ranges[0]?.topic, "setup"); - assert.equal(ranges[0]?.summaryMaxChars, 5000); + const input = { + content: [ + { + startRef: "m00001", + endRef: "m00002", + summary: "intro", + topic: "setup", + summaryMaxChars: 5000, + }, + ], + }; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "ok"); + assert.equal(diagnostics.ok, true); + assert.equal(diagnostics.invalidItems, 0); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.startRef, "m00001"); + assert.equal(ranges[0]?.endRef, "m00002"); + assert.equal(ranges[0]?.summary, "intro"); + assert.equal(ranges[0]?.topic, "setup"); + assert.equal(ranges[0]?.summaryMaxChars, 5000); }); test("parseCompressArgs parses a raw JSON document string", () => { - const input = JSON.stringify({ - content: [{ startRef: "m00001", endRef: "m00002", summary: "intro" }], - }); - const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "ok"); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]?.summary, "intro"); - assert.equal(diagnostics.length, input.length); - assert.equal(diagnostics.rawPrefix, input.slice(0, 800)); + const input = JSON.stringify({ + content: [{ startRef: "m00001", endRef: "m00002", summary: "intro" }], + }); + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "intro"); + assert.equal(diagnostics.length, input.length); + assert.equal(diagnostics.rawPrefix, input.slice(0, 800)); }); test("parseCompressArgs unwraps one level of double-stringification", () => { - const input = JSON.stringify( - JSON.stringify({ - content: [{ startRef: "m00001", endRef: "m00002", summary: "doubled" }], - }), - ); - const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "ok"); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]?.summary, "doubled"); + const input = JSON.stringify( + JSON.stringify({ + content: [{ startRef: "m00001", endRef: "m00002", summary: "doubled" }], + }), + ); + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "doubled"); }); test("parseCompressArgs accepts a stringified content array (vLLM shape)", () => { - const input = { - content: JSON.stringify([{ startRef: "m00001", endRef: "m00002", summary: "vllm" }]), - }; - const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "ok"); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]?.startRef, "m00001"); - assert.equal(ranges[0]?.summary, "vllm"); + const input = { + content: JSON.stringify([ + { startRef: "m00001", endRef: "m00002", summary: "vllm" }, + ]), + }; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.startRef, "m00001"); + assert.equal(ranges[0]?.summary, "vllm"); }); test("parseCompressArgs salvages a stringified content array truncated mid-entry", () => { - const full = [ - { startRef: "m00001", endRef: "m00002", summary: "first entry complete" }, - { startRef: "m00003", endRef: "m00004", summary: "second entry cut off mid" }, - ]; - const cut = JSON.stringify(full).slice(0, JSON.stringify(full).length - 12); - const input = { content: cut }; - const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "truncated"); - assert.equal(diagnostics.ok, true); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]?.summary, "first entry complete"); - assert.equal(diagnostics.invalidItems, 0); + const full = [ + { startRef: "m00001", endRef: "m00002", summary: "first entry complete" }, + { + startRef: "m00003", + endRef: "m00004", + summary: "second entry cut off mid", + }, + ]; + const cut = JSON.stringify(full).slice(0, JSON.stringify(full).length - 12); + const input = { content: cut }; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "truncated"); + assert.equal(diagnostics.ok, true); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "first entry complete"); + assert.equal(diagnostics.invalidItems, 0); }); test("parseCompressArgs strips markdown fences (with and without language tag)", () => { - const doc = JSON.stringify({ - content: [{ startRef: "m00001", endRef: "m00002", summary: "fenced" }], - }); - for (const input of ["```json\n" + doc + "\n```", "```\n" + doc + "\n```"]) { - const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "ok", "expected ok for: " + input.slice(0, 20)); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]?.summary, "fenced"); - } + const doc = JSON.stringify({ + content: [{ startRef: "m00001", endRef: "m00002", summary: "fenced" }], + }); + for (const input of ["```json\n" + doc + "\n```", "```\n" + doc + "\n```"]) { + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal( + diagnostics.kind, + "ok", + "expected ok for: " + input.slice(0, 20), + ); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "fenced"); + } }); test("parseCompressArgs repairs trailing commas", () => { - const input = - '{"content": [{"startRef": "m00001", "endRef": "m00002", "summary": "S",}]}'; - const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "ok"); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]?.summary, "S"); + const input = + '{"content": [{"startRef": "m00001", "endRef": "m00002", "summary": "S",}]}'; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "S"); }); test("parseCompressArgs escapes raw newlines inside JSON string values", () => { - // A provider serialized a real newline inside the summary string. - const input = - '{"content": [{"startRef": "m00001", "endRef": "m00002", "summary": "line1\nline2"}]}'; - const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "ok"); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]?.summary, "line1\nline2"); + // A provider serialized a real newline inside the summary string. + const input = + '{"content": [{"startRef": "m00001", "endRef": "m00002", "summary": "line1\nline2"}]}'; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "line1\nline2"); }); test("parseCompressArgs preserves escaped quotes in salvaged entries", () => { - const entry = { startRef: "m00001", endRef: "m00002", summary: 'has "quotes" inside' }; - const doc = JSON.stringify({ content: [entry, { startRef: "m00003", endRef: "m00004", summary: "gone" }] }); - // Cut after the first entry's closing brace (the comma follows it). - const firstClose = doc.indexOf("}") + 1; - const input = doc.slice(0, firstClose) + ', {"startRef": "m00003", "endRef": "m0'; - const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "truncated"); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]?.summary, 'has "quotes" inside'); + const entry = { + startRef: "m00001", + endRef: "m00002", + summary: 'has "quotes" inside', + }; + const doc = JSON.stringify({ + content: [entry, { startRef: "m00003", endRef: "m00004", summary: "gone" }], + }); + // Cut after the first entry's closing brace (the comma follows it). + const firstClose = doc.indexOf("}") + 1; + const input = + doc.slice(0, firstClose) + ', {"startRef": "m00003", "endRef": "m0'; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "truncated"); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, 'has "quotes" inside'); }); // --------------------------------------------------------------------------- @@ -126,89 +149,97 @@ test("parseCompressArgs preserves escaped quotes in salvaged entries", () => { // --------------------------------------------------------------------------- test("parseCompressArgs salvages single-quoted JSON args (#603)", () => { - const input = `{'content':[{'startId':'m00010','endId':'m00020','summary':'first'},{'startId':'m00030','endId':'m00040','summary':'second','topic':'mid'}],'topic':'intro'}`; - const { ranges, diagnostics } = parseCompressArgs(input, { callId: "call-q" }); - assert.equal(diagnostics.kind, "ok"); - assert.equal(diagnostics.quoteSalvage, true); - assert.equal(ranges.length, 2); - assert.equal(ranges[0]?.startRef, "m00010"); - assert.equal(ranges[0]?.endRef, "m00020"); - assert.equal(ranges[0]?.summary, "first"); - assert.equal(ranges[0]?.topic, "intro"); - assert.equal(ranges[0]?.compressCallId, "call-q"); - assert.equal(ranges[1]?.startRef, "m00030"); - assert.equal(ranges[1]?.topic, "mid"); + const input = `{'content':[{'startId':'m00010','endId':'m00020','summary':'first'},{'startId':'m00030','endId':'m00040','summary':'second','topic':'mid'}],'topic':'intro'}`; + const { ranges, diagnostics } = parseCompressArgs(input, { + callId: "call-q", + }); + assert.equal(diagnostics.kind, "ok"); + assert.equal(diagnostics.quoteSalvage, true); + assert.equal(ranges.length, 2); + assert.equal(ranges[0]?.startRef, "m00010"); + assert.equal(ranges[0]?.endRef, "m00020"); + assert.equal(ranges[0]?.summary, "first"); + assert.equal(ranges[0]?.topic, "intro"); + assert.equal(ranges[0]?.compressCallId, "call-q"); + assert.equal(ranges[1]?.startRef, "m00030"); + assert.equal(ranges[1]?.topic, "mid"); }); test("parseCompressArgs salvages mixed single/double quotes, keeps apostrophes in data (#603)", () => { - const input = `{'content': [{'startId': "m00001", 'endId': "m00009", 'summary': "it's done, really"}]}`; - const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "ok"); - assert.equal(diagnostics.quoteSalvage, true); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]?.startRef, "m00001"); - assert.equal(ranges[0]?.endRef, "m00009"); - assert.equal(ranges[0]?.summary, "it's done, really"); + const input = `{'content': [{'startId': "m00001", 'endId': "m00009", 'summary': "it's done, really"}]}`; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "ok"); + assert.equal(diagnostics.quoteSalvage, true); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.startRef, "m00001"); + assert.equal(ranges[0]?.endRef, "m00009"); + assert.equal(ranges[0]?.summary, "it's done, really"); }); test("parseCompressArgs salvages truncated single-quoted content array (#603)", () => { - const input = `{'content':[{'startId':'m00010','endId':'m00020','summary':'first'},{'startId':'m00030','endId':'m00040','summary':'secon`; - const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "truncated"); - assert.equal(diagnostics.ok, true); - assert.equal(diagnostics.quoteSalvage, true); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]?.startRef, "m00010"); - assert.equal(ranges[0]?.endRef, "m00020"); - assert.equal(ranges[0]?.summary, "first"); + const input = `{'content':[{'startId':'m00010','endId':'m00020','summary':'first'},{'startId':'m00030','endId':'m00040','summary':'secon`; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "truncated"); + assert.equal(diagnostics.ok, true); + assert.equal(diagnostics.quoteSalvage, true); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.startRef, "m00010"); + assert.equal(ranges[0]?.endRef, "m00020"); + assert.equal(ranges[0]?.summary, "first"); }); test("parseCompressArgs salvages object input whose content string is single-quoted (#603)", () => { - const input = { content: `[{'startId':'m00005','endId':'m00006','summary':'ok'}]` }; - const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "ok"); - assert.equal(diagnostics.quoteSalvage, true); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]?.startRef, "m00005"); - assert.equal(ranges[0]?.endRef, "m00006"); - assert.equal(ranges[0]?.summary, "ok"); + const input = { + content: `[{'startId':'m00005','endId':'m00006','summary':'ok'}]`, + }; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "ok"); + assert.equal(diagnostics.quoteSalvage, true); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.startRef, "m00005"); + assert.equal(ranges[0]?.endRef, "m00006"); + assert.equal(ranges[0]?.summary, "ok"); }); test("parseCompressArgs still rejects prose with apostrophes unchanged (#603)", () => { - const { ranges, diagnostics } = parseCompressArgs("I couldn't believe {that} would 'work'"); - assert.deepEqual(ranges, []); - assert.equal(diagnostics.ok, false); - assert.equal(diagnostics.kind, "malformed-json"); - assert.equal(diagnostics.quoteSalvage, undefined); + const { ranges, diagnostics } = parseCompressArgs( + "I couldn't believe {that} would 'work'", + ); + assert.deepEqual(ranges, []); + assert.equal(diagnostics.ok, false); + assert.equal(diagnostics.kind, "malformed-json"); + assert.equal(diagnostics.quoteSalvage, undefined); }); test("parseCompressArgs leaves valid double-quoted args untouched by salvage (#603)", () => { - const input = JSON.stringify({ content: [{ startId: "m00001", endId: "m00002", summary: "don't panic" }] }); - const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "ok"); - assert.equal(diagnostics.quoteSalvage, undefined); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]?.summary, "don't panic"); + const input = JSON.stringify({ + content: [{ startId: "m00001", endId: "m00002", summary: "don't panic" }], + }); + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "ok"); + assert.equal(diagnostics.quoteSalvage, undefined); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "don't panic"); }); test("parseCompressArgs repairs single quotes combined with trailing commas and raw newlines", () => { - const input = `{'content':[{'startId':'m00001','endId':'m00002','summary':'line1\nline2',}]}`; - const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "ok"); - assert.equal(diagnostics.quoteSalvage, true); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]?.summary, "line1\nline2"); + const input = `{'content':[{'startId':'m00001','endId':'m00002','summary':'line1\nline2',}]}`; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "ok"); + assert.equal(diagnostics.quoteSalvage, true); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "line1\nline2"); }); test("parseCompressArgs does not fabricate fields for a single-quoted entry missing bounds", () => { - // Quote repair makes the entry parseable, but validation still applies: - // no endId/summary means no range — reported, not invented. The retry - // wins only on strictly more ranges, so first-pass diagnostics stand. - const input = `{'content':[{'startId':'m00010'}]}`; - const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(ranges.length, 0); - assert.equal(diagnostics.ok, false); - assert.equal(diagnostics.quoteSalvage, undefined); + // Quote repair makes the entry parseable, but validation still applies: + // no endId/summary means no range — reported, not invented. The retry + // wins only on strictly more ranges, so first-pass diagnostics stand. + const input = `{'content':[{'startId':'m00010'}]}`; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(ranges.length, 0); + assert.equal(diagnostics.ok, false); + assert.equal(diagnostics.quoteSalvage, undefined); }); // --------------------------------------------------------------------------- @@ -216,44 +247,49 @@ test("parseCompressArgs does not fabricate fields for a single-quoted entry miss // --------------------------------------------------------------------------- test("parseCompressArgs salvages complete entries from a truncated JSON prefix", () => { - const doc = JSON.stringify({ - content: [ - { startRef: "m00001", endRef: "m00002", summary: "first entry complete" }, - { startRef: "m00003", endRef: "m00004", summary: "second entry cut off mid" }, - ], - }); - const input = doc.slice(0, doc.length - 12); - const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "truncated"); - assert.equal(diagnostics.ok, true); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]?.summary, "first entry complete"); + const doc = JSON.stringify({ + content: [ + { startRef: "m00001", endRef: "m00002", summary: "first entry complete" }, + { + startRef: "m00003", + endRef: "m00004", + summary: "second entry cut off mid", + }, + ], + }); + const input = doc.slice(0, doc.length - 12); + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "truncated"); + assert.equal(diagnostics.ok, true); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "first entry complete"); }); test("parseCompressArgs salvages entries when the closing brackets are missing entirely", () => { - const entry = '{"startRef": "m00001", "endRef": "m00002", "summary": "no brackets after me"}'; - const input = '{"content": [' + entry + ", {\"startRef\": \"m00003\""; - const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "truncated"); - assert.equal(diagnostics.ok, true); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]?.summary, "no brackets after me"); + const entry = + '{"startRef": "m00001", "endRef": "m00002", "summary": "no brackets after me"}'; + const input = '{"content": [' + entry + ', {"startRef": "m00003"'; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "truncated"); + assert.equal(diagnostics.ok, true); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "no brackets after me"); }); test("parseCompressArgs reports truncated with no ranges when the first entry is cut", () => { - const input = '{"content": [{"startRef": "m00001", "endRef": "m0'; - const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "truncated"); - assert.equal(diagnostics.ok, false); - assert.equal(ranges.length, 0); + const input = '{"content": [{"startRef": "m00001", "endRef": "m0'; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "truncated"); + assert.equal(diagnostics.ok, false); + assert.equal(ranges.length, 0); }); test("parseCompressArgs does not fabricate partial entries", () => { - // startRef present, endRef/summary missing and cut: entry is incomplete, must be dropped. - const input = '{"content": [{"startRef": "m00001", "summary": "par'; - const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "truncated"); - assert.equal(ranges.length, 0); + // startRef present, endRef/summary missing and cut: entry is incomplete, must be dropped. + const input = '{"content": [{"startRef": "m00001", "summary": "par'; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "truncated"); + assert.equal(ranges.length, 0); }); // --------------------------------------------------------------------------- @@ -261,108 +297,108 @@ test("parseCompressArgs does not fabricate partial entries", () => { // --------------------------------------------------------------------------- test("parseCompressArgs classifies empty inputs", () => { - for (const input of [null, undefined, "", " "]) { - const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "empty-input", JSON.stringify(input)); - assert.equal(diagnostics.ok, false); - assert.equal(ranges.length, 0); - } + for (const input of [null, undefined, "", " "]) { + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "empty-input", JSON.stringify(input)); + assert.equal(diagnostics.ok, false); + assert.equal(ranges.length, 0); + } }); test("parseCompressArgs classifies non-object parsed values", () => { - for (const input of [[], 42, '"just a string"', "true"]) { - const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "not-object", JSON.stringify(input)); - assert.equal(diagnostics.ok, false); - assert.equal(ranges.length, 0); - } + for (const input of [[], 42, '"just a string"', "true"]) { + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "not-object", JSON.stringify(input)); + assert.equal(diagnostics.ok, false); + assert.equal(ranges.length, 0); + } }); test("parseCompressArgs classifies an object without a content key", () => { - const { diagnostics } = parseCompressArgs({ foo: 1, bar: "x" }); - assert.equal(diagnostics.kind, "missing-content"); - assert.equal(diagnostics.ok, false); - assert.deepEqual(diagnostics.keys, ["foo", "bar"]); + const { diagnostics } = parseCompressArgs({ foo: 1, bar: "x" }); + assert.equal(diagnostics.kind, "missing-content"); + assert.equal(diagnostics.ok, false); + assert.deepEqual(diagnostics.keys, ["foo", "bar"]); }); test("parseCompressArgs classifies a content value that is neither array nor array-string", () => { - const { diagnostics } = parseCompressArgs({ content: "123" }); - assert.equal(diagnostics.kind, "content-not-array"); - assert.equal(diagnostics.ok, false); + const { diagnostics } = parseCompressArgs({ content: "123" }); + assert.equal(diagnostics.kind, "content-not-array"); + assert.equal(diagnostics.ok, false); }); test("parseCompressArgs classifies balanced garbage as malformed-json", () => { - const { ranges, diagnostics } = parseCompressArgs("hello"); - assert.equal(diagnostics.kind, "malformed-json"); - assert.equal(diagnostics.ok, false); - assert.equal(ranges.length, 0); + const { ranges, diagnostics } = parseCompressArgs("hello"); + assert.equal(diagnostics.kind, "malformed-json"); + assert.equal(diagnostics.ok, false); + assert.equal(ranges.length, 0); }); test("parseCompressArgs counts invalid entries and keeps the valid ones", () => { - const input = { - content: [ - { startRef: "m00001", endRef: "m00002", summary: "good" }, - { summary: "missing refs" }, - { startRef: "m00003", endRef: "m00004" }, - { startRef: "m00005", endRef: "m00006", summary: 42 }, - "garbage", - null, - ], - }; - const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "ok"); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]?.summary, "good"); - assert.equal(diagnostics.invalidItems, 5); + const input = { + content: [ + { startRef: "m00001", endRef: "m00002", summary: "good" }, + { summary: "missing refs" }, + { startRef: "m00003", endRef: "m00004" }, + { startRef: "m00005", endRef: "m00006", summary: 42 }, + "garbage", + null, + ], + }; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "good"); + assert.equal(diagnostics.invalidItems, 5); }); test("parseCompressArgs reports per-entry invalid reasons", () => { - const input = { - content: [ - { startRef: "m00001", endRef: "m00002", summary: "good" }, - { summary: "missing refs" }, - { startRef: "m00003", endRef: "m00004" }, - "garbage", - ], - }; - const { diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "ok"); - assert.equal(diagnostics.invalidItems, 3); - assert.deepEqual(diagnostics.invalidReasons, [ - "entry 1: missing range bounds (need startRef/startId and endRef/endId)", - "entry 2: missing summary", - "entry 3: line entry: no mNNNNN/bN refs in header", - ]); + const input = { + content: [ + { startRef: "m00001", endRef: "m00002", summary: "good" }, + { summary: "missing refs" }, + { startRef: "m00003", endRef: "m00004" }, + "garbage", + ], + }; + const { diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "ok"); + assert.equal(diagnostics.invalidItems, 3); + assert.deepEqual(diagnostics.invalidReasons, [ + "entry 1: missing range bounds (need startRef/startId and endRef/endId)", + "entry 2: missing summary", + "entry 3: line entry: no mNNNNN/bN refs in header", + ]); }); test("parseCompressArgs reports invalid reasons for no-valid-ranges", () => { - const input = { - content: [ - { startRef: "m00001", endRef: "m00002" }, - { summary: "no bounds" }, - ], - }; - const { diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "no-valid-ranges"); - assert.equal(diagnostics.invalidItems, 2); - assert.deepEqual(diagnostics.invalidReasons, [ - "entry 0: missing summary", - "entry 1: missing range bounds (need startRef/startId and endRef/endId)", - ]); + const input = { + content: [ + { startRef: "m00001", endRef: "m00002" }, + { summary: "no bounds" }, + ], + }; + const { diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "no-valid-ranges"); + assert.equal(diagnostics.invalidItems, 2); + assert.deepEqual(diagnostics.invalidReasons, [ + "entry 0: missing summary", + "entry 1: missing range bounds (need startRef/startId and endRef/endId)", + ]); }); test("parseCompressArgs reports no-valid-ranges for an empty content array", () => { - const { ranges, diagnostics } = parseCompressArgs({ content: [] }); - assert.equal(diagnostics.kind, "no-valid-ranges"); - assert.equal(diagnostics.ok, false); - assert.equal(ranges.length, 0); - assert.equal(diagnostics.invalidItems, 0); + const { ranges, diagnostics } = parseCompressArgs({ content: [] }); + assert.equal(diagnostics.kind, "no-valid-ranges"); + assert.equal(diagnostics.ok, false); + assert.equal(ranges.length, 0); + assert.equal(diagnostics.invalidItems, 0); }); test("parseCompressArgs reports no-valid-ranges for an empty stringified content", () => { - const { ranges, diagnostics } = parseCompressArgs({ content: "" }); - assert.equal(diagnostics.kind, "no-valid-ranges"); - assert.equal(ranges.length, 0); + const { ranges, diagnostics } = parseCompressArgs({ content: "" }); + assert.equal(diagnostics.kind, "no-valid-ranges"); + assert.equal(ranges.length, 0); }); // --------------------------------------------------------------------------- @@ -370,26 +406,28 @@ test("parseCompressArgs reports no-valid-ranges for an empty stringified content // --------------------------------------------------------------------------- test("parseCompressArgs accepts startId/endId and messageId name variants", () => { - const { ranges, diagnostics } = parseCompressArgs({ - content: [ - { startId: "m00001", endId: "m00002", summary: "variant-a" }, - { messageId: "m00003", summary: "variant-b" }, - ], - }); - assert.equal(diagnostics.kind, "ok"); - assert.equal(ranges.length, 2); - assert.equal(ranges[0]?.startRef, "m00001"); - assert.equal(ranges[0]?.endRef, "m00002"); - assert.equal(ranges[1]?.startRef, "m00003"); - assert.equal(ranges[1]?.endRef, "m00003"); + const { ranges, diagnostics } = parseCompressArgs({ + content: [ + { startId: "m00001", endId: "m00002", summary: "variant-a" }, + { messageId: "m00003", summary: "variant-b" }, + ], + }); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 2); + assert.equal(ranges[0]?.startRef, "m00001"); + assert.equal(ranges[0]?.endRef, "m00002"); + assert.equal(ranges[1]?.startRef, "m00003"); + assert.equal(ranges[1]?.endRef, "m00003"); }); test("parseCompressArgs stamps compressCallId when a callId is provided", () => { - const input = { content: [{ startRef: "m00001", endRef: "m00002", summary: "stamped" }] }; - const stamped = parseCompressArgs(input, { callId: "call-9" }); - assert.equal(stamped.ranges[0]?.compressCallId, "call-9"); - const plain = parseCompressArgs(input); - assert.equal(plain.ranges[0]?.compressCallId, undefined); + const input = { + content: [{ startRef: "m00001", endRef: "m00002", summary: "stamped" }], + }; + const stamped = parseCompressArgs(input, { callId: "call-9" }); + assert.equal(stamped.ranges[0]?.compressCallId, "call-9"); + const plain = parseCompressArgs(input); + assert.equal(plain.ranges[0]?.compressCallId, undefined); }); // --------------------------------------------------------------------------- @@ -397,339 +435,421 @@ test("parseCompressArgs stamps compressCallId when a callId is provided", () => // --------------------------------------------------------------------------- test("parseCompressArgs caps rawPrefix at 800 chars and reports full length", () => { - const input = "x".repeat(1000); - const { diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "malformed-json"); - assert.equal(diagnostics.length, 1000); - assert.equal(diagnostics.rawPrefix?.length, 800); + const input = "x".repeat(1000); + const { diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "malformed-json"); + assert.equal(diagnostics.length, 1000); + assert.equal(diagnostics.rawPrefix?.length, 800); }); test("parseCompressArgs leaves rawPrefix/length undefined for object input", () => { - const { diagnostics } = parseCompressArgs({ content: [] }); - assert.equal(diagnostics.rawPrefix, undefined); - assert.equal(diagnostics.length, undefined); + const { diagnostics } = parseCompressArgs({ content: [] }); + assert.equal(diagnostics.rawPrefix, undefined); + assert.equal(diagnostics.length, undefined); }); // --------------------------------------------------------------------------- // rebuildCompressionState regression (fork-recovery path) // --------------------------------------------------------------------------- -function rebuildMessages(toolCallText: string, toolCallId: string): CoreMessage[] { - const messages: CoreMessage[] = [ - { id: "raw1", role: "user", contentType: "text", text: "first message" }, - { id: "raw2", role: "assistant", contentType: "text", text: "second message" }, - ]; - if (toolCallText !== "") { - messages.push({ - id: "raw3", - role: "assistant", - contentType: "tool-call", - toolName: "compress", - toolCallId, - text: toolCallText, - }); - } - return messages; +function rebuildMessages( + toolCallText: string, + toolCallId: string, +): CoreMessage[] { + const messages: CoreMessage[] = [ + { id: "raw1", role: "user", contentType: "text", text: "first message" }, + { + id: "raw2", + role: "assistant", + contentType: "text", + text: "second message", + }, + ]; + if (toolCallText !== "") { + messages.push({ + id: "raw3", + role: "assistant", + contentType: "tool-call", + toolName: "compress", + toolCallId, + text: toolCallText, + }); + } + return messages; } function rebuildConfig() { - return defaultConfig(200000, { - compress: { minCompressRange: 0, maxSummaryLength: 0, minSummaryLength: 0 }, - preserveRecentMessages: 0, - preserveRecentTokens: 0, - }); + return defaultConfig(200000, { + compress: { minCompressRange: 0, maxSummaryLength: 0, minSummaryLength: 0 }, + preserveRecentMessages: 0, + preserveRecentTokens: 0, + }); } test("rebuildCompressionState rebuilds from a strict JSON object (pre-existing behavior)", () => { - const args = JSON.stringify({ - content: [{ startId: "m00001", endId: "m00002", summary: "strict object" }], - }); - const result = rebuildCompressionState(createInitialState(), rebuildMessages(args, "call1"), rebuildConfig()); - assert.equal(result.blocksRebuilt, 1); - const block = result.state.blocks.find((b) => b.summary.includes("strict object")); - assert.ok(block); - assert.equal(block.compressCallId, "call1"); + const args = JSON.stringify({ + content: [{ startId: "m00001", endId: "m00002", summary: "strict object" }], + }); + const result = rebuildCompressionState( + createInitialState(), + rebuildMessages(args, "call1"), + rebuildConfig(), + ); + assert.equal(result.blocksRebuilt, 1); + const block = result.state.blocks.find((b) => + b.summary.includes("strict object"), + ); + assert.ok(block); + assert.equal(block.compressCallId, "call1"); }); test("rebuildCompressionState recovers vLLM stringified content (fork-recovery gap fix)", () => { - // vLLM hosts stringify the nested content array; the old strict - // Array.isArray check dropped these silently. - const args = JSON.stringify({ - content: JSON.stringify([{ startId: "m00001", endId: "m00002", summary: "vllm stringified" }]), - }); - const result = rebuildCompressionState(createInitialState(), rebuildMessages(args, "call1"), rebuildConfig()); - assert.equal(result.blocksRebuilt, 1); - const block = result.state.blocks.find((b) => b.summary.includes("vllm stringified")); - assert.ok(block); - assert.equal(block.compressCallId, "call1"); + // vLLM hosts stringify the nested content array; the old strict + // Array.isArray check dropped these silently. + const args = JSON.stringify({ + content: JSON.stringify([ + { startId: "m00001", endId: "m00002", summary: "vllm stringified" }, + ]), + }); + const result = rebuildCompressionState( + createInitialState(), + rebuildMessages(args, "call1"), + rebuildConfig(), + ); + assert.equal(result.blocksRebuilt, 1); + const block = result.state.blocks.find((b) => + b.summary.includes("vllm stringified"), + ); + assert.ok(block); + assert.equal(block.compressCallId, "call1"); }); test("rebuildCompressionState salvages a truncated compress tool-call text", () => { - const doc = JSON.stringify({ - content: [ - { startId: "m00001", endId: "m00002", summary: "survives truncation" }, - { startId: "m00003", endId: "m00004", summary: "cut off here" }, - ], - }); - const result = rebuildCompressionState( - createInitialState(), - rebuildMessages(doc.slice(0, doc.length - 12), "call1"), - rebuildConfig(), - ); - assert.equal(result.blocksRebuilt, 1); - assert.ok(result.state.blocks.find((b) => b.summary.includes("survives truncation"))); + const doc = JSON.stringify({ + content: [ + { startId: "m00001", endId: "m00002", summary: "survives truncation" }, + { startId: "m00003", endId: "m00004", summary: "cut off here" }, + ], + }); + const result = rebuildCompressionState( + createInitialState(), + rebuildMessages(doc.slice(0, doc.length - 12), "call1"), + rebuildConfig(), + ); + assert.equal(result.blocksRebuilt, 1); + assert.ok( + result.state.blocks.find((b) => b.summary.includes("survives truncation")), + ); }); test("rebuildCompressionState survives garbage tool-call text without rebuilding", () => { - const result = rebuildCompressionState( - createInitialState(), - rebuildMessages("total garbage not json", "call1"), - rebuildConfig(), - ); - assert.equal(result.blocksRebuilt, 0); - assert.equal(result.state.blocks.length, 0); + const result = rebuildCompressionState( + createInitialState(), + rebuildMessages("total garbage not json", "call1"), + rebuildConfig(), + ); + assert.equal(result.blocksRebuilt, 0); + assert.equal(result.state.blocks.length, 0); }); test("rebuildCompressionState skips compress calls with no valid ranges", () => { - const result = rebuildCompressionState( - createInitialState(), - rebuildMessages(JSON.stringify({ content: [] }), "call1"), - rebuildConfig(), - ); - assert.equal(result.blocksRebuilt, 0); - assert.equal(result.state.blocks.length, 0); + const result = rebuildCompressionState( + createInitialState(), + rebuildMessages(JSON.stringify({ content: [] }), "call1"), + rebuildConfig(), + ); + assert.equal(result.blocksRebuilt, 0); + assert.equal(result.state.blocks.length, 0); }); test("parseCompressArgs returns ranges typed as CompressRangeSpec", () => { - // Compile-time contract: ranges feed core.applyCompression directly. - const { ranges } = parseCompressArgs({ - content: [{ startRef: "m00001", endRef: "m00002", summary: "typed" }], - }); - const spec: CompressRangeSpec | undefined = ranges[0]; - assert.ok(spec); - assert.equal(typeof spec.startRef, "string"); + // Compile-time contract: ranges feed core.applyCompression directly. + const { ranges } = parseCompressArgs({ + content: [{ startRef: "m00001", endRef: "m00002", summary: "typed" }], + }); + const spec: CompressRangeSpec | undefined = ranges[0]; + assert.ok(spec); + assert.equal(typeof spec.startRef, "string"); }); // --- top-level shapes the adapters observe (single range, topic, summaryMaxChars) --- test("parseCompressArgs accepts a single range at the top level (no content array)", () => { - const { ranges, diagnostics } = parseCompressArgs({ startRef: "m00001", endRef: "m00002", summary: "single" }); - assert.equal(diagnostics.kind, "ok"); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]!.startRef, "m00001"); - assert.equal(ranges[0]!.endRef, "m00002"); - assert.equal(ranges[0]!.summary, "single"); + const { ranges, diagnostics } = parseCompressArgs({ + startRef: "m00001", + endRef: "m00002", + summary: "single", + }); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]!.startRef, "m00001"); + assert.equal(ranges[0]!.endRef, "m00002"); + assert.equal(ranges[0]!.summary, "single"); }); test("parseCompressArgs accepts a top-level single range with startId/endId variants", () => { - const { ranges, diagnostics } = parseCompressArgs({ startId: "m00001", endId: "m00002", summary: "single" }); - assert.equal(diagnostics.kind, "ok"); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]!.startRef, "m00001"); + const { ranges, diagnostics } = parseCompressArgs({ + startId: "m00001", + endId: "m00002", + summary: "single", + }); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]!.startRef, "m00001"); }); test("parseCompressArgs reports missing-content when there is no content and no valid single range", () => { - const { ranges, diagnostics } = parseCompressArgs({ foo: "bar" }); - assert.equal(diagnostics.kind, "missing-content"); - assert.equal(ranges.length, 0); + const { ranges, diagnostics } = parseCompressArgs({ foo: "bar" }); + assert.equal(diagnostics.kind, "missing-content"); + assert.equal(ranges.length, 0); }); test("parseCompressArgs applies a top-level topic to ranges without their own", () => { - const { ranges } = parseCompressArgs({ - topic: "Top", - content: [ - { startRef: "m00001", endRef: "m00002", summary: "a" }, - { startRef: "m00003", endRef: "m00004", summary: "b", topic: "Own" }, - ], - }); - assert.equal(ranges[0]!.topic, "Top"); - assert.equal(ranges[1]!.topic, "Own"); + const { ranges } = parseCompressArgs({ + topic: "Top", + content: [ + { startRef: "m00001", endRef: "m00002", summary: "a" }, + { startRef: "m00003", endRef: "m00004", summary: "b", topic: "Own" }, + ], + }); + assert.equal(ranges[0]!.topic, "Top"); + assert.equal(ranges[1]!.topic, "Own"); }); test("parseCompressArgs applies a top-level summaryMaxChars to ranges without their own", () => { - const { ranges } = parseCompressArgs({ - summaryMaxChars: 5000, - content: [ - { startRef: "m00001", endRef: "m00002", summary: "a" }, - { startRef: "m00003", endRef: "m00004", summary: "b", summaryMaxChars: 999 }, - ], - }); - assert.equal(ranges[0]!.summaryMaxChars, 5000); - assert.equal(ranges[1]!.summaryMaxChars, 999); + const { ranges } = parseCompressArgs({ + summaryMaxChars: 5000, + content: [ + { startRef: "m00001", endRef: "m00002", summary: "a" }, + { + startRef: "m00003", + endRef: "m00004", + summary: "b", + summaryMaxChars: 999, + }, + ], + }); + assert.equal(ranges[0]!.summaryMaxChars, 5000); + assert.equal(ranges[1]!.summaryMaxChars, 999); }); // --- line-form entries (#non-strict-tool providers) ------------------------ test("line form: array of strings — refs header, optional topic, verbatim markdown summary", () => { - const { ranges, diagnostics } = parseCompressArgs({ - content: [ - "m00150–m00220 debug part 1\n## Fixed the loop\nbody with | pipes | and \"quotes\" and \\backslashes\\", - "m00300-m00350\nplain summary without heading", - ], - }); - assert.equal(diagnostics.kind, "ok"); - assert.equal(ranges.length, 2); - assert.equal(ranges[0]!.startRef, "m00150"); - assert.equal(ranges[0]!.endRef, "m00220"); - assert.equal(ranges[0]!.topic, "debug part 1"); - assert.equal(ranges[0]!.summary, '## Fixed the loop\nbody with | pipes | and "quotes" and \\backslashes\\'); - assert.equal(ranges[1]!.startRef, "m00300"); - assert.equal(ranges[1]!.endRef, "m00350"); - assert.equal(ranges[1]!.topic, "plain summary without heading"); + const { ranges, diagnostics } = parseCompressArgs({ + content: [ + 'm00150–m00220 debug part 1\n## Fixed the loop\nbody with | pipes | and "quotes" and \\backslashes\\', + "m00300-m00350\nplain summary without heading", + ], + }); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 2); + assert.equal(ranges[0]!.startRef, "m00150"); + assert.equal(ranges[0]!.endRef, "m00220"); + assert.equal(ranges[0]!.topic, "debug part 1"); + assert.equal( + ranges[0]!.summary, + '## Fixed the loop\nbody with | pipes | and "quotes" and \\backslashes\\', + ); + assert.equal(ranges[1]!.startRef, "m00300"); + assert.equal(ranges[1]!.endRef, "m00350"); + assert.equal(ranges[1]!.topic, "plain summary without heading"); }); test("line form: topic derived from first markdown heading, else first line, truncated to 60", () => { - const { ranges } = parseCompressArgs({ - content: [ - "m00010–m00020\n## The Heading Wins\ndetail", - "m00030–m00040\nfirst plain line is the topic\nmore", - "m00050–m00060\n" + "x".repeat(80) + "\ntail", - ], - }); - assert.equal(ranges[0]!.topic, "The Heading Wins"); - assert.equal(ranges[1]!.topic, "first plain line is the topic"); - assert.equal(ranges[2]!.topic!.length, 60); + const { ranges } = parseCompressArgs({ + content: [ + "m00010–m00020\n## The Heading Wins\ndetail", + "m00030–m00040\nfirst plain line is the topic\nmore", + "m00050–m00060\n" + "x".repeat(80) + "\ntail", + ], + }); + assert.equal(ranges[0]!.topic, "The Heading Wins"); + assert.equal(ranges[1]!.topic, "first plain line is the topic"); + assert.equal(ranges[2]!.topic!.length, 60); }); test("line form: mixed with legacy object entries — each validated independently", () => { - const { ranges, diagnostics } = parseCompressArgs({ - content: [ - "m00001–m00002\nstring entry summary", - { startId: "m00003", endId: "m00004", summary: "object entry", topic: "Obj" }, - "no refs here at all\nsummary", - { startId: "m00005" }, - ], - }); - assert.equal(ranges.length, 2); - assert.equal(diagnostics.invalidItems, 2); - assert.equal(ranges[0]!.summary, "string entry summary"); - assert.equal(ranges[1]!.topic, "Obj"); + const { ranges, diagnostics } = parseCompressArgs({ + content: [ + "m00001–m00002\nstring entry summary", + { + startId: "m00003", + endId: "m00004", + summary: "object entry", + topic: "Obj", + }, + "no refs here at all\nsummary", + { startId: "m00005" }, + ], + }); + assert.equal(ranges.length, 2); + assert.equal(diagnostics.invalidItems, 2); + assert.equal(ranges[0]!.summary, "string entry summary"); + assert.equal(ranges[1]!.topic, "Obj"); }); test("line form: single string entry (header + summary) and single bare-ref entry", () => { - const a = parseCompressArgs({ content: ["m00007—m00009\nsummary here"] }); - assert.equal(a.ranges.length, 1); - assert.equal(a.ranges[0]!.startRef, "m00007"); - assert.equal(a.ranges[0]!.endRef, "m00009"); - const b = parseCompressArgs({ content: ["m00042 lone ref line\nsummary"] }); - assert.equal(b.ranges[0]!.startRef, "m00042"); - assert.equal(b.ranges[0]!.endRef, "m00042"); + const a = parseCompressArgs({ content: ["m00007—m00009\nsummary here"] }); + assert.equal(a.ranges.length, 1); + assert.equal(a.ranges[0]!.startRef, "m00007"); + assert.equal(a.ranges[0]!.endRef, "m00009"); + const b = parseCompressArgs({ content: ["m00042 lone ref line\nsummary"] }); + assert.equal(b.ranges[0]!.startRef, "m00042"); + assert.equal(b.ranges[0]!.endRef, "m00042"); }); test("line form: all spec separators — tilde, to, unicode ellipsis, three dots", () => { - for (const header of ["m00150~m00220", "m00150 to m00220", "m00150…m00220", "m00150...m00220"]) { - const { ranges, diagnostics } = parseCompressArgs({ content: [`${header}\nsum`] }); - assert.equal(diagnostics.kind, "ok", `separator in ${JSON.stringify(header)}`); - assert.equal(ranges[0]!.startRef, "m00150", `separator in ${JSON.stringify(header)}`); - assert.equal(ranges[0]!.endRef, "m00220", `separator in ${JSON.stringify(header)}`); - } + for (const header of [ + "m00150~m00220", + "m00150 to m00220", + "m00150…m00220", + "m00150...m00220", + ]) { + const { ranges, diagnostics } = parseCompressArgs({ + content: [`${header}\nsum`], + }); + assert.equal( + diagnostics.kind, + "ok", + `separator in ${JSON.stringify(header)}`, + ); + assert.equal( + ranges[0]!.startRef, + "m00150", + `separator in ${JSON.stringify(header)}`, + ); + assert.equal( + ranges[0]!.endRef, + "m00220", + `separator in ${JSON.stringify(header)}`, + ); + } }); test("line form: header line with no summary after it is dropped, others survive", () => { - const { ranges, diagnostics } = parseCompressArgs({ - content: [ - "m00001–m00005 topic\nfirst summary", - "m00010–m00012", - ], - }); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]!.summary, "first summary"); - assert.equal(diagnostics.invalidItems, 1); - assert.ok(diagnostics.invalidReasons?.[0]?.includes("missing summary")); + const { ranges, diagnostics } = parseCompressArgs({ + content: ["m00001–m00005 topic\nfirst summary", "m00010–m00012"], + }); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]!.summary, "first summary"); + assert.equal(diagnostics.invalidItems, 1); + assert.ok(diagnostics.invalidReasons?.[0]?.includes("missing summary")); }); test("line form: bare string content (not JSON) splits on ref-pair header lines", () => { - const { ranges, diagnostics } = parseCompressArgs({ - content: "m00100–m00110 part A\n## A\nsummary A\nm00200–m00220 part B\nsummary B", - }); - assert.equal(diagnostics.kind, "ok"); - assert.equal(ranges.length, 2); - assert.equal(ranges[0]!.topic, "part A"); - assert.equal(ranges[0]!.summary, "## A\nsummary A"); - assert.equal(ranges[1]!.summary, "summary B"); + const { ranges, diagnostics } = parseCompressArgs({ + content: + "m00100–m00110 part A\n## A\nsummary A\nm00200–m00220 part B\nsummary B", + }); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 2); + assert.equal(ranges[0]!.topic, "part A"); + assert.equal(ranges[0]!.summary, "## A\nsummary A"); + assert.equal(ranges[1]!.summary, "summary B"); }); test("line form: stringified array of line-form strings salvages through the string path", () => { - const { ranges } = parseCompressArgs( - JSON.stringify({ content: ["m00111–m00122\nsum one", "m00333–m00344 t2\nsum two"] }), - ); - assert.equal(ranges.length, 2); - assert.equal(ranges[0]!.summary, "sum one"); - assert.equal(ranges[1]!.topic, "t2"); + const { ranges } = parseCompressArgs( + JSON.stringify({ + content: ["m00111–m00122\nsum one", "m00333–m00344 t2\nsum two"], + }), + ); + assert.equal(ranges.length, 2); + assert.equal(ranges[0]!.summary, "sum one"); + assert.equal(ranges[1]!.topic, "t2"); }); test("line form: an empty string entry is invalid, not a crash", () => { - const { ranges, diagnostics } = parseCompressArgs({ content: ["", "m00001–m00002\nok"] }); - assert.equal(ranges.length, 1); - assert.equal(diagnostics.invalidItems, 1); + const { ranges, diagnostics } = parseCompressArgs({ + content: ["", "m00001–m00002\nok"], + }); + assert.equal(ranges.length, 1); + assert.equal(diagnostics.invalidItems, 1); }); test("line form: bN block refs (multi-tier folds) with short ids", () => { - const { ranges } = parseCompressArgs({ - content: ["b3–b15 tier fold\n## Folded\nsummary of blocks", "b2 lone block\nsummary"], - }); - assert.equal(ranges.length, 2); - assert.equal(ranges[0]!.startRef, "b3"); - assert.equal(ranges[0]!.endRef, "b15"); - assert.equal(ranges[1]!.startRef, "b2"); - assert.equal(ranges[1]!.endRef, "b2"); + const { ranges } = parseCompressArgs({ + content: [ + "b3–b15 tier fold\n## Folded\nsummary of blocks", + "b2 lone block\nsummary", + ], + }); + assert.equal(ranges.length, 2); + assert.equal(ranges[0]!.startRef, "b3"); + assert.equal(ranges[0]!.endRef, "b15"); + assert.equal(ranges[1]!.startRef, "b2"); + assert.equal(ranges[1]!.endRef, "b2"); }); test("line form: m refs zero-pad like the object form", () => { - const { ranges } = parseCompressArgs({ content: ["m150–m220 pad\nsummary"] }); - assert.equal(ranges[0]!.startRef, "m00150"); - assert.equal(ranges[0]!.endRef, "m00220"); + const { ranges } = parseCompressArgs({ content: ["m150–m220 pad\nsummary"] }); + assert.equal(ranges[0]!.startRef, "m00150"); + assert.equal(ranges[0]!.endRef, "m00220"); }); test("line form: refs cited inside the summary body never become the range", () => { - const { ranges, diagnostics } = parseCompressArgs({ - content: ["m00042 lone ref line\nEarlier I compressed m00300–m00400 for the auth work.\nDetails here."], - }); - assert.equal(diagnostics.kind, "ok"); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]!.startRef, "m00042"); - assert.equal(ranges[0]!.endRef, "m00042"); - assert.equal( - ranges[0]!.summary, - "Earlier I compressed m00300–m00400 for the auth work.\nDetails here.", - ); + const { ranges, diagnostics } = parseCompressArgs({ + content: [ + "m00042 lone ref line\nEarlier I compressed m00300–m00400 for the auth work.\nDetails here.", + ], + }); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]!.startRef, "m00042"); + assert.equal(ranges[0]!.endRef, "m00042"); + assert.equal( + ranges[0]!.summary, + "Earlier I compressed m00300–m00400 for the auth work.\nDetails here.", + ); }); test("line form: entry whose first line has no refs is dropped, not salvaged from later lines", () => { - const { ranges, diagnostics } = parseCompressArgs({ - content: [ - "Summary of work\nm00150–m00220 were discussed before\nmore detail", - "m00001–m00002 ok entry\nfine", - ], - }); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]!.startRef, "m00001"); - assert.equal(diagnostics.invalidItems, 1); - assert.ok(diagnostics.invalidReasons?.[0]?.includes("no mNNNNN/bN refs in header")); + const { ranges, diagnostics } = parseCompressArgs({ + content: [ + "Summary of work\nm00150–m00220 were discussed before\nmore detail", + "m00001–m00002 ok entry\nfine", + ], + }); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]!.startRef, "m00001"); + assert.equal(diagnostics.invalidItems, 1); + assert.ok( + diagnostics.invalidReasons?.[0]?.includes("no mNNNNN/bN refs in header"), + ); }); test("line form: header matches on the first line only — body ref citations cannot hijack the range", () => { - const { ranges, diagnostics } = parseCompressArgs({ - content: [ - "## Auth exploration\nFound token in m00042. Decisions made:\n- chose X because Y", - "Explored the range m00150–m00220 earlier.\nSummary body here", - "m00007–m00009 real header\ncites m00065 and m00150–m00220 in the body", - ], - }); - assert.equal(diagnostics.kind, "ok"); - assert.equal(ranges.length, 1); - assert.equal(diagnostics.invalidItems, 2); - assert.ok(diagnostics.invalidReasons?.every((r) => r.includes("no mNNNNN/bN refs in header"))); - assert.equal(ranges[0]!.startRef, "m00007"); - assert.equal(ranges[0]!.endRef, "m00009"); - assert.equal(ranges[0]!.summary, "cites m00065 and m00150–m00220 in the body"); + const { ranges, diagnostics } = parseCompressArgs({ + content: [ + "## Auth exploration\nFound token in m00042. Decisions made:\n- chose X because Y", + "Explored the range m00150–m00220 earlier.\nSummary body here", + "m00007–m00009 real header\ncites m00065 and m00150–m00220 in the body", + ], + }); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 1); + assert.equal(diagnostics.invalidItems, 2); + assert.ok( + diagnostics.invalidReasons?.every((r) => + r.includes("no mNNNNN/bN refs in header"), + ), + ); + assert.equal(ranges[0]!.startRef, "m00007"); + assert.equal(ranges[0]!.endRef, "m00009"); + assert.equal( + ranges[0]!.summary, + "cites m00065 and m00150–m00220 in the body", + ); }); test("line form: CRLF header line parses; trailing \\r trimmed from topic", () => { - const { ranges } = parseCompressArgs({ content: ["m00001–m00005 crlf topic\r\nbody line"] }); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]!.topic, "crlf topic"); - assert.equal(ranges[0]!.summary, "body line"); + const { ranges } = parseCompressArgs({ + content: ["m00001–m00005 crlf topic\r\nbody line"], + }); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]!.topic, "crlf topic"); + assert.equal(ranges[0]!.summary, "body line"); }); // --------------------------------------------------------------------------- @@ -737,54 +857,70 @@ test("line form: CRLF header line parses; trailing \\r trimmed from topic", () = // --------------------------------------------------------------------------- test("parseCompressArgs repairs compound damage: raw newlines + invalid escapes in one stringified payload", () => { - // Production shape (01a09a0b): vLLM-stringified content array whose summary - // has REAL raw newlines AND markdown backslash-escapes, unescaped. - const summary = - "m00066-m00081 Investigation reads: exact pre-edit code\n\n" + - "## Files to modify (`src/x.ts`)\n" + - "and some \\*markdown\\* with a \\`tick\\`"; - const arrStr = '[{"startRef": "m00066", "endRef": "m00081", "summary": "' + summary + '"}]'; - const { ranges, diagnostics } = parseCompressArgs({ content: arrStr }); - assert.equal(diagnostics.kind, "ok"); - assert.equal(diagnostics.ok, true); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]?.startRef, "m00066"); - assert.equal(ranges[0]?.endRef, "m00081"); - assert.equal(ranges[0]?.summary, summary); + // Production shape (01a09a0b): vLLM-stringified content array whose summary + // has REAL raw newlines AND markdown backslash-escapes, unescaped. + const summary = + "m00066-m00081 Investigation reads: exact pre-edit code\n\n" + + "## Files to modify (`src/x.ts`)\n" + + "and some \\*markdown\\* with a \\`tick\\`"; + const arrStr = + '[{"startRef": "m00066", "endRef": "m00081", "summary": "' + + summary + + '"}]'; + const { ranges, diagnostics } = parseCompressArgs({ content: arrStr }); + assert.equal(diagnostics.kind, "ok"); + assert.equal(diagnostics.ok, true); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.startRef, "m00066"); + assert.equal(ranges[0]?.endRef, "m00081"); + assert.equal(ranges[0]?.summary, summary); }); test("parseCompressArgs repairs multiple scattered control chars in one pass", () => { - const summary = "line one\ttabbed\rreturn\nnewline then a \\*star\\*"; - const doc = '{"content": [{"startRef": "m00001", "endRef": "m00002", "summary": "' + summary + '"}]}'; - const { ranges, diagnostics } = parseCompressArgs(doc); - assert.equal(diagnostics.kind, "ok"); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]?.summary, summary); + const summary = "line one\ttabbed\rreturn\nnewline then a \\*star\\*"; + const doc = + '{"content": [{"startRef": "m00001", "endRef": "m00002", "summary": "' + + summary + + '"}]}'; + const { ranges, diagnostics } = parseCompressArgs(doc); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, summary); }); test("parseCompressArgs keeps an invalid escape's backslash literally in the value", () => { - // `\X` is invalid JSON; repair doubles it so the parsed value keeps `\X`. - const doc = '{"content": [{"startRef": "m00001", "endRef": "m00002", "summary": "a \\* b"}]}'; - const { ranges, diagnostics } = parseCompressArgs(doc); - assert.equal(diagnostics.kind, "ok"); - assert.equal(ranges[0]?.summary, "a \\* b"); + // `\X` is invalid JSON; repair doubles it so the parsed value keeps `\X`. + const doc = + '{"content": [{"startRef": "m00001", "endRef": "m00002", "summary": "a \\* b"}]}'; + const { ranges, diagnostics } = parseCompressArgs(doc); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges[0]?.summary, "a \\* b"); }); test("parseCompressArgs leaves valid JSON and structural garbage untouched by the repair", () => { - const valid = JSON.stringify({ content: [{ startRef: "m00001", endRef: "m00002", summary: "clean" }] }); - assert.equal(parseCompressArgs(valid).diagnostics.kind, "ok"); - const garbage = '{"content": [{"startRef": "m00001", "summary": "unterminated'; - const g = parseCompressArgs(garbage); - assert.ok(g.diagnostics.kind === "truncated" || g.diagnostics.kind === "malformed-json"); - assert.equal(g.ranges.length, 0); + const valid = JSON.stringify({ + content: [{ startRef: "m00001", endRef: "m00002", summary: "clean" }], + }); + assert.equal(parseCompressArgs(valid).diagnostics.kind, "ok"); + const garbage = + '{"content": [{"startRef": "m00001", "summary": "unterminated'; + const g = parseCompressArgs(garbage); + assert.ok( + g.diagnostics.kind === "truncated" || + g.diagnostics.kind === "malformed-json", + ); + assert.equal(g.ranges.length, 0); }); test("parseCompressArgs maps non-named control chars through \\uXXXX escapes", () => { - const summary = "a\u0001b"; - const doc = '{"content": [{"startRef": "m00001", "endRef": "m00002", "summary": "' + summary + '"}]}'; - const { ranges, diagnostics } = parseCompressArgs(doc); - assert.equal(diagnostics.kind, "ok"); - assert.equal(ranges[0]?.summary, summary); + const summary = "a\u0001b"; + const doc = + '{"content": [{"startRef": "m00001", "endRef": "m00002", "summary": "' + + summary + + '"}]}'; + const { ranges, diagnostics } = parseCompressArgs(doc); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges[0]?.summary, summary); }); // --------------------------------------------------------------------------- @@ -794,58 +930,70 @@ test("parseCompressArgs maps non-named control chars through \\uXXXX escapes", ( // --------------------------------------------------------------------------- test("parseCompressArgs coalesces a split line-form entry (header + summary siblings) (#1001)", () => { - const input = { - content: [ - "m01588\u2013m01712 走bili UI move + proxy 调试", - "## TASK AS OF THIS BLOCK\n- user goal\n- decisions with reasons", - ], - }; - const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "ok"); - assert.equal(diagnostics.invalidItems, 0); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]?.startRef, "m01588"); - assert.equal(ranges[0]?.endRef, "m01712"); - assert.equal(ranges[0]?.summary, "## TASK AS OF THIS BLOCK\n- user goal\n- decisions with reasons"); - assert.equal(ranges[0]?.topic, "走bili UI move + proxy 调试"); + const input = { + content: [ + "m01588\u2013m01712 走bili UI move + proxy 调试", + "## TASK AS OF THIS BLOCK\n- user goal\n- decisions with reasons", + ], + }; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "ok"); + assert.equal(diagnostics.invalidItems, 0); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.startRef, "m01588"); + assert.equal(ranges[0]?.endRef, "m01712"); + assert.equal( + ranges[0]?.summary, + "## TASK AS OF THIS BLOCK\n- user goal\n- decisions with reasons", + ); + assert.equal(ranges[0]?.topic, "走bili UI move + proxy 调试"); }); test("parseCompressArgs coalesces multiple summary siblings onto one header-only entry (#1001)", () => { - const input = { - content: ["m00150\u2013m00220", "first summary chunk", "second chunk"], - }; - const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "ok"); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]?.summary, "first summary chunk\nsecond chunk"); + const input = { + content: ["m00150\u2013m00220", "first summary chunk", "second chunk"], + }; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "first summary chunk\nsecond chunk"); }); test("parseCompressArgs keeps a stray refless string invalid after a complete line entry (#1001)", () => { - const input = { - content: ["m00001\u2013m00002 topic\nreal summary", "## stray with no pending header"], - }; - const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "ok"); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]?.summary, "real summary"); - assert.equal(diagnostics.invalidItems, 1); + const input = { + content: [ + "m00001\u2013m00002 topic\nreal summary", + "## stray with no pending header", + ], + }; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "real summary"); + assert.equal(diagnostics.invalidItems, 1); }); test("parseCompressArgs still rejects a header-only entry with nothing to adopt (#1001)", () => { - const input = { content: ["m00001\u2013m00002 topic"] }; - const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(ranges.length, 0); - assert.equal(diagnostics.kind, "no-valid-ranges"); - assert.match(String(diagnostics.invalidReasons?.[0] ?? ""), /missing summary/); + const input = { content: ["m00001\u2013m00002 topic"] }; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(ranges.length, 0); + assert.equal(diagnostics.kind, "no-valid-ranges"); + assert.match( + String(diagnostics.invalidReasons?.[0] ?? ""), + /missing summary/, + ); }); test("parseCompressArgs resets the adoption pairing at an object entry (#1001)", () => { - const input = { - content: [{ startRef: "m00001", endRef: "m00002", summary: "good" }, "orphan summary body"], - }; - const { ranges, diagnostics } = parseCompressArgs(input); - assert.equal(diagnostics.kind, "ok"); - assert.equal(ranges.length, 1); - assert.equal(ranges[0]?.summary, "good"); - assert.equal(diagnostics.invalidItems, 1); + const input = { + content: [ + { startRef: "m00001", endRef: "m00002", summary: "good" }, + "orphan summary body", + ], + }; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "good"); + assert.equal(diagnostics.invalidItems, 1); }); diff --git a/tests/persist.test.ts b/tests/persist.test.ts index 8997dd3..b652c44 100644 --- a/tests/persist.test.ts +++ b/tests/persist.test.ts @@ -1,7 +1,15 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import fs from "node:fs"; -import { mkdtempSync, rmSync, writeFileSync, mkdirSync, readdirSync, readFileSync, existsSync } from "node:fs"; +import { + mkdtempSync, + rmSync, + writeFileSync, + mkdirSync, + readdirSync, + readFileSync, + existsSync, +} from "node:fs"; import fsp from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; @@ -12,16 +20,19 @@ import { createInitialState } from "../src/state.js"; import type { CompressionState } from "../src/types.js"; interface Payload { - label: string; - count: number; + label: string; + count: number; } function tmpDir(): string { - return mkdtempSync(path.join(tmpdir(), "acp-kernel-persist-")); + return mkdtempSync(path.join(tmpdir(), "acp-kernel-persist-")); } -function store(dir: string, opts: Partial>[0]> = {}): StateStore { - return new StateStore({ dir, version: 1, ...opts }); +function store( + dir: string, + opts: Partial>[0]> = {}, +): StateStore { + return new StateStore({ dir, version: 1, ...opts }); } /** @@ -30,925 +41,1119 @@ function store(dir: string, opts: Partial boolean, what: string, deadlineMs = 4000): Promise { - const start = Date.now(); - while (!cond()) { - if (Date.now() - start > deadlineMs) { - assert.fail(`timed out waiting for: ${what}`); - } - await new Promise((r) => setTimeout(r, 5)); +async function until( + cond: () => boolean, + what: string, + deadlineMs = 4000, +): Promise { + const start = Date.now(); + while (!cond()) { + if (Date.now() - start > deadlineMs) { + assert.fail(`timed out waiting for: ${what}`); } + await new Promise((r) => setTimeout(r, 5)); + } } test("writeNow persists a round-trippable envelope", async () => { - const dir = tmpDir(); - try { - const s = store(dir); - await s.writeNow("sid-1", () => ({ label: "hello", count: 42 })); - const loaded = s.loadSync("sid-1"); - assert.ok(loaded); - assert.equal(loaded.id, "sid-1"); - assert.equal(loaded.version, 1); - assert.equal(typeof loaded.savedAt, "number"); - assert.deepEqual(loaded.payload, { label: "hello", count: 42 }); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const s = store(dir); + await s.writeNow("sid-1", () => ({ label: "hello", count: 42 })); + const loaded = s.loadSync("sid-1"); + assert.ok(loaded); + assert.equal(loaded.id, "sid-1"); + assert.equal(loaded.version, 1); + assert.equal(typeof loaded.savedAt, "number"); + assert.deepEqual(loaded.payload, { label: "hello", count: 42 }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); test("writeNow builds the payload at write time, not call time", async () => { - const dir = tmpDir(); - try { - const s = store(dir); - const live = { label: "before", count: 0 }; - const pending = s.writeNow("sid-live", () => live); - live.label = "after"; - live.count = 7; - await pending; - const loaded = s.loadSync("sid-live"); - assert.ok(loaded); - assert.deepEqual(loaded.payload, { label: "after", count: 7 }); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const s = store(dir); + const live = { label: "before", count: 0 }; + const pending = s.writeNow("sid-live", () => live); + live.label = "after"; + live.count = 7; + await pending; + const loaded = s.loadSync("sid-live"); + assert.ok(loaded); + assert.deepEqual(loaded.payload, { label: "after", count: 7 }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); test("loadSync returns null for unknown ids and when disabled", async () => { - const dir = tmpDir(); - try { - const s = store(dir); - assert.equal(s.loadSync("missing"), null); - const off = store(dir, { enabled: false }); - await off.writeNow("sid-off", () => ({ label: "x", count: 0 })); - assert.equal(off.loadSync("sid-off"), null); - assert.equal((await off.loadAll()).size, 0); - // disabled flushSync reports success (nothing was at risk) - assert.equal(off.flushSync("sid-off", () => ({ label: "x", count: 0 })), true); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const s = store(dir); + assert.equal(s.loadSync("missing"), null); + const off = store(dir, { enabled: false }); + await off.writeNow("sid-off", () => ({ label: "x", count: 0 })); + assert.equal(off.loadSync("sid-off"), null); + assert.equal((await off.loadAll()).size, 0); + // disabled flushSync reports success (nothing was at risk) + assert.equal( + off.flushSync("sid-off", () => ({ label: "x", count: 0 })), + true, + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); test("scheduleSave coalesces bursts into one freshest write", async () => { - const dir = tmpDir(); - try { - const s = store(dir, { debounceMs: 20 }); - let builds = 0; - const live = { label: "v1", count: 1 }; - const build = (): Payload => { - builds += 1; - return live; - }; - s.scheduleSave("sid-d", build); - live.label = "v2"; - live.count = 2; - s.scheduleSave("sid-d", build); - assert.equal(s.hasPending("sid-d"), true); - assert.deepEqual(s.pendingIds(), ["sid-d"]); - // await the settled disk state: the debounced timer fired, the write - // landed, and it carries the freshest payload - await until(() => s.loadSync("sid-d")?.payload.label === "v2", "debounced write to settle"); - assert.equal(s.hasPending("sid-d"), false); - // builder ran exactly once — the second scheduleSave only replaced it - assert.equal(builds, 1); - assert.deepEqual(s.loadSync("sid-d")?.payload, { label: "v2", count: 2 }); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const s = store(dir, { debounceMs: 20 }); + let builds = 0; + const live = { label: "v1", count: 1 }; + const build = (): Payload => { + builds += 1; + return live; + }; + s.scheduleSave("sid-d", build); + live.label = "v2"; + live.count = 2; + s.scheduleSave("sid-d", build); + assert.equal(s.hasPending("sid-d"), true); + assert.deepEqual(s.pendingIds(), ["sid-d"]); + // await the settled disk state: the debounced timer fired, the write + // landed, and it carries the freshest payload + await until( + () => s.loadSync("sid-d")?.payload.label === "v2", + "debounced write to settle", + ); + assert.equal(s.hasPending("sid-d"), false); + // builder ran exactly once — the second scheduleSave only replaced it + assert.equal(builds, 1); + assert.deepEqual(s.loadSync("sid-d")?.payload, { label: "v2", count: 2 }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); test("scheduleSave builder failures are contained, not unhandled rejections", async () => { - const dir = tmpDir(); - try { - const errors: string[] = []; - const s = store(dir, { - debounceMs: 10, - log: (level, msg) => { - if (level === "error") errors.push(msg); - }, - }); - s.scheduleSave("sid-bad", () => { - throw new Error("boom"); - }); - await until(() => !s.hasPending("sid-bad") && errors.length > 0, "failed builder to be logged"); - assert.equal(errors.length, 1); - assert.match(errors[0] ?? "", /boom/); - // store still usable afterwards - await s.writeNow("sid-ok", () => ({ label: "ok", count: 0 })); - assert.ok(s.loadSync("sid-ok")); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const errors: string[] = []; + const s = store(dir, { + debounceMs: 10, + log: (level, msg) => { + if (level === "error") errors.push(msg); + }, + }); + s.scheduleSave("sid-bad", () => { + throw new Error("boom"); + }); + await until( + () => !s.hasPending("sid-bad") && errors.length > 0, + "failed builder to be logged", + ); + assert.equal(errors.length, 1); + assert.match(errors[0] ?? "", /boom/); + // store still usable afterwards + await s.writeNow("sid-ok", () => ({ label: "ok", count: 0 })); + assert.ok(s.loadSync("sid-ok")); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); test("flushSync writes immediately and cancels the debounce", async () => { - const dir = tmpDir(); - try { - const s = store(dir, { debounceMs: 30 }); - let builds = 0; - s.scheduleSave("sid-f", () => { - builds += 1; - return { label: "debounced", count: 0 }; - }); - assert.equal(s.flushSync("sid-f", () => ({ label: "flushed", count: 3 })), true); - assert.equal(s.hasPending("sid-f"), false); - const loaded = s.loadSync("sid-f"); - assert.ok(loaded); - assert.deepEqual(loaded.payload, { label: "flushed", count: 3 }); - // Proving the cancelled timer never fires is a negative over time. - // Deterministic clock control can't cover it: node:test mock timers - // don't support the unref() the store's debounce relies on, so this - // one assertion deliberately outruns the real 30ms window. - const start = Date.now(); - while (Date.now() - start < 60) { - await new Promise((r) => setTimeout(r, 5)); - } - assert.equal(builds, 0); // timer cancelled before it could fire - } finally { - rmSync(dir, { recursive: true, force: true }); + const dir = tmpDir(); + try { + const s = store(dir, { debounceMs: 30 }); + let builds = 0; + s.scheduleSave("sid-f", () => { + builds += 1; + return { label: "debounced", count: 0 }; + }); + assert.equal( + s.flushSync("sid-f", () => ({ label: "flushed", count: 3 })), + true, + ); + assert.equal(s.hasPending("sid-f"), false); + const loaded = s.loadSync("sid-f"); + assert.ok(loaded); + assert.deepEqual(loaded.payload, { label: "flushed", count: 3 }); + // Proving the cancelled timer never fires is a negative over time. + // Deterministic clock control can't cover it: node:test mock timers + // don't support the unref() the store's debounce relies on, so this + // one assertion deliberately outruns the real 30ms window. + const start = Date.now(); + while (Date.now() - start < 60) { + await new Promise((r) => setTimeout(r, 5)); } + assert.equal(builds, 0); // timer cancelled before it could fire + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); test("flushAll flushes pending builders and drains in-flight writes", async () => { - const dir = tmpDir(); - try { - const s = store(dir, { debounceMs: 5000 }); - s.scheduleSave("sid-a", () => ({ label: "a", count: 0 })); - s.scheduleSave("sid-b", () => ({ label: "b", count: 0 })); - assert.equal(s.pendingIds().length, 2); - await s.flushAll(); - assert.equal(s.pendingIds().length, 0); - assert.ok(s.loadSync("sid-a")); - assert.ok(s.loadSync("sid-b")); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const s = store(dir, { debounceMs: 5000 }); + s.scheduleSave("sid-a", () => ({ label: "a", count: 0 })); + s.scheduleSave("sid-b", () => ({ label: "b", count: 0 })); + assert.equal(s.pendingIds().length, 2); + await s.flushAll(); + assert.equal(s.pendingIds().length, 0); + assert.ok(s.loadSync("sid-a")); + assert.ok(s.loadSync("sid-b")); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); test("concurrent writeNow calls on one id serialize and leave a valid file", async () => { - const dir = tmpDir(); - try { - const s = store(dir); - const writes = Array.from({ length: 12 }, (_, i) => - s.writeNow("sid-race", () => ({ label: `w${i}`, count: i })).then(() => i), - ); - const settled = await Promise.all(writes); - assert.deepEqual(settled, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); - const loaded = s.loadSync("sid-race"); - assert.ok(loaded); - assert.equal(loaded.payload.count, 11); // last write wins - // no orphan temp files left behind - const leftovers = readdirSync(dir).filter((f) => f.startsWith(".tmp-")); - assert.deepEqual(leftovers, []); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const s = store(dir); + const writes = Array.from({ length: 12 }, (_, i) => + s + .writeNow("sid-race", () => ({ label: `w${i}`, count: i })) + .then(() => i), + ); + const settled = await Promise.all(writes); + assert.deepEqual(settled, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); + const loaded = s.loadSync("sid-race"); + assert.ok(loaded); + assert.equal(loaded.payload.count, 11); // last write wins + // no orphan temp files left behind + const leftovers = readdirSync(dir).filter((f) => f.startsWith(".tmp-")); + assert.deepEqual(leftovers, []); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); test("loadAll skips corrupt files, tmp orphans, and id/filename mismatches", async () => { - const dir = tmpDir(); - try { - const s = store(dir); - await s.writeNow("good-1", () => ({ label: "good", count: 1 })); - writeFileSync(path.join(dir, flatFileNameFor("corrupt-1")), "{ not json", "utf8"); - writeFileSync(path.join(dir, ".tmp-orphan-123"), JSON.stringify({ version: 1, savedAt: 0, id: "orph", payload: {} }), "utf8"); - // envelope with a valid shape but a filename from a different id - writeFileSync(path.join(dir, flatFileNameFor("other-id")), JSON.stringify({ version: 1, savedAt: 0, id: "impostor", payload: { label: "x", count: 0 } }), "utf8"); - const all = await s.loadAll(); - assert.deepEqual([...all.keys()], ["good-1"]); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const s = store(dir); + await s.writeNow("good-1", () => ({ label: "good", count: 1 })); + writeFileSync( + path.join(dir, flatFileNameFor("corrupt-1")), + "{ not json", + "utf8", + ); + writeFileSync( + path.join(dir, ".tmp-orphan-123"), + JSON.stringify({ version: 1, savedAt: 0, id: "orph", payload: {} }), + "utf8", + ); + // envelope with a valid shape but a filename from a different id + writeFileSync( + path.join(dir, flatFileNameFor("other-id")), + JSON.stringify({ + version: 1, + savedAt: 0, + id: "impostor", + payload: { label: "x", count: 0 }, + }), + "utf8", + ); + const all = await s.loadAll(); + assert.deepEqual([...all.keys()], ["good-1"]); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); test("loadAll rejects records failing the downstream validate hook", async () => { - const dir = tmpDir(); - try { - const s = store(dir, { validate: (env) => typeof env.payload.count === "number" && env.payload.count > 0 }); - await s.writeNow("pos", () => ({ label: "p", count: 5 })); - await s.writeNow("zero", () => ({ label: "z", count: 0 })); - const all = await s.loadAll(); - assert.deepEqual([...all.keys()], ["pos"]); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const s = store(dir, { + validate: (env) => + typeof env.payload.count === "number" && env.payload.count > 0, + }); + await s.writeNow("pos", () => ({ label: "p", count: 5 })); + await s.writeNow("zero", () => ({ label: "z", count: 0 })); + const all = await s.loadAll(); + assert.deepEqual([...all.keys()], ["pos"]); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); test("custom relPath namespaces into subdirectories and loadAll discovers them", async () => { - const dir = tmpDir(); - try { - const s = store(dir, { - relPath: (id, payload) => path.join(payload.label, `${id}.json`), - }); - await s.writeNow("ns-1", () => ({ label: "openai", count: 1 })); - // namespaced layout on disk - const nested = readdirSync(dir).filter((f) => f === "openai"); - assert.equal(nested.length, 1); - // same process: the write populated the discovery map - assert.ok(s.loadSync("ns-1")); - // a fresh store (next process) cannot know the namespaced path - // before discovery — loadSync probes only the flat default name - const next = store(dir, { - relPath: (id, payload) => path.join(payload.label, `${id}.json`), - }); - assert.equal(next.loadSync("ns-1"), null); - // …but loadAll discovers it and subsequent loadSync resolves it - const all = await next.loadAll(); - assert.deepEqual([...all.keys()], ["ns-1"]); - assert.ok(next.loadSync("ns-1")); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const s = store(dir, { + relPath: (id, payload) => path.join(payload.label, `${id}.json`), + }); + await s.writeNow("ns-1", () => ({ label: "openai", count: 1 })); + // namespaced layout on disk + const nested = readdirSync(dir).filter((f) => f === "openai"); + assert.equal(nested.length, 1); + // same process: the write populated the discovery map + assert.ok(s.loadSync("ns-1")); + // a fresh store (next process) cannot know the namespaced path + // before discovery — loadSync probes only the flat default name + const next = store(dir, { + relPath: (id, payload) => path.join(payload.label, `${id}.json`), + }); + assert.equal(next.loadSync("ns-1"), null); + // …but loadAll discovers it and subsequent loadSync resolves it + const all = await next.loadAll(); + assert.deepEqual([...all.keys()], ["ns-1"]); + assert.ok(next.loadSync("ns-1")); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); test("flat legacy name is tolerated in loadAll alongside namespaced relPath", async () => { - const dir = tmpDir(); - try { - // a record written flat by an older version of the downstream - writeFileSync( - path.join(dir, flatFileNameFor("legacy-1")), - JSON.stringify({ version: 1, savedAt: 0, id: "legacy-1", payload: { label: "old", count: 9 } }), - "utf8", - ); - const s = store(dir, { - relPath: (id, payload) => path.join(payload.label, `${id}.json`), - }); - const all = await s.loadAll(); - assert.deepEqual([...all.keys()], ["legacy-1"]); - assert.deepEqual(all.get("legacy-1")?.payload, { label: "old", count: 9 }); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + // a record written flat by an older version of the downstream + writeFileSync( + path.join(dir, flatFileNameFor("legacy-1")), + JSON.stringify({ + version: 1, + savedAt: 0, + id: "legacy-1", + payload: { label: "old", count: 9 }, + }), + "utf8", + ); + const s = store(dir, { + relPath: (id, payload) => path.join(payload.label, `${id}.json`), + }); + const all = await s.loadAll(); + assert.deepEqual([...all.keys()], ["legacy-1"]); + assert.deepEqual(all.get("legacy-1")?.payload, { label: "old", count: 9 }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); test("relPath escaping the dir is rejected to the safe flat name", async () => { - const dir = tmpDir(); - try { - const warned: string[] = []; - const s = store(dir, { - log: (level, msg) => { - if (level === "warn") warned.push(msg); - }, - relPath: () => "../../escape.json", - }); - await s.writeNow("esc-1", () => ({ label: "x", count: 0 })); - // landed flat inside dir, not outside - assert.ok(s.loadSync("esc-1")); - const outsideExists = readdirSync(path.dirname(dir)).includes(flatFileNameFor("esc-1")); - assert.equal(outsideExists, false); - assert.ok(warned.some((w) => w.includes("escapes dir"))); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const warned: string[] = []; + const s = store(dir, { + log: (level, msg) => { + if (level === "warn") warned.push(msg); + }, + relPath: () => "../../escape.json", + }); + await s.writeNow("esc-1", () => ({ label: "x", count: 0 })); + // landed flat inside dir, not outside + assert.ok(s.loadSync("esc-1")); + const outsideExists = readdirSync(path.dirname(dir)).includes( + flatFileNameFor("esc-1"), + ); + assert.equal(outsideExists, false); + assert.ok(warned.some((w) => w.includes("escapes dir"))); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); test("loadAll walks nested namespaces recursively", async () => { - const dir = tmpDir(); - try { - const s = store(dir, { - relPath: (id, payload) => path.join("proto", payload.label, `${id}.json`), - }); - await s.writeNow("deep-1", () => ({ label: "hostA", count: 1 })); - await s.writeNow("deep-2", () => ({ label: "hostB", count: 2 })); - mkdirSync(path.join(dir, "proto", "hostC"), { recursive: true }); - writeFileSync( - path.join(dir, "proto", "hostC", "deep-3.json"), - JSON.stringify({ version: 1, savedAt: 0, id: "deep-3", payload: { label: "hostC", count: 3 } }), - "utf8", - ); - const all = await s.loadAll(); - assert.deepEqual([...all.keys()].sort(), ["deep-1", "deep-2", "deep-3"]); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const s = store(dir, { + relPath: (id, payload) => path.join("proto", payload.label, `${id}.json`), + }); + await s.writeNow("deep-1", () => ({ label: "hostA", count: 1 })); + await s.writeNow("deep-2", () => ({ label: "hostB", count: 2 })); + mkdirSync(path.join(dir, "proto", "hostC"), { recursive: true }); + writeFileSync( + path.join(dir, "proto", "hostC", "deep-3.json"), + JSON.stringify({ + version: 1, + savedAt: 0, + id: "deep-3", + payload: { label: "hostC", count: 3 }, + }), + "utf8", + ); + const all = await s.loadAll(); + assert.deepEqual([...all.keys()].sort(), ["deep-1", "deep-2", "deep-3"]); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); test("cancelAll drops pending writes without flushing", async () => { - const dir = tmpDir(); - try { - const s = store(dir, { debounceMs: 5000 }); - s.scheduleSave("sid-c", () => ({ label: "gone", count: 0 })); - s.cancelAll(); - assert.equal(s.hasPending("sid-c"), false); - // deterministic: the debounce timer was the only write path, and it - // was cleared before firing — no file exists - assert.equal(s.loadSync("sid-c"), null); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const s = store(dir, { debounceMs: 5000 }); + s.scheduleSave("sid-c", () => ({ label: "gone", count: 0 })); + s.cancelAll(); + assert.equal(s.hasPending("sid-c"), false); + // deterministic: the debounce timer was the only write path, and it + // was cleared before firing — no file exists + assert.equal(s.loadSync("sid-c"), null); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); test("a store on a missing dir loads empty and creates the dir on first write", async () => { - const dir = path.join(tmpDir(), "not-created-yet"); - try { - const s = store(dir); - assert.equal((await s.loadAll()).size, 0); - await s.writeNow("sid-m", () => ({ label: "m", count: 0 })); - assert.ok(s.loadSync("sid-m")); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const dir = path.join(tmpDir(), "not-created-yet"); + try { + const s = store(dir); + assert.equal((await s.loadAll()).size, 0); + await s.writeNow("sid-m", () => ({ label: "m", count: 0 })); + assert.ok(s.loadSync("sid-m")); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); test("mergeCompressionState fills missing fields from a fresh state", () => { - const fresh = createInitialState(); - const partial = { blocks: [] } as CompressionState; - const merged = mergeCompressionState(partial); - assert.deepEqual(merged.messageRefs, fresh.messageRefs); - assert.deepEqual(merged.nudge, fresh.nudge); - assert.deepEqual(merged.stats, fresh.stats); - assert.equal(merged.nextBlockId, fresh.nextBlockId); - assert.equal(merged.nextRunId, fresh.nextRunId); - assert.deepEqual(merged.tokenSnapshot, fresh.tokenSnapshot); + const fresh = createInitialState(); + const partial = { blocks: [] } as CompressionState; + const merged = mergeCompressionState(partial); + assert.deepEqual(merged.messageRefs, fresh.messageRefs); + assert.deepEqual(merged.nudge, fresh.nudge); + assert.deepEqual(merged.stats, fresh.stats); + assert.equal(merged.nextBlockId, fresh.nextBlockId); + assert.equal(merged.nextRunId, fresh.nextRunId); + assert.deepEqual(merged.tokenSnapshot, fresh.tokenSnapshot); }); test("mergeCompressionState keeps parsed values over defaults", () => { - const parsed: CompressionState = { - ...createInitialState(), - nextBlockId: 41, - stats: { tokensCompressed: 1234, compressionCount: 7 }, - }; - const merged = mergeCompressionState(parsed); - assert.equal(merged.nextBlockId, 41); - assert.deepEqual(merged.stats, { tokensCompressed: 1234, compressionCount: 7, absorbedTokens: 0, imagesShrunk: 0, imageBytesSaved: 0, imageTokensSaved: 0, storedCount: 0, retrievalCount: 0 }); + const parsed: CompressionState = { + ...createInitialState(), + nextBlockId: 41, + stats: { tokensCompressed: 1234, compressionCount: 7 }, + }; + const merged = mergeCompressionState(parsed); + assert.equal(merged.nextBlockId, 41); + assert.deepEqual(merged.stats, { + tokensCompressed: 1234, + compressionCount: 7, + absorbedTokens: 0, + imagesShrunk: 0, + imageBytesSaved: 0, + imageTokensSaved: 0, + storedCount: 0, + retrievalCount: 0, + }); }); test("legacy hook adopts pre-envelope records on loadAll and loadSync", async () => { - const dir = tmpDir(); - try { - // A proxy-style flat record: no `payload` wrapper, id at top level. - const flat = { - version: 3, - savedAt: 12345, - id: "flat-1", - label: "adopted", - count: 9, - }; - const flatFile = path.join(dir, flatFileNameFor("flat-1")); - writeFileSync(flatFile, JSON.stringify(flat), "utf8"); - const s = store(dir, { - legacy: (parsed) => { - const p = parsed as { id?: unknown; label?: unknown; count?: unknown }; - if (typeof p.id !== "string" || typeof p.label !== "string" || typeof p.count !== "number") return null; - return { id: p.id, payload: { label: p.label, count: p.count } }; - }, - }); - const all = await s.loadAll(); - assert.equal(all.size, 1); - const env = all.get("flat-1"); - assert.ok(env); - assert.equal(env.id, "flat-1"); - assert.deepEqual(env.payload, { label: "adopted", count: 9 }); - // The source record's own stamps are preserved, not stamped fresh. - assert.equal(env.version, 3); - assert.equal(env.savedAt, 12345); - // loadSync finds it too (discovered by loadAll). - const direct = s.loadSync("flat-1"); - assert.ok(direct); - assert.deepEqual(direct.payload, { label: "adopted", count: 9 }); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + // A proxy-style flat record: no `payload` wrapper, id at top level. + const flat = { + version: 3, + savedAt: 12345, + id: "flat-1", + label: "adopted", + count: 9, + }; + const flatFile = path.join(dir, flatFileNameFor("flat-1")); + writeFileSync(flatFile, JSON.stringify(flat), "utf8"); + const s = store(dir, { + legacy: (parsed) => { + const p = parsed as { id?: unknown; label?: unknown; count?: unknown }; + if ( + typeof p.id !== "string" || + typeof p.label !== "string" || + typeof p.count !== "number" + ) + return null; + return { id: p.id, payload: { label: p.label, count: p.count } }; + }, + }); + const all = await s.loadAll(); + assert.equal(all.size, 1); + const env = all.get("flat-1"); + assert.ok(env); + assert.equal(env.id, "flat-1"); + assert.deepEqual(env.payload, { label: "adopted", count: 9 }); + // The source record's own stamps are preserved, not stamped fresh. + assert.equal(env.version, 3); + assert.equal(env.savedAt, 12345); + // loadSync finds it too (discovered by loadAll). + const direct = s.loadSync("flat-1"); + assert.ok(direct); + assert.deepEqual(direct.payload, { label: "adopted", count: 9 }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); test("legacy adoption is skipped when the hook returns null", async () => { - const dir = tmpDir(); - try { - writeFileSync( - path.join(dir, flatFileNameFor("foreign-1")), - JSON.stringify({ totally: "unrelated", shape: true }), - "utf8", - ); - const s = store(dir, { legacy: () => null }); - assert.equal((await s.loadAll()).size, 0); - assert.equal(s.loadSync("foreign-1"), null); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + writeFileSync( + path.join(dir, flatFileNameFor("foreign-1")), + JSON.stringify({ totally: "unrelated", shape: true }), + "utf8", + ); + const s = store(dir, { legacy: () => null }); + assert.equal((await s.loadAll()).size, 0); + assert.equal(s.loadSync("foreign-1"), null); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); test("adopted legacy record re-persists as an envelope on the next write", async () => { - const dir = tmpDir(); - try { - writeFileSync( - path.join(dir, flatFileNameFor("migrate-1")), - JSON.stringify({ version: 3, savedAt: 1, id: "migrate-1", label: "old", count: 0 }), - "utf8", - ); - const s = store(dir, { - version: 4, - legacy: (parsed) => { - const p = parsed as { id?: string; label?: string; count?: number }; - if (typeof p.id !== "string" || typeof p.label !== "string" || typeof p.count !== "number") return null; - return { id: p.id, payload: { label: p.label, count: p.count } }; - }, - }); - await s.loadAll(); - // Dirty write after adoption: the file on disk becomes an envelope - // stamped with the store's current version. - await s.writeNow("migrate-1", () => ({ label: "new", count: 5 })); - const raw = JSON.parse(readFileSync(path.join(dir, flatFileNameFor("migrate-1")), "utf8")) as { - version: number; - payload: { label: string }; - }; - assert.equal(raw.version, 4); - assert.deepEqual(raw.payload, { label: "new", count: 5 }); - // And a FRESH store (no legacy hook) can read it — migration done. - const plain = store(dir); - assert.deepEqual(plain.loadSync("migrate-1")?.payload, { label: "new", count: 5 }); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + writeFileSync( + path.join(dir, flatFileNameFor("migrate-1")), + JSON.stringify({ + version: 3, + savedAt: 1, + id: "migrate-1", + label: "old", + count: 0, + }), + "utf8", + ); + const s = store(dir, { + version: 4, + legacy: (parsed) => { + const p = parsed as { id?: string; label?: string; count?: number }; + if ( + typeof p.id !== "string" || + typeof p.label !== "string" || + typeof p.count !== "number" + ) + return null; + return { id: p.id, payload: { label: p.label, count: p.count } }; + }, + }); + await s.loadAll(); + // Dirty write after adoption: the file on disk becomes an envelope + // stamped with the store's current version. + await s.writeNow("migrate-1", () => ({ label: "new", count: 5 })); + const raw = JSON.parse( + readFileSync(path.join(dir, flatFileNameFor("migrate-1")), "utf8"), + ) as { + version: number; + payload: { label: string }; + }; + assert.equal(raw.version, 4); + assert.deepEqual(raw.payload, { label: "new", count: 5 }); + // And a FRESH store (no legacy hook) can read it — migration done. + const plain = store(dir); + assert.deepEqual(plain.loadSync("migrate-1")?.payload, { + label: "new", + count: 5, + }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); test("loadSync hint probes a namespaced path without loadAll", async () => { - const dir = tmpDir(); - try { - const s = store(dir, { relPath: (id, payload) => path.join("proto", payload.label, `${id}.json`) }); - await s.writeNow("hint-1", () => ({ label: "hostA", count: 1 })); - // A second store instance has NOT discovered the namespaced file — - // without the hint, loadSync misses (flat fallback only). - const s2 = store(dir, { relPath: (id, payload) => path.join("proto", payload.label, `${id}.json`) }); - assert.equal(s2.loadSync("hint-1"), null); + const dir = tmpDir(); + try { + const s = store(dir, { + relPath: (id, payload) => path.join("proto", payload.label, `${id}.json`), + }); + await s.writeNow("hint-1", () => ({ label: "hostA", count: 1 })); + // A second store instance has NOT discovered the namespaced file — + // without the hint, loadSync misses (flat fallback only). + const s2 = store(dir, { + relPath: (id, payload) => path.join("proto", payload.label, `${id}.json`), + }); + assert.equal(s2.loadSync("hint-1"), null); // With the relative-path hint, the namespaced file resolves. - const hit = s2.loadSync("hint-1", path.join("proto", "hostA", "hint-1.json")); + const hit = s2.loadSync( + "hint-1", + path.join("proto", "hostA", "hint-1.json"), + ); assert.ok(hit); assert.deepEqual(hit.payload, { label: "hostA", count: 1 }); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); function spillNameFor(id: string): string { - return flatFileNameFor(id).replace(/\.json$/, ".fb.json"); + return flatFileNameFor(id).replace(/\.json$/, ".fb.json"); } function eperm(): NodeJS.ErrnoException { - const e = new Error("EPERM: operation not permitted, rename") as NodeJS.ErrnoException; - e.code = "EPERM"; - return e; + const e = new Error( + "EPERM: operation not permitted, rename", + ) as NodeJS.ErrnoException; + e.code = "EPERM"; + return e; } test("rename failure spills to a side file instead of dropping data", async (t) => { - const dir = tmpDir(); + const dir = tmpDir(); + try { + const s = store(dir, { retryAttempts: 2, retryBaseMs: 1, retryMaxMs: 2 }); + t.mock.method(fs, "renameSync", () => { + throw eperm(); + }); + let threw = false; try { - const s = store(dir, { retryAttempts: 2, retryBaseMs: 1, retryMaxMs: 2 }); - t.mock.method(fs, "renameSync", () => { - throw eperm(); - }); - let threw = false; - try { - await s.writeNow("sid-spill", () => ({ label: "kept", count: 1 })); - } catch { - threw = true; - } - assert.equal(threw, true); - const spillFile = path.join(dir, spillNameFor("sid-spill")); - assert.ok(existsSync(spillFile), "spill file exists"); - const env = JSON.parse(readFileSync(spillFile, "utf8")) as { id: string; payload: Payload }; - assert.equal(env.id, "sid-spill"); - assert.deepEqual(env.payload, { label: "kept", count: 1 }); - } finally { - t.mock.restoreAll(); - rmSync(dir, { recursive: true, force: true }); + await s.writeNow("sid-spill", () => ({ label: "kept", count: 1 })); + } catch { + threw = true; } + assert.equal(threw, true); + const spillFile = path.join(dir, spillNameFor("sid-spill")); + assert.ok(existsSync(spillFile), "spill file exists"); + const env = JSON.parse(readFileSync(spillFile, "utf8")) as { + id: string; + payload: Payload; + }; + assert.equal(env.id, "sid-spill"); + assert.deepEqual(env.payload, { label: "kept", count: 1 }); + } finally { + t.mock.restoreAll(); + rmSync(dir, { recursive: true, force: true }); + } }); test("loadAll discovers a spilled record and reconciles against the canonical by savedAt", async (t) => { - const dir = tmpDir(); - try { - const s = store(dir, { retryAttempts: 2, retryBaseMs: 1, retryMaxMs: 2 }); - await s.writeNow("sid-r", () => ({ label: "old", count: 1 })); - await new Promise((r) => setTimeout(r, 5)); - t.mock.method(fs, "renameSync", () => { - throw eperm(); - }); - await s.writeNow("sid-r", () => ({ label: "new", count: 2 })).catch(() => {}); - t.mock.restoreAll(); - const next = store(dir); - const all = await next.loadAll(); - assert.deepEqual(all.get("sid-r")?.payload, { label: "new", count: 2 }); - assert.deepEqual(next.loadSync("sid-r")?.payload, { label: "new", count: 2 }); - } finally { - t.mock.restoreAll(); - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const s = store(dir, { retryAttempts: 2, retryBaseMs: 1, retryMaxMs: 2 }); + await s.writeNow("sid-r", () => ({ label: "old", count: 1 })); + await new Promise((r) => setTimeout(r, 5)); + t.mock.method(fs, "renameSync", () => { + throw eperm(); + }); + await s + .writeNow("sid-r", () => ({ label: "new", count: 2 })) + .catch(() => {}); + t.mock.restoreAll(); + const next = store(dir); + const all = await next.loadAll(); + assert.deepEqual(all.get("sid-r")?.payload, { label: "new", count: 2 }); + assert.deepEqual(next.loadSync("sid-r")?.payload, { + label: "new", + count: 2, + }); + } finally { + t.mock.restoreAll(); + rmSync(dir, { recursive: true, force: true }); + } }); test("a canonical newer than the spill wins on load, and the stale spill is cleaned on success", async () => { - const dir = tmpDir(); - try { - const spillFile = path.join(dir, spillNameFor("sid-c")); - writeFileSync(spillFile, JSON.stringify({ version: 1, savedAt: 1, id: "sid-c", payload: { label: "stale", count: 0 } }), "utf8"); - const s = store(dir); - await s.writeNow("sid-c", () => ({ label: "fresh", count: 5 })); - assert.equal(existsSync(spillFile), false, "stale spill removed after canonical success"); - assert.deepEqual(s.loadSync("sid-c")?.payload, { label: "fresh", count: 5 }); - const next = store(dir); - assert.deepEqual((await next.loadAll()).get("sid-c")?.payload, { label: "fresh", count: 5 }); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const spillFile = path.join(dir, spillNameFor("sid-c")); + writeFileSync( + spillFile, + JSON.stringify({ + version: 1, + savedAt: 1, + id: "sid-c", + payload: { label: "stale", count: 0 }, + }), + "utf8", + ); + const s = store(dir); + await s.writeNow("sid-c", () => ({ label: "fresh", count: 5 })); + assert.equal( + existsSync(spillFile), + false, + "stale spill removed after canonical success", + ); + assert.deepEqual(s.loadSync("sid-c")?.payload, { + label: "fresh", + count: 5, + }); + const next = store(dir); + assert.deepEqual((await next.loadAll()).get("sid-c")?.payload, { + label: "fresh", + count: 5, + }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); test("write failures alert on the first and power-of-two counts, not every write", async (t) => { - const dir = tmpDir(); - try { - const logs: { level: string; msg: string }[] = []; - const s = store(dir, { - retryAttempts: 2, - retryBaseMs: 1, - retryMaxMs: 2, - log: (level, msg) => logs.push({ level, msg }), - }); - t.mock.method(fs, "renameSync", () => { - throw eperm(); - }); - for (let i = 0; i < 5; i++) { - await s.writeNow("sid-f", () => ({ label: "x", count: i })).catch(() => {}); - } - t.mock.restoreAll(); - const failLogs = logs.filter((l) => l.msg.includes("write failed for sid-f")); - assert.equal(failLogs.length, 3); - assert.equal(failLogs[0].level, "error"); - assert.ok(failLogs[0].msg.includes("total 1x")); - assert.ok(failLogs[0].msg.includes("data spilled to")); - assert.equal(failLogs[1].level, "warn"); - assert.ok(failLogs[1].msg.includes("total 2x")); - assert.equal(failLogs[2].level, "warn"); - assert.ok(failLogs[2].msg.includes("total 4x")); - } finally { - t.mock.restoreAll(); - rmSync(dir, { recursive: true, force: true }); + const dir = tmpDir(); + try { + const logs: { level: string; msg: string }[] = []; + const s = store(dir, { + retryAttempts: 2, + retryBaseMs: 1, + retryMaxMs: 2, + log: (level, msg) => logs.push({ level, msg }), + }); + t.mock.method(fs, "renameSync", () => { + throw eperm(); + }); + for (let i = 0; i < 5; i++) { + await s + .writeNow("sid-f", () => ({ label: "x", count: i })) + .catch(() => {}); } + t.mock.restoreAll(); + const failLogs = logs.filter((l) => + l.msg.includes("write failed for sid-f"), + ); + assert.equal(failLogs.length, 3); + assert.equal(failLogs[0].level, "error"); + assert.ok(failLogs[0].msg.includes("total 1x")); + assert.ok(failLogs[0].msg.includes("data spilled to")); + assert.equal(failLogs[1].level, "warn"); + assert.ok(failLogs[1].msg.includes("total 2x")); + assert.equal(failLogs[2].level, "warn"); + assert.ok(failLogs[2].msg.includes("total 4x")); + } finally { + t.mock.restoreAll(); + rmSync(dir, { recursive: true, force: true }); + } }); test("flushSync spills to a side file when the rename keeps failing", (t) => { - const dir = tmpDir(); - try { - const s = store(dir, { retryAttempts: 2, retryBaseMs: 1, retryMaxMs: 2 }); - t.mock.method(fs, "renameSync", () => { - throw eperm(); - }); - const ok = s.flushSync("sid-fs", () => ({ label: "sync", count: 9 })); - assert.equal(ok, true); - const spillFile = path.join(dir, spillNameFor("sid-fs")); - assert.ok(existsSync(spillFile), "spill file exists"); - const env = JSON.parse(readFileSync(spillFile, "utf8")) as { id: string; payload: Payload }; - assert.equal(env.id, "sid-fs"); - assert.deepEqual(env.payload, { label: "sync", count: 9 }); - } finally { - t.mock.restoreAll(); - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const s = store(dir, { retryAttempts: 2, retryBaseMs: 1, retryMaxMs: 2 }); + t.mock.method(fs, "renameSync", () => { + throw eperm(); + }); + const ok = s.flushSync("sid-fs", () => ({ label: "sync", count: 9 })); + assert.equal(ok, true); + const spillFile = path.join(dir, spillNameFor("sid-fs")); + assert.ok(existsSync(spillFile), "spill file exists"); + const env = JSON.parse(readFileSync(spillFile, "utf8")) as { + id: string; + payload: Payload; + }; + assert.equal(env.id, "sid-fs"); + assert.deepEqual(env.payload, { label: "sync", count: 9 }); + } finally { + t.mock.restoreAll(); + rmSync(dir, { recursive: true, force: true }); + } }); test("a successful write clears the failure counter so alerting restarts", async (t) => { - const dir = tmpDir(); - try { - const logs: { level: string; msg: string }[] = []; - const s = store(dir, { - retryAttempts: 2, - retryBaseMs: 1, - retryMaxMs: 2, - log: (level, msg) => logs.push({ level, msg }), - }); - t.mock.method(fs, "renameSync", () => { - throw eperm(); - }); - await s.writeNow("sid-reset", () => ({ label: "a", count: 0 })).catch(() => {}); - t.mock.restoreAll(); - await s.writeNow("sid-reset", () => ({ label: "b", count: 1 })); - t.mock.method(fs, "renameSync", () => { - throw eperm(); - }); - await s.writeNow("sid-reset", () => ({ label: "c", count: 2 })).catch(() => {}); - t.mock.restoreAll(); - const failLogs = logs.filter((l) => l.msg.includes("write failed for sid-reset")); - assert.equal(failLogs.length, 2); - assert.ok(failLogs[0].msg.includes("total 1x")); - assert.ok(failLogs[1].msg.includes("total 1x")); - } finally { - t.mock.restoreAll(); - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const logs: { level: string; msg: string }[] = []; + const s = store(dir, { + retryAttempts: 2, + retryBaseMs: 1, + retryMaxMs: 2, + log: (level, msg) => logs.push({ level, msg }), + }); + t.mock.method(fs, "renameSync", () => { + throw eperm(); + }); + await s + .writeNow("sid-reset", () => ({ label: "a", count: 0 })) + .catch(() => {}); + t.mock.restoreAll(); + await s.writeNow("sid-reset", () => ({ label: "b", count: 1 })); + t.mock.method(fs, "renameSync", () => { + throw eperm(); + }); + await s + .writeNow("sid-reset", () => ({ label: "c", count: 2 })) + .catch(() => {}); + t.mock.restoreAll(); + const failLogs = logs.filter((l) => + l.msg.includes("write failed for sid-reset"), + ); + assert.equal(failLogs.length, 2); + assert.ok(failLogs[0].msg.includes("total 1x")); + assert.ok(failLogs[1].msg.includes("total 1x")); + } finally { + t.mock.restoreAll(); + rmSync(dir, { recursive: true, force: true }); + } }); test("flushSync retries use a fresh temp name per attempt (no tombstoned reuse)", (t) => { - const dir = tmpDir(); - try { - const s = store(dir, { retryAttempts: 3, retryBaseMs: 1, retryMaxMs: 1 }); - const written: string[] = []; - t.mock.method(fs, "writeFileSync", ((p: string, _data: string) => { - written.push(p); - }) as typeof fs.writeFileSync); - t.mock.method(fs, "renameSync", (() => { - throw eperm(); - }) as typeof fs.renameSync); - const ok = s.flushSync("sid-tmp", () => ({ label: "x", count: 1 })); - // The write spilled, so the caller still sees success. - assert.equal(ok, true); - const tmps = written.filter((p) => path.basename(p).startsWith(".tmp-")); - assert.equal(tmps.length, 3, "one temp write per attempt"); - assert.equal(new Set(tmps).size, 3, "each attempt used a distinct temp name"); - assert.ok(written.some((p) => p.endsWith(".fb.json")), "spill file written"); - } finally { - t.mock.restoreAll(); - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const s = store(dir, { retryAttempts: 3, retryBaseMs: 1, retryMaxMs: 1 }); + const written: string[] = []; + t.mock.method(fs, "writeFileSync", ((p: string, _data: string) => { + written.push(p); + }) as typeof fs.writeFileSync); + t.mock.method(fs, "renameSync", (() => { + throw eperm(); + }) as typeof fs.renameSync); + const ok = s.flushSync("sid-tmp", () => ({ label: "x", count: 1 })); + // The write spilled, so the caller still sees success. + assert.equal(ok, true); + const tmps = written.filter((p) => path.basename(p).startsWith(".tmp-")); + assert.equal(tmps.length, 3, "one temp write per attempt"); + assert.equal( + new Set(tmps).size, + 3, + "each attempt used a distinct temp name", + ); + assert.ok( + written.some((p) => p.endsWith(".fb.json")), + "spill file written", + ); + } finally { + t.mock.restoreAll(); + rmSync(dir, { recursive: true, force: true }); + } }); function enoent(): NodeJS.ErrnoException { - const e = new Error("ENOENT: no such file or directory, rename") as NodeJS.ErrnoException; - e.code = "ENOENT"; - return e; + const e = new Error( + "ENOENT: no such file or directory, rename", + ) as NodeJS.ErrnoException; + e.code = "ENOENT"; + return e; } test("a temp file vanishing before rename (ENOENT) is healed by whole-cycle retry", async (t) => { - const dir = tmpDir(); - try { - const s = store(dir, { retryAttempts: 3, retryBaseMs: 1, retryMaxMs: 1 }); - const realRenameSync = fs.renameSync.bind(fs); - let calls = 0; - t.mock.method(fs, "renameSync", ((src: string, dest: string) => { - calls++; - if (calls === 1) throw enoent(); - realRenameSync(src, dest); - }) as typeof fs.renameSync); - await s.writeNow("sid-sweep", () => ({ label: "healed", count: 3 })); - const canonical = path.join(dir, flatFileNameFor("sid-sweep")); - const env = JSON.parse(readFileSync(canonical, "utf8")) as { id: string; payload: Payload }; - assert.equal(env.id, "sid-sweep"); - assert.deepEqual(env.payload, { label: "healed", count: 3 }); - assert.equal(existsSync(path.join(dir, spillNameFor("sid-sweep"))), false, "no spill — canonical healed"); - assert.ok(calls >= 2, "rename was retried"); - } finally { - t.mock.restoreAll(); - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const s = store(dir, { retryAttempts: 3, retryBaseMs: 1, retryMaxMs: 1 }); + const realRenameSync = fs.renameSync.bind(fs); + let calls = 0; + t.mock.method(fs, "renameSync", ((src: string, dest: string) => { + calls++; + if (calls === 1) throw enoent(); + realRenameSync(src, dest); + }) as typeof fs.renameSync); + await s.writeNow("sid-sweep", () => ({ label: "healed", count: 3 })); + const canonical = path.join(dir, flatFileNameFor("sid-sweep")); + const env = JSON.parse(readFileSync(canonical, "utf8")) as { + id: string; + payload: Payload; + }; + assert.equal(env.id, "sid-sweep"); + assert.deepEqual(env.payload, { label: "healed", count: 3 }); + assert.equal( + existsSync(path.join(dir, spillNameFor("sid-sweep"))), + false, + "no spill — canonical healed", + ); + assert.ok(calls >= 2, "rename was retried"); + } finally { + t.mock.restoreAll(); + rmSync(dir, { recursive: true, force: true }); + } }); test("the target directory vanishing (writeFile ENOENT) is recreated on retry", async (t) => { - const dir = tmpDir(); - try { - const s = store(dir, { retryAttempts: 3, retryBaseMs: 1, retryMaxMs: 1 }); - const realWriteFile = fsp.writeFile.bind(fsp); - let calls = 0; - t.mock.method(fsp, "writeFile", (async (p: string, data: string) => { - calls++; - if (calls === 1) throw enoent(); - await realWriteFile(p, data); - }) as typeof fsp.writeFile); - await s.writeNow("sid-gone-dir", () => ({ label: "recreated", count: 7 })); - const canonical = path.join(dir, flatFileNameFor("sid-gone-dir")); - const env = JSON.parse(readFileSync(canonical, "utf8")) as { payload: Payload }; - assert.deepEqual(env.payload, { label: "recreated", count: 7 }); - } finally { - t.mock.restoreAll(); - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const s = store(dir, { retryAttempts: 3, retryBaseMs: 1, retryMaxMs: 1 }); + const realWriteFile = fsp.writeFile.bind(fsp); + let calls = 0; + t.mock.method(fsp, "writeFile", (async (p: string, data: string) => { + calls++; + if (calls === 1) throw enoent(); + await realWriteFile(p, data); + }) as typeof fsp.writeFile); + await s.writeNow("sid-gone-dir", () => ({ label: "recreated", count: 7 })); + const canonical = path.join(dir, flatFileNameFor("sid-gone-dir")); + const env = JSON.parse(readFileSync(canonical, "utf8")) as { + payload: Payload; + }; + assert.deepEqual(env.payload, { label: "recreated", count: 7 }); + } finally { + t.mock.restoreAll(); + rmSync(dir, { recursive: true, force: true }); + } }); test("persistent ENOENT still spills so data is never dropped", async (t) => { - const dir = tmpDir(); + const dir = tmpDir(); + try { + const s = store(dir, { retryAttempts: 2, retryBaseMs: 1, retryMaxMs: 2 }); + t.mock.method(fs, "renameSync", (() => { + throw enoent(); + }) as typeof fsp.rename); + let threw = false; try { - const s = store(dir, { retryAttempts: 2, retryBaseMs: 1, retryMaxMs: 2 }); - t.mock.method(fs, "renameSync", (() => { - throw enoent(); - }) as typeof fsp.rename); - let threw = false; - try { - await s.writeNow("sid-enoent", () => ({ label: "kept", count: 1 })); - } catch { - threw = true; - } - assert.equal(threw, true); - const spillFile = path.join(dir, spillNameFor("sid-enoent")); - const env = JSON.parse(readFileSync(spillFile, "utf8")) as { id: string; payload: Payload }; - assert.equal(env.id, "sid-enoent"); - assert.deepEqual(env.payload, { label: "kept", count: 1 }); - } finally { - t.mock.restoreAll(); - rmSync(dir, { recursive: true, force: true }); + await s.writeNow("sid-enoent", () => ({ label: "kept", count: 1 })); + } catch { + threw = true; } + assert.equal(threw, true); + const spillFile = path.join(dir, spillNameFor("sid-enoent")); + const env = JSON.parse(readFileSync(spillFile, "utf8")) as { + id: string; + payload: Payload; + }; + assert.equal(env.id, "sid-enoent"); + assert.deepEqual(env.payload, { label: "kept", count: 1 }); + } finally { + t.mock.restoreAll(); + rmSync(dir, { recursive: true, force: true }); + } }); test("flushSync heals an ENOENT sweep the same way", (t) => { - const dir = tmpDir(); - try { - const s = store(dir, { retryAttempts: 3, retryBaseMs: 1, retryMaxMs: 1 }); - const realRenameSync = fs.renameSync.bind(fs); - let calls = 0; - t.mock.method(fs, "renameSync", ((src: string, dest: string) => { - calls++; - if (calls === 1) throw enoent(); - return realRenameSync(src, dest); - }) as typeof fs.renameSync); - const ok = s.flushSync("sid-fs-sweep", () => ({ label: "sync", count: 4 })); - assert.equal(ok, true); - const canonical = path.join(dir, flatFileNameFor("sid-fs-sweep")); - const env = JSON.parse(readFileSync(canonical, "utf8")) as { payload: Payload }; - assert.deepEqual(env.payload, { label: "sync", count: 4 }); - assert.ok(calls >= 2, "renameSync was retried"); - } finally { - t.mock.restoreAll(); - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const s = store(dir, { retryAttempts: 3, retryBaseMs: 1, retryMaxMs: 1 }); + const realRenameSync = fs.renameSync.bind(fs); + let calls = 0; + t.mock.method(fs, "renameSync", ((src: string, dest: string) => { + calls++; + if (calls === 1) throw enoent(); + return realRenameSync(src, dest); + }) as typeof fs.renameSync); + const ok = s.flushSync("sid-fs-sweep", () => ({ label: "sync", count: 4 })); + assert.equal(ok, true); + const canonical = path.join(dir, flatFileNameFor("sid-fs-sweep")); + const env = JSON.parse(readFileSync(canonical, "utf8")) as { + payload: Payload; + }; + assert.deepEqual(env.payload, { label: "sync", count: 4 }); + assert.ok(calls >= 2, "renameSync was retried"); + } finally { + t.mock.restoreAll(); + rmSync(dir, { recursive: true, force: true }); + } }); function xorCodec(): StateStoreCodec { - const x = (buf: Buffer): Buffer => { - const out = Buffer.alloc(buf.length); - for (let i = 0; i < buf.length; i++) out[i] = buf[i] ^ 0x5a; - return out; - }; - return { encode: (data) => x(Buffer.from(data, "utf8")), decode: (buf) => x(buf).toString("utf8") }; + const x = (buf: Buffer): Buffer => { + const out = Buffer.alloc(buf.length); + for (let i = 0; i < buf.length; i++) out[i] = buf[i] ^ 0x5a; + return out; + }; + return { + encode: (data) => x(Buffer.from(data, "utf8")), + decode: (buf) => x(buf).toString("utf8"), + }; } const ENC_MAGIC = Buffer.from("ENC1", "utf8"); function magicCodec(): StateStoreCodec { - const inner = xorCodec(); - return { - encode: (data) => Buffer.concat([ENC_MAGIC, inner.encode(data)]), - decode: (buf) => (buf.subarray(0, 4).equals(ENC_MAGIC) ? inner.decode(buf.subarray(4)) : buf.toString("utf8")), - }; + const inner = xorCodec(); + return { + encode: (data) => Buffer.concat([ENC_MAGIC, inner.encode(data)]), + decode: (buf) => + buf.subarray(0, 4).equals(ENC_MAGIC) + ? inner.decode(buf.subarray(4)) + : buf.toString("utf8"), + }; } test("codec round-trips records and keeps files opaque on disk", async () => { - const dir = tmpDir(); - try { - const s = store(dir, { codec: xorCodec() }); - await s.writeNow("sid-enc", () => ({ label: "secret", count: 7 })); - const raw = readFileSync(path.join(dir, flatFileNameFor("sid-enc"))); - assert.ok(!raw.includes(Buffer.from('"label"')), "on-disk bytes are not plaintext JSON"); - const hit = s.loadSync("sid-enc"); - assert.ok(hit); - assert.deepEqual(hit.payload, { label: "secret", count: 7 }); - const all = await s.loadAll(); - assert.deepEqual(all.get("sid-enc")?.payload, { label: "secret", count: 7 }); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const s = store(dir, { codec: xorCodec() }); + await s.writeNow("sid-enc", () => ({ label: "secret", count: 7 })); + const raw = readFileSync(path.join(dir, flatFileNameFor("sid-enc"))); + assert.ok( + !raw.includes(Buffer.from('"label"')), + "on-disk bytes are not plaintext JSON", + ); + const hit = s.loadSync("sid-enc"); + assert.ok(hit); + assert.deepEqual(hit.payload, { label: "secret", count: 7 }); + const all = await s.loadAll(); + assert.deepEqual(all.get("sid-enc")?.payload, { + label: "secret", + count: 7, + }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); test("codec applies to debounced scheduleSave writes", async () => { - const dir = tmpDir(); - try { - const s = store(dir, { codec: xorCodec(), debounceMs: 10 }); - s.scheduleSave("sid-deb", () => ({ label: "deb", count: 3 })); - await s.flushAll(); - const all = await s.loadAll(); - assert.deepEqual(all.get("sid-deb")?.payload, { label: "deb", count: 3 }); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const s = store(dir, { codec: xorCodec(), debounceMs: 10 }); + s.scheduleSave("sid-deb", () => ({ label: "deb", count: 3 })); + await s.flushAll(); + const all = await s.loadAll(); + assert.deepEqual(all.get("sid-deb")?.payload, { label: "deb", count: 3 }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); test("codec applies to spill writes as well", async (t) => { - const dir = tmpDir(); + const dir = tmpDir(); + try { + const s = store(dir, { + codec: xorCodec(), + retryAttempts: 2, + retryBaseMs: 1, + retryMaxMs: 2, + }); + t.mock.method(fs, "renameSync", () => { + throw eperm(); + }); + let threw = false; try { - const s = store(dir, { codec: xorCodec(), retryAttempts: 2, retryBaseMs: 1, retryMaxMs: 2 }); - t.mock.method(fs, "renameSync", () => { - throw eperm(); - }); - let threw = false; - try { - await s.writeNow("sid-espill", () => ({ label: "kept", count: 1 })); - } catch { - threw = true; - } - assert.equal(threw, true); - const raw = readFileSync(path.join(dir, spillNameFor("sid-espill"))); - assert.ok(!raw.includes(Buffer.from('"label"')), "spill is encoded"); - const env = JSON.parse(xorCodec().decode(raw)) as { id: string; payload: Payload }; - assert.equal(env.id, "sid-espill"); - assert.deepEqual(env.payload, { label: "kept", count: 1 }); - } finally { - t.mock.restoreAll(); - rmSync(dir, { recursive: true, force: true }); + await s.writeNow("sid-espill", () => ({ label: "kept", count: 1 })); + } catch { + threw = true; } + assert.equal(threw, true); + const raw = readFileSync(path.join(dir, spillNameFor("sid-espill"))); + assert.ok(!raw.includes(Buffer.from('"label"')), "spill is encoded"); + const env = JSON.parse(xorCodec().decode(raw)) as { + id: string; + payload: Payload; + }; + assert.equal(env.id, "sid-espill"); + assert.deepEqual(env.payload, { label: "kept", count: 1 }); + } finally { + t.mock.restoreAll(); + rmSync(dir, { recursive: true, force: true }); + } }); test("flushSync spill is encoded too", (t) => { - const dir = tmpDir(); - try { - const s = store(dir, { codec: xorCodec(), retryAttempts: 2, retryBaseMs: 1, retryMaxMs: 2 }); - t.mock.method(fs, "renameSync", () => { - throw eperm(); - }); - const ok = s.flushSync("sid-fsenc", () => ({ label: "sync", count: 9 })); - assert.equal(ok, true); - const raw = readFileSync(path.join(dir, spillNameFor("sid-fsenc"))); - assert.ok(!raw.includes(Buffer.from('"label"')), "spill is encoded"); - const env = JSON.parse(xorCodec().decode(raw)) as { payload: Payload }; - assert.deepEqual(env.payload, { label: "sync", count: 9 }); - } finally { - t.mock.restoreAll(); - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const s = store(dir, { + codec: xorCodec(), + retryAttempts: 2, + retryBaseMs: 1, + retryMaxMs: 2, + }); + t.mock.method(fs, "renameSync", () => { + throw eperm(); + }); + const ok = s.flushSync("sid-fsenc", () => ({ label: "sync", count: 9 })); + assert.equal(ok, true); + const raw = readFileSync(path.join(dir, spillNameFor("sid-fsenc"))); + assert.ok(!raw.includes(Buffer.from('"label"')), "spill is encoded"); + const env = JSON.parse(xorCodec().decode(raw)) as { payload: Payload }; + assert.deepEqual(env.payload, { label: "sync", count: 9 }); + } finally { + t.mock.restoreAll(); + rmSync(dir, { recursive: true, force: true }); + } }); test("codec decode failure is treated as a corrupt file, not fatal", async () => { - const dir = tmpDir(); - try { - const plain = store(dir); - await plain.writeNow("sid-corrupt", () => ({ label: "x", count: 1 })); - const logs: Array<{ level: string; msg: string }> = []; - const strict = store(dir, { - codec: { encode: (d) => d, decode: () => { throw new Error("auth failed"); } }, - log: (level, msg) => logs.push({ level, msg }), - }); - assert.equal(strict.loadSync("sid-corrupt"), null); - const all = await strict.loadAll(); - assert.equal(all.size, 0); - assert.ok(logs.some((l) => l.level === "warn" && l.msg.includes("corrupt")), "logged as corrupt"); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const plain = store(dir); + await plain.writeNow("sid-corrupt", () => ({ label: "x", count: 1 })); + const logs: Array<{ level: string; msg: string }> = []; + const strict = store(dir, { + codec: { + encode: (d) => d, + decode: () => { + throw new Error("auth failed"); + }, + }, + log: (level, msg) => logs.push({ level, msg }), + }); + assert.equal(strict.loadSync("sid-corrupt"), null); + const all = await strict.loadAll(); + assert.equal(all.size, 0); + assert.ok( + logs.some((l) => l.level === "warn" && l.msg.includes("corrupt")), + "logged as corrupt", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); test("mixed tree: unencoded legacy files and encoded files both load", async () => { - const dir = tmpDir(); - try { - const plain = store(dir); - await plain.writeNow("sid-old", () => ({ label: "legacy", count: 1 })); - const enc = store(dir, { codec: magicCodec() }); - await enc.writeNow("sid-new", () => ({ label: "encoded", count: 2 })); - const fresh = store(dir, { codec: magicCodec() }); - const all = await fresh.loadAll(); - assert.deepEqual(all.get("sid-old")?.payload, { label: "legacy", count: 1 }); - assert.deepEqual(all.get("sid-new")?.payload, { label: "encoded", count: 2 }); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const plain = store(dir); + await plain.writeNow("sid-old", () => ({ label: "legacy", count: 1 })); + const enc = store(dir, { codec: magicCodec() }); + await enc.writeNow("sid-new", () => ({ label: "encoded", count: 2 })); + const fresh = store(dir, { codec: magicCodec() }); + const all = await fresh.loadAll(); + assert.deepEqual(all.get("sid-old")?.payload, { + label: "legacy", + count: 1, + }); + assert.deepEqual(all.get("sid-new")?.payload, { + label: "encoded", + count: 2, + }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); test("#373: a flushSync landing while an async write is mid-flight cannot be clobbered by the stale rename", async () => { - const dir = tmpDir(); - try { - const s = store(dir); - // Reentrant interleave: while writeInner is between its build() and - // its commit, a flushSync commits a NEWER payload. Deterministic — - // the reentrant call runs inside build() itself. - const inFlight = s.writeNow("sid-fence", () => { - const snap: Payload = { label: "stale", count: 1 }; - const ok = s.flushSync("sid-fence", () => ({ label: "fresh", count: 2 })); - assert.equal(ok, true, "reentrant flushSync must succeed"); - return snap; - }); - await inFlight; - const loaded = s.loadSync("sid-fence"); - assert.ok(loaded, "record must exist"); - assert.deepEqual( - loaded.payload, - { label: "fresh", count: 2 }, - "disk must hold the flushSync payload, not the stale in-flight snapshot", - ); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const s = store(dir); + // Reentrant interleave: while writeInner is between its build() and + // its commit, a flushSync commits a NEWER payload. Deterministic — + // the reentrant call runs inside build() itself. + const inFlight = s.writeNow("sid-fence", () => { + const snap: Payload = { label: "stale", count: 1 }; + const ok = s.flushSync("sid-fence", () => ({ label: "fresh", count: 2 })); + assert.equal(ok, true, "reentrant flushSync must succeed"); + return snap; + }); + await inFlight; + const loaded = s.loadSync("sid-fence"); + assert.ok(loaded, "record must exist"); + assert.deepEqual( + loaded.payload, + { label: "fresh", count: 2 }, + "disk must hold the flushSync payload, not the stale in-flight snapshot", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); test("#373: fenced async writes that lose the race settle quietly and never throw into the chain", async () => { - const dir = tmpDir(); - try { - const s = store(dir); - let observed: Payload | null = null; - const p1 = s.writeNow("sid-chain", () => { - const snap: Payload = { label: "older", count: 1 }; - s.flushSync("sid-chain", () => ({ label: "newer", count: 5 })); - return snap; - }); - await p1; - // A queued write built AFTER the flushSync commits is NOT fenced — - // it snapshots later state and must land. - await s.writeNow("sid-chain", () => { - observed = { label: "latest", count: 9 }; - return observed; - }); - const loaded = s.loadSync("sid-chain"); - assert.ok(loaded); - assert.deepEqual(loaded.payload, { label: "latest", count: 9 }); - // The fenced older write left no tmp orphans behind. - const leftovers = readdirSync(dir).filter((f) => f.startsWith(".tmp-")); - assert.deepEqual(leftovers, []); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const s = store(dir); + let observed: Payload | null = null; + const p1 = s.writeNow("sid-chain", () => { + const snap: Payload = { label: "older", count: 1 }; + s.flushSync("sid-chain", () => ({ label: "newer", count: 5 })); + return snap; + }); + await p1; + // A queued write built AFTER the flushSync commits is NOT fenced — + // it snapshots later state and must land. + await s.writeNow("sid-chain", () => { + observed = { label: "latest", count: 9 }; + return observed; + }); + const loaded = s.loadSync("sid-chain"); + assert.ok(loaded); + assert.deepEqual(loaded.payload, { label: "latest", count: 9 }); + // The fenced older write left no tmp orphans behind. + const leftovers = readdirSync(dir).filter((f) => f.startsWith(".tmp-")); + assert.deepEqual(leftovers, []); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); test("#373: flushSync after a fully settled async write simply supersedes it", async () => { - const dir = tmpDir(); - try { - const s = store(dir); - await s.writeNow("sid-order", () => ({ label: "async-first", count: 1 })); - const ok = s.flushSync("sid-order", () => ({ label: "sync-second", count: 2 })); - assert.equal(ok, true); - const loaded = s.loadSync("sid-order"); - assert.ok(loaded); - assert.deepEqual(loaded.payload, { label: "sync-second", count: 2 }); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const dir = tmpDir(); + try { + const s = store(dir); + await s.writeNow("sid-order", () => ({ label: "async-first", count: 1 })); + const ok = s.flushSync("sid-order", () => ({ + label: "sync-second", + count: 2, + })); + assert.equal(ok, true); + const loaded = s.loadSync("sid-order"); + assert.ok(loaded); + assert.deepEqual(loaded.payload, { label: "sync-second", count: 2 }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); diff --git a/tests/prompts.test.ts b/tests/prompts.test.ts index b65dc1b..bbe0c58 100644 --- a/tests/prompts.test.ts +++ b/tests/prompts.test.ts @@ -138,13 +138,19 @@ test("resolvePrompts drops undefined/null/non-string overrides (never clobbers d }); test("resolvePrompts does not throw when only non-string overrides are present", () => { - const p = resolvePrompts({ compressPhilosophy: undefined, howToCompressRules: null as unknown as string }); + const p = resolvePrompts({ + compressPhilosophy: undefined, + howToCompressRules: null as unknown as string, + }); assert.equal(p.compressPhilosophy, COMPRESS_PHILOSOPHY); assert.equal(p.howToCompressRules, HOW_TO_COMPRESS_RULES); }); test("resolvePrompts never aliases or mutates defaultPrompts", () => { - const p = resolvePrompts({ compressPhilosophy: "CUSTOM" }, { acknowledgeRisk: true }); + const p = resolvePrompts( + { compressPhilosophy: "CUSTOM" }, + { acknowledgeRisk: true }, + ); p.compressPhilosophy = "MUTATED-BY-CALLER"; assert.equal(defaultPrompts.compressPhilosophy, COMPRESS_PHILOSOPHY); }); diff --git a/tests/protect-recent.test.ts b/tests/protect-recent.test.ts index 9043f36..c26b351 100644 --- a/tests/protect-recent.test.ts +++ b/tests/protect-recent.test.ts @@ -74,7 +74,12 @@ test("applyCompression fails when the range is ENTIRELY within the recent zone", // is nothing left to compress, so the range must still fail. const result = core.applyCompression({ ranges: [ - { startRef: "m00006", endRef: "m00008", summary: "trying to compress the recent zone", topic: "bad" }, + { + startRef: "m00006", + endRef: "m00008", + summary: "trying to compress the recent zone", + topic: "bad", + }, ], messages, state, @@ -101,22 +106,44 @@ test("applyCompression excludes protected tail and compresses the rest (partial // and surfaced as a warning rather than failing the whole range. const result = core.applyCompression({ ranges: [ - { startRef: "m00005", endRef: "m00007", summary: "overlapping the recent zone", topic: "partial" }, + { + startRef: "m00005", + endRef: "m00007", + summary: "overlapping the recent zone", + topic: "partial", + }, ], messages, state, config: cfg, }); - assert.equal(result.result.blocksCreated, 1, "unprotected head still compressed"); - assert.equal(result.result.errors.length, 0, "no error — overlap is non-fatal"); + assert.equal( + result.result.blocksCreated, + 1, + "unprotected head still compressed", + ); + assert.equal( + result.result.errors.length, + 0, + "no error — overlap is non-fatal", + ); assert.equal(result.result.warnings.length, 1, "a warning is surfaced"); assert.match(result.result.warnings[0]!, /Excluded.*protected.*m0000[67]/i); // The created block must NOT cover the protected messages. const block = result.state.blocks[result.state.blocks.length - 1]!; - assert.ok(!block.effectiveMessageIds.includes("f"), "m00006 raw id not covered"); - assert.ok(!block.effectiveMessageIds.includes("g"), "m00007 raw id not covered"); - assert.ok(block.directMessageIds.includes("e"), "m00005 raw id IS compressed"); + assert.ok( + !block.effectiveMessageIds.includes("f"), + "m00006 raw id not covered", + ); + assert.ok( + !block.effectiveMessageIds.includes("g"), + "m00007 raw id not covered", + ); + assert.ok( + block.directMessageIds.includes("e"), + "m00005 raw id IS compressed", + ); }); test("applyCompression excludes the most recent user message and compresses the rest", () => { @@ -133,14 +160,23 @@ test("applyCompression excludes the most recent user message and compresses the const result = core.applyCompression({ ranges: [ - { startRef: "m00002", endRef: "m00003", summary: "grabbing the last user msg", topic: "partial" }, + { + startRef: "m00002", + endRef: "m00003", + summary: "grabbing the last user msg", + topic: "partial", + }, ], messages, state, config: cfg, }); - assert.equal(result.result.blocksCreated, 1, "assistant msg still compressed"); + assert.equal( + result.result.blocksCreated, + 1, + "assistant msg still compressed", + ); assert.equal(result.result.errors.length, 0); assert.equal(result.result.warnings.length, 1); assert.match(result.result.warnings[0]!, /Excluded.*protected.*m00003/i); @@ -158,16 +194,29 @@ test("applyCompression still allows compressing messages strictly before the rec // m00001..m00003 are well before the last 3 (m00006..m00008) → must succeed. const result = core.applyCompression({ ranges: [ - { startRef: "m00001", endRef: "m00003", summary: "compressing older messages is allowed", topic: "ok" }, + { + startRef: "m00001", + endRef: "m00003", + summary: "compressing older messages is allowed", + topic: "ok", + }, ], messages, state, config: cfg, }); - assert.equal(result.result.blocksCreated, 1, "older range still compressible"); + assert.equal( + result.result.blocksCreated, + 1, + "older range still compressible", + ); assert.equal(result.result.errors.length, 0); - assert.equal(result.result.warnings.length, 0, "no warnings for a clean range"); + assert.equal( + result.result.warnings.length, + 0, + "no warnings for a clean range", + ); }); test("applyCompression fails by default when the whole range is protected (no explicit set)", () => { @@ -185,7 +234,12 @@ test("applyCompression fails by default when the whole range is protected (no ex const result = core.applyCompression({ ranges: [ - { startRef: "m00002", endRef: "m00003", summary: "should be refused by default protection", topic: "bad" }, + { + startRef: "m00002", + endRef: "m00003", + summary: "should be refused by default protection", + topic: "bad", + }, ], messages, state, @@ -193,7 +247,11 @@ test("applyCompression fails by default when the whole range is protected (no ex // intentionally NO protectedMessageIds }); - assert.equal(result.result.blocksCreated, 0, "default protection applies without explicit set"); + assert.equal( + result.result.blocksCreated, + 0, + "default protection applies without explicit set", + ); assert.match(result.result.errors[0]!, /protected/i); }); @@ -205,7 +263,14 @@ function toolResult( text: string, toolCallId = "tc-" + id, ): CoreMessage { - return { id, role: "tool", contentType: "tool-result", toolName, toolCallId, text }; + return { + id, + role: "tool", + contentType: "tool-result", + toolName, + toolCallId, + text, + }; } test("computeProtectedRefs excludes decompress tool results from the recent zone", async () => { @@ -223,7 +288,10 @@ test("computeProtectedRefs excludes decompress tool results from the recent zone const protectedRefs = computeProtectedRefs(messages, state, cfg); // m00004 is the decompress result — must NOT be in the protected zone. - assert.ok(!protectedRefs.has("m00004"), "decompress result not protected by recent zone"); + assert.ok( + !protectedRefs.has("m00004"), + "decompress result not protected by recent zone", + ); // The last USER message (m00003) is still protected by Rule 3. assert.ok(protectedRefs.has("m00003"), "last user message still protected"); }); @@ -243,16 +311,29 @@ test("applyCompression can compress a decompress tool result in the recent tail" const result = core.applyCompression({ ranges: [ - { startRef: "m00004", endRef: "m00004", summary: "compressing the decompress result", topic: "reclaim" }, + { + startRef: "m00004", + endRef: "m00004", + summary: "compressing the decompress result", + topic: "reclaim", + }, ], messages, state, config: cfg, }); - assert.equal(result.result.blocksCreated, 1, "decompress result is compressible despite being in the tail"); + assert.equal( + result.result.blocksCreated, + 1, + "decompress result is compressible despite being in the tail", + ); assert.equal(result.result.errors.length, 0); - assert.equal(result.result.warnings.length, 0, "no warning — it is genuinely outside the protected zone"); + assert.equal( + result.result.warnings.length, + 0, + "no warning — it is genuinely outside the protected zone", + ); }); test("warnings accumulate across multiple ranges in one batch", () => { @@ -269,15 +350,28 @@ test("warnings accumulate across multiple ranges in one batch", () => { // Both should produce warnings, and both unprotected heads should compress. const result = core.applyCompression({ ranges: [ - { startRef: "m00004", endRef: "m00009", summary: "first partial range summary", topic: "a" }, - { startRef: "m00010", endRef: "m00010", summary: "second range fully protected tail", topic: "b" }, + { + startRef: "m00004", + endRef: "m00009", + summary: "first partial range summary", + topic: "a", + }, + { + startRef: "m00010", + endRef: "m00010", + summary: "second range fully protected tail", + topic: "b", + }, ], messages, state, config: cfg, }); - assert.ok(result.result.blocksCreated >= 1, "at least the unprotected head compresses"); + assert.ok( + result.result.blocksCreated >= 1, + "at least the unprotected head compresses", + ); assert.ok(result.result.warnings.length >= 1, "warnings surfaced"); }); @@ -285,10 +379,22 @@ test("warnings accumulate across multiple ranges in one batch", () => { test("isNeverPreserveRecent defaults to the built-in list when patterns is omitted", () => { for (const name of ["decompress", "search_context", "read", "bash"]) { - assert.equal(isNeverPreserveRecent(toolResult("x", name, "body")), true, `${name} excluded by default`); + assert.equal( + isNeverPreserveRecent(toolResult("x", name, "body")), + true, + `${name} excluded by default`, + ); } - assert.equal(isNeverPreserveRecent(toolResult("y", "grep", "hits")), false, "other tools not excluded by default"); - assert.equal(isNeverPreserveRecent(msg("z", "plain text")), false, "non-tool messages never excluded"); + assert.equal( + isNeverPreserveRecent(toolResult("y", "grep", "hits")), + false, + "other tools not excluded by default", + ); + assert.equal( + isNeverPreserveRecent(msg("z", "plain text")), + false, + "non-tool messages never excluded", + ); }); test("isNeverPreserveRecent honors an explicit pattern list (glob suffix included)", () => { @@ -298,7 +404,11 @@ test("isNeverPreserveRecent honors an explicit pattern list (glob suffix include // An explicit list replaces the built-in default verbatim. assert.equal(isNeverPreserveRecent(bashResult, ["bash"]), true); - assert.equal(isNeverPreserveRecent(readResult, ["bash"]), false, "read no longer excluded once the list is explicit"); + assert.equal( + isNeverPreserveRecent(readResult, ["bash"]), + false, + "read no longer excluded once the list is explicit", + ); // The empty list excludes nothing. assert.equal(isNeverPreserveRecent(readResult, []), false); assert.equal(isNeverPreserveRecent(bashResult, []), false); @@ -319,8 +429,15 @@ test("computeProtectedRefs flips recent-zone membership per configured list", as const state = seededState(messages); // Default (field unset): read stays out of the zone → compressible in place. - let refs = computeProtectedRefs(messages, state, config({ preserveRecentMessages: 3 })); - assert.ok(!refs.has("m00004"), "default list keeps the read result outside the zone"); + let refs = computeProtectedRefs( + messages, + state, + config({ preserveRecentMessages: 3 }), + ); + assert.ok( + !refs.has("m00004"), + "default list keeps the read result outside the zone", + ); // Explicit list WITHOUT read: the replacement drops read's exclusion, so the // fresh read result is back inside the zone (this is the #1198 trade-off @@ -330,15 +447,24 @@ test("computeProtectedRefs flips recent-zone membership per configured list", as state, config({ preserveRecentMessages: 3, neverPreserveRecentTools: ["bash"] }), ); - assert.ok(refs.has("m00004"), "explicit list without read gives the read result recent-zone protection"); + assert.ok( + refs.has("m00004"), + "explicit list without read gives the read result recent-zone protection", + ); // Explicit list WITH read: same as default for this message. refs = computeProtectedRefs( messages, state, - config({ preserveRecentMessages: 3, neverPreserveRecentTools: ["read", "bash"] }), + config({ + preserveRecentMessages: 3, + neverPreserveRecentTools: ["read", "bash"], + }), + ); + assert.ok( + !refs.has("m00004"), + "explicit list containing read keeps it compressible", ); - assert.ok(!refs.has("m00004"), "explicit list containing read keeps it compressible"); // Empty list: nothing excluded → the fresh read result sits IN the zone. refs = computeProtectedRefs( @@ -346,7 +472,10 @@ test("computeProtectedRefs flips recent-zone membership per configured list", as state, config({ preserveRecentMessages: 3, neverPreserveRecentTools: [] }), ); - assert.ok(refs.has("m00004"), "empty list gives the fresh read result full recent-zone protection"); + assert.ok( + refs.has("m00004"), + "empty list gives the fresh read result full recent-zone protection", + ); }); test("applyCompression A/B: empty list protects a fresh read result; default reclaims it immediately", () => { @@ -363,26 +492,44 @@ test("applyCompression A/B: empty list protects a fresh read result; default rec // → m00004 is inside the protected zone under the empty list. const refused = core.applyCompression({ ranges: [ - { startRef: "m00004", endRef: "m00004", summary: "should be refused while in the recent zone", topic: "fresh" }, + { + startRef: "m00004", + endRef: "m00004", + summary: "should be refused while in the recent zone", + topic: "fresh", + }, ], messages, state: seededState(messages), config: config({ preserveRecentMessages: 3, neverPreserveRecentTools: [] }), }); - assert.equal(refused.result.blocksCreated, 0, "fresh read result protected with empty exclusion list"); + assert.equal( + refused.result.blocksCreated, + 0, + "fresh read result protected with empty exclusion list", + ); assert.equal(refused.result.errors.length, 1); assert.match(refused.result.errors[0]!, /protected/i); // Same session shape under the default list: read is out of the zone → reclaimable now. const allowed = core.applyCompression({ ranges: [ - { startRef: "m00004", endRef: "m00004", summary: "reclaiming the spent read result", topic: "reclaim" }, + { + startRef: "m00004", + endRef: "m00004", + summary: "reclaiming the spent read result", + topic: "reclaim", + }, ], messages, state: seededState(messages), config: config({ preserveRecentMessages: 3 }), }); - assert.equal(allowed.result.blocksCreated, 1, "default list still reclaims the read result immediately"); + assert.equal( + allowed.result.blocksCreated, + 1, + "default list still reclaims the read result immediately", + ); assert.equal(allowed.result.errors.length, 0); }); @@ -402,24 +549,49 @@ test("empty list: a read result ages OUT of the recent zone and becomes compress // visible tail = f,g,h → m00004 has aged out of the last-3 window. const result = core.applyCompression({ ranges: [ - { startRef: "m00004", endRef: "m00004", summary: "aging out of the zone", topic: "aged" }, + { + startRef: "m00004", + endRef: "m00004", + summary: "aging out of the zone", + topic: "aged", + }, ], messages, state, config: config({ preserveRecentMessages: 3, neverPreserveRecentTools: [] }), }); - assert.equal(result.result.blocksCreated, 1, "aged-out read result compressible even with empty exclusion list"); + assert.equal( + result.result.blocksCreated, + 1, + "aged-out read result compressible even with empty exclusion list", + ); assert.equal(result.result.errors.length, 0); }); test("validateConfig rejects a non-string-array neverPreserveRecentTools", () => { assert.deepEqual( - validateConfig(defaultConfig(200000, { neverPreserveRecentTools: [1] as unknown as string[] })), + validateConfig( + defaultConfig(200000, { + neverPreserveRecentTools: [1] as unknown as string[], + }), + ), ["neverPreserveRecentTools must be a string array"], ); - assert.deepEqual(validateConfig(defaultConfig(200000, { neverPreserveRecentTools: ["read"] })), []); - assert.deepEqual(validateConfig(defaultConfig(200000, { neverPreserveRecentTools: [] })), []); - assert.deepEqual(validateConfig(defaultConfig(200000)), [], "unset stays valid (built-in default applies)"); + assert.deepEqual( + validateConfig( + defaultConfig(200000, { neverPreserveRecentTools: ["read"] }), + ), + [], + ); + assert.deepEqual( + validateConfig(defaultConfig(200000, { neverPreserveRecentTools: [] })), + [], + ); + assert.deepEqual( + validateConfig(defaultConfig(200000)), + [], + "unset stays valid (built-in default applies)", + ); }); // --- preserveRecentTools: positive-facing subtraction knob (#1198/#1277) ---- @@ -437,8 +609,15 @@ test("preserveRecentTools subtracts from the built-in list without restating it" const state = seededState(messages); // Default: read is excluded from the zone → not protected. - let refs = computeProtectedRefs(messages, state, config({ preserveRecentMessages: 3 })); - assert.ok(!refs.has("m00004"), "default list keeps the fresh read result compressible"); + let refs = computeProtectedRefs( + messages, + state, + config({ preserveRecentMessages: 3 }), + ); + assert.ok( + !refs.has("m00004"), + "default list keeps the fresh read result compressible", + ); // One-entry positive override: protect read, rest of the built-in untouched. refs = computeProtectedRefs( @@ -446,7 +625,10 @@ test("preserveRecentTools subtracts from the built-in list without restating it" state, config({ preserveRecentMessages: 3, preserveRecentTools: ["read"] }), ); - assert.ok(refs.has("m00004"), "preserveRecentTools:[\"read\"] gives the read result zone protection"); + assert.ok( + refs.has("m00004"), + 'preserveRecentTools:["read"] gives the read result zone protection', + ); // A bash result at the same position stays excluded — only read was subtracted. const bashMessages = [ @@ -462,7 +644,10 @@ test("preserveRecentTools subtracts from the built-in list without restating it" seededState(bashMessages), config({ preserveRecentMessages: 3, preserveRecentTools: ["read"] }), ); - assert.ok(!refs.has("m00004"), "bash stays excluded — the subtraction is per-tool"); + assert.ok( + !refs.has("m00004"), + "bash stays excluded — the subtraction is per-tool", + ); }); test("preserveRecentTools combines with an explicit neverPreserveRecentTools list", async () => { @@ -482,8 +667,14 @@ test("preserveRecentTools combines with an explicit neverPreserveRecentTools lis preserveRecentTools: ["read"], }); const refs = computeProtectedRefs(messages, state, cfg); - assert.ok(refs.has("m00004"), "read protected: subtracted from the explicit list"); - assert.ok(!refs.has("m00005"), "bash still excluded: not in preserveRecentTools"); + assert.ok( + refs.has("m00004"), + "read protected: subtracted from the explicit list", + ); + assert.ok( + !refs.has("m00005"), + "bash still excluded: not in preserveRecentTools", + ); }); test("preserveRecentTools supports glob suffixes and tolerates empty/no-match", async () => { @@ -502,12 +693,18 @@ test("preserveRecentTools supports glob suffixes and tolerates empty/no-match", state, config({ preserveRecentMessages: 3, preserveRecentTools: ["bash*"] }), ); - assert.ok(refs.has("m00004"), "glob pattern bash* removes the built-in bash entry"); + assert.ok( + refs.has("m00004"), + "glob pattern bash* removes the built-in bash entry", + ); refs = computeProtectedRefs( messages, state, - config({ preserveRecentMessages: 3, preserveRecentTools: ["nonexistent_tool"] }), + config({ + preserveRecentMessages: 3, + preserveRecentTools: ["nonexistent_tool"], + }), ); assert.ok(!refs.has("m00004"), "no-match preserve pattern is a no-op"); @@ -516,10 +713,13 @@ test("preserveRecentTools supports glob suffixes and tolerates empty/no-match", state, config({ preserveRecentMessages: 3, preserveRecentTools: [] }), ); - assert.ok(!refs.has("m00004"), "empty preserve array is a no-op, not protect-everything"); + assert.ok( + !refs.has("m00004"), + "empty preserve array is a no-op, not protect-everything", + ); }); -test("applyCompression: preserveRecentTools [\"read\"] is the one-line #1198 remedy", () => { +test('applyCompression: preserveRecentTools ["read"] is the one-line #1198 remedy', () => { const core = createCore(); const messages: CoreMessage[] = [ msg("a", "first message alpha"), @@ -531,19 +731,37 @@ test("applyCompression: preserveRecentTools [\"read\"] is the one-line #1198 rem ]; const refused = core.applyCompression({ ranges: [ - { startRef: "m00004", endRef: "m00004", summary: "fold the fresh read", topic: "read" }, + { + startRef: "m00004", + endRef: "m00004", + summary: "fold the fresh read", + topic: "read", + }, ], messages, state: seededState(messages), - config: config({ preserveRecentMessages: 3, preserveRecentTools: ["read"] }), + config: config({ + preserveRecentMessages: 3, + preserveRecentTools: ["read"], + }), }); - assert.equal(refused.result.blocksCreated, 0, "fresh read result protected by the subtraction knob"); + assert.equal( + refused.result.blocksCreated, + 0, + "fresh read result protected by the subtraction knob", + ); assert.match(refused.result.errors[0]!, /protected/i); }); test("validateConfig checks preserveRecentTools shape and accepts []", () => { const bad = config({ preserveRecentTools: [42] as unknown as string[] }); - assert.ok(validateConfig(bad).some((e) => /preserveRecentTools/.test(e)), "non-string array rejected"); + assert.ok( + validateConfig(bad).some((e) => /preserveRecentTools/.test(e)), + "non-string array rejected", + ); const empty = config({ preserveRecentTools: [] }); - assert.ok(!validateConfig(empty).some((e) => /preserveRecentTools/.test(e)), "empty array is a tolerated no-op"); + assert.ok( + !validateConfig(empty).some((e) => /preserveRecentTools/.test(e)), + "empty array is a tolerated no-op", + ); }); diff --git a/tests/protected-content.test.ts b/tests/protected-content.test.ts index 0c50888..e0da23b 100644 --- a/tests/protected-content.test.ts +++ b/tests/protected-content.test.ts @@ -6,16 +6,38 @@ import { assignRefs } from "../src/refs.js"; import { defaultConfig } from "../src/config.js"; import type { Config, CoreMessage } from "../src/types.js"; -function msg(id: string, text: string, role: CoreMessage["role"] = "user"): CoreMessage { +function msg( + id: string, + text: string, + role: CoreMessage["role"] = "user", +): CoreMessage { return { id, role, contentType: "text", text }; } -function toolCall(id: string, toolName: string, callId: string, args: string): CoreMessage { - return { id, role: "assistant", contentType: "tool-call", toolName, toolCallId: callId, text: args }; +function toolCall( + id: string, + toolName: string, + callId: string, + args: string, +): CoreMessage { + return { + id, + role: "assistant", + contentType: "tool-call", + toolName, + toolCallId: callId, + text: args, + }; } function toolResult(id: string, callId: string, text: string): CoreMessage { - return { id, role: "tool", contentType: "tool-result", toolCallId: callId, text }; + return { + id, + role: "tool", + contentType: "tool-result", + toolCallId: callId, + text, + }; } function setupRefs(messages: CoreMessage[]) { @@ -28,12 +50,14 @@ function setupRefs(messages: CoreMessage[]) { } const longText = "x".repeat(6000); -const validSummary = "A meaningful summary that captures the key information of the compressed range including file paths and decisions."; +const validSummary = + "A meaningful summary that captures the key information of the compressed range including file paths and decisions."; function cfg(overrides: Partial = {}): Config { return defaultConfig(200000, { compress: { minCompressRange: 0, maxSummaryLength: 0, minSummaryLength: 0 }, - preserveRecentMessages: 0, preserveRecentTokens: 0, + preserveRecentMessages: 0, + preserveRecentTokens: 0, ...overrides, }); } @@ -60,16 +84,40 @@ test("Feature 2: protected tool-call is excluded from compression range", () => assert.equal(result.result.errors.length, 0); const block = result.state.blocks[0]!; - assert.ok(!block.directMessageIds.includes("b"), "tool-call 'b' should be excluded"); - assert.ok(!block.directMessageIds.includes("c"), "tool-result 'c' should be excluded"); - assert.ok(block.directMessageIds.includes("a"), "regular msg 'a' should remain"); - assert.ok(block.directMessageIds.includes("d"), "regular msg 'd' should remain"); + assert.ok( + !block.directMessageIds.includes("b"), + "tool-call 'b' should be excluded", + ); + assert.ok( + !block.directMessageIds.includes("c"), + "tool-result 'c' should be excluded", + ); + assert.ok( + block.directMessageIds.includes("a"), + "regular msg 'a' should remain", + ); + assert.ok( + block.directMessageIds.includes("d"), + "regular msg 'd' should remain", + ); // Bug 39 regression: effectiveMessageIds must also exclude protected tool // messages, otherwise the block would mark them as covered and hide them. - assert.ok(!block.effectiveMessageIds.includes("b"), "tool-call 'b' excluded from effective coverage"); - assert.ok(!block.effectiveMessageIds.includes("c"), "tool-result 'c' excluded from effective coverage"); - assert.ok(block.effectiveMessageIds.includes("a"), "regular msg 'a' in effective coverage"); - assert.ok(block.effectiveMessageIds.includes("d"), "regular msg 'd' in effective coverage"); + assert.ok( + !block.effectiveMessageIds.includes("b"), + "tool-call 'b' excluded from effective coverage", + ); + assert.ok( + !block.effectiveMessageIds.includes("c"), + "tool-result 'c' excluded from effective coverage", + ); + assert.ok( + block.effectiveMessageIds.includes("a"), + "regular msg 'a' in effective coverage", + ); + assert.ok( + block.effectiveMessageIds.includes("d"), + "regular msg 'd' in effective coverage", + ); }); test("Feature 2: protected tool messages are filtered out, not appended", () => { @@ -91,15 +139,37 @@ test("Feature 2: protected tool messages are filtered out, not appended", () => }); const block = result.state.blocks[0]!; - assert.ok(!block.directMessageIds.includes("b"), "protected skill tool-call excluded from compressed set"); - assert.ok(!block.directMessageIds.includes("c"), "protected skill tool-result excluded"); + assert.ok( + !block.directMessageIds.includes("b"), + "protected skill tool-call excluded from compressed set", + ); + assert.ok( + !block.directMessageIds.includes("c"), + "protected skill tool-result excluded", + ); // Bug 39 regression: effectiveMessageIds must match directMessageIds here // (no consumed blocks), so protected messages are excluded from both. - assert.ok(!block.effectiveMessageIds.includes("b"), "protected skill tool-call excluded from effective coverage"); - assert.ok(!block.effectiveMessageIds.includes("c"), "protected skill tool-result excluded from effective coverage"); - assert.ok(!block.summary.includes("Protected:"), "no protected content folded into summary"); - assert.ok(!block.summary.includes('{"name":"git-master"}'), "protected tool content not leaked into summary"); - assert.equal(block.summary, validSummary, "summary is exactly what the author wrote"); + assert.ok( + !block.effectiveMessageIds.includes("b"), + "protected skill tool-call excluded from effective coverage", + ); + assert.ok( + !block.effectiveMessageIds.includes("c"), + "protected skill tool-result excluded from effective coverage", + ); + assert.ok( + !block.summary.includes("Protected:"), + "no protected content folded into summary", + ); + assert.ok( + !block.summary.includes('{"name":"git-master"}'), + "protected tool content not leaked into summary", + ); + assert.equal( + block.summary, + validSummary, + "summary is exactly what the author wrote", + ); }); test("Feature 2: non-protected tool-call is NOT excluded", () => { @@ -121,16 +191,30 @@ test("Feature 2: non-protected tool-call is NOT excluded", () => { }); const block = result.state.blocks[0]!; - assert.ok(block.directMessageIds.includes("b"), "non-protected bash tool-call should be compressed"); - assert.ok(block.directMessageIds.includes("c"), "non-protected bash tool-result should be compressed"); - assert.ok(!block.summary.includes("Protected:"), "no protected content appended for non-protected tools"); + assert.ok( + block.directMessageIds.includes("b"), + "non-protected bash tool-call should be compressed", + ); + assert.ok( + block.directMessageIds.includes("c"), + "non-protected bash tool-result should be compressed", + ); + assert.ok( + !block.summary.includes("Protected:"), + "no protected content appended for non-protected tools", + ); }); test("Feature 2: compress tool itself is always protected", () => { const core = createCore(); const messages = [ msg("a", longText), - toolCall("b", "compress", "call1", '{"content":[{"startId":"m00001","endId":"m00001","summary":"test"}]}'), + toolCall( + "b", + "compress", + "call1", + '{"content":[{"startId":"m00001","endId":"m00001","summary":"test"}]}', + ), toolResult("c", "call1", "compressed"), msg("d", longText), ]; @@ -145,8 +229,14 @@ test("Feature 2: compress tool itself is always protected", () => { }); const block = result.state.blocks[0]!; - assert.ok(!block.directMessageIds.includes("b"), "compress tool-call should be excluded"); - assert.ok(!block.directMessageIds.includes("c"), "compress tool-result should be excluded"); + assert.ok( + !block.directMessageIds.includes("b"), + "compress tool-call should be excluded", + ); + assert.ok( + !block.directMessageIds.includes("c"), + "compress tool-result should be excluded", + ); }); test("Feature 3: isToolProtected custom predicate excludes matching tools", () => { @@ -174,9 +264,18 @@ test("Feature 3: isToolProtected custom predicate excludes matching tools", () = }); const block = result.state.blocks[0]!; - assert.ok(!block.directMessageIds.includes("b"), "write to .env should be excluded by predicate"); - assert.ok(!block.directMessageIds.includes("c"), "write result should be excluded too"); - assert.ok(!block.summary.includes("Protected:"), "no protected content folded into summary"); + assert.ok( + !block.directMessageIds.includes("b"), + "write to .env should be excluded by predicate", + ); + assert.ok( + !block.directMessageIds.includes("c"), + "write result should be excluded too", + ); + assert.ok( + !block.summary.includes("Protected:"), + "no protected content folded into summary", + ); }); test("Feature 3: wildcard pattern in protectedTools works", () => { @@ -198,7 +297,10 @@ test("Feature 3: wildcard pattern in protectedTools works", () => { }); const block = result.state.blocks[0]!; - assert.ok(!block.directMessageIds.includes("b"), "skill-* pattern should match skill-git-master"); + assert.ok( + !block.directMessageIds.includes("b"), + "skill-* pattern should match skill-git-master", + ); }); test("Feature 3: predicate returning false does not protect", () => { @@ -223,7 +325,10 @@ test("Feature 3: predicate returning false does not protect", () => { }); const block = result.state.blocks[0]!; - assert.ok(block.directMessageIds.includes("b"), "predicate=false should not protect"); + assert.ok( + block.directMessageIds.includes("b"), + "predicate=false should not protect", + ); }); test("Feature 2+3: both protectedTools and isToolProtected work together", () => { @@ -239,7 +344,8 @@ test("Feature 2+3: both protectedTools and isToolProtected work together", () => const state = setupRefs(messages); const config = cfg({ protectedTools: ["skill"], - isToolProtected: (name, input) => name === "edit" && !!input?.includes(".pem"), + isToolProtected: (name, input) => + name === "edit" && !!input?.includes(".pem"), }); const result = core.applyCompression({ @@ -250,21 +356,42 @@ test("Feature 2+3: both protectedTools and isToolProtected work together", () => }); const block = result.state.blocks[0]!; - assert.ok(!block.directMessageIds.includes("b"), "skill excluded by protectedTools"); + assert.ok( + !block.directMessageIds.includes("b"), + "skill excluded by protectedTools", + ); assert.ok(!block.directMessageIds.includes("c"), "skill result excluded"); - assert.ok(!block.directMessageIds.includes("d"), "edit excluded by predicate"); + assert.ok( + !block.directMessageIds.includes("d"), + "edit excluded by predicate", + ); assert.ok(!block.directMessageIds.includes("e"), "edit result excluded"); assert.ok(block.directMessageIds.includes("a"), "regular msg remains"); assert.ok(block.directMessageIds.includes("f"), "regular msg remains"); - assert.ok(!block.summary.includes("Protected: skill"), "skill content NOT folded into summary"); - assert.ok(!block.summary.includes("Protected: edit"), "edit content NOT folded into summary"); + assert.ok( + !block.summary.includes("Protected: skill"), + "skill content NOT folded into summary", + ); + assert.ok( + !block.summary.includes("Protected: edit"), + "edit content NOT folded into summary", + ); }); test("compress tool is ALWAYS protected, even with empty protectedTools", () => { const core = createCore(); const messages = [ msg("a", longText), - toolCall("b", "compress", "call1", JSON.stringify({ content: [{ startId: "m00001", endId: "m00001", summary: "x".repeat(60) }] })), + toolCall( + "b", + "compress", + "call1", + JSON.stringify({ + content: [ + { startId: "m00001", endId: "m00001", summary: "x".repeat(60) }, + ], + }), + ), toolResult("c", "call1", "compressed"), msg("d", longText), ]; @@ -280,8 +407,14 @@ test("compress tool is ALWAYS protected, even with empty protectedTools", () => }); const block = result.state.blocks[0]!; - assert.ok(!block.directMessageIds.includes("b"), "compress tool-call excluded regardless of config"); - assert.ok(!block.directMessageIds.includes("c"), "compress tool-result excluded regardless of config"); + assert.ok( + !block.directMessageIds.includes("b"), + "compress tool-call excluded regardless of config", + ); + assert.ok( + !block.directMessageIds.includes("c"), + "compress tool-result excluded regardless of config", + ); assert.ok(block.directMessageIds.includes("a"), "regular msg remains"); assert.ok(block.directMessageIds.includes("d"), "regular msg remains"); }); @@ -293,7 +426,13 @@ test("compress tool-result is protected by toolCallId pairing even without toolN const messages = [ msg("a", longText), toolCall("b", "compress", "callX", "{}"), - { id: "c", role: "tool", contentType: "tool-result", toolCallId: "callX", text: "result".repeat(50) }, + { + id: "c", + role: "tool", + contentType: "tool-result", + toolCallId: "callX", + text: "result".repeat(50), + }, msg("d", longText), ]; const state = setupRefs(messages); @@ -307,8 +446,14 @@ test("compress tool-result is protected by toolCallId pairing even without toolN }); const block = result.state.blocks[0]!; - assert.ok(!block.directMessageIds.includes("b"), "compress tool-call excluded"); - assert.ok(!block.directMessageIds.includes("c"), "compress tool-result (no toolName) excluded via pairing"); + assert.ok( + !block.directMessageIds.includes("b"), + "compress tool-call excluded", + ); + assert.ok( + !block.directMessageIds.includes("c"), + "compress tool-result (no toolName) excluded via pairing", + ); assert.ok(block.directMessageIds.includes("a"), "regular msg remains"); assert.ok(block.directMessageIds.includes("d"), "regular msg remains"); }); diff --git a/tests/protected-latest.test.ts b/tests/protected-latest.test.ts index 48e0a60..01fd18a 100644 --- a/tests/protected-latest.test.ts +++ b/tests/protected-latest.test.ts @@ -196,8 +196,14 @@ test("applyCompression: latest pair excluded from block, older pair folded", () assert.ok(block.effectiveMessageIds.includes("r1")); assert.ok(!block.directMessageIds.includes("c2"), "latest call excluded"); assert.ok(!block.directMessageIds.includes("r2"), "latest result excluded"); - assert.ok(!block.effectiveMessageIds.includes("c2"), "latest call not covered"); - assert.ok(!block.effectiveMessageIds.includes("r2"), "latest result not covered"); + assert.ok( + !block.effectiveMessageIds.includes("c2"), + "latest call not covered", + ); + assert.ok( + !block.effectiveMessageIds.includes("r2"), + "latest result not covered", + ); }); test("applyCompression: superseded pair is folded once a newer pair exists", () => { @@ -253,14 +259,29 @@ test("applyCompression: protectedTools (hard) excludes ALL pairs, unlike protect ranges: [{ startRef: "m00001", endRef: "m00006", summary: validSummary }], messages, state, - config: latestCfg({ protectedLatestTools: [], protectedTools: ["todo_list"] }), + config: latestCfg({ + protectedLatestTools: [], + protectedTools: ["todo_list"], + }), }); assert.equal(result.result.blocksCreated, 1); const block = result.state.blocks[0]!; - assert.ok(!block.directMessageIds.includes("c1"), "hard: first call excluded"); - assert.ok(!block.directMessageIds.includes("r1"), "hard: first result excluded"); - assert.ok(!block.directMessageIds.includes("c2"), "hard: latest call excluded"); - assert.ok(!block.directMessageIds.includes("r2"), "hard: latest result excluded"); + assert.ok( + !block.directMessageIds.includes("c1"), + "hard: first call excluded", + ); + assert.ok( + !block.directMessageIds.includes("r1"), + "hard: first result excluded", + ); + assert.ok( + !block.directMessageIds.includes("c2"), + "hard: latest call excluded", + ); + assert.ok( + !block.directMessageIds.includes("r2"), + "hard: latest result excluded", + ); }); test("buildCompressibleRanges: latest pair protected, older pair compressible", () => { @@ -288,7 +309,11 @@ test("buildCompressibleRanges: latest pair protected, older pair compressible", assert.equal(ranges.compressible[1]!.startRef, "m00004"); assert.equal(ranges.compressible[1]!.endRef, "m00004"); assert.equal(ranges.compressible[1]!.count, 1); - assert.equal(ranges.protected.length, 1, "latest pair is one protected range"); + assert.equal( + ranges.protected.length, + 1, + "latest pair is one protected range", + ); const pr = ranges.protected[0]!; assert.equal(pr.startRef, "m00005"); assert.equal(pr.endRef, "m00006"); diff --git a/tests/recommend-merge.test.ts b/tests/recommend-merge.test.ts index 5f90f3c..a98c283 100644 --- a/tests/recommend-merge.test.ts +++ b/tests/recommend-merge.test.ts @@ -173,11 +173,7 @@ test("count-weighted toolPct: A(count2,tool100) + B(count3,tool0) → 40 / 60", ]; const out = mergeRangesToThreshold(ranges, 5000); assert.equal(out.length, 1); - assert.equal( - out[0]!.toolPct, - 40, - "round((100×2 + 0×3)/5) = 40", - ); + assert.equal(out[0]!.toolPct, 40, "round((100×2 + 0×3)/5) = 40"); assert.equal(out[0]!.textPct, 60, "textPct = 100 - toolPct"); }); diff --git a/tests/recommend.test.ts b/tests/recommend.test.ts index 4e980c3..06cb2da 100644 --- a/tests/recommend.test.ts +++ b/tests/recommend.test.ts @@ -33,12 +33,22 @@ function config(overrides: Partial = {}): Config { }; } -function msg(id: string, text: string, role: CoreMessage["role"] = "user"): CoreMessage { +function msg( + id: string, + text: string, + role: CoreMessage["role"] = "user", +): CoreMessage { return { id, role, contentType: "text", text }; } function toolMsg(id: string, toolName: string): CoreMessage { - return { id, role: "assistant", contentType: "tool-call", toolName, text: `call ${toolName}` }; + return { + id, + role: "assistant", + contentType: "tool-call", + toolName, + text: `call ${toolName}`, + }; } function assignAll( @@ -59,21 +69,35 @@ function assignAll( test("computeProtectedRefs: last user message is protected when preserveRecentMessages > 0", () => { const messages = [msg("a", "x"), msg("b", "y", "assistant")]; const state = assignAll(messages); - const refs = computeProtectedRefs(messages, state, config({ preserveRecentMessages: 5 })); - assert.ok(refs.has("m00001"), "last user message (a) protected by Rule 3 when recent protection is on"); + const refs = computeProtectedRefs( + messages, + state, + config({ preserveRecentMessages: 5 }), + ); + assert.ok( + refs.has("m00001"), + "last user message (a) protected by Rule 3 when recent protection is on", + ); }); test("computeProtectedRefs: last user message is NOT protected when preserveRecentMessages = 0 (full opt-out)", () => { const messages = [msg("a", "x"), msg("b", "y", "assistant")]; const state = assignAll(messages); const refs = computeProtectedRefs(messages, state, config()); - assert.ok(!refs.has("m00001"), "Rule 3 follows preserveRecentMessages — 0 opts out of all recent protection"); + assert.ok( + !refs.has("m00001"), + "Rule 3 follows preserveRecentMessages — 0 opts out of all recent protection", + ); }); test("computeProtectedRefs: preserves last N messages by count", () => { const messages = [msg("a", "x"), msg("b", "y"), msg("c", "z")]; const state = assignAll(messages); - const refs = computeProtectedRefs(messages, state, config({ preserveRecentMessages: 2 })); + const refs = computeProtectedRefs( + messages, + state, + config({ preserveRecentMessages: 2 }), + ); assert.ok(refs.has("m00002")); assert.ok(refs.has("m00003")); assert.ok(!refs.has("m00001")); @@ -86,7 +110,11 @@ test("computeProtectedRefs: preserves last N tokens expanding backward", () => { msg("c", "z".repeat(1200)), ]; const state = assignAll(messages); - const refs = computeProtectedRefs(messages, state, config({ preserveRecentTokens: 500 })); + const refs = computeProtectedRefs( + messages, + state, + config({ preserveRecentTokens: 500 }), + ); assert.ok(refs.has("m00003")); assert.ok(refs.has("m00002")); assert.ok(!refs.has("m00001"), "a is outside the 500-token window"); @@ -119,9 +147,19 @@ test("buildCompressibleRanges: range tokens follow injected countTokens (feeds p const messages = [msg("a", "x".repeat(100)), msg("b", "y".repeat(100))]; const state = assignAll(messages); const mock = (t: string): number => t.length * 7; - const ranges = buildCompressibleRanges(messages, state, config(), undefined, mock); + const ranges = buildCompressibleRanges( + messages, + state, + config(), + undefined, + mock, + ); assert.equal(ranges.compressible.length, 1); - assert.equal(ranges.compressible[0]!.tokens, 1400, "range.tokens must follow injected countTokens, not chars/4"); + assert.equal( + ranges.compressible[0]!.tokens, + 1400, + "range.tokens must follow injected countTokens, not chars/4", + ); }); test("computeProtectedRefs: combines count + token rules (union)", () => { @@ -172,7 +210,11 @@ test("buildCompressibleRanges: protected tools get BLOCKED, excluded from compre state, config({ protectedTools: ["skill"] }), ); - assert.equal(ranges.compressible.length, 1, "a and c form one contiguous range"); + assert.equal( + ranges.compressible.length, + 1, + "a and c form one contiguous range", + ); assert.equal(ranges.compressible[0]!.startRef, "m00001"); assert.equal(ranges.compressible[0]!.endRef, "m00002"); }); @@ -207,7 +249,10 @@ test("buildCompressibleRanges: tool/text percentage computed", () => { const ranges = buildCompressibleRanges(messages, state, config()); assert.ok(ranges.compressible[0]!.toolPct > 0); assert.ok(ranges.compressible[0]!.textPct > 0); - assert.equal(ranges.compressible[0]!.toolPct + ranges.compressible[0]!.textPct, 100); + assert.equal( + ranges.compressible[0]!.toolPct + ranges.compressible[0]!.textPct, + 100, + ); }); test("buildCompressibleRanges: splits at user-turn boundaries once a group has >= 3 messages", () => { @@ -218,11 +263,21 @@ test("buildCompressibleRanges: splits at user-turn boundaries once a group has > for (let i = 0; i < 4; i++) { messages.push(msg("u" + i, "user turn " + i + " content".repeat(50))); messages.push(toolMsg("a" + i, "bash")); - messages.push({ id: "t" + i, role: "tool", contentType: "tool-result", toolCallId: "c" + i, text: "result".repeat(50) }); + messages.push({ + id: "t" + i, + role: "tool", + contentType: "tool-result", + toolCallId: "c" + i, + text: "result".repeat(50), + }); } const state = assignAll(messages); const ranges = buildCompressibleRanges(messages, state, config()); - assert.equal(ranges.compressible.length, 4, "one compressible block per user turn"); + assert.equal( + ranges.compressible.length, + 4, + "one compressible block per user turn", + ); // Each block covers exactly one turn (3 msgs): u0/a0/t0, u1/a1/t1, ... assert.equal(ranges.compressible[0]!.startRef, "m00001"); assert.equal(ranges.compressible[0]!.endRef, "m00003"); @@ -236,12 +291,22 @@ test("buildCompressibleRanges: does NOT split before a group reaches 3 messages" // so the user message should join the existing group rather than start a new one. const messages = [ toolMsg("a", "bash"), - { id: "t", role: "tool", contentType: "tool-result", toolCallId: "c", text: "result".repeat(50) }, + { + id: "t", + role: "tool", + contentType: "tool-result", + toolCallId: "c", + text: "result".repeat(50), + }, msg("u", "user message"), ]; const state = assignAll(messages); const ranges = buildCompressibleRanges(messages, state, config()); - assert.equal(ranges.compressible.length, 1, "short group not split at user boundary"); + assert.equal( + ranges.compressible.length, + 1, + "short group not split at user boundary", + ); assert.equal(ranges.compressible[0]!.count, 3); }); @@ -257,7 +322,11 @@ test("buildCompressibleRanges: ref-map holes do NOT fragment ranges (surface-rep const e = msg("e", "v".repeat(2000)); const state = assignAll([a, msg("b", "y"), msg("c", "z"), msg("d", "w"), e]); const ranges = buildCompressibleRanges([a, e], state, config()); - assert.equal(ranges.compressible.length, 1, "holes in the ref map must not split the range"); + assert.equal( + ranges.compressible.length, + 1, + "holes in the ref map must not split the range", + ); assert.equal(ranges.compressible[0]!.startRef, "m00001"); assert.equal(ranges.compressible[0]!.endRef, "m00005"); assert.equal(ranges.compressible[0]!.count, 2); @@ -272,12 +341,24 @@ test("buildCompressibleRanges: mid-array summary node extends the range, never a const a = msg("a", "x".repeat(2000), "assistant"); const d = msg("d", "w".repeat(2000), "assistant"); const e = msg("e", "v".repeat(2000), "assistant"); - const summary = msg("s", "Summary of the compressed span: did the work.", "assistant"); + const summary = msg( + "s", + "Summary of the compressed span: did the work.", + "assistant", + ); const s1 = assignAll([a, msg("b", "y"), msg("c", "z"), d, e]); const state = assignAll([a, summary, d, e], s1); - assert.equal(state.messageRefs.byRaw["s"], "m00006", "summary node gets a fresh high ref"); + assert.equal( + state.messageRefs.byRaw["s"], + "m00006", + "summary node gets a fresh high ref", + ); const ranges = buildCompressibleRanges([a, summary, d, e], state, config()); - assert.equal(ranges.compressible.length, 1, "mid-array summary node must not flush the range"); + assert.equal( + ranges.compressible.length, + 1, + "mid-array summary node must not flush the range", + ); assert.equal(ranges.compressible[0]!.startRef, "m00001"); assert.equal(ranges.compressible[0]!.endRef, "m00005"); assert.equal(ranges.compressible[0]!.count, 4); @@ -290,11 +371,19 @@ test("buildCompressibleRanges: synthetic (covered) summary node still splits ran // where nothing is physically present between them. const a = msg("a", "x".repeat(2000)); const e = msg("e", "v".repeat(2000)); - const synthetic = msg("s", "[Compressed conversation section] earlier work summarized.", "assistant"); + const synthetic = msg( + "s", + "[Compressed conversation section] earlier work summarized.", + "assistant", + ); const s1 = assignAll([a, msg("b", "y"), msg("c", "z"), msg("d", "w"), e]); const state = assignAll([a, synthetic, e], s1); const ranges = buildCompressibleRanges([a, synthetic, e], state, config()); - assert.equal(ranges.compressible.length, 2, "covered span must not be folded into the new range"); + assert.equal( + ranges.compressible.length, + 2, + "covered span must not be folded into the new range", + ); assert.equal(ranges.compressible[0]!.startRef, "m00001"); assert.equal(ranges.compressible[0]!.endRef, "m00001"); assert.equal(ranges.compressible[1]!.startRef, "m00005"); @@ -316,15 +405,27 @@ test("buildCompressibleRanges: protected groups segment by array adjacency too", dense, config({ protectedTools: ["skill"] }), ); - assert.equal(denseRanges.protected.length, 2, "interleaved non-protected message splits protected groups"); + assert.equal( + denseRanges.protected.length, + 2, + "interleaved non-protected message splits protected groups", + ); const holed = createInitialState(); holed.messageRefs = { byRaw: { p1: "m00001", p2: "m00005" }, byRef: { m00001: "p1", m00005: "p2" }, }; - const holedRanges = buildCompressibleRanges([p1, p2], holed, config({ protectedTools: ["skill"] })); - assert.equal(holedRanges.protected.length, 1, "hole in the ref map must not split the protected group"); + const holedRanges = buildCompressibleRanges( + [p1, p2], + holed, + config({ protectedTools: ["skill"] }), + ); + assert.equal( + holedRanges.protected.length, + 1, + "hole in the ref map must not split the protected group", + ); assert.equal(holedRanges.protected[0]!.count, 2); assert.equal(holedRanges.protected[0]!.startRef, "m00001"); assert.equal(holedRanges.protected[0]!.endRef, "m00005"); @@ -335,10 +436,7 @@ test("buildCompressibleRanges: protected groups segment by array adjacency too", test("integration: tiny ranges are suppressed — fixes the 19-token compression bug", () => { const core = createCore(); const state = createInitialState(); - const messages = [ - msg("a", "hello"), - msg("b", "world"), - ]; + const messages = [msg("a", "hello"), msg("b", "world")]; const result = core.processTurn({ messages, state, @@ -346,24 +444,43 @@ test("integration: tiny ranges are suppressed — fixes the 19-token compression tokenCount: 5000, }); assert.equal(result.nudge!.shouldInject, false, "turn 1: growth=0, no nudge"); - assert.ok(result.nudge!.reason.includes("growth"), `reason: ${result.nudge!.reason}`); + assert.ok( + result.nudge!.reason.includes("growth"), + `reason: ${result.nudge!.reason}`, + ); }); // ─── countTokens injection (Phase 1: T1 pending uses injected countTokens) ──── test("buildCompressibleRanges: CJK-aware countTokens inflates Chinese pending vs chars/4", () => { const zh = "这是一个中文测试消息用于验证token校准的效果。".repeat(20); // ~340 chars - const messages = [msg("a", zh), toolMsg("b", "bash"), msg("c", "tail", "tool")]; + const messages = [ + msg("a", zh), + toolMsg("b", "bash"), + msg("c", "tail", "tool"), + ]; const state = assignAll(messages); const chars4 = buildCompressibleRanges(messages, state, config()); - const cjk = buildCompressibleRanges(messages, state, config(), undefined, (t) => t.length); + const cjk = buildCompressibleRanges( + messages, + state, + config(), + undefined, + (t) => t.length, + ); const pendingDefault = chars4.compressible.reduce((s, r) => s + r.tokens, 0); const pendingCjk = cjk.compressible.reduce((s, r) => s + r.tokens, 0); - assert.ok(pendingCjk > pendingDefault, `cjk ${pendingCjk} should exceed chars/4 ${pendingDefault}`); + assert.ok( + pendingCjk > pendingDefault, + `cjk ${pendingCjk} should exceed chars/4 ${pendingDefault}`, + ); // ~340 chars: chars/4 ≈ 85, CJK-aware ≈ 340+ → multiple of ~3+ - assert.ok(pendingCjk >= pendingDefault * 3, `expected >=3x inflation, got ${pendingCjk} vs ${pendingDefault}`); + assert.ok( + pendingCjk >= pendingDefault * 3, + `expected >=3x inflation, got ${pendingCjk} vs ${pendingDefault}`, + ); }); test("buildCompressibleRanges: default countTokens preserves chars/4 legacy behavior", () => { @@ -378,13 +495,38 @@ test("computeProtectedRefs: injected countTokens sizes the recent-token zone", ( // three fit in 100 tokens (zone {a,b,c}); CJK-aware (1 tok/char) → b alone // blows past 100 (zone {b,c}). If countTokens were ignored, both calls would // protect a and the cjk assertion below would fail. - const messages = [msg("a", "x".repeat(200)), msg("b", "x".repeat(200)), msg("c", "y")]; + const messages = [ + msg("a", "x".repeat(200)), + msg("b", "x".repeat(200)), + msg("c", "y"), + ]; const state = assignAll(messages); - const chars4 = computeProtectedRefs(messages, state, config({ preserveRecentTokens: 100, preserveRecentMessages: 0 })); - assert.equal(chars4.has("m00001"), true, "chars/4: a (50 tok) fits within 100-token zone"); - assert.equal(chars4.has("m00003"), true, "chars/4: c is most recent, in zone"); - const cjk = computeProtectedRefs(messages, state, config({ preserveRecentTokens: 100, preserveRecentMessages: 0 }), (t) => t.length); - assert.equal(cjk.has("m00001"), false, "cjk: a excluded (b alone exceeds 100-token budget)"); + const chars4 = computeProtectedRefs( + messages, + state, + config({ preserveRecentTokens: 100, preserveRecentMessages: 0 }), + ); + assert.equal( + chars4.has("m00001"), + true, + "chars/4: a (50 tok) fits within 100-token zone", + ); + assert.equal( + chars4.has("m00003"), + true, + "chars/4: c is most recent, in zone", + ); + const cjk = computeProtectedRefs( + messages, + state, + config({ preserveRecentTokens: 100, preserveRecentMessages: 0 }), + (t) => t.length, + ); + assert.equal( + cjk.has("m00001"), + false, + "cjk: a excluded (b alone exceeds 100-token budget)", + ); assert.equal(cjk.has("m00002"), true, "cjk: b in zone"); assert.equal(cjk.has("m00003"), true, "cjk: c most recent, in zone"); }); diff --git a/tests/regression-bugfixes.test.ts b/tests/regression-bugfixes.test.ts index cbed5d6..7a6d011 100644 --- a/tests/regression-bugfixes.test.ts +++ b/tests/regression-bugfixes.test.ts @@ -136,7 +136,11 @@ test("compressibleRanges use real assigned refs, not array-index arithmetic", () messages, state, // minPressureBenefitTokens: 0 = legacy any-pending pressure injection; this test exercises ref assignment, not the #198 benefit gate. - config: config({ protectedTools: ["skill"], preserveRecentMessages: 2, nudge: { ...config().nudge, growthRatio: 0, minPressureBenefitTokens: 0 } }), + config: config({ + protectedTools: ["skill"], + preserveRecentMessages: 2, + nudge: { ...config().nudge, growthRatio: 0, minPressureBenefitTokens: 0 }, + }), tokenCount: 99000, }); @@ -153,7 +157,11 @@ test("compressibleRanges use real assigned refs, not array-index arithmetic", () "m00001", "startRef must be a's real assigned ref", ); - assert.equal(range.endRef, "m00004", "endRef must be e's real assigned ref (f protected)"); + assert.equal( + range.endRef, + "m00004", + "endRef must be e's real assigned ref (f protected)", + ); }); // Bug C: processTurn mutated the caller's state.nudge because syncBlocks only @@ -172,7 +180,9 @@ test("processTurn does not mutate the caller's input state.nudge", () => { const result = core.processTurn({ messages, state, - config: config({ nudge: { ...config().nudge, growthRatio: 0, minPressureBenefitTokens: 0 } }), + config: config({ + nudge: { ...config().nudge, growthRatio: 0, minPressureBenefitTokens: 0 }, + }), tokenCount: 99000, }); @@ -276,11 +286,11 @@ test("composable pipeline excluding sync-blocks must not leak nudge mutations to const state = createInitialState(); const messages = [msg("a", "hello"), msg("b", "world")]; // minPressureBenefitTokens: 0 = legacy any-pending pressure injection; this test exercises pipeline composition, not the #198 benefit gate. - const cfg = config({ nudge: { ...config().nudge, growthRatio: 0, minPressureBenefitTokens: 0 } }); + const cfg = config({ + nudge: { ...config().nudge, growthRatio: 0, minPressureBenefitTokens: 0 }, + }); - const nodes = core - .defaultNodes() - .filter((n) => n.name !== "sync-blocks"); + const nodes = core.defaultNodes().filter((n) => n.name !== "sync-blocks"); const initial = makeIO(messages, state); const ctx = { config: cfg, @@ -323,9 +333,18 @@ test("render-refs preserves user content that starts with a ref-like token", () const rendered = renderVisibleRefs(messages, state); - assert.match(rendered[0]!.text!, /^m00001<\/acp>\n\[m1\] is my first point$/); - assert.match(rendered[1]!.text!, /^m00002<\/acp>\n\[m00005\] please review$/); - assert.match(rendered[2]!.text!, /^m00003<\/acp>\n\[m00003\] \[m00003\] dup tags$/); + assert.match( + rendered[0]!.text!, + /^m00001<\/acp>\n\[m1\] is my first point$/, + ); + assert.match( + rendered[1]!.text!, + /^m00002<\/acp>\n\[m00005\] please review$/, + ); + assert.match( + rendered[2]!.text!, + /^m00003<\/acp>\n\[m00003\] \[m00003\] dup tags$/, + ); }); test("render-refs is idempotent across repeated renders", () => { @@ -342,8 +361,14 @@ test("render-refs is idempotent across repeated renders", () => { once.map((m) => m.text), twice.map((m) => m.text), ); - assert.match(once[0]!.text!, /^m00001<\/acp>\nalpha$/); - assert.match(once[1]!.text!, /^m00002<\/acp>\n\[m00002\] beta$/); + assert.match( + once[0]!.text!, + /^m00001<\/acp>\nalpha$/, + ); + assert.match( + once[1]!.text!, + /^m00002<\/acp>\n\[m00002\] beta$/, + ); }); test("batch: overlapping ranges warn and skip (earliest wins) instead of failing", () => { @@ -369,8 +394,16 @@ test("batch: overlapping ranges warn and skip (earliest wins) instead of failing config: config(), }); - assert.equal(result.result.errors.length, 0, "overlap is non-fatal — no errors"); - assert.equal(result.result.blocksCreated, 1, "the earlier range is still compressed"); + assert.equal( + result.result.errors.length, + 0, + "overlap is non-fatal — no errors", + ); + assert.equal( + result.result.blocksCreated, + 1, + "the earlier range is still compressed", + ); assert.ok( result.result.warnings.some((w) => /overlaps an earlier range/i.test(w)), "a warning is recorded for the skipped range", @@ -423,8 +456,16 @@ test("batch: disjoint range still compresses when two others overlap", () => { const result = core.applyCompression({ ranges: [ { startRef: "m00001", endRef: "m00002", summary: "ab summary" }, - { startRef: "m00002", endRef: "m00003", summary: "bc summary (overlaps ab)" }, - { startRef: "m00004", endRef: "m00005", summary: "de summary (disjoint)" }, + { + startRef: "m00002", + endRef: "m00003", + summary: "bc summary (overlaps ab)", + }, + { + startRef: "m00004", + endRef: "m00005", + summary: "de summary (disjoint)", + }, ], messages, state, @@ -453,26 +494,56 @@ test("batch: overlap resolved deterministically regardless of input order", () = nextIndex: 1, }).map; - const run = (ranges: Parameters[0]["ranges"]) => { + const run = ( + ranges: Parameters[0]["ranges"], + ) => { const fresh = createInitialState(); fresh.messageRefs = state.messageRefs; - return core.applyCompression({ ranges, messages, state: fresh, config: config() }); + return core.applyCompression({ + ranges, + messages, + state: fresh, + config: config(), + }); }; const sorted = run([ { startRef: "m00001", endRef: "m00002", summary: "ab summary" }, - { startRef: "m00002", endRef: "m00003", summary: "bc summary (overlaps ab)" }, - { startRef: "m00003", endRef: "m00004", summary: "cd summary (disjoint from ab)" }, + { + startRef: "m00002", + endRef: "m00003", + summary: "bc summary (overlaps ab)", + }, + { + startRef: "m00003", + endRef: "m00004", + summary: "cd summary (disjoint from ab)", + }, ]); const shuffled = run([ - { startRef: "m00003", endRef: "m00004", summary: "cd summary (disjoint from ab)" }, - { startRef: "m00002", endRef: "m00003", summary: "bc summary (overlaps ab)" }, + { + startRef: "m00003", + endRef: "m00004", + summary: "cd summary (disjoint from ab)", + }, + { + startRef: "m00002", + endRef: "m00003", + summary: "bc summary (overlaps ab)", + }, { startRef: "m00001", endRef: "m00002", summary: "ab summary" }, ]); - for (const [label, result] of [["sorted", sorted], ["shuffled", shuffled]] as const) { + for (const [label, result] of [ + ["sorted", sorted], + ["shuffled", shuffled], + ] as const) { assert.equal(result.result.errors.length, 0, `${label}: no errors`); - assert.equal(result.result.blocksCreated, 2, `${label}: earliest range + disjoint range compressed`); + assert.equal( + result.result.blocksCreated, + 2, + `${label}: earliest range + disjoint range compressed`, + ); assert.ok( result.result.warnings.some((w) => /overlaps an earlier range/i.test(w)), `${label}: overlap warning present`, @@ -495,16 +566,27 @@ test("block-boundary distillation counts consumed block tokens", () => { }).map; const t1 = core.applyCompression({ - ranges: [{ startRef: "m00001", endRef: "m00002", summary: "first block summary with enough detail" }], + ranges: [ + { + startRef: "m00001", + endRef: "m00002", + summary: "first block summary with enough detail", + }, + ], messages, state, config: config(), }); - assert.ok(t1.result.tokensCompressed > 0, "T1 should count compressed tokens"); + assert.ok( + t1.result.tokensCompressed > 0, + "T1 should count compressed tokens", + ); const t2 = core.applyCompression({ - ranges: [{ startRef: "b1", endRef: "b1", summary: "distilled tier 2 summary" }], + ranges: [ + { startRef: "b1", endRef: "b1", summary: "distilled tier 2 summary" }, + ], messages, state: t1.state, config: config(), diff --git a/tests/regression-empty-rewrap.test.ts b/tests/regression-empty-rewrap.test.ts index e2eca52..25414f0 100644 --- a/tests/regression-empty-rewrap.test.ts +++ b/tests/regression-empty-rewrap.test.ts @@ -38,12 +38,14 @@ function toolCallMsg( }; } -function toolResultMsg( - id: string, - callId: string, - text: string, -): CoreMessage { - return { id, role: "tool", contentType: "tool-result", toolCallId: callId, text }; +function toolResultMsg(id: string, callId: string, text: string): CoreMessage { + return { + id, + role: "tool", + contentType: "tool-result", + toolCallId: callId, + text, + }; } function config(overrides: Partial = {}): Config { @@ -74,10 +76,12 @@ function config(overrides: Partial = {}): Config { }; } -function makeState(specs: { - blockId: string; - effectiveMessageIds: string[]; -}[]): CompressionState { +function makeState( + specs: { + blockId: string; + effectiveMessageIds: string[]; + }[], +): CompressionState { const state = createInitialState(); state.blocks = specs.map((spec) => ({ blockId: spec.blockId, @@ -111,7 +115,10 @@ function loopedView(): CoreMessage[] { ]; } -function withRefs(messages: CoreMessage[], state: CompressionState): CompressionState { +function withRefs( + messages: CoreMessage[], + state: CompressionState, +): CompressionState { state.messageRefs = assignRefs(messages, { existing: state.messageRefs, nextIndex: 1, @@ -125,7 +132,12 @@ test("#199: message-ref range that only re-wraps an active block is rejected, no const messages = loopedView(); const state = withRefs( messages, - makeState([{ blockId: "b1", effectiveMessageIds: ["raw-2", "raw-3", "raw-4", "raw-5"] }]), + makeState([ + { + blockId: "b1", + effectiveMessageIds: ["raw-2", "raw-3", "raw-4", "raw-5"], + }, + ]), ); const startRef = state.messageRefs.byRaw["raw-1"]!; const endRef = state.messageRefs.byRaw["tool-2"]!; @@ -152,7 +164,12 @@ test("#199: promote via block-boundary refs (b1..b1) still works after the guard const messages = loopedView(); const state = withRefs( messages, - makeState([{ blockId: "b1", effectiveMessageIds: ["raw-2", "raw-3", "raw-4", "raw-5"] }]), + makeState([ + { + blockId: "b1", + effectiveMessageIds: ["raw-2", "raw-3", "raw-4", "raw-5"], + }, + ]), ); const applied = core.applyCompression({ @@ -167,7 +184,11 @@ test("#199: promote via block-boundary refs (b1..b1) still works after the guard const newBlock = applied.state.blocks[applied.state.blocks.length - 1]!; assert.equal(newBlock.tier, 2); assert.deepEqual(newBlock.directBlockIds, ["b1"]); - assert.equal(applied.state.blocks[0]!.active, false, "b1 consumed by promote"); + assert.equal( + applied.state.blocks[0]!.active, + false, + "b1 consumed by promote", + ); }); test("#199: message-ref range that pulls in NEW messages plus a nested block is still allowed", () => { @@ -180,7 +201,12 @@ test("#199: message-ref range that pulls in NEW messages plus a nested block is ]; const state = withRefs( messages, - makeState([{ blockId: "b1", effectiveMessageIds: ["raw-2", "raw-3", "raw-4", "raw-5"] }]), + makeState([ + { + blockId: "b1", + effectiveMessageIds: ["raw-2", "raw-3", "raw-4", "raw-5"], + }, + ]), ); const startRef = state.messageRefs.byRaw["raw-1"]!; const endRef = state.messageRefs.byRaw["tool-2"]!; diff --git a/tests/regression-features.test.ts b/tests/regression-features.test.ts index 5126cd9..e9e1b38 100644 --- a/tests/regression-features.test.ts +++ b/tests/regression-features.test.ts @@ -38,11 +38,7 @@ function toolCall( }; } -function toolResult( - id: string, - callId: string, - text: string, -): CoreMessage { +function toolResult(id: string, callId: string, text: string): CoreMessage { return { id, role: "tool", @@ -85,22 +81,31 @@ test("Bug 1: protected tool messages are filtered (no summary inflation)", () => ]; const config = defaultConfig(200000, { protectedTools: ["skill"], - compress: { minCompressRange: 0, maxSummaryLength: 100, minSummaryLength: 0 }, - preserveRecentMessages: 0, preserveRecentTokens: 0, + compress: { + minCompressRange: 0, + maxSummaryLength: 100, + minSummaryLength: 0, + }, + preserveRecentMessages: 0, + preserveRecentTokens: 0, }); - let state = setupRefs(messages, { - blocks: [], - prune: { byMessageId: {}, activeBlockIds: [] }, - messageRefs: { byRaw: {}, byRef: {} }, - nudge: { - lastPerMessageNudgeTokens: 0, - lastNudgeShownTokens: 0, - pendingNudgeTurn: null, - baselineTokens: 0, + let state = setupRefs( + messages, + { + blocks: [], + prune: { byMessageId: {}, activeBlockIds: [] }, + messageRefs: { byRaw: {}, byRef: {} }, + nudge: { + lastPerMessageNudgeTokens: 0, + lastNudgeShownTokens: 0, + pendingNudgeTurn: null, + baselineTokens: 0, + }, + stats: { totalTokensCompressed: 0 }, + compressionTiming: {}, }, - stats: { totalTokensCompressed: 0 }, - compressionTiming: {}, - }, config); + config, + ); const result = core.applyCompression({ ranges: [{ startRef: "m00001", endRef: "m00003", summary: "short" }], @@ -110,12 +115,22 @@ test("Bug 1: protected tool messages are filtered (no summary inflation)", () => countTokens: defaultCountTokens, }); - assert.equal(result.result.blocksCreated, 1, "compression succeeds — no length error"); + assert.equal( + result.result.blocksCreated, + 1, + "compression succeeds — no length error", + ); assert.equal(result.result.errors.length, 0, "no errors"); const block = result.state.blocks[0]!; assert.equal(block.summary, "short", "summary is exactly the author's text"); - assert.ok(!block.directMessageIds.includes("b"), "protected skill call excluded"); - assert.ok(!block.directMessageIds.includes("c"), "protected skill result excluded"); + assert.ok( + !block.directMessageIds.includes("b"), + "protected skill call excluded", + ); + assert.ok( + !block.directMessageIds.includes("c"), + "protected skill result excluded", + ); assert.ok(block.directMessageIds.includes("a"), "regular msg compressed"); assert.ok(block.directMessageIds.includes("d"), "regular msg compressed"); }); @@ -131,21 +146,26 @@ test("Bug 2: orphaned tool-call — protected call outside range, result inside" const config = defaultConfig(200000, { protectedTools: ["skill"], compress: { minCompressRange: 0, maxSummaryLength: 0, minSummaryLength: 0 }, - preserveRecentMessages: 0, preserveRecentTokens: 0, + preserveRecentMessages: 0, + preserveRecentTokens: 0, }); - let state = setupRefs(messages, { - blocks: [], - prune: { byMessageId: {}, activeBlockIds: [] }, - messageRefs: { byRaw: {}, byRef: {} }, - nudge: { - lastPerMessageNudgeTokens: 0, - lastNudgeShownTokens: 0, - pendingNudgeTurn: null, - baselineTokens: 0, + let state = setupRefs( + messages, + { + blocks: [], + prune: { byMessageId: {}, activeBlockIds: [] }, + messageRefs: { byRaw: {}, byRef: {} }, + nudge: { + lastPerMessageNudgeTokens: 0, + lastNudgeShownTokens: 0, + pendingNudgeTurn: null, + baselineTokens: 0, + }, + stats: { totalTokensCompressed: 0 }, + compressionTiming: {}, }, - stats: { totalTokensCompressed: 0 }, - compressionTiming: {}, - }, config); + config, + ); // 'a' (protected skill call) is BLOCKED, so refs are: b=m00001, c=m00002, d=m00003 // Compress m00001->m00003 covers b(result), c, d — b's call 'a' is outside range diff --git a/tests/report-accounting.test.ts b/tests/report-accounting.test.ts index 5c8a7f6..3b9f4f6 100644 --- a/tests/report-accounting.test.ts +++ b/tests/report-accounting.test.ts @@ -48,9 +48,22 @@ function breakdownLine(report: string): string { test("breakdown counts tool-results in the tool bucket (issue #390)", () => { const messages: CoreMessage[] = [ { id: "m1", role: "user", contentType: "text", text: "run the build" }, - { id: "m2", role: "assistant", contentType: "tool-call", toolName: "bash", toolCallId: "c1", text: "npm test" }, + { + id: "m2", + role: "assistant", + contentType: "tool-call", + toolName: "bash", + toolCallId: "c1", + text: "npm test", + }, // wire converters never set toolName on the result side - { id: "m3", role: "tool", contentType: "tool-result", toolCallId: "c1", text: "x".repeat(51_361) }, + { + id: "m3", + role: "tool", + contentType: "tool-result", + toolCallId: "c1", + text: "x".repeat(51_361), + }, ]; const state = withRefs(messages); const report = buildStatusReport(state, messages, defaultCountTokens); @@ -62,30 +75,64 @@ test("breakdown counts tool-results in the tool bucket (issue #390)", () => { test("breakdown tool split corroborates compressible-range toolPct (issue #390)", () => { const messages: CoreMessage[] = [ - { id: "m1", role: "assistant", contentType: "tool-call", toolName: "bash", toolCallId: "c1", text: "ls" }, - { id: "m2", role: "tool", contentType: "tool-result", toolCallId: "c1", text: "y".repeat(40_000) }, + { + id: "m1", + role: "assistant", + contentType: "tool-call", + toolName: "bash", + toolCallId: "c1", + text: "ls", + }, + { + id: "m2", + role: "tool", + contentType: "tool-result", + toolCallId: "c1", + text: "y".repeat(40_000), + }, ]; const state = withRefs(messages); - const ranges = buildCompressibleRanges(messages, state, config(), new Set(), defaultCountTokens); + const ranges = buildCompressibleRanges( + messages, + state, + config(), + new Set(), + defaultCountTokens, + ); assert.equal(ranges.compressible.length, 1); const range = ranges.compressible[0]!; // same message set, both surfaces must agree: all-tool assert.equal(range.toolPct, 100); assert.equal(range.textPct, 0); - assert.match(breakdownLine(buildStatusReport(state, messages, defaultCountTokens)), /100%\)/); + assert.match( + breakdownLine(buildStatusReport(state, messages, defaultCountTokens)), + /100%\)/, + ); }); test("pct has no 1% floor — tiny buckets print 0% and sums stay <= 100 (issue #390)", () => { const messages: CoreMessage[] = [ { id: "m1", role: "user", contentType: "text", text: "z".repeat(4000) }, // 1000 tokens - { id: "m2", role: "assistant", contentType: "tool-call", toolName: "bash", toolCallId: "c1", text: "ls" }, // 1 token + { + id: "m2", + role: "assistant", + contentType: "tool-call", + toolName: "bash", + toolCallId: "c1", + text: "ls", + }, // 1 token ]; const state = withRefs(messages); - const line = breakdownLine(buildStatusReport(state, messages, defaultCountTokens)); + const line = breakdownLine( + buildStatusReport(state, messages, defaultCountTokens), + ); assert.match(line, /1 tool \(0%\)/); assert.match(line, /1\.0K text \(100%\)/); const pcts = [...line.matchAll(/(\d+)%/g)].map((m) => Number(m[1]!)); - assert.ok(pcts.reduce((s, n) => s + n, 0) <= 100, `bucket percents must not exceed 100: ${line}`); + assert.ok( + pcts.reduce((s, n) => s + n, 0) <= 100, + `bucket percents must not exceed 100: ${line}`, + ); }); test("breakdown uses the injected estimator — CJK is not 4x-underestimated (issue #390)", () => { @@ -93,13 +140,25 @@ test("breakdown uses the injected estimator — CJK is not 4x-underestimated (is { id: "m1", role: "user", contentType: "text", text: "中".repeat(4000) }, ]; const state = withRefs(messages); - assert.match(breakdownLine(buildStatusReport(state, messages, defaultCountTokens)), /4\.0K text \(100%\)/); - assert.match(breakdownLine(buildStatusReport(state, messages, estimateTokensFast)), /1\.0K text \(100%\)/); + assert.match( + breakdownLine(buildStatusReport(state, messages, defaultCountTokens)), + /4\.0K text \(100%\)/, + ); + assert.match( + breakdownLine(buildStatusReport(state, messages, estimateTokensFast)), + /1\.0K text \(100%\)/, + ); }); test("tool-result without a resolvable call stays in the tool bucket", () => { const messages: CoreMessage[] = [ - { id: "m1", role: "tool", contentType: "tool-result", toolCallId: "orphan", text: "w".repeat(4000) }, + { + id: "m1", + role: "tool", + contentType: "tool-result", + toolCallId: "orphan", + text: "w".repeat(4000), + }, ]; const state = withRefs(messages); const report = buildStatusReport(state, messages, defaultCountTokens); @@ -109,8 +168,21 @@ test("tool-result without a resolvable call stays in the tool bucket", () => { test("message drilldown filters results by resolved tool name", () => { const messages: CoreMessage[] = [ - { id: "m1", role: "assistant", contentType: "tool-call", toolName: "bash", toolCallId: "c1", text: "ls" }, - { id: "m2", role: "tool", contentType: "tool-result", toolCallId: "c1", text: "y".repeat(4000) }, + { + id: "m1", + role: "assistant", + contentType: "tool-call", + toolName: "bash", + toolCallId: "c1", + text: "ls", + }, + { + id: "m2", + role: "tool", + contentType: "tool-result", + toolCallId: "c1", + text: "y".repeat(4000), + }, ]; const state = withRefs(messages); const report = buildStatusReport(state, messages, defaultCountTokens, { diff --git a/tests/report-tool-accounting.test.ts b/tests/report-tool-accounting.test.ts index eb63db4..33a9d0b 100644 --- a/tests/report-tool-accounting.test.ts +++ b/tests/report-tool-accounting.test.ts @@ -11,135 +11,165 @@ import type { CompressionState, CoreMessage } from "../src/types.js"; const pad = (n: number): string => "a".repeat(n); interface Pair { - callId: string; - toolName: string; - resultChars: number; + callId: string; + toolName: string; + resultChars: number; } function toolHeavySession(pairs: Pair[], textChars = 200): CoreMessage[] { - const messages: CoreMessage[] = [ - { id: "m1", role: "user", contentType: "text", text: pad(textChars) }, - ]; - for (const p of pairs) { - messages.push({ - id: `call-${p.callId}`, - role: "assistant", - contentType: "tool-call", - toolName: p.toolName, - toolCallId: p.callId, - text: "{}", - }); - messages.push({ - id: `res-${p.callId}`, - role: "user", - contentType: "tool-result", - toolCallId: p.callId, - text: pad(p.resultChars), - }); - } - return messages; + const messages: CoreMessage[] = [ + { id: "m1", role: "user", contentType: "text", text: pad(textChars) }, + ]; + for (const p of pairs) { + messages.push({ + id: `call-${p.callId}`, + role: "assistant", + contentType: "tool-call", + toolName: p.toolName, + toolCallId: p.callId, + text: "{}", + }); + messages.push({ + id: `res-${p.callId}`, + role: "user", + contentType: "tool-result", + toolCallId: p.callId, + text: pad(p.resultChars), + }); + } + return messages; } function stateFor(messages: CoreMessage[]): CompressionState { - const state = createInitialState(); - messages.forEach((m, i) => { - const ref = indexToRef(i + 1); - state.messageRefs.byRaw[m.id] = ref; - state.messageRefs.byRef[ref] = m.id; - }); - return state; + const state = createInitialState(); + messages.forEach((m, i) => { + const ref = indexToRef(i + 1); + state.messageRefs.byRaw[m.id] = ref; + state.messageRefs.byRef[ref] = m.id; + }); + return state; } function breakdownLine(report: string): string { - const line = report.split("\n").find((l) => l.includes(" tool (") && l.includes(" text (")); - assert.ok(line, `no breakdown line in:\n${report}`); - return line; + const line = report + .split("\n") + .find((l) => l.includes(" tool (") && l.includes(" text (")); + assert.ok(line, `no breakdown line in:\n${report}`); + return line; } function bucketPct(line: string, label: string): number { - const m = new RegExp(`\\((\\d+)%\\) \\| ${label}`).exec(line) ?? new RegExp(`${label} \\((\\d+)%\\)`).exec(line); - assert.ok(m, `no ${label} pct in: ${line}`); - return Number(m[1]); + const m = + new RegExp(`\\((\\d+)%\\) \\| ${label}`).exec(line) ?? + new RegExp(`${label} \\((\\d+)%\\)`).exec(line); + assert.ok(m, `no ${label} pct in: ${line}`); + return Number(m[1]); } test("tool-result volume is attributed to the tool bucket (#386)", () => { - // 5 tools x 10_000 chars ASCII = 5 x 2_500 = 12_500 tokens of tool - // RESULTS. Wire adapters set toolName only on the call, never the result. - const pairs: Pair[] = [ - { callId: "c1", toolName: "bash", resultChars: 10_000 }, - { callId: "c2", toolName: "read", resultChars: 10_000 }, - { callId: "c3", toolName: "edit", resultChars: 10_000 }, - { callId: "c4", toolName: "grep", resultChars: 10_000 }, - { callId: "c5", toolName: "bash", resultChars: 10_000 }, - ]; - const messages = toolHeavySession(pairs); - const report = buildStatusReport(stateFor(messages), messages, defaultCountTokens); - const line = breakdownLine(report); - // 12_500 tool vs 50 text + 5 calls x 1 = true ratio must be ~99%. - assert.ok(line.includes("12.5K tool"), `tool bucket wrong: ${line}`); - assert.ok(bucketPct(line, "tool") >= 99, `tool pct wrong: ${line}`); + // 5 tools x 10_000 chars ASCII = 5 x 2_500 = 12_500 tokens of tool + // RESULTS. Wire adapters set toolName only on the call, never the result. + const pairs: Pair[] = [ + { callId: "c1", toolName: "bash", resultChars: 10_000 }, + { callId: "c2", toolName: "read", resultChars: 10_000 }, + { callId: "c3", toolName: "edit", resultChars: 10_000 }, + { callId: "c4", toolName: "grep", resultChars: 10_000 }, + { callId: "c5", toolName: "bash", resultChars: 10_000 }, + ]; + const messages = toolHeavySession(pairs); + const report = buildStatusReport( + stateFor(messages), + messages, + defaultCountTokens, + ); + const line = breakdownLine(report); + // 12_500 tool vs 50 text + 5 calls x 1 = true ratio must be ~99%. + assert.ok(line.includes("12.5K tool"), `tool bucket wrong: ${line}`); + assert.ok(bucketPct(line, "tool") >= 99, `tool pct wrong: ${line}`); }); test("tool-results resolve back to their calling tool's name (#386)", () => { - const pairs: Pair[] = [ - { callId: "c1", toolName: "bash", resultChars: 8_000 }, - { callId: "c2", toolName: "read", resultChars: 2_000 }, - ]; - const messages = toolHeavySession(pairs); - const state = stateFor(messages); - const report = buildStatusReport(state, messages, defaultCountTokens); - const top = report.split("\n").find((l) => l.startsWith(" Top tools:")); - assert.ok(top, `no Top tools line in:\n${report}`); - assert.ok(top.includes("bash"), `bash missing: ${top}`); - assert.ok(top.includes("read"), `read missing: ${top}`); - const textPct = /text \((\d+)%\)/.exec(top); - assert.ok(!textPct || Number(textPct[1]) <= 5, `text dominates Top tools: ${top}`); - // Message drilldown by resolved name lists the result refs. - const byName = buildStatusReport(state, messages, defaultCountTokens, { - scope: "uncompressed", - view: "messages", - tool: "bash", - }); - assert.ok(byName.includes("m00003"), `bash drilldown should include result ref:\n${byName}`); + const pairs: Pair[] = [ + { callId: "c1", toolName: "bash", resultChars: 8_000 }, + { callId: "c2", toolName: "read", resultChars: 2_000 }, + ]; + const messages = toolHeavySession(pairs); + const state = stateFor(messages); + const report = buildStatusReport(state, messages, defaultCountTokens); + const top = report.split("\n").find((l) => l.startsWith(" Top tools:")); + assert.ok(top, `no Top tools line in:\n${report}`); + assert.ok(top.includes("bash"), `bash missing: ${top}`); + assert.ok(top.includes("read"), `read missing: ${top}`); + const textPct = /text \((\d+)%\)/.exec(top); + assert.ok( + !textPct || Number(textPct[1]) <= 5, + `text dominates Top tools: ${top}`, + ); + // Message drilldown by resolved name lists the result refs. + const byName = buildStatusReport(state, messages, defaultCountTokens, { + scope: "uncompressed", + view: "messages", + tool: "bash", + }); + assert.ok( + byName.includes("m00003"), + `bash drilldown should include result ref:\n${byName}`, + ); }); test("orphan tool-results fall into the generic tool bucket, not text (#386)", () => { - const messages: CoreMessage[] = [ - { id: "m1", role: "user", contentType: "text", text: pad(200) }, - { - id: "m2", - role: "user", - contentType: "tool-result", - toolCallId: "missing-call", - text: pad(8_000), - }, - ]; - const state = stateFor(messages); - const report = buildStatusReport(state, messages, defaultCountTokens); - const line = breakdownLine(report); - assert.ok(line.includes("2.0K tool"), `orphan result should stay in tool bucket: ${line}`); - assert.ok(bucketPct(line, "tool") >= 97, `tool pct wrong: ${line}`); + const messages: CoreMessage[] = [ + { id: "m1", role: "user", contentType: "text", text: pad(200) }, + { + id: "m2", + role: "user", + contentType: "tool-result", + toolCallId: "missing-call", + text: pad(8_000), + }, + ]; + const state = stateFor(messages); + const report = buildStatusReport(state, messages, defaultCountTokens); + const line = breakdownLine(report); + assert.ok( + line.includes("2.0K tool"), + `orphan result should stay in tool bucket: ${line}`, + ); + assert.ok(bucketPct(line, "tool") >= 97, `tool pct wrong: ${line}`); }); test("pct() has no artificial floor: tiny buckets show 0% and sum <= 100 (#386)", () => { - const pairs: Pair[] = [{ callId: "c1", toolName: "bash", resultChars: 40_000 }]; - const messages = toolHeavySession(pairs, 4); - const report = buildStatusReport(stateFor(messages), messages, defaultCountTokens); - const line = breakdownLine(report); - const toolPct = bucketPct(line, "tool"); - const textPct = bucketPct(line, "text"); - const sumPct = bucketPct(line, "summaries"); - assert.equal(textPct, 0, `1 token of 10_002 must round to 0%, got: ${line}`); - assert.ok(toolPct + textPct + sumPct <= 100, `bucket sum exceeds 100: ${line}`); + const pairs: Pair[] = [ + { callId: "c1", toolName: "bash", resultChars: 40_000 }, + ]; + const messages = toolHeavySession(pairs, 4); + const report = buildStatusReport( + stateFor(messages), + messages, + defaultCountTokens, + ); + const line = breakdownLine(report); + const toolPct = bucketPct(line, "tool"); + const textPct = bucketPct(line, "text"); + const sumPct = bucketPct(line, "summaries"); + assert.equal(textPct, 0, `1 token of 10_002 must round to 0%, got: ${line}`); + assert.ok( + toolPct + textPct + sumPct <= 100, + `bucket sum exceeds 100: ${line}`, + ); }); test("ranges view attributes resolved tool names consistently with the breakdown (#386)", () => { - const pairs: Pair[] = [{ callId: "c1", toolName: "bash", resultChars: 8_000 }]; - const messages = toolHeavySession(pairs); - const state = stateFor(messages); - const ranges = buildStatusReport(state, messages, defaultCountTokens, { scope: "uncompressed" }); - const rangeLines = ranges.split("\n").filter((l) => /^ m\d/.test(l)); - assert.ok(rangeLines.length > 0, `no range lines:\n${ranges}`); - const resultRange = rangeLines.find((l) => l.trimEnd().endsWith("bash")); - assert.ok(resultRange, `result range should be labeled bash:\n${ranges}`); + const pairs: Pair[] = [ + { callId: "c1", toolName: "bash", resultChars: 8_000 }, + ]; + const messages = toolHeavySession(pairs); + const state = stateFor(messages); + const ranges = buildStatusReport(state, messages, defaultCountTokens, { + scope: "uncompressed", + }); + const rangeLines = ranges.split("\n").filter((l) => /^ m\d/.test(l)); + assert.ok(rangeLines.length > 0, `no range lines:\n${ranges}`); + const resultRange = rangeLines.find((l) => l.trimEnd().endsWith("bash")); + assert.ok(resultRange, `result range should be labeled bash:\n${ranges}`); }); diff --git a/tests/report-truncation.test.ts b/tests/report-truncation.test.ts index 73ceb3c..05c9229 100644 --- a/tests/report-truncation.test.ts +++ b/tests/report-truncation.test.ts @@ -4,125 +4,148 @@ import { buildStatusReport } from "../src/report.js"; import { renderNudgeText } from "../src/nudge-text.js"; import { createInitialState } from "../src/state.js"; import { defaultCountTokens } from "../src/tokenize.js"; -import type { CompressionBlock, CompressionState, NudgeDecision } from "../src/types.js"; +import type { + CompressionBlock, + CompressionState, + NudgeDecision, +} from "../src/types.js"; function block(overrides: Partial): CompressionBlock { - return { - blockId: "b0", - runId: "r0", - tier: 1, - summary: "summary", - directMessageIds: [], - effectiveMessageIds: [], - directBlockIds: [], - createdAt: 1000, - survivedCount: 0, - generation: "young", - active: true, - ...overrides, - }; + return { + blockId: "b0", + runId: "r0", + tier: 1, + summary: "summary", + directMessageIds: [], + effectiveMessageIds: [], + directBlockIds: [], + createdAt: 1000, + survivedCount: 0, + generation: "young", + active: true, + ...overrides, + }; } // Mirrors bcp#330 session 01a07b3c: 41 active blocks (35 T1 + 6 T2) — the // exact condition under which a tier-2 distillation nudge fires AND the // overview's 30-entry cap starts hiding active target-tier blocks. function makeSessionState(): CompressionState { - const blocks: CompressionBlock[] = []; - for (let i = 1; i <= 35; i++) { - blocks.push( - block({ - blockId: `b${i}`, - tier: 1, - summary: `tier-1 summary ${i}`, - topic: `topic-${i}`, - compressedTokens: 8000 + i * 100, - effectiveMessageIds: Array.from({ length: 5 }, (_, j) => `m${i}_${j}`), - createdAt: 1000 + i, - }), - ); - } - for (let i = 36; i <= 41; i++) { - blocks.push( - block({ - blockId: `b${i}`, - tier: 2, - summary: `tier-2 summary ${i}`, - topic: `distilled-${i}`, - compressedTokens: 40000 + i * 100, - effectiveMessageIds: Array.from({ length: 40 }, (_, j) => `m${i}_${j}`), - directBlockIds: [`b${i - 3}`], - createdAt: 5000 + i, - }), - ); - } - return { ...createInitialState(), blocks }; + const blocks: CompressionBlock[] = []; + for (let i = 1; i <= 35; i++) { + blocks.push( + block({ + blockId: `b${i}`, + tier: 1, + summary: `tier-1 summary ${i}`, + topic: `topic-${i}`, + compressedTokens: 8000 + i * 100, + effectiveMessageIds: Array.from({ length: 5 }, (_, j) => `m${i}_${j}`), + createdAt: 1000 + i, + }), + ); + } + for (let i = 36; i <= 41; i++) { + blocks.push( + block({ + blockId: `b${i}`, + tier: 2, + summary: `tier-2 summary ${i}`, + topic: `distilled-${i}`, + compressedTokens: 40000 + i * 100, + effectiveMessageIds: Array.from({ length: 40 }, (_, j) => `m${i}_${j}`), + directBlockIds: [`b${i - 3}`], + createdAt: 5000 + i, + }), + ); + } + return { ...createInitialState(), blocks }; } function tier2Decision(state: CompressionState): NudgeDecision { - const targets = state.blocks.filter((b) => b.active && b.tier === 1); - return { - shouldInject: true, - reason: "tier-2 distillation pending", - compressibleRanges: [], - contextUsage: 0.7, - tier: 2, - tierTargetBlocks: targets, - breakdown: { - usage: 0.7, - growth: 0, - growthReference: 0, - effectiveThreshold: 0, - nudgeGrowthTokens: 0, - growthFloor: 0, - hasPendingNudge: 1, - overLimit: 0, - emergencyOverride: 0, - pendingT1: targets.length, - pendingT2: 0, - pendingT3: 0, - }, - }; + const targets = state.blocks.filter((b) => b.active && b.tier === 1); + return { + shouldInject: true, + reason: "tier-2 distillation pending", + compressibleRanges: [], + contextUsage: 0.7, + tier: 2, + tierTargetBlocks: targets, + breakdown: { + usage: 0.7, + growth: 0, + growthReference: 0, + effectiveThreshold: 0, + nudgeGrowthTokens: 0, + growthFloor: 0, + hasPendingNudge: 1, + overLimit: 0, + emergencyOverride: 0, + pendingT1: targets.length, + pendingT2: 0, + pendingT3: 0, + }, + }; } test("overview announces truncated block lists instead of hiding them (#221)", () => { - const state = makeSessionState(); - const report = buildStatusReport(state, [], defaultCountTokens); - assert.ok(report.includes("41 active"), "header keeps the full active count"); - const shownIds = state.blocks - .filter((b) => b.active) - .map((b) => b.blockId) - .filter((id) => report.includes(`${id} (`)); - assert.ok(shownIds.length < 41, "the list itself is capped below the full count"); - assert.match(report, /11 more blocks not shown/, "states the exact hidden count"); - assert.ok( - report.includes('scope:"compressed"'), - "points at the view that shows the full list", - ); + const state = makeSessionState(); + const report = buildStatusReport(state, [], defaultCountTokens); + assert.ok(report.includes("41 active"), "header keeps the full active count"); + const shownIds = state.blocks + .filter((b) => b.active) + .map((b) => b.blockId) + .filter((id) => report.includes(`${id} (`)); + assert.ok( + shownIds.length < 41, + "the list itself is capped below the full count", + ); + assert.match( + report, + /11 more blocks not shown/, + "states the exact hidden count", + ); + assert.ok( + report.includes('scope:"compressed"'), + "points at the view that shows the full list", + ); }); test("nudge target list vs status overview reconcile once truncation is announced (#221)", () => { - const state = makeSessionState(); - const decision = tier2Decision(state); - const nudge = renderNudgeText(decision); - assert.ok(nudge.text.includes("Target tier-1 blocks to distill (35)"), "nudge lists all 35 targets"); - for (const id of ["b1", "b20", "b35"]) { - assert.ok(nudge.text.includes(`${id} `), `nudge names ${id}`); - } + const state = makeSessionState(); + const decision = tier2Decision(state); + const nudge = renderNudgeText(decision); + assert.ok( + nudge.text.includes("Target tier-1 blocks to distill (35)"), + "nudge lists all 35 targets", + ); + for (const id of ["b1", "b20", "b35"]) { + assert.ok(nudge.text.includes(`${id} `), `nudge names ${id}`); + } - const report = buildStatusReport(state, [], defaultCountTokens); - const t1Shown = state.blocks.filter((b) => b.active && b.tier === 1 && report.includes(`${b.blockId} (T1)`)).length; - assert.ok(t1Shown < 35, "size-sorted cap hides some T1 targets from the overview"); - assert.ok(report.includes("41 active")); - assert.match(report, /11 more blocks not shown/, "shown + announced-hidden reconciles with the nudge count"); - assert.match(report, /T1: .*\(35 blocks\)/); - assert.match(report, /T2: .*\(6 blocks\)/); + const report = buildStatusReport(state, [], defaultCountTokens); + const t1Shown = state.blocks.filter( + (b) => b.active && b.tier === 1 && report.includes(`${b.blockId} (T1)`), + ).length; + assert.ok( + t1Shown < 35, + "size-sorted cap hides some T1 targets from the overview", + ); + assert.ok(report.includes("41 active")); + assert.match( + report, + /11 more blocks not shown/, + "shown + announced-hidden reconciles with the nudge count", + ); + assert.match(report, /T1: .*\(35 blocks\)/); + assert.match(report, /T2: .*\(6 blocks\)/); }); test("overview does not announce truncation when the list fits", () => { - const state: CompressionState = { - ...createInitialState(), - blocks: [block({ blockId: "b1" }), block({ blockId: "b2" })], - }; - const report = buildStatusReport(state, [], defaultCountTokens); - assert.ok(!report.includes("more blocks not shown")); + const state: CompressionState = { + ...createInitialState(), + blocks: [block({ blockId: "b1" }), block({ blockId: "b2" })], + }; + const report = buildStatusReport(state, [], defaultCountTokens); + assert.ok(!report.includes("more blocks not shown")); }); diff --git a/tests/search.test.ts b/tests/search.test.ts index 3a2166f..c7c131b 100644 --- a/tests/search.test.ts +++ b/tests/search.test.ts @@ -1,21 +1,38 @@ import test from "node:test"; import assert from "node:assert"; -import { searchBlocks, searchBlocksAsync, blockDocs, messageDocs, clearDocFeatures } from "../src/search.js"; -import { registerSearchAlgorithm, listSearchAlgorithms } from "../src/search.js"; +import { + searchBlocks, + searchBlocksAsync, + blockDocs, + messageDocs, + clearDocFeatures, +} from "../src/search.js"; +import { + registerSearchAlgorithm, + listSearchAlgorithms, +} from "../src/search.js"; import { createInitialState } from "../src/state.js"; import type { CompressionState, CompressionBlock } from "../src/types.js"; import type { SearchDoc } from "../src/search/types.js"; function makeBlock(overrides: Partial): CompressionBlock { - return { - blockId: "b1", runId: "r1", tier: 1, active: true, topic: "", summary: "", - directMessageIds: [], effectiveMessageIds: [], survivedCount: 0, createdAt: Date.now(), - ...overrides, - }; + return { + blockId: "b1", + runId: "r1", + tier: 1, + active: true, + topic: "", + summary: "", + directMessageIds: [], + effectiveMessageIds: [], + survivedCount: 0, + createdAt: Date.now(), + ...overrides, + }; } function stateWithBlocks(...blocks: CompressionBlock[]): CompressionState { - return { ...createInitialState(), blocks }; + return { ...createInitialState(), blocks }; } // ───────────────────────────────────────────────────────────────────────── @@ -23,36 +40,58 @@ function stateWithBlocks(...blocks: CompressionBlock[]): CompressionState { // ───────────────────────────────────────────────────────────────────────── test("blockDocs: includes BOTH active and inactive blocks", () => { - const state = stateWithBlocks( - makeBlock({ blockId: "b1", active: true, topic: "active", summary: "live content" }), - makeBlock({ blockId: "b2", active: false, topic: "archived", summary: "old content" }), - ); - const docs = blockDocs(state); - assert.equal(docs.length, 2); - assert.ok(docs.some((d) => d.ref === "b1")); - assert.ok(docs.some((d) => d.ref === "b2")); + const state = stateWithBlocks( + makeBlock({ + blockId: "b1", + active: true, + topic: "active", + summary: "live content", + }), + makeBlock({ + blockId: "b2", + active: false, + topic: "archived", + summary: "old content", + }), + ); + const docs = blockDocs(state); + assert.equal(docs.length, 2); + assert.ok(docs.some((d) => d.ref === "b1")); + assert.ok(docs.some((d) => d.ref === "b2")); }); test("searchBlocks: finds match in active block", () => { - const docs = blockDocs(stateWithBlocks( - makeBlock({ blockId: "b1", summary: "Auth token refresh", topic: "Auth" }), - makeBlock({ blockId: "b2", summary: "database pool", topic: "DB" }), - )); - const r = searchBlocks(docs, "auth"); - assert.equal(r.length, 1); - assert.equal(r[0].ref, "b1"); - assert.equal(r[0].kind, "block"); - assert.ok(r[0].score > 0); + const docs = blockDocs( + stateWithBlocks( + makeBlock({ + blockId: "b1", + summary: "Auth token refresh", + topic: "Auth", + }), + makeBlock({ blockId: "b2", summary: "database pool", topic: "DB" }), + ), + ); + const r = searchBlocks(docs, "auth"); + assert.equal(r.length, 1); + assert.equal(r[0].ref, "b1"); + assert.equal(r[0].kind, "block"); + assert.ok(r[0].score > 0); }); test("searchBlocks: finds match in INACTIVE block (the bug fix)", () => { - const docs = blockDocs(stateWithBlocks( - makeBlock({ blockId: "b1", active: true, summary: "current work" }), - makeBlock({ blockId: "b2", active: false, summary: "old auth token logic" }), - )); - const r = searchBlocks(docs, "auth"); - assert.equal(r.length, 1); - assert.equal(r[0].ref, "b2"); + const docs = blockDocs( + stateWithBlocks( + makeBlock({ blockId: "b1", active: true, summary: "current work" }), + makeBlock({ + blockId: "b2", + active: false, + summary: "old auth token logic", + }), + ), + ); + const r = searchBlocks(docs, "auth"); + assert.equal(r.length, 1); + assert.equal(r[0].ref, "b2"); }); // ───────────────────────────────────────────────────────────────────────── @@ -60,59 +99,99 @@ test("searchBlocks: finds match in INACTIVE block (the bug fix)", () => { // ───────────────────────────────────────────────────────────────────────── test("messageDocs: builds message docs from inputs", () => { - const docs = messageDocs([ - { ref: "m00100", role: "user", text: "how does auth work", blockId: "b1" }, - { ref: "m00200", role: "tool", text: "auth.ts: 401 handler", blockId: "b1" }, - ]); - assert.equal(docs.length, 2); - assert.equal(docs[0].kind, "message"); - assert.equal(docs[0].role, "user"); - assert.equal(docs[0].blockId, "b1"); + const docs = messageDocs([ + { ref: "m00100", role: "user", text: "how does auth work", blockId: "b1" }, + { + ref: "m00200", + role: "tool", + text: "auth.ts: 401 handler", + blockId: "b1", + }, + ]); + assert.equal(docs.length, 2); + assert.equal(docs[0].kind, "message"); + assert.equal(docs[0].role, "user"); + assert.equal(docs[0].blockId, "b1"); }); test("searchBlocks: searches messages by original text", () => { - const docs = messageDocs([ - { ref: "m00100", role: "user", text: "implement jwt refresh endpoint", blockId: "b1" }, - { ref: "m00200", role: "tool", text: "postgres connection string", blockId: "b2" }, - ]); - const r = searchBlocks(docs, "jwt"); - assert.equal(r.length, 1); - assert.equal(r[0].ref, "m00100"); - assert.equal(r[0].kind, "message"); - assert.equal(r[0].blockId, "b1", "result carries owning block for decompress"); + const docs = messageDocs([ + { + ref: "m00100", + role: "user", + text: "implement jwt refresh endpoint", + blockId: "b1", + }, + { + ref: "m00200", + role: "tool", + text: "postgres connection string", + blockId: "b2", + }, + ]); + const r = searchBlocks(docs, "jwt"); + assert.equal(r.length, 1); + assert.equal(r[0].ref, "m00100"); + assert.equal(r[0].kind, "message"); + assert.equal( + r[0].blockId, + "b1", + "result carries owning block for decompress", + ); }); test("searchBlocks: user role outranks tool role at equal text match (role weighting)", () => { - const docs = messageDocs([ - { ref: "m-tool", role: "tool", text: "match match here", blockId: "b1" }, - { ref: "m-user", role: "user", text: "match match here", blockId: "b1" }, - ]); - const r = searchBlocks(docs, "match"); - assert.equal(r[0].ref, "m-user", "user (1.5x) beats tool (0.6x) on same content"); - assert.ok(r[0].score > r[1].score); + const docs = messageDocs([ + { ref: "m-tool", role: "tool", text: "match match here", blockId: "b1" }, + { ref: "m-user", role: "user", text: "match match here", blockId: "b1" }, + ]); + const r = searchBlocks(docs, "match"); + assert.equal( + r[0].ref, + "m-user", + "user (1.5x) beats tool (0.6x) on same content", + ); + assert.ok(r[0].score > r[1].score); }); test("searchBlocks: roleWeights option overrides defaults", () => { - const docs = messageDocs([ - { ref: "m-tool", role: "tool", text: "match match here", blockId: "b1" }, - { ref: "m-user", role: "user", text: "match match here", blockId: "b1" }, - ]); - // equalize weights → scores tie, order falls back to original - const r = searchBlocks(docs, "match", { roleWeights: { user: 1, tool: 1, assistant: 1, block: 1 } }); - assert.ok(Math.abs(r[0].score - r[1].score) < 1e-9); + const docs = messageDocs([ + { ref: "m-tool", role: "tool", text: "match match here", blockId: "b1" }, + { ref: "m-user", role: "user", text: "match match here", blockId: "b1" }, + ]); + // equalize weights → scores tie, order falls back to original + const r = searchBlocks(docs, "match", { + roleWeights: { user: 1, tool: 1, assistant: 1, block: 1 }, + }); + assert.ok(Math.abs(r[0].score - r[1].score) < 1e-9); }); test("searchBlocks: mixed blocks + messages ranked together", () => { - const docs = [ - ...blockDocs(stateWithBlocks(makeBlock({ blockId: "b1", summary: "auth token refresh", topic: "auth" }))), - ...messageDocs([{ ref: "m00500", role: "assistant", text: "auth token refresh detail", blockId: "b1" }]), - ]; - const r = searchBlocks(docs, "auth"); - assert.equal(r.length, 2); - // both should appear; refs preserved - const refs = r.map((x) => x.ref); - assert.ok(refs.includes("b1")); - assert.ok(refs.includes("m00500")); + const docs = [ + ...blockDocs( + stateWithBlocks( + makeBlock({ + blockId: "b1", + summary: "auth token refresh", + topic: "auth", + }), + ), + ), + ...messageDocs([ + { + ref: "m00500", + role: "assistant", + text: "auth token refresh detail", + blockId: "b1", + }, + ]), + ]; + const r = searchBlocks(docs, "auth"); + assert.equal(r.length, 2); + // both should appear; refs preserved + const refs = r.map((x) => x.ref); + assert.ok(refs.includes("b1")); + assert.ok(refs.includes("m00500")); }); // ───────────────────────────────────────────────────────────────────────── @@ -120,23 +199,34 @@ test("searchBlocks: mixed blocks + messages ranked together", () => { // ───────────────────────────────────────────────────────────────────────── test("searchBlocks: message result includes tokens + blockId (for decompress hint)", () => { - const docs = messageDocs([{ ref: "m00420", role: "user", text: "login flow design".padEnd(500, "."), blockId: "b7", tokens: 150 }]); - const r = searchBlocks(docs, "login"); - assert.equal(r[0].ref, "m00420"); - assert.equal(r[0].blockId, "b7"); - assert.equal(r[0].tokens, 150); - assert.match(r[0].preview, /login/); + const docs = messageDocs([ + { + ref: "m00420", + role: "user", + text: "login flow design".padEnd(500, "."), + blockId: "b7", + tokens: 150, + }, + ]); + const r = searchBlocks(docs, "login"); + assert.equal(r[0].ref, "m00420"); + assert.equal(r[0].blockId, "b7"); + assert.equal(r[0].tokens, 150); + assert.match(r[0].preview, /login/); }); test("searchBlocks: preview centers on the matched term", () => { - const docs = messageDocs([{ - ref: "m1", role: "assistant", - text: "padding ".repeat(20) + " NEEDLE found here " + "more ".repeat(20), - blockId: "b1", - }]); - const r = searchBlocks(docs, "NEEDLE", { previewLength: 40 }); - assert.match(r[0].preview, /NEEDLE/); - assert.ok(!r[0].preview.startsWith("padding padding")); + const docs = messageDocs([ + { + ref: "m1", + role: "assistant", + text: "padding ".repeat(20) + " NEEDLE found here " + "more ".repeat(20), + blockId: "b1", + }, + ]); + const r = searchBlocks(docs, "NEEDLE", { previewLength: 40 }); + assert.match(r[0].preview, /NEEDLE/); + assert.ok(!r[0].preview.startsWith("padding padding")); }); // ───────────────────────────────────────────────────────────────────────── @@ -144,69 +234,81 @@ test("searchBlocks: preview centers on the matched term", () => { // ───────────────────────────────────────────────────────────────────────── test("searchBlocks: empty query returns nothing", () => { - const docs = blockDocs(stateWithBlocks(makeBlock({ summary: "content" }))); - assert.equal(searchBlocks(docs, "").length, 0); - assert.equal(searchBlocks(docs, " ").length, 0); + const docs = blockDocs(stateWithBlocks(makeBlock({ summary: "content" }))); + assert.equal(searchBlocks(docs, "").length, 0); + assert.equal(searchBlocks(docs, " ").length, 0); }); test("searchBlocks: respects limit", () => { - const docs = blockDocs(stateWithBlocks( - ...Array.from({ length: 20 }, (_, i) => makeBlock({ blockId: `b${i}`, summary: "match match match" })), - )); - assert.equal(searchBlocks(docs, "match", { limit: 5 }).length, 5); + const docs = blockDocs( + stateWithBlocks( + ...Array.from({ length: 20 }, (_, i) => + makeBlock({ blockId: `b${i}`, summary: "match match match" }), + ), + ), + ); + assert.equal(searchBlocks(docs, "match", { limit: 5 }).length, 5); }); test("searchBlocks: sorts by score descending", () => { - const docs = blockDocs(stateWithBlocks( - makeBlock({ blockId: "b1", summary: "one match" }), - makeBlock({ blockId: "b2", summary: "match match match match" }), - makeBlock({ blockId: "b3", summary: "match match" }), - )); - const r = searchBlocks(docs, "match"); - assert.equal(r[0].ref, "b2"); - assert.ok(r[0].score >= r[1].score); + const docs = blockDocs( + stateWithBlocks( + makeBlock({ blockId: "b1", summary: "one match" }), + makeBlock({ blockId: "b2", summary: "match match match match" }), + makeBlock({ blockId: "b3", summary: "match match" }), + ), + ); + const r = searchBlocks(docs, "match"); + assert.equal(r[0].ref, "b2"); + assert.ok(r[0].score >= r[1].score); }); test("searchBlocks: case insensitive", () => { - const docs = blockDocs(stateWithBlocks(makeBlock({ summary: "Auth Token Refresh" }))); - assert.equal(searchBlocks(docs, "AUTH token").length, 1); + const docs = blockDocs( + stateWithBlocks(makeBlock({ summary: "Auth Token Refresh" })), + ); + assert.equal(searchBlocks(docs, "AUTH token").length, 1); }); test("searchBlocks: algorithm option selects algorithm", () => { - const docs = blockDocs(stateWithBlocks(makeBlock({ summary: "auth token" }))); - const hybrid = searchBlocks(docs, "auth", { algorithm: "hybrid" }); - const substr = searchBlocks(docs, "auth", { algorithm: "substring" }); - assert.ok(hybrid.length > 0 && substr.length > 0); - assert.ok(substr[0].score >= 1, "substring score is occurrence count"); - assert.ok(hybrid[0].score <= 1.0001, "hybrid score is normalized"); + const docs = blockDocs(stateWithBlocks(makeBlock({ summary: "auth token" }))); + const hybrid = searchBlocks(docs, "auth", { algorithm: "hybrid" }); + const substr = searchBlocks(docs, "auth", { algorithm: "substring" }); + assert.ok(hybrid.length > 0 && substr.length > 0); + assert.ok(substr[0].score >= 1, "substring score is occurrence count"); + assert.ok(hybrid[0].score <= 1.0001, "hybrid score is normalized"); }); test("searchBlocks: unknown algorithm returns empty", () => { - const docs = blockDocs(stateWithBlocks(makeBlock({ summary: "match" }))); - assert.equal(searchBlocks(docs, "match", { algorithm: "nope" }).length, 0); + const docs = blockDocs(stateWithBlocks(makeBlock({ summary: "match" }))); + assert.equal(searchBlocks(docs, "match", { algorithm: "nope" }).length, 0); }); test("listSearchAlgorithms: includes all builtins", () => { - const names = listSearchAlgorithms().map((a) => a.name); - for (const n of ["hybrid", "bm25", "fuzzy", "substring"]) assert.ok(names.includes(n)); + const names = listSearchAlgorithms().map((a) => a.name); + for (const n of ["hybrid", "bm25", "fuzzy", "substring"]) + assert.ok(names.includes(n)); }); test("registerSearchAlgorithm: custom algorithm usable by name", () => { - registerSearchAlgorithm({ - name: "test-prefix", - description: "prefix-only scorer", - score(docs, query) { - const q = query.toLowerCase(); - return docs.map((d) => ({ ref: d.ref, score: d.text.toLowerCase().startsWith(q) ? 1 : 0 })); - }, - }); - const docs = messageDocs([ - { ref: "m1", role: "user", text: "prefix match", blockId: "b1" }, - { ref: "m2", role: "user", text: "no prefix here", blockId: "b2" }, - ]); - const r = searchBlocks(docs, "prefix", { algorithm: "test-prefix" }); - assert.equal(r.length, 1); - assert.equal(r[0].ref, "m1"); + registerSearchAlgorithm({ + name: "test-prefix", + description: "prefix-only scorer", + score(docs, query) { + const q = query.toLowerCase(); + return docs.map((d) => ({ + ref: d.ref, + score: d.text.toLowerCase().startsWith(q) ? 1 : 0, + })); + }, + }); + const docs = messageDocs([ + { ref: "m1", role: "user", text: "prefix match", blockId: "b1" }, + { ref: "m2", role: "user", text: "no prefix here", blockId: "b2" }, + ]); + const r = searchBlocks(docs, "prefix", { algorithm: "test-prefix" }); + assert.equal(r.length, 1); + assert.equal(r[0].ref, "m1"); }); // ───────────────────────────────────────────────────────────────────────── @@ -214,113 +316,190 @@ test("registerSearchAlgorithm: custom algorithm usable by name", () => { // ───────────────────────────────────────────────────────────────────────── test("hybrid: CJK query matches CJK content", () => { - const docs = [ - ...blockDocs(stateWithBlocks(makeBlock({ blockId: "b1", topic: "用户认证", summary: "实现了用户登录认证流程" }))), - ...messageDocs([{ ref: "m1", role: "assistant", text: "database postgres pool", blockId: "b2" }]), - ]; - const r = searchBlocks(docs, "登录"); - assert.equal(r[0].ref, "b1"); + const docs = [ + ...blockDocs( + stateWithBlocks( + makeBlock({ + blockId: "b1", + topic: "用户认证", + summary: "实现了用户登录认证流程", + }), + ), + ), + ...messageDocs([ + { + ref: "m1", + role: "assistant", + text: "database postgres pool", + blockId: "b2", + }, + ]), + ]; + const r = searchBlocks(docs, "登录"); + assert.equal(r[0].ref, "b1"); }); test("hybrid: typo tolerance via fuzzy", () => { - const docs = messageDocs([{ ref: "m1", role: "assistant", text: "authentication token refresh", blockId: "b1" }]); - const r = searchBlocks(docs, "tokan"); - assert.equal(r[0].ref, "m1"); + const docs = messageDocs([ + { + ref: "m1", + role: "assistant", + text: "authentication token refresh", + blockId: "b1", + }, + ]); + const r = searchBlocks(docs, "tokan"); + assert.equal(r[0].ref, "m1"); }); test("hybrid: stemming matches morphological variants", () => { - const docs = blockDocs(stateWithBlocks( - makeBlock({ blockId: "b1", topic: "compress", summary: "the compress utility" }), - makeBlock({ blockId: "b2", topic: "other", summary: "completely different caching topic" }), - )); - const r = searchBlocks(docs, "compressed"); - assert.equal(r[0].ref, "b1"); + const docs = blockDocs( + stateWithBlocks( + makeBlock({ + blockId: "b1", + topic: "compress", + summary: "the compress utility", + }), + makeBlock({ + blockId: "b2", + topic: "other", + summary: "completely different caching topic", + }), + ), + ); + const r = searchBlocks(docs, "compressed"); + assert.equal(r[0].ref, "b1"); }); test("tokenize: CJK runs become dictionary words, no single-char noise", async () => { - const { tokenize } = await import("../src/search/tokenizer.js"); - assert.deepEqual(tokenize("身份验证流程"), ["身份", "验证", "流程"]); - assert.equal(tokenize("身份验证流程").includes("份"), false); - assert.equal(tokenize("身份验证流程").includes("证流"), false); + const { tokenize } = await import("../src/search/tokenizer.js"); + assert.deepEqual(tokenize("身份验证流程"), ["身份", "验证", "流程"]); + assert.equal(tokenize("身份验证流程").includes("份"), false); + assert.equal(tokenize("身份验证流程").includes("证流"), false); }); test("tokenize: all-OOV run falls back to bigrams (recall preserved)", async () => { - const { tokenize } = await import("../src/search/tokenizer.js"); - const toks = tokenize("可视化"); - assert.ok(toks.includes("可视") || toks.includes("视化"), `got ${JSON.stringify(toks)}`); + const { tokenize } = await import("../src/search/tokenizer.js"); + const toks = tokenize("可视化"); + assert.ok( + toks.includes("可视") || toks.includes("视化"), + `got ${JSON.stringify(toks)}`, + ); }); test("tokenize: Japanese katakana becomes dictionary words, not char-run fragments", async () => { - const { tokenize } = await import("../src/search/tokenizer.js"); - const toks = tokenize("テスト用の文章です"); - assert.ok(toks.includes("テスト"), `got ${JSON.stringify(toks)}`); - assert.ok(toks.includes("文章"), `got ${JSON.stringify(toks)}`); - assert.equal(toks.includes("テス"), false, "katakana must not be split into char runs"); - assert.equal(toks.includes("スト"), false, "katakana must not be split into char runs"); + const { tokenize } = await import("../src/search/tokenizer.js"); + const toks = tokenize("テスト用の文章です"); + assert.ok(toks.includes("テスト"), `got ${JSON.stringify(toks)}`); + assert.ok(toks.includes("文章"), `got ${JSON.stringify(toks)}`); + assert.equal( + toks.includes("テス"), + false, + "katakana must not be split into char runs", + ); + assert.equal( + toks.includes("スト"), + false, + "katakana must not be split into char runs", + ); }); test("tokenize: Japanese kanji compounds are kept whole, no cross-word fragments", async () => { - const { tokenize } = await import("../src/search/tokenizer.js"); - const toks = tokenize("日本語の検索テスト"); - assert.ok(toks.includes("日本語"), `got ${JSON.stringify(toks)}`); - assert.equal(toks.includes("索テ"), false, "no cross-word '索テ' fragment"); + const { tokenize } = await import("../src/search/tokenizer.js"); + const toks = tokenize("日本語の検索テスト"); + assert.ok(toks.includes("日本語"), `got ${JSON.stringify(toks)}`); + assert.equal(toks.includes("索テ"), false, "no cross-word '索テ' fragment"); }); test("tokenize: Korean hangul becomes dictionary words, no cross-word fragments", async () => { - const { tokenize } = await import("../src/search/tokenizer.js"); - const toks = tokenize("테스트 문장입니다"); - assert.ok(toks.includes("테스트"), `got ${JSON.stringify(toks)}`); - assert.equal(toks.includes("니다"), false, "no cross-word '니다' fragment"); + const { tokenize } = await import("../src/search/tokenizer.js"); + const toks = tokenize("테스트 문장입니다"); + assert.ok(toks.includes("테스트"), `got ${JSON.stringify(toks)}`); + assert.equal(toks.includes("니다"), false, "no cross-word '니다' fragment"); }); test("tokenize: single-char query survives via fallback", async () => { - const { tokenize } = await import("../src/search/tokenizer.js"); - assert.deepEqual(tokenize("验"), ["验"]); + const { tokenize } = await import("../src/search/tokenizer.js"); + assert.deepEqual(tokenize("验"), ["验"]); }); test("hybrid: CJK query no longer matches across word boundaries (试验证明 vs 验证)", () => { - const docs = blockDocs(stateWithBlocks( - makeBlock({ blockId: "b1", topic: "auth", summary: "用户身份验证通过" }), - makeBlock({ blockId: "b2", topic: "experiment", summary: "试验数据采集已完成" }), - )); - const r = searchBlocks(docs, "试验证明"); - assert.equal(r[0].ref, "b2", "word-segmented docs must not rank 验证 above 试验"); + const docs = blockDocs( + stateWithBlocks( + makeBlock({ blockId: "b1", topic: "auth", summary: "用户身份验证通过" }), + makeBlock({ + blockId: "b2", + topic: "experiment", + summary: "试验数据采集已完成", + }), + ), + ); + const r = searchBlocks(docs, "试验证明"); + assert.equal( + r[0].ref, + "b2", + "word-segmented docs must not rank 验证 above 试验", + ); }); test("hybrid: dictionary word query hits the doc containing the whole word", () => { - const docs = blockDocs(stateWithBlocks( - makeBlock({ blockId: "b1", topic: "i18n", summary: "i18n 国际化 setup with locales" }), - makeBlock({ blockId: "b2", topic: "logs", summary: "ELK 日志栈 采集 过滤 存储。结构化日志 JSON。" }), - )); - const r = searchBlocks(docs, "国际化"); - assert.equal(r[0].ref, "b1"); + const docs = blockDocs( + stateWithBlocks( + makeBlock({ + blockId: "b1", + topic: "i18n", + summary: "i18n 国际化 setup with locales", + }), + makeBlock({ + blockId: "b2", + topic: "logs", + summary: "ELK 日志栈 采集 过滤 存储。结构化日志 JSON。", + }), + ), + ); + const r = searchBlocks(docs, "国际化"); + assert.equal(r[0].ref, "b1"); }); test("hybrid: OOV doc still recallable via bigram fallback", () => { - const docs = blockDocs(stateWithBlocks( - makeBlock({ blockId: "b1", topic: "dash", summary: "可视化" }), - makeBlock({ blockId: "b2", topic: "cache", summary: "缓存策略 redis 层" }), - )); - const r = searchBlocks(docs, "可视化"); - assert.equal(r[0].ref, "b1"); + const docs = blockDocs( + stateWithBlocks( + makeBlock({ blockId: "b1", topic: "dash", summary: "可视化" }), + makeBlock({ + blockId: "b2", + topic: "cache", + summary: "缓存策略 redis 层", + }), + ), + ); + const r = searchBlocks(docs, "可视化"); + assert.equal(r[0].ref, "b1"); }); test("hybrid: no RangeError at large doc counts (argument-spread → reduce, #227)", () => { - // The old `Math.max(...scores)` spread threw RangeError past V8's argument - // limit (~65K–125K docs). 160K is the scale that crashed on Node 22. - clearDocFeatures(); - try { - const n = 160_000; - const docs: SearchDoc[] = new Array(n); - for (let i = 0; i < n; i++) { - docs[i] = { kind: "block", ref: `b${i}`, text: `doc ${i} alpha beta`, title: `b${i}`, blockId: `b${i}`, tier: 1, tokens: 10 }; - } - const r = searchBlocks(docs, "alpha", { algorithm: "hybrid", limit: 10 }); - assert.equal(r.length, 10); - assert.ok(r[0].score > 0); - } finally { - clearDocFeatures(); + // The old `Math.max(...scores)` spread threw RangeError past V8's argument + // limit (~65K–125K docs). 160K is the scale that crashed on Node 22. + clearDocFeatures(); + try { + const n = 160_000; + const docs: SearchDoc[] = new Array(n); + for (let i = 0; i < n; i++) { + docs[i] = { + kind: "block", + ref: `b${i}`, + text: `doc ${i} alpha beta`, + title: `b${i}`, + blockId: `b${i}`, + tier: 1, + tokens: 10, + }; } + const r = searchBlocks(docs, "alpha", { algorithm: "hybrid", limit: 10 }); + assert.equal(r.length, 10); + assert.ok(r[0].score > 0); + } finally { + clearDocFeatures(); + } }); // ───────────────────────────────────────────────────────────────────────── @@ -328,31 +507,44 @@ test("hybrid: no RangeError at large doc counts (argument-spread → reduce, #22 // ───────────────────────────────────────────────────────────────────────── test("searchBlocks: throws for async algorithm when sync entry used", async () => { - const { createSemanticAlgorithm } = await import("../src/search/algorithms/semantic.js"); - registerSearchAlgorithm(createSemanticAlgorithm({ embed: async (t) => t.map(() => [1, 0]) })); - const docs = blockDocs(stateWithBlocks(makeBlock({ summary: "auth login" }))); - assert.throws(() => searchBlocks(docs, "login", { algorithm: "semantic" }), /searchBlocksAsync/); + const { createSemanticAlgorithm } = + await import("../src/search/algorithms/semantic.js"); + registerSearchAlgorithm( + createSemanticAlgorithm({ embed: async (t) => t.map(() => [1, 0]) }), + ); + const docs = blockDocs(stateWithBlocks(makeBlock({ summary: "auth login" }))); + assert.throws( + () => searchBlocks(docs, "login", { algorithm: "semantic" }), + /searchBlocksAsync/, + ); }); test("searchBlocksAsync: semantic ranks synonyms by cosine similarity", async () => { - const { createSemanticAlgorithm } = await import("../src/search/algorithms/semantic.js"); - const vocab: Record = { - auth: [1, 0], login: [0.95, 0.05], signin: [0.9, 0.1], database: [0, 1], - }; - const semantic = createSemanticAlgorithm({ - embed: async (texts: string[]) => texts.map((t) => { - const hit = Object.keys(vocab).find((k) => t.toLowerCase().includes(k)); - return hit ? vocab[hit]! : [0, 0, 1]; - }), - name: "semantic-syn", - }); - registerSearchAlgorithm(semantic); - const docs = messageDocs([ - { ref: "m1", role: "user", text: "login authentication", blockId: "b1" }, - { ref: "m2", role: "user", text: "database postgres", blockId: "b2" }, - ]); - const r = await searchBlocksAsync(docs, "signin", { algorithm: "semantic-syn" }); - assert.equal(r[0].ref, "m1", "semantic catches synonym signin≈login"); + const { createSemanticAlgorithm } = + await import("../src/search/algorithms/semantic.js"); + const vocab: Record = { + auth: [1, 0], + login: [0.95, 0.05], + signin: [0.9, 0.1], + database: [0, 1], + }; + const semantic = createSemanticAlgorithm({ + embed: async (texts: string[]) => + texts.map((t) => { + const hit = Object.keys(vocab).find((k) => t.toLowerCase().includes(k)); + return hit ? vocab[hit]! : [0, 0, 1]; + }), + name: "semantic-syn", + }); + registerSearchAlgorithm(semantic); + const docs = messageDocs([ + { ref: "m1", role: "user", text: "login authentication", blockId: "b1" }, + { ref: "m2", role: "user", text: "database postgres", blockId: "b2" }, + ]); + const r = await searchBlocksAsync(docs, "signin", { + algorithm: "semantic-syn", + }); + assert.equal(r[0].ref, "m1", "semantic catches synonym signin≈login"); }); // ───────────────────────────────────────────────────────────────────────── @@ -366,167 +558,270 @@ test("searchBlocksAsync: semantic ranks synonyms by cosine similarity", async () // ───────────────────────────────────────────────────────────────────────── test("stem: authenticate family does not converge (pinned, defect #2)", async () => { - const { stem } = await import("../src/search/stemmer.js"); - assert.equal(stem("authenticate"), "authenticate"); - assert.equal(stem("authenticated"), "authenticat"); - assert.equal(stem("authentication"), "authenticat"); + const { stem } = await import("../src/search/stemmer.js"); + assert.equal(stem("authenticate"), "authenticate"); + assert.equal(stem("authenticated"), "authenticat"); + assert.equal(stem("authentication"), "authenticat"); }); test("searchBlocks: morphology family only rescued by fuzzy cap (pinned)", () => { - const docs = blockDocs(stateWithBlocks(makeBlock({ blockId: "b1", summary: "authenticate the user account" }))); - const r = searchBlocks(docs, "authentication"); - assert.equal(r[0].ref, "b1", "fuzzy bigram overlap still rescues the hit"); - assert.ok(r[0].score < 0.7, `fuzzy-only rescue must stay under BM25 cap, got ${r[0].score}`); + const docs = blockDocs( + stateWithBlocks( + makeBlock({ blockId: "b1", summary: "authenticate the user account" }), + ), + ); + const r = searchBlocks(docs, "authentication"); + assert.equal(r[0].ref, "b1", "fuzzy bigram overlap still rescues the hit"); + assert.ok( + r[0].score < 0.7, + `fuzzy-only rescue must stay under BM25 cap, got ${r[0].score}`, + ); }); test("fuzzy: 2-char CJK query reaches the scorer, full bigram overlap scores 1 (defect #1 fixed)", () => { - // 缓存 has no dictionary word in the CLDR zh segmenter — the query would - // have been dropped by the old >= 4 gate. The CJK-only gate admits it. - const docs = blockDocs(stateWithBlocks(makeBlock({ blockId: "b1", summary: "缓存策略 redis 层" }))); - const r = searchBlocks(docs, "缓存", { algorithm: "fuzzy" }); - assert.equal(r.length, 1, "2-char CJK query must reach the fuzzy scorer"); - assert.equal(r[0].ref, "b1"); - assert.equal(r[0].score, 1, "full bigram overlap scores 1.0"); + // 缓存 has no dictionary word in the CLDR zh segmenter — the query would + // have been dropped by the old >= 4 gate. The CJK-only gate admits it. + const docs = blockDocs( + stateWithBlocks(makeBlock({ blockId: "b1", summary: "缓存策略 redis 层" })), + ); + const r = searchBlocks(docs, "缓存", { algorithm: "fuzzy" }); + assert.equal(r.length, 1, "2-char CJK query must reach the fuzzy scorer"); + assert.equal(r[0].ref, "b1"); + assert.equal(r[0].score, 1, "full bigram overlap scores 1.0"); }); test("fuzzy: 1-char CJK and 2-char Latin queries still filtered (gate is CJK-only)", () => { - const docs = blockDocs(stateWithBlocks(makeBlock({ blockId: "b1", summary: "缓存策略 redis 层" }))); - assert.equal(searchBlocks(docs, "验", { algorithm: "fuzzy" }).length, 0, "single char cannot form a bigram"); - assert.equal(searchBlocks(docs, "ok", { algorithm: "fuzzy" }).length, 0, "2-char Latin stays filtered (noise)"); - assert.equal(searchBlocks(docs, "to", { algorithm: "fuzzy" }).length, 0, "2-char Latin stays filtered (noise)"); + const docs = blockDocs( + stateWithBlocks(makeBlock({ blockId: "b1", summary: "缓存策略 redis 层" })), + ); + assert.equal( + searchBlocks(docs, "验", { algorithm: "fuzzy" }).length, + 0, + "single char cannot form a bigram", + ); + assert.equal( + searchBlocks(docs, "ok", { algorithm: "fuzzy" }).length, + 0, + "2-char Latin stays filtered (noise)", + ); + assert.equal( + searchBlocks(docs, "to", { algorithm: "fuzzy" }).length, + 0, + "2-char Latin stays filtered (noise)", + ); }); test("searchBlocks: 2-char CJK typo (登入 vs 登录) still returns nothing — gram mismatch, not gate", () => { - const docs = blockDocs(stateWithBlocks(makeBlock({ blockId: "b1", topic: "用户认证", summary: "实现了用户登录认证流程" }))); - const r = searchBlocks(docs, "登入"); - assert.equal(r.length, 0, "BM25 sees no shared token; 登入 is now admitted to fuzzy but shares no bigram with 登录"); + const docs = blockDocs( + stateWithBlocks( + makeBlock({ + blockId: "b1", + topic: "用户认证", + summary: "实现了用户登录认证流程", + }), + ), + ); + const r = searchBlocks(docs, "登入"); + assert.equal( + r.length, + 0, + "BM25 sees no shared token; 登入 is now admitted to fuzzy but shares no bigram with 登录", + ); }); test("searchBlocks: 2-char CJK query rescued end-to-end by fuzzy (缓存 → 缓存策略 doc)", () => { - // BM25 misses on purpose: 缓存 is not a dictionary word, and the doc's own - // tokens are its other dictionary words (策略/redis/层). Only the fuzzy - // channel sees the raw-text bigram 缓存 — which is exactly the recall gap - // the CJK gate repairs. - const docs = blockDocs(stateWithBlocks( - makeBlock({ blockId: "b1", topic: "viz", summary: "可视化 dashboards" }), - makeBlock({ blockId: "b2", topic: "cache", summary: "缓存策略 redis 层" }), - )); - const r = searchBlocks(docs, "缓存"); - assert.equal(r.length, 1); - assert.equal(r[0].ref, "b2", "fuzzy bigram overlap must rescue the cache block"); - assert.ok(r[0].score > 0, "rescued result must not be filtered"); - // Fuzzy-only rescue scores exactly W_FUZZY (0.3) — but ONLY while the ICU - // dictionary keeps 缓存 a non-word and BM25 misses. That is deliberate: - // if a Node/ICU bump starts segmenting 缓存 as a dictionary word, BM25 - // takes over and this assertion turns red. The red IS the signal that the - // dictionary changed — don't paper over it; decide then (e.g. swap the - // fixture to a word the dictionary still doesn't know). - assert.ok(r[0].score <= 0.31, `fuzzy-only rescue caps at W_FUZZY, got ${r[0].score}`); + // BM25 misses on purpose: 缓存 is not a dictionary word, and the doc's own + // tokens are its other dictionary words (策略/redis/层). Only the fuzzy + // channel sees the raw-text bigram 缓存 — which is exactly the recall gap + // the CJK gate repairs. + const docs = blockDocs( + stateWithBlocks( + makeBlock({ blockId: "b1", topic: "viz", summary: "可视化 dashboards" }), + makeBlock({ + blockId: "b2", + topic: "cache", + summary: "缓存策略 redis 层", + }), + ), + ); + const r = searchBlocks(docs, "缓存"); + assert.equal(r.length, 1); + assert.equal( + r[0].ref, + "b2", + "fuzzy bigram overlap must rescue the cache block", + ); + assert.ok(r[0].score > 0, "rescued result must not be filtered"); + // Fuzzy-only rescue scores exactly W_FUZZY (0.3) — but ONLY while the ICU + // dictionary keeps 缓存 a non-word and BM25 misses. That is deliberate: + // if a Node/ICU bump starts segmenting 缓存 as a dictionary word, BM25 + // takes over and this assertion turns red. The red IS the signal that the + // dictionary changed — don't paper over it; decide then (e.g. swap the + // fixture to a word the dictionary still doesn't know). + assert.ok( + r[0].score <= 0.31, + `fuzzy-only rescue caps at W_FUZZY, got ${r[0].score}`, + ); }); test("hybrid: CJK phrase still ranks whole-word doc above char-run doc (图表可视化, no leftover noise)", () => { - // Regression guard: the pr-rework guarantee must survive the fuzzy gate — - // a phrase must rank the fully-matching doc first and must not let the - // OOV-run doc's bigram leftovers (可视/视化) outrank it via fuzzy. - const docs = blockDocs(stateWithBlocks( - makeBlock({ blockId: "b1", topic: "dash", summary: "可视化" }), - makeBlock({ blockId: "b2", topic: "charts", summary: "图表可视化" }), - )); - const r = searchBlocks(docs, "图表可视化"); - assert.equal(r[0].ref, "b2", "whole-word doc must win"); - assert.ok(r[0].score > 0.5, `BM25 carries the full match, got ${r[0].score}`); - assert.ok(r.every((x) => x.ref !== "b1") || (r[0].ref === "b2" && r[1]?.ref !== "b2"), "OOV leftovers must not rank first"); + // Regression guard: the pr-rework guarantee must survive the fuzzy gate — + // a phrase must rank the fully-matching doc first and must not let the + // OOV-run doc's bigram leftovers (可视/视化) outrank it via fuzzy. + const docs = blockDocs( + stateWithBlocks( + makeBlock({ blockId: "b1", topic: "dash", summary: "可视化" }), + makeBlock({ blockId: "b2", topic: "charts", summary: "图表可视化" }), + ), + ); + const r = searchBlocks(docs, "图表可视化"); + assert.equal(r[0].ref, "b2", "whole-word doc must win"); + assert.ok(r[0].score > 0.5, `BM25 carries the full match, got ${r[0].score}`); + assert.ok( + r.every((x) => x.ref !== "b1") || (r[0].ref === "b2" && r[1]?.ref !== "b2"), + "OOV leftovers must not rank first", + ); }); test("searchBlocks: single-char query misses dict-word docs, hits OOV docs (pinned, defect #3)", () => { - const docs = blockDocs(stateWithBlocks( - makeBlock({ blockId: "b1", topic: "auth", summary: "身份验证流程" }), - makeBlock({ blockId: "b2", topic: "viz", summary: "可视化" }), - )); - const r = searchBlocks(docs, "验"); - assert.ok(r.every((x) => x.ref !== "b1"), `dict-word doc has no single-char token, got ${JSON.stringify(r)}`); - const oov = blockDocs(stateWithBlocks(makeBlock({ blockId: "b3", summary: "验" }))); - const r2 = searchBlocks(oov, "验"); - assert.equal(r2[0].ref, "b3", "OOV fallback docs keep single chars, still recallable"); + const docs = blockDocs( + stateWithBlocks( + makeBlock({ blockId: "b1", topic: "auth", summary: "身份验证流程" }), + makeBlock({ blockId: "b2", topic: "viz", summary: "可视化" }), + ), + ); + const r = searchBlocks(docs, "验"); + assert.ok( + r.every((x) => x.ref !== "b1"), + `dict-word doc has no single-char token, got ${JSON.stringify(r)}`, + ); + const oov = blockDocs( + stateWithBlocks(makeBlock({ blockId: "b3", summary: "验" })), + ); + const r2 = searchBlocks(oov, "验"); + assert.equal( + r2[0].ref, + "b3", + "OOV fallback docs keep single chars, still recallable", + ); }); test("tokenize: camelCase is not split (pinned, defect #3)", async () => { - const { tokenize } = await import("../src/search/tokenizer.js"); - assert.deepEqual(tokenize("syncBlocks"), ["syncblocks"]); + const { tokenize } = await import("../src/search/tokenizer.js"); + assert.deepEqual(tokenize("syncBlocks"), ["syncblocks"]); }); test("searchBlocks: space-separated camelCase query rescued by fuzzy cap (pinned)", () => { - const docs = blockDocs(stateWithBlocks(makeBlock({ blockId: "b1", summary: "syncBlocks registry walk" }))); - const r = searchBlocks(docs, "sync blocks"); - assert.equal(r[0].ref, "b1"); - assert.ok(r[0].score < 0.7, `single camelCase token can't be BM25-hit via 'sync', got ${r[0].score}`); + const docs = blockDocs( + stateWithBlocks( + makeBlock({ blockId: "b1", summary: "syncBlocks registry walk" }), + ), + ); + const r = searchBlocks(docs, "sync blocks"); + assert.equal(r[0].ref, "b1"); + assert.ok( + r[0].score < 0.7, + `single camelCase token can't be BM25-hit via 'sync', got ${r[0].score}`, + ); }); test("hybrid: fuzzy-only full overlap caps at 0.3 (pinned weight split)", () => { - const docs = messageDocs([ - { ref: "m1", role: "assistant", text: "auth token refresh", blockId: "b1" }, - { ref: "m2", role: "assistant", text: "tokanxyz", blockId: "b2" }, - ]); - const r = searchBlocks(docs, "tokan"); - assert.equal(r[0].ref, "m2", "fuzzy full overlap wins"); - assert.ok(r[0].score <= 0.31, `fuzzy-only score stuck at W_FUZZY cap, got ${r[0].score}`); + const docs = messageDocs([ + { ref: "m1", role: "assistant", text: "auth token refresh", blockId: "b1" }, + { ref: "m2", role: "assistant", text: "tokanxyz", blockId: "b2" }, + ]); + const r = searchBlocks(docs, "tokan"); + assert.equal(r[0].ref, "m2", "fuzzy full overlap wins"); + assert.ok( + r[0].score <= 0.31, + `fuzzy-only score stuck at W_FUZZY cap, got ${r[0].score}`, + ); }); test("searchBlocks: exact BM25 term hit dominates fuzzy-only overlap (weights pinned)", () => { - const docs = messageDocs([ - { ref: "m1", role: "assistant", text: "tokenized", blockId: "b1" }, - { ref: "m2", role: "assistant", text: "token", blockId: "b2" }, - ]); - const r = searchBlocks(docs, "token"); - assert.equal(r[0].ref, "m2", "exact 'token' beats 'tokenized' (stem splits -ized, pinned)"); - assert.ok(r[0].score >= 0.7, `BM25 channel carries exact hit, got ${r[0].score}`); + const docs = messageDocs([ + { ref: "m1", role: "assistant", text: "tokenized", blockId: "b1" }, + { ref: "m2", role: "assistant", text: "token", blockId: "b2" }, + ]); + const r = searchBlocks(docs, "token"); + assert.equal( + r[0].ref, + "m2", + "exact 'token' beats 'tokenized' (stem splits -ized, pinned)", + ); + assert.ok( + r[0].score >= 0.7, + `BM25 channel carries exact hit, got ${r[0].score}`, + ); }); test("preview: anchored on FIRST hit term only, later terms ignored (pinned, defect #6)", () => { - const text = "a".repeat(20) + " cache strategy " + "b".repeat(40) + " redis config"; - const r = searchBlocks(messageDocs([{ ref: "m1", role: "assistant", text, blockId: "b1" }]), "cache redis", { previewLength: 40 }); - const preview = r[0].preview; - assert.ok(preview.includes("cache"), `preview centers first hit: ${preview}`); - assert.ok(!preview.includes("redis"), `second hit term left out of window: ${preview}`); + const text = + "a".repeat(20) + " cache strategy " + "b".repeat(40) + " redis config"; + const r = searchBlocks( + messageDocs([{ ref: "m1", role: "assistant", text, blockId: "b1" }]), + "cache redis", + { previewLength: 40 }, + ); + const preview = r[0].preview; + assert.ok(preview.includes("cache"), `preview centers first hit: ${preview}`); + assert.ok( + !preview.includes("redis"), + `second hit term left out of window: ${preview}`, + ); }); test("tokenize: empty / punctuation / pure digits (edge)", async () => { - const { tokenize } = await import("../src/search/tokenizer.js"); - assert.deepEqual(tokenize(""), []); - assert.deepEqual(tokenize("!! ... "), []); - assert.deepEqual(tokenize("4096"), [], "leading-digit run fails LATIN_WORD first alt, single digits filtered"); - assert.deepEqual(tokenize("value=4096"), ["value"]); + const { tokenize } = await import("../src/search/tokenizer.js"); + assert.deepEqual(tokenize(""), []); + assert.deepEqual(tokenize("!! ... "), []); + assert.deepEqual( + tokenize("4096"), + [], + "leading-digit run fails LATIN_WORD first alt, single digits filtered", + ); + assert.deepEqual(tokenize("value=4096"), ["value"]); }); test("tokenize: digits inside CJK run dropped (edge)", async () => { - const { tokenize } = await import("../src/search/tokenizer.js"); - const toks = tokenize("检测到3个异常"); - assert.ok(toks.includes("检测") || toks.includes("检测到"), `got ${JSON.stringify(toks)}`); - assert.ok(toks.includes("异常"), `got ${JSON.stringify(toks)}`); - assert.equal(toks.includes("3"), false, "digit is not CJK, never tokenized"); + const { tokenize } = await import("../src/search/tokenizer.js"); + const toks = tokenize("检测到3个异常"); + assert.ok( + toks.includes("检测") || toks.includes("检测到"), + `got ${JSON.stringify(toks)}`, + ); + assert.ok(toks.includes("异常"), `got ${JSON.stringify(toks)}`); + assert.equal(toks.includes("3"), false, "digit is not CJK, never tokenized"); }); test("tokenize: lone single char survives, in-word single chars drop (asymmetry pinned)", async () => { - const { tokenize } = await import("../src/search/tokenizer.js"); - assert.deepEqual(tokenize("验"), ["验"], "fallback keeps single-char query"); - assert.equal(tokenize("身份验证流程").includes("份"), false, "segmented words drop interior chars"); + const { tokenize } = await import("../src/search/tokenizer.js"); + assert.deepEqual(tokenize("验"), ["验"], "fallback keeps single-char query"); + assert.equal( + tokenize("身份验证流程").includes("份"), + false, + "segmented words drop interior chars", + ); }); test("tokenize: stem flag leaves CJK untouched (invariance)", async () => { - const { tokenize } = await import("../src/search/tokenizer.js"); - assert.deepEqual(tokenize("身份验证", { stem: true }), tokenize("身份验证", { stem: false })); + const { tokenize } = await import("../src/search/tokenizer.js"); + assert.deepEqual( + tokenize("身份验证", { stem: true }), + tokenize("身份验证", { stem: false }), + ); }); test("charBigrams: pairs, whitespace filtered, short input empty", async () => { - const { charBigrams } = await import("../src/search/tokenizer.js"); - assert.deepEqual(charBigrams("登录"), ["登录"]); - assert.deepEqual(charBigrams("可视 化"), ["可视"]); - assert.deepEqual(charBigrams("a"), []); + const { charBigrams } = await import("../src/search/tokenizer.js"); + assert.deepEqual(charBigrams("登录"), ["登录"]); + assert.deepEqual(charBigrams("可视 化"), ["可视"]); + assert.deepEqual(charBigrams("a"), []); }); test("tfMap: counts per token, stems by flag", async () => { - const { tfMap } = await import("../src/search/tokenizer.js"); - const m = tfMap("token token 身份", true); - assert.equal(m.get("token"), 2); - assert.equal(m.get("身份"), 1); + const { tfMap } = await import("../src/search/tokenizer.js"); + const m = tfMap("token token 身份", true); + assert.equal(m.get("token"), 2); + assert.equal(m.get("身份"), 1); }); diff --git a/tests/state-prune.test.ts b/tests/state-prune.test.ts index 6581592..1f59292 100644 --- a/tests/state-prune.test.ts +++ b/tests/state-prune.test.ts @@ -189,7 +189,10 @@ test("prune covers sub-id projections when block recorded plain base id (#231)", ]; const result = prune(messages, state, { injectSummaries: false }); - assert.deepEqual(result.map((m) => m.id), ["h_pre", "h_post"]); + assert.deepEqual( + result.map((m) => m.id), + ["h_pre", "h_post"], + ); }); test("prune covers sibling sub-id projections of the same base (#231)", () => { @@ -200,7 +203,10 @@ test("prune covers sibling sub-id projections of the same base (#231)", () => { const messages = [msg("h_abc#call1", "assistant"), msg("h_post")]; const result = prune(messages, state, { injectSummaries: false }); - assert.deepEqual(result.map((m) => m.id), ["h_post"]); + assert.deepEqual( + result.map((m) => m.id), + ["h_post"], + ); }); test("prune does not cover a distinct id that extends a covered base (#231)", () => { @@ -211,7 +217,10 @@ test("prune does not cover a distinct id that extends a covered base (#231)", () const messages = [msg("h_abcde"), msg("h_post")]; const result = prune(messages, state, { injectSummaries: false }); - assert.deepEqual(result.map((m) => m.id), ["h_abcde", "h_post"]); + assert.deepEqual( + result.map((m) => m.id), + ["h_abcde", "h_post"], + ); }); test("prune anchors summary at base id when block recorded sub-ids (#231)", () => { diff --git a/tests/status-surface-meta.test.ts b/tests/status-surface-meta.test.ts index 8d61328..2f31bdc 100644 --- a/tests/status-surface-meta.test.ts +++ b/tests/status-surface-meta.test.ts @@ -7,95 +7,136 @@ import { indexToRef } from "../src/refs.js"; import type { CoreMessage } from "../src/types.js"; const messages: CoreMessage[] = [ - { id: "m1", role: "user", contentType: "text", text: "hello world, enough text to register" }, + { + id: "m1", + role: "user", + contentType: "text", + text: "hello world, enough text to register", + }, ]; function stateWithBlock(): ReturnType { - const state = createInitialState(); - messages.forEach((m, i) => { - const ref = indexToRef(i + 1); - state.messageRefs.byRaw[m.id] = ref; - state.messageRefs.byRef[ref] = m.id; - }); - state.blocks.push({ - blockId: "b1", - runId: "r1", - tier: 2, - summary: "folded history", - directMessageIds: ["m1"], - effectiveMessageIds: ["m1"], - directBlockIds: [], - compressedTokens: 40, - createdAt: 1, - survivedCount: 0, - generation: "young", - active: true, - }); - return state; + const state = createInitialState(); + messages.forEach((m, i) => { + const ref = indexToRef(i + 1); + state.messageRefs.byRaw[m.id] = ref; + state.messageRefs.byRef[ref] = m.id; + }); + state.blocks.push({ + blockId: "b1", + runId: "r1", + tier: 2, + summary: "folded history", + directMessageIds: ["m1"], + effectiveMessageIds: ["m1"], + directBlockIds: [], + compressedTokens: 40, + createdAt: 1, + survivedCount: 0, + generation: "young", + active: true, + }); + return state; } -const noMeta = buildStatusReport(createInitialState(), messages, defaultCountTokens); +const noMeta = buildStatusReport( + createInitialState(), + messages, + defaultCountTokens, +); test("overview renders a leading ACTIVE SURFACE line when meta is declared", () => { - const report = buildStatusReport(createInitialState(), messages, defaultCountTokens, { - meta: { pack: "lean", packVersion: "3", host: "billion-context-pi 0.1.69" }, - }); - const lines = report.split("\n"); - assert.equal(lines[0], "ACTIVE SURFACE: pack=lean v3 | host=billion-context-pi 0.1.69"); - assert.equal(lines[1], ""); - assert.equal(lines[2], "CONTEXT BREAKDOWN"); + const report = buildStatusReport( + createInitialState(), + messages, + defaultCountTokens, + { + meta: { + pack: "lean", + packVersion: "3", + host: "billion-context-pi 0.1.69", + }, + }, + ); + const lines = report.split("\n"); + assert.equal( + lines[0], + "ACTIVE SURFACE: pack=lean v3 | host=billion-context-pi 0.1.69", + ); + assert.equal(lines[1], ""); + assert.equal(lines[2], "CONTEXT BREAKDOWN"); }); test("overview stays byte-identical when no meta is passed", () => { - const report = buildStatusReport(createInitialState(), messages, defaultCountTokens, { - meta: undefined, - }); - assert.equal(report, noMeta); - assert.ok(!report.includes("ACTIVE SURFACE")); + const report = buildStatusReport( + createInitialState(), + messages, + defaultCountTokens, + { + meta: undefined, + }, + ); + assert.equal(report, noMeta); + assert.ok(!report.includes("ACTIVE SURFACE")); }); test("meta without host or version renders pack only", () => { - const report = buildStatusReport(createInitialState(), messages, defaultCountTokens, { - meta: { pack: "default" }, - }); - assert.ok(report.includes("ACTIVE SURFACE: pack=default\n")); + const report = buildStatusReport( + createInitialState(), + messages, + defaultCountTokens, + { + meta: { pack: "default" }, + }, + ); + assert.ok(report.includes("ACTIVE SURFACE: pack=default\n")); }); test("meta with only host renders host only", () => { - const report = buildStatusReport(createInitialState(), messages, defaultCountTokens, { - meta: { host: "billion-context" }, - }); - assert.ok(report.includes("ACTIVE SURFACE: host=billion-context\n")); + const report = buildStatusReport( + createInitialState(), + messages, + defaultCountTokens, + { + meta: { host: "billion-context" }, + }, + ); + assert.ok(report.includes("ACTIVE SURFACE: host=billion-context\n")); }); test("empty meta object renders nothing", () => { - const report = buildStatusReport(createInitialState(), messages, defaultCountTokens, { meta: {} }); - assert.equal(report, noMeta); + const report = buildStatusReport( + createInitialState(), + messages, + defaultCountTokens, + { meta: {} }, + ); + assert.equal(report, noMeta); }); test("compressed drilldown renders a leading ACTIVE SURFACE line", () => { - const state = stateWithBlock(); - const report = buildStatusReport(state, messages, defaultCountTokens, { - scope: "compressed", - meta: { pack: "lean", packVersion: "3" }, - }); - const lines = report.split("\n"); - assert.equal(lines[0], "ACTIVE SURFACE: pack=lean v3"); - assert.equal(lines[1], ""); - assert.ok((lines[2] ?? "").startsWith("COMPRESSED — 1 blocks")); + const state = stateWithBlock(); + const report = buildStatusReport(state, messages, defaultCountTokens, { + scope: "compressed", + meta: { pack: "lean", packVersion: "3" }, + }); + const lines = report.split("\n"); + assert.equal(lines[0], "ACTIVE SURFACE: pack=lean v3"); + assert.equal(lines[1], ""); + assert.ok((lines[2] ?? "").startsWith("COMPRESSED — 1 blocks")); }); test("uncompressed views never get the surface line", () => { - const state = stateWithBlock(); - const options = [ - { scope: "uncompressed" as const }, - { scope: "uncompressed" as const, view: "messages" as const }, - ]; - for (const o of options) { - const report = buildStatusReport(state, messages, defaultCountTokens, { - ...o, - meta: { pack: "lean", packVersion: "3", host: "billion-context" }, - }); - assert.ok(!report.includes("ACTIVE SURFACE")); - } + const state = stateWithBlock(); + const options = [ + { scope: "uncompressed" as const }, + { scope: "uncompressed" as const, view: "messages" as const }, + ]; + for (const o of options) { + const report = buildStatusReport(state, messages, defaultCountTokens, { + ...o, + meta: { pack: "lean", packVersion: "3", host: "billion-context" }, + }); + assert.ok(!report.includes("ACTIVE SURFACE")); + } }); diff --git a/tests/surface-config.test.ts b/tests/surface-config.test.ts index 6c77b05..d2d9efd 100644 --- a/tests/surface-config.test.ts +++ b/tests/surface-config.test.ts @@ -32,7 +32,11 @@ test("applySectionOverrides with no overrides returns all defaults in order", () test("applySectionOverrides replaces a section with the given string", () => { const out = applySectionOverrides(SECTIONS, { beta: "BETA\n\nCUSTOM" }); - assert.deepEqual(out, ["ALPHA\n\nalpha body", "BETA\n\nCUSTOM", "GAMMA\n\ngamma body"]); + assert.deepEqual(out, [ + "ALPHA\n\nalpha body", + "BETA\n\nCUSTOM", + "GAMMA\n\ngamma body", + ]); }); test("applySectionOverrides removes a section on null (no empty gap)", () => { @@ -42,7 +46,10 @@ test("applySectionOverrides removes a section on null (no empty gap)", () => { test("applySectionOverrides ignores unknown keys", () => { const out = applySectionOverrides(SECTIONS, { nope: "X" }); - assert.deepEqual(out, SECTIONS.map(([, t]) => t)); + assert.deepEqual( + out, + SECTIONS.map(([, t]) => t), + ); }); test("applySectionOverrides ignores malformed (non-string, non-null) values", () => { @@ -60,9 +67,18 @@ test("applySectionOverrides ignores malformed (non-string, non-null) values", () test("default builders are byte-identical to the published 0.0.46 output", () => { const fixture = (name: string) => readFileSync(new URL(`./fixtures/${name}`, import.meta.url), "utf8"); - assert.equal(buildCompressSystemPrompt(), fixture("prompt-function-default.txt")); - assert.equal(buildCompressTextSystemPrompt(), fixture("prompt-text-default.txt")); - assert.equal(buildCompressHybridSystemPrompt(), fixture("prompt-hybrid-default.txt")); + assert.equal( + buildCompressSystemPrompt(), + fixture("prompt-function-default.txt"), + ); + assert.equal( + buildCompressTextSystemPrompt(), + fixture("prompt-text-default.txt"), + ); + assert.equal( + buildCompressHybridSystemPrompt(), + fixture("prompt-hybrid-default.txt"), + ); }); test("function builder honors section replace and remove", () => { @@ -70,16 +86,22 @@ test("function builder honors section replace and remove", () => { acpTags: "ACP TAGS\n\nCUSTOM TAG TEXT", }); assert.ok(replaced.includes("CUSTOM TAG TEXT")); - assert.ok(!replaced.includes("NEVER echo, repeat, or reference these XML tags")); + assert.ok( + !replaced.includes("NEVER echo, repeat, or reference these XML tags"), + ); assert.ok(replaced.includes("You have five context-management tools")); - const removed = buildCompressSystemPrompt(undefined, { summariesInContext: null }); + const removed = buildCompressSystemPrompt(undefined, { + summariesInContext: null, + }); assert.ok(!removed.includes("COMPRESSION SUMMARIES IN CONTEXT")); assert.ok(!removed.includes("MODEL-GENERATED summaries")); assert.ok(removed.endsWith("Use to find what to compress next.")); assert.ok(!removed.includes("next.\n\n\n")); - const crossIgnored = buildCompressSystemPrompt(undefined, { textProtocol: "NOPE" }); + const crossIgnored = buildCompressSystemPrompt(undefined, { + textProtocol: "NOPE", + }); assert.equal(crossIgnored, buildCompressSystemPrompt()); }); @@ -127,8 +149,14 @@ test("cloneWithDescriptions replaces top-level and nested parameter descriptions startId: "NEW-START", }) as ReturnType; assert.equal(out.properties.topic.description, "NEW-TOPIC"); - assert.equal(out.properties.content.items.properties.startId.description, "NEW-START"); - assert.equal(out.properties.content.items.properties.endId.description, "END-DESC"); + assert.equal( + out.properties.content.items.properties.startId.description, + "NEW-START", + ); + assert.equal( + out.properties.content.items.properties.endId.description, + "END-DESC", + ); }); test("cloneWithDescriptions never mutates the input", () => { @@ -167,11 +195,23 @@ test("applyAcpToolOverrides handles the anthropic shape (name + input_schema)", assert.ok(compress); assert.equal(compress.description, "NEW-COMPRESS-DESC"); const schema = compress.input_schema as { - properties: Record }[] } }[] }>; + properties: Record< + string, + { + anyOf: { + items?: { + anyOf: { properties?: Record }[]; + }; + }[]; + } + >; }; - const objectForms = schema.properties.content.anyOf[0].items!.anyOf.filter((v) => v.properties !== undefined); + const objectForms = schema.properties.content.anyOf[0].items!.anyOf.filter( + (v) => v.properties !== undefined, + ); assert.equal(objectForms.length, 2); - for (const objectForm of objectForms) assert.equal(objectForm.properties!.startId!.description, "NEW-START"); + for (const objectForm of objectForms) + assert.equal(objectForm.properties!.startId!.description, "NEW-START"); const untouched = out.find((t) => t.name === "decompress"); assert.equal(untouched, ACP_TOOLS_ANTHROPIC[1]); }); @@ -206,7 +246,8 @@ test("applyAcpToolOverrides without overrides copies the array but keeps tool re const out = applyAcpToolOverrides(ACP_TOOLS_ANTHROPIC); assert.notEqual(out, ACP_TOOLS_ANTHROPIC); assert.deepEqual(out, ACP_TOOLS_ANTHROPIC); - for (let i = 0; i < out.length; i++) assert.equal(out[i], ACP_TOOLS_ANTHROPIC[i]); + for (let i = 0; i < out.length; i++) + assert.equal(out[i], ACP_TOOLS_ANTHROPIC[i]); }); test("applyAcpToolOverrides never mutates the shared tool constants", () => { diff --git a/tests/sync-config.test.ts b/tests/sync-config.test.ts index 29c17a7..b376e43 100644 --- a/tests/sync-config.test.ts +++ b/tests/sync-config.test.ts @@ -53,20 +53,41 @@ test("syncBlocks leaves blocks intact when at least one message remains", () => test("syncBlocks keeps expanded (user-decompressed) blocks deactivated when present", () => { const state = createInitialState(); state.blocks.push( - makeBlock({ blockId: "b1", effectiveMessageIds: ["a", "b"], active: false, expanded: true }), + makeBlock({ + blockId: "b1", + effectiveMessageIds: ["a", "b"], + active: false, + expanded: true, + }), ); const result = syncBlocks([msg("a"), msg("b")], state); - assert.equal(result.state.blocks[0]!.active, false, "expanded block must stay deactivated"); - assert.deepEqual(result.deactivated, [], "expanded block is not reported as orphaned"); + assert.equal( + result.state.blocks[0]!.active, + false, + "expanded block must stay deactivated", + ); + assert.deepEqual( + result.deactivated, + [], + "expanded block is not reported as orphaned", + ); }); test("syncBlocks re-activates a deactivated (non-expanded) block when its messages return", () => { const state = createInitialState(); state.blocks.push( - makeBlock({ blockId: "b1", effectiveMessageIds: ["a", "b"], active: false }), + makeBlock({ + blockId: "b1", + effectiveMessageIds: ["a", "b"], + active: false, + }), ); const result = syncBlocks([msg("a"), msg("b")], state); - assert.equal(result.state.blocks[0]!.active, true, "legacy resurrection preserved for non-expanded blocks"); + assert.equal( + result.state.blocks[0]!.active, + true, + "legacy resurrection preserved for non-expanded blocks", + ); }); test("syncBlocks keeps block active when view emits base id for sub-id block (#231)", () => { @@ -146,9 +167,13 @@ test("validateConfig flags min > max nudge thresholds", () => { test("validateConfig flags negative or non-finite minPressureBenefitTokens", () => { const base = defaultConfig(200000); base.nudge.minPressureBenefitTokens = -1; - assert.ok(validateConfig(base).some((e) => e.includes("minPressureBenefitTokens"))); + assert.ok( + validateConfig(base).some((e) => e.includes("minPressureBenefitTokens")), + ); base.nudge.minPressureBenefitTokens = Number.NaN; - assert.ok(validateConfig(base).some((e) => e.includes("minPressureBenefitTokens"))); + assert.ok( + validateConfig(base).some((e) => e.includes("minPressureBenefitTokens")), + ); }); test("validateConfig accepts explicit minPressureBenefitTokens override", () => { diff --git a/tests/tier-scale-separation.test.ts b/tests/tier-scale-separation.test.ts index 1225546..88f6cc2 100644 --- a/tests/tier-scale-separation.test.ts +++ b/tests/tier-scale-separation.test.ts @@ -47,13 +47,21 @@ function buildConfig(overrides: Partial = {}): Config { }; } -function textMessage(role: CoreMessage["role"], id: string, text: string): CoreMessage { +function textMessage( + role: CoreMessage["role"], + id: string, + text: string, +): CoreMessage { return { id, role, contentType: "text", text }; } function makeMessages(count: number): CoreMessage[] { return Array.from({ length: count }, (_, i) => - textMessage(i % 2 === 0 ? "user" : "assistant", `m${i}`, `message ${i} content`), + textMessage( + i % 2 === 0 ? "user" : "assistant", + `m${i}`, + `message ${i} content`, + ), ); } @@ -81,7 +89,12 @@ function grownState(blocks: CompressionBlock[], messageCount: number) { const core = createCore(); const config = buildConfig(); const messages = makeMessages(messageCount); - const first = core.processTurn({ messages, state: createInitialState(), config, tokenCount: 50_000 }).state; + const first = core.processTurn({ + messages, + state: createInitialState(), + config, + tokenCount: 50_000, + }).state; return { core, config, messages, state: { ...first, blocks } }; } @@ -90,15 +103,32 @@ test("T2 count boundary: 999 tier-1 blocks stay silent, 1000 fires (#383)", () = // (nudgeGrowthTokens 6000, tier2Threshold 9000), so ONLY the count path can // speak. 999 blocks: silent. 1000: fires. const silent = grownState(microBlocks(999, 1), 1030); - const quiet = silent.core.processTurn({ messages: silent.messages, state: silent.state, config: silent.config, tokenCount: 60_000 }); - assert.equal(quiet.nudge.shouldInject, false, `reason: ${quiet.nudge.reason}`); + const quiet = silent.core.processTurn({ + messages: silent.messages, + state: silent.state, + config: silent.config, + tokenCount: 60_000, + }); + assert.equal( + quiet.nudge.shouldInject, + false, + `reason: ${quiet.nudge.reason}`, + ); assert.doesNotMatch(quiet.nudge.reason ?? "", /T2 distill ready/); const ready = grownState(microBlocks(1000, 1), 1031); - const turn = ready.core.processTurn({ messages: ready.messages, state: ready.state, config: ready.config, tokenCount: 60_000 }); + const turn = ready.core.processTurn({ + messages: ready.messages, + state: ready.state, + config: ready.config, + tokenCount: 60_000, + }); assert.equal(turn.nudge.shouldInject, true, `reason: ${turn.nudge.reason}`); assert.equal(turn.nudge.tier, 2); - assert.match(turn.nudge.reason ?? "", /1000 tier-1 blocks >= tier2Trigger 1000/); + assert.match( + turn.nudge.reason ?? "", + /1000 tier-1 blocks >= tier2Trigger 1000/, + ); }); test("T3 count boundary: 1999 tier-2 blocks stay silent, 2000 fires (#383)", () => { @@ -106,15 +136,32 @@ test("T3 count boundary: 1999 tier-2 blocks stay silent, 2000 fires (#383)", () // so the mass path stays silent and the T2 branch cannot preempt // (t2Count = 0 active tier-1 blocks). const silent = grownState(microBlocks(1999, 2), 2030); - const quiet = silent.core.processTurn({ messages: silent.messages, state: silent.state, config: silent.config, tokenCount: 60_000 }); - assert.equal(quiet.nudge.shouldInject, false, `reason: ${quiet.nudge.reason}`); + const quiet = silent.core.processTurn({ + messages: silent.messages, + state: silent.state, + config: silent.config, + tokenCount: 60_000, + }); + assert.equal( + quiet.nudge.shouldInject, + false, + `reason: ${quiet.nudge.reason}`, + ); assert.doesNotMatch(quiet.nudge.reason ?? "", /T3 condense ready/); const ready = grownState(microBlocks(2000, 2), 2031); - const turn = ready.core.processTurn({ messages: ready.messages, state: ready.state, config: ready.config, tokenCount: 60_000 }); + const turn = ready.core.processTurn({ + messages: ready.messages, + state: ready.state, + config: ready.config, + tokenCount: 60_000, + }); assert.equal(turn.nudge.shouldInject, true, `reason: ${turn.nudge.reason}`); assert.equal(turn.nudge.tier, 3); - assert.match(turn.nudge.reason ?? "", /2000 tier-2 blocks >= tier3Trigger 2000/); + assert.match( + turn.nudge.reason ?? "", + /2000 tier-2 blocks >= tier3Trigger 2000/, + ); }); test("scale contract: default triggers encode 亿/几十亿 cumulative folded mass (#383)", () => { @@ -129,7 +176,16 @@ test("scale contract: default triggers encode 亿/几十亿 cumulative folded ma assert.equal(config.tiers.tier3Trigger, 2000); const minFoldPerT1 = 5000; // compress.minCompressRange floor per tier-1 block at creation const typicalFoldPerT1 = 50_000; // default nudge growthFloor cadence - assert.ok(config.tiers.tier2Trigger * minFoldPerT1 >= 5_000_000, "T2 count floor must be >= 5M cumulative folded tokens"); - assert.ok(config.tiers.tier2Trigger * typicalFoldPerT1 >= 50_000_000, "T2 count typical scale must reach 亿-level (50M+)"); - assert.ok(config.tiers.tier3Trigger * 75_000 >= 150_000_000, "T3 count floor must be >= 1.5亿 cumulative (2000 x 75K distill mass)"); + assert.ok( + config.tiers.tier2Trigger * minFoldPerT1 >= 5_000_000, + "T2 count floor must be >= 5M cumulative folded tokens", + ); + assert.ok( + config.tiers.tier2Trigger * typicalFoldPerT1 >= 50_000_000, + "T2 count typical scale must reach 亿-level (50M+)", + ); + assert.ok( + config.tiers.tier3Trigger * 75_000 >= 150_000_000, + "T3 count floor must be >= 1.5亿 cumulative (2000 x 75K distill mass)", + ); }); diff --git a/tests/token-snapshot.test.ts b/tests/token-snapshot.test.ts index 999480b..7e95cd1 100644 --- a/tests/token-snapshot.test.ts +++ b/tests/token-snapshot.test.ts @@ -30,7 +30,10 @@ function stateWithRefs(messages: CoreMessage[]): CompressionState { // point of tokenSnapshot (issue #96, 方案 A). First render writes, later // renders reuse the recorded count even when countTokens changes. test("renderWithSnapshot: token count is stable across countTokens changes", () => { - const messages = [msg("a", "你好世界这是一段中文消息"), msg("b", "hello world")]; + const messages = [ + msg("a", "你好世界这是一段中文消息"), + msg("b", "hello world"), + ]; const state = stateWithRefs(messages); const slow = (t: string) => Math.ceil(t.length / 2); // CJK-ish density 2 @@ -41,7 +44,12 @@ test("renderWithSnapshot: token count is stable across countTokens changes", () assert.ok(Object.keys(snapshot).length >= 2, "first render records entries"); // Second render with a different countTokens: same tokens from snapshot. - const r2 = renderWithSnapshot(messages, { ...state, tokenSnapshot: snapshot }, fast, "all"); + const r2 = renderWithSnapshot( + messages, + { ...state, tokenSnapshot: snapshot }, + fast, + "all", + ); assert.deepEqual(r2.tokenSnapshot, snapshot, "snapshot unchanged on hit"); const tokens1 = [...r1.messages.map((m) => m.text ?? "")].map((t) => Number(/\stokens="(\d+)"/.exec(t)?.[1]), @@ -52,7 +60,12 @@ test("renderWithSnapshot: token count is stable across countTokens changes", () assert.deepEqual(tokens2, tokens1, "rendered tag token counts identical"); // Sanity: the recorded count came from the FIRST renderer, not the second. - const fresh = renderWithSnapshot(messages, { ...state, tokenSnapshot: {} }, fast, "all"); + const fresh = renderWithSnapshot( + messages, + { ...state, tokenSnapshot: {} }, + fast, + "all", + ); assert.notDeepEqual( fresh.messages.map((m) => /\stokens="(\d+)"/.exec(m.text ?? "")?.[1]), tokens1.map(String), @@ -79,7 +92,9 @@ test("renderVisibleRefs keeps live recompute (no snapshot)", () => { // otherwise render-refs (which runs AFTER sync-blocks) sees an empty snapshot // every turn and the whole scheme collapses. test("syncBlocks preserves tokenSnapshot", () => { - const messages = [msg("a", "hello world this is a longer message for tokens")]; + const messages = [ + msg("a", "hello world this is a longer message for tokens"), + ]; const state = stateWithRefs(messages); const { messages: rendered, tokenSnapshot } = renderWithSnapshot( messages, @@ -102,7 +117,10 @@ test("syncBlocks preserves tokenSnapshot", () => { // G2 regression: cloneState runs inside applyCompression — a compression must // not drop the snapshot. test("compression preserves tokenSnapshot", () => { - const messages = [msg("a", "first message content here"), msg("b", "second message content")]; + const messages = [ + msg("a", "first message content here"), + msg("b", "second message content"), + ]; const state = stateWithRefs(messages); const { messages: rendered, tokenSnapshot } = renderWithSnapshot( messages, @@ -133,13 +151,19 @@ test("compression preserves tokenSnapshot", () => { test("processTurn: snapshot persists across turns and stabilizes tags", () => { const core = createCore(); let state = createInitialState(); - let messages = [msg("a", "第一轮消息内容比较长一些"), msg("b", "第二轮 hello world")]; + let messages = [ + msg("a", "第一轮消息内容比较长一些"), + msg("b", "第二轮 hello world"), + ]; let firstTagTokens: (string | undefined)[]; for (let turn = 0; turn < 4; turn++) { const ctx = { config: defaultConfig(), - countTokens: turn % 2 === 0 ? (t: string) => Math.ceil(t.length / 2) : (t: string) => Math.ceil(t.length / 4), + countTokens: + turn % 2 === 0 + ? (t: string) => Math.ceil(t.length / 2) + : (t: string) => Math.ceil(t.length / 4), }; const io = makeIO(messages, state); state = io.state; @@ -147,7 +171,9 @@ test("processTurn: snapshot persists across turns and stabilizes tags", () => { state = result.state; messages = result.messages; - const tags = messages.map((m) => /\stokens="(\d+)"/.exec(m.text ?? "")?.[1]); + const tags = messages.map( + (m) => /\stokens="(\d+)"/.exec(m.text ?? "")?.[1], + ); if (turn === 0) { firstTagTokens = tags; assert.ok( @@ -201,11 +227,18 @@ test("renderRefsNode: steady-state hit does not churn the state object", () => { const io = makeIO(rendered, filled); const ctx = { config: defaultConfig(), countTokens: (t: string) => t.length }; const out = renderRefsNode.run(io, ctx); - assert.equal(out.state, filled, "all-hit render must reuse the same state object"); + assert.equal( + out.state, + filled, + "all-hit render must reuse the same state object", + ); }); test("syncBlocks prunes tokenSnapshot entries for absent messages", () => { - const messages = [msg("a", "first message here"), msg("b", "second message here")]; + const messages = [ + msg("a", "first message here"), + msg("b", "second message here"), + ]; const state = stateWithRefs(messages); const { tokenSnapshot } = renderWithSnapshot( messages, @@ -213,7 +246,18 @@ test("syncBlocks prunes tokenSnapshot entries for absent messages", () => { (t) => Math.ceil(t.length / 2), "all", ); - const synced = syncBlocks([msg("b", "second message here")], { ...state, tokenSnapshot }); - assert.equal("m00001" in synced.state.tokenSnapshot, false, "absent 'a' entry must be pruned"); - assert.equal("m00002" in synced.state.tokenSnapshot, true, "present 'b' entry must be retained"); + const synced = syncBlocks([msg("b", "second message here")], { + ...state, + tokenSnapshot, + }); + assert.equal( + "m00001" in synced.state.tokenSnapshot, + false, + "absent 'a' entry must be pruned", + ); + assert.equal( + "m00002" in synced.state.tokenSnapshot, + true, + "present 'b' entry must be retained", + ); }); diff --git a/tests/tool-pairs.test.ts b/tests/tool-pairs.test.ts index 099aa5a..730f86e 100644 --- a/tests/tool-pairs.test.ts +++ b/tests/tool-pairs.test.ts @@ -6,16 +6,38 @@ import { createCore } from "../src/compress.js"; import { createInitialState } from "../src/state.js"; import { defaultConfig } from "../src/config.js"; -function msg(id: string, contentType: CoreMessage["contentType"], extra: Partial = {}): CoreMessage { +function msg( + id: string, + contentType: CoreMessage["contentType"], + extra: Partial = {}, +): CoreMessage { return { id, role: "user", contentType, text: `msg-${id}`, ...extra }; } function toolCall(id: string, callId: string, toolName = "read"): CoreMessage { - return { id, role: "assistant", contentType: "tool-call", toolName, toolCallId: callId, text: `call-${callId}` }; + return { + id, + role: "assistant", + contentType: "tool-call", + toolName, + toolCallId: callId, + text: `call-${callId}`, + }; } -function toolResult(id: string, callId: string, toolName = "read"): CoreMessage { - return { id, role: "user", contentType: "tool-result", toolName, toolCallId: callId, text: `result-${callId}` }; +function toolResult( + id: string, + callId: string, + toolName = "read", +): CoreMessage { + return { + id, + role: "user", + contentType: "tool-result", + toolName, + toolCallId: callId, + text: `result-${callId}`, + }; } function textMsg(id: string): CoreMessage { @@ -114,7 +136,12 @@ describe("adjustBoundariesForToolPairs", () => { }); describe("compression with tool-pair protection", () => { - const cfg = { ...defaultConfig(100000), compress: { ...defaultConfig(100000).compress, minCompressRange: 0 }, preserveRecentMessages: 0, preserveRecentTokens: 0 }; + const cfg = { + ...defaultConfig(100000), + compress: { ...defaultConfig(100000).compress, minCompressRange: 0 }, + preserveRecentMessages: 0, + preserveRecentTokens: 0, + }; it("compression auto-includes orphaned tool-result", () => { const core = createCore(); @@ -126,14 +153,22 @@ describe("compression with tool-pair protection", () => { textMsg("m5"), ]; const state = createInitialState(); - const stateWithRefs = core.processTurn({ messages, state, config: cfg, tokenCount: 1000 }).state; + const stateWithRefs = core.processTurn({ + messages, + state, + config: cfg, + tokenCount: 1000, + }).state; const result = core.applyCompression({ - ranges: [{ - startRef: "m00001", - endRef: "m00003", - summary: "Compressed range with tool-call but not its result — extension should pull the result in.", - }], + ranges: [ + { + startRef: "m00001", + endRef: "m00003", + summary: + "Compressed range with tool-call but not its result — extension should pull the result in.", + }, + ], messages, state: stateWithRefs, config: cfg, @@ -142,8 +177,14 @@ describe("compression with tool-pair protection", () => { assert.equal(result.result.blocksCreated, 1); const block = result.state.blocks.find((b) => b.active); assert.ok(block); - assert.ok(block!.effectiveMessageIds.includes("m4"), "tool-result m4 should be auto-included"); - assert.ok(block!.effectiveMessageIds.includes("m2"), "tool-call m2 should be in block"); + assert.ok( + block!.effectiveMessageIds.includes("m4"), + "tool-result m4 should be auto-included", + ); + assert.ok( + block!.effectiveMessageIds.includes("m2"), + "tool-call m2 should be in block", + ); }); it("compression auto-includes orphaned tool-call", () => { @@ -156,14 +197,22 @@ describe("compression with tool-pair protection", () => { textMsg("m5"), ]; const state = createInitialState(); - const stateWithRefs = core.processTurn({ messages, state, config: cfg, tokenCount: 1000 }).state; + const stateWithRefs = core.processTurn({ + messages, + state, + config: cfg, + tokenCount: 1000, + }).state; const result = core.applyCompression({ - ranges: [{ - startRef: "m00003", - endRef: "m00004", - summary: "Compressed range with tool-result but not its call — extension pulls the call in.", - }], + ranges: [ + { + startRef: "m00003", + endRef: "m00004", + summary: + "Compressed range with tool-result but not its call — extension pulls the call in.", + }, + ], messages, state: stateWithRefs, config: cfg, @@ -172,8 +221,14 @@ describe("compression with tool-pair protection", () => { assert.equal(result.result.blocksCreated, 1); const block = result.state.blocks.find((b) => b.active); assert.ok(block); - assert.ok(block!.effectiveMessageIds.includes("m2"), "tool-call m2 should be auto-included"); - assert.ok(block!.effectiveMessageIds.includes("m4"), "tool-result m4 should be in block"); + assert.ok( + block!.effectiveMessageIds.includes("m2"), + "tool-call m2 should be auto-included", + ); + assert.ok( + block!.effectiveMessageIds.includes("m4"), + "tool-result m4 should be in block", + ); }); it("block-boundary range skips tool-pair adjustment (tier detection preserved)", () => { @@ -188,15 +243,23 @@ describe("compression with tool-pair protection", () => { textMsg("m7"), ]; const state = createInitialState(); - const s1 = core.processTurn({ messages, state, config: cfg, tokenCount: 1000 }).state; + const s1 = core.processTurn({ + messages, + state, + config: cfg, + tokenCount: 1000, + }).state; const r1 = core.applyCompression({ - ranges: [{ - startRef: "m00001", - endRef: "m00004", - summary: "First tier-1 block covering initial text and a complete tool pair for reading configuration.", - topic: "Phase 1", - }], + ranges: [ + { + startRef: "m00001", + endRef: "m00004", + summary: + "First tier-1 block covering initial text and a complete tool pair for reading configuration.", + topic: "Phase 1", + }, + ], messages, state: s1, config: cfg, @@ -207,23 +270,34 @@ describe("compression with tool-pair protection", () => { assert.equal(b1!.tier, 1); const r2 = core.applyCompression({ - ranges: [{ - startRef: "b1", - endRef: "b1", - summary: "Tier-2 distillation of the first phase, condensing the initial setup and configuration reads.", - topic: "Distilled Phase 1", - }], + ranges: [ + { + startRef: "b1", + endRef: "b1", + summary: + "Tier-2 distillation of the first phase, condensing the initial setup and configuration reads.", + topic: "Distilled Phase 1", + }, + ], messages, state: r1.state, config: cfg, }); - assert.equal(r2.result.blocksCreated, 1, `b1→b1 should create b2, errors: ${JSON.stringify(r2.result.errors)}`); + assert.equal( + r2.result.blocksCreated, + 1, + `b1→b1 should create b2, errors: ${JSON.stringify(r2.result.errors)}`, + ); const b2 = r2.state.blocks.find((b) => b.blockId === "b2"); assert.ok(b2, "tier-2 block b2 should be created"); assert.equal(b2!.tier, 2, "b2 should be tier-2"); const b1After = r2.state.blocks.find((b) => b.blockId === "b1"); assert.ok(!b1After!.active, "b1 should be consumed by b2"); - assert.deepEqual(b2!.directBlockIds, ["b1"], "b2 should list b1 as consumed"); + assert.deepEqual( + b2!.directBlockIds, + ["b1"], + "b2 should list b1 as consumed", + ); }); it("tool-pair extension re-scans for nested blocks in adjusted range", () => { @@ -237,15 +311,23 @@ describe("compression with tool-pair protection", () => { textMsg("m6"), ]; const state = createInitialState(); - const s1 = core.processTurn({ messages, state, config: cfg, tokenCount: 1000 }).state; + const s1 = core.processTurn({ + messages, + state, + config: cfg, + tokenCount: 1000, + }).state; const r1 = core.applyCompression({ - ranges: [{ - startRef: "m00001", - endRef: "m00002", - summary: "Block covering the first complete tool pair for initial file read operations.", - topic: "Pair 1", - }], + ranges: [ + { + startRef: "m00001", + endRef: "m00002", + summary: + "Block covering the first complete tool pair for initial file read operations.", + topic: "Pair 1", + }, + ], messages, state: s1, config: cfg, @@ -253,12 +335,15 @@ describe("compression with tool-pair protection", () => { assert.equal(r1.result.blocksCreated, 1); const r2 = core.applyCompression({ - ranges: [{ - startRef: "m00004", - endRef: "m00005", - summary: "Second complete tool pair covering the second file read and its result output.", - topic: "Pair 2", - }], + ranges: [ + { + startRef: "m00004", + endRef: "m00005", + summary: + "Second complete tool pair covering the second file read and its result output.", + topic: "Pair 2", + }, + ], messages, state: r1.state, config: cfg, @@ -266,12 +351,20 @@ describe("compression with tool-pair protection", () => { assert.equal(r2.result.blocksCreated, 1); const b2 = r2.state.blocks.find((b) => b.blockId === "b2"); assert.ok(b2); - assert.ok(!b2!.directBlockIds.includes("b1"), "b1 should NOT be consumed (its anchor is outside the range)"); + assert.ok( + !b2!.directBlockIds.includes("b1"), + "b1 should NOT be consumed (its anchor is outside the range)", + ); }); }); describe("prune stripOrphanedToolCalls (defense-in-depth)", () => { - const cfg = { ...defaultConfig(100000), compress: { ...defaultConfig(100000).compress, minCompressRange: 0 }, preserveRecentMessages: 0, preserveRecentTokens: 0 }; + const cfg = { + ...defaultConfig(100000), + compress: { ...defaultConfig(100000).compress, minCompressRange: 0 }, + preserveRecentMessages: 0, + preserveRecentTokens: 0, + }; it("complete pair survives when other pair is compressed", () => { const core = createCore(); @@ -284,14 +377,22 @@ describe("prune stripOrphanedToolCalls (defense-in-depth)", () => { textMsg("m6"), ]; const state = createInitialState(); - const stateWithRefs = core.processTurn({ messages, state, config: cfg, tokenCount: 1000 }).state; + const stateWithRefs = core.processTurn({ + messages, + state, + config: cfg, + tokenCount: 1000, + }).state; const result = core.applyCompression({ - ranges: [{ - startRef: "m00004", - endRef: "m00005", - summary: "Second tool pair compressed. First pair should survive intact.", - }], + ranges: [ + { + startRef: "m00004", + endRef: "m00005", + summary: + "Second tool pair compressed. First pair should survive intact.", + }, + ], messages, state: stateWithRefs, config: cfg, @@ -318,16 +419,19 @@ describe("prune stripOrphanedToolCalls (defense-in-depth)", () => { // builds and for block-boundary rewrites — so construct the split state // directly: a block whose coverage contains only the tool-call. const core = createCore(); - const messages: CoreMessage[] = [ - toolCall("m1", "c1"), - ]; + const messages: CoreMessage[] = [toolCall("m1", "c1")]; for (let i = 0; i < 25; i++) { messages.push(textMsg(`gap${i}`)); } messages.push(toolResult("result", "c1")); const state = createInitialState(); - const stateWithRefs = core.processTurn({ messages, state, config: cfg, tokenCount: 5000 }).state; + const stateWithRefs = core.processTurn({ + messages, + state, + config: cfg, + tokenCount: 5000, + }).state; const refOf = (raw: string) => stateWithRefs.messageRefs.byRaw[raw]!; stateWithRefs.blocks.push({ @@ -356,6 +460,9 @@ describe("prune stripOrphanedToolCalls (defense-in-depth)", () => { const ids = pruned.map((m) => m.id); assert.ok(!ids.includes("m1"), "blocked tool-call m1 hidden"); - assert.ok(!ids.includes("result"), "orphaned tool-result stripped by defense-in-depth"); + assert.ok( + !ids.includes("result"), + "orphaned tool-result stripped by defense-in-depth", + ); }); }); diff --git a/tests/truncate-keep-filter.test.ts b/tests/truncate-keep-filter.test.ts index f59f6f8..e391da1 100644 --- a/tests/truncate-keep-filter.test.ts +++ b/tests/truncate-keep-filter.test.ts @@ -2,160 +2,194 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { truncateLargeToolOutputs } from "../src/truncate-tools.js"; import { - applyMessageFilters, - clearMessageFilters, - registerMessageFilter, + applyMessageFilters, + clearMessageFilters, + registerMessageFilter, } from "../src/filter/index.js"; import { createCore } from "../src/compress.js"; import { createInitialState } from "../src/state.js"; import { defaultConfig } from "../src/config.js"; import type { CoreMessage } from "../src/types.js"; -function msg(id: string, text: string, type: CoreMessage["contentType"] = "text"): CoreMessage { - return { id, role: "user", contentType: type, text }; +function msg( + id: string, + text: string, + type: CoreMessage["contentType"] = "text", +): CoreMessage { + return { id, role: "user", contentType: type, text }; } test("truncateLargeToolOutputs is a no-op below threshold", () => { - const cfg = defaultConfig(100000, { truncate: { threshold: 0.8 } }); - const messages = [msg("t1", "x".repeat(5000), "tool-result")]; - const result = truncateLargeToolOutputs(messages, 1000, cfg, (t) => t.length, { - minOutputTokens: 100, - }); - assert.equal(result.truncatedCount, 0); + const cfg = defaultConfig(100000, { truncate: { threshold: 0.8 } }); + const messages = [msg("t1", "x".repeat(5000), "tool-result")]; + const result = truncateLargeToolOutputs( + messages, + 1000, + cfg, + (t) => t.length, + { + minOutputTokens: 100, + }, + ); + assert.equal(result.truncatedCount, 0); }); test("truncateLargeToolOutputs truncates large tool outputs over threshold", () => { - const cfg = defaultConfig(100000, { truncate: { threshold: 0.5 } }); - const big = "L".repeat(10000); - const messages = [ - msg("a", big, "tool-result"), - msg("b", big, "tool-result"), - msg("c", "recent", "tool-result"), - msg("d", "recent", "tool-result"), - msg("e", "recent", "tool-result"), - ]; - const result = truncateLargeToolOutputs(messages, 90000, cfg, (t) => t.length, { - minOutputTokens: 100, - keepPrefixChars: 100, - keepSuffixChars: 100, - protectRecentMessages: 3, - }); - assert.ok(result.truncatedCount >= 1); - assert.ok(result.savedTokens > 0); - assert.ok(result.messages[0]!.text!.includes("[truncated for context space")); + const cfg = defaultConfig(100000, { truncate: { threshold: 0.5 } }); + const big = "L".repeat(10000); + const messages = [ + msg("a", big, "tool-result"), + msg("b", big, "tool-result"), + msg("c", "recent", "tool-result"), + msg("d", "recent", "tool-result"), + msg("e", "recent", "tool-result"), + ]; + const result = truncateLargeToolOutputs( + messages, + 90000, + cfg, + (t) => t.length, + { + minOutputTokens: 100, + keepPrefixChars: 100, + keepSuffixChars: 100, + protectRecentMessages: 3, + }, + ); + assert.ok(result.truncatedCount >= 1); + assert.ok(result.savedTokens > 0); + assert.ok(result.messages[0]!.text!.includes("[truncated for context space")); }); test("truncateLargeToolOutputs protects recent messages", () => { - const cfg = defaultConfig(100000, { truncate: { threshold: 0.1 } }); - const big = "L".repeat(10000); - const messages = [msg("only", big, "tool-result")]; - const result = truncateLargeToolOutputs(messages, 90000, cfg, (t) => t.length, { - minOutputTokens: 100, - protectRecentMessages: 3, - }); - assert.equal(result.truncatedCount, 0); + const cfg = defaultConfig(100000, { truncate: { threshold: 0.1 } }); + const big = "L".repeat(10000); + const messages = [msg("only", big, "tool-result")]; + const result = truncateLargeToolOutputs( + messages, + 90000, + cfg, + (t) => t.length, + { + minOutputTokens: 100, + protectRecentMessages: 3, + }, + ); + assert.equal(result.truncatedCount, 0); }); test("applyMessageFilters is a no-op when disabled", () => { - clearMessageFilters(); - const messages = [msg("a", "hello")]; - const result = applyMessageFilters(messages, { enabled: false, filters: {} }); - assert.equal(result.partsFiltered, 0); - assert.equal(result.messages[0]!.text, "hello"); + clearMessageFilters(); + const messages = [msg("a", "hello")]; + const result = applyMessageFilters(messages, { enabled: false, filters: {} }); + assert.equal(result.partsFiltered, 0); + assert.equal(result.messages[0]!.text, "hello"); }); test("applyMessageFilters drops text matching an immediate filter", () => { - clearMessageFilters(); - registerMessageFilter({ - name: "drop-noise", - version: "1.0.0", - description: "", - filter: (ctx) => (ctx.text.includes("NOISE") ? { action: "drop", reason: "noise" } : { action: "keep" }), - }); - const messages = [msg("a", "clean"), msg("b", "has NOISE inside")]; - const result = applyMessageFilters(messages, { enabled: true, filters: { "drop-noise": { enabled: true } } }); - assert.equal(result.partsDropped, 1); - assert.equal(result.messages[1]!.text, ""); - assert.equal(result.messages[0]!.text, "clean"); - clearMessageFilters(); + clearMessageFilters(); + registerMessageFilter({ + name: "drop-noise", + version: "1.0.0", + description: "", + filter: (ctx) => + ctx.text.includes("NOISE") + ? { action: "drop", reason: "noise" } + : { action: "keep" }, + }); + const messages = [msg("a", "clean"), msg("b", "has NOISE inside")]; + const result = applyMessageFilters(messages, { + enabled: true, + filters: { "drop-noise": { enabled: true } }, + }); + assert.equal(result.partsDropped, 1); + assert.equal(result.messages[1]!.text, ""); + assert.equal(result.messages[0]!.text, "clean"); + clearMessageFilters(); }); test("applyMessageFilters keepLastOnly keeps only the last match", () => { - clearMessageFilters(); - registerMessageFilter({ - name: "dedup", - version: "1.0.0", - description: "", - keepLastOnly: true, - filter: (ctx) => (ctx.text.startsWith("REPEAT:") ? { action: "modify", text: ctx.text } : { action: "keep" }), - }); - const messages = [ - msg("a", "REPEAT:directive"), - msg("b", "normal"), - msg("c", "REPEAT:directive-updated"), - ]; - const result = applyMessageFilters(messages, { enabled: true, filters: { dedup: { enabled: true } } }); - assert.equal(result.messages[0]!.text, ""); - assert.equal(result.messages[1]!.text, "normal"); - assert.equal(result.messages[2]!.text, "REPEAT:directive-updated"); - clearMessageFilters(); + clearMessageFilters(); + registerMessageFilter({ + name: "dedup", + version: "1.0.0", + description: "", + keepLastOnly: true, + filter: (ctx) => + ctx.text.startsWith("REPEAT:") + ? { action: "modify", text: ctx.text } + : { action: "keep" }, + }); + const messages = [ + msg("a", "REPEAT:directive"), + msg("b", "normal"), + msg("c", "REPEAT:directive-updated"), + ]; + const result = applyMessageFilters(messages, { + enabled: true, + filters: { dedup: { enabled: true } }, + }); + assert.equal(result.messages[0]!.text, ""); + assert.equal(result.messages[1]!.text, "normal"); + assert.equal(result.messages[2]!.text, "REPEAT:directive-updated"); + clearMessageFilters(); }); test("emergency-truncate node fires at default 0.95 threshold via processTurn", () => { - const core = createCore(); - const cfg = defaultConfig(100000); - const big = "L".repeat(40000); - const messages: CoreMessage[] = [ - msg("a", big, "tool-result"), - msg("b", big, "tool-result"), - msg("c", big, "tool-result"), - msg("d", "recent", "tool-result"), - msg("e", "recent", "tool-result"), - msg("f", "recent", "tool-result"), - msg("g", "recent", "tool-result"), - msg("h", "recent", "tool-result"), - ]; - const result = core.processTurn({ - messages, - state: createInitialState(), - config: cfg, - tokenCount: 96000, - }); - const truncated = result.messages.filter((m) => - m.text?.includes("[truncated for context space"), - ); - assert.ok( - truncated.length >= 1, - `expected >= 1 truncated msg at 96% usage (default truncate.threshold 0.95), got ${truncated.length}`, - ); + const core = createCore(); + const cfg = defaultConfig(100000); + const big = "L".repeat(40000); + const messages: CoreMessage[] = [ + msg("a", big, "tool-result"), + msg("b", big, "tool-result"), + msg("c", big, "tool-result"), + msg("d", "recent", "tool-result"), + msg("e", "recent", "tool-result"), + msg("f", "recent", "tool-result"), + msg("g", "recent", "tool-result"), + msg("h", "recent", "tool-result"), + ]; + const result = core.processTurn({ + messages, + state: createInitialState(), + config: cfg, + tokenCount: 96000, + }); + const truncated = result.messages.filter((m) => + m.text?.includes("[truncated for context space"), + ); + assert.ok( + truncated.length >= 1, + `expected >= 1 truncated msg at 96% usage (default truncate.threshold 0.95), got ${truncated.length}`, + ); }); test("emergency-truncate node is a no-op below default 0.95 threshold", () => { - const core = createCore(); - const cfg = defaultConfig(100000); - const big = "L".repeat(40000); - const messages: CoreMessage[] = [ - msg("a", big, "tool-result"), - msg("b", big, "tool-result"), - msg("c", big, "tool-result"), - msg("d", "recent", "tool-result"), - msg("e", "recent", "tool-result"), - msg("f", "recent", "tool-result"), - msg("g", "recent", "tool-result"), - msg("h", "recent", "tool-result"), - ]; - const result = core.processTurn({ - messages, - state: createInitialState(), - config: cfg, - tokenCount: 80000, - }); - const truncated = result.messages.filter((m) => - m.text?.includes("[truncated for context space"), - ); - assert.equal( - truncated.length, - 0, - `expected 0 truncated msgs at 80% usage (below default 0.95), got ${truncated.length}`, - ); + const core = createCore(); + const cfg = defaultConfig(100000); + const big = "L".repeat(40000); + const messages: CoreMessage[] = [ + msg("a", big, "tool-result"), + msg("b", big, "tool-result"), + msg("c", big, "tool-result"), + msg("d", "recent", "tool-result"), + msg("e", "recent", "tool-result"), + msg("f", "recent", "tool-result"), + msg("g", "recent", "tool-result"), + msg("h", "recent", "tool-result"), + ]; + const result = core.processTurn({ + messages, + state: createInitialState(), + config: cfg, + tokenCount: 80000, + }); + const truncated = result.messages.filter((m) => + m.text?.includes("[truncated for context space"), + ); + assert.equal( + truncated.length, + 0, + `expected 0 truncated msgs at 80% usage (below default 0.95), got ${truncated.length}`, + ); }); diff --git a/tests/truncate-surrogates.test.ts b/tests/truncate-surrogates.test.ts index a981160..83f1318 100644 --- a/tests/truncate-surrogates.test.ts +++ b/tests/truncate-surrogates.test.ts @@ -2,196 +2,286 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { clampPrefix, clampWindow } from "../src/truncate.js"; import { hideConsumedCompressCalls } from "../src/hide-consumed.js"; -import { deriveTopicFromSummary, parseCompressArgs } from "../src/parse-compress-input.js"; +import { + deriveTopicFromSummary, + parseCompressArgs, +} from "../src/parse-compress-input.js"; import { buildRecap } from "../src/report.js"; -import { searchBlocks, messageDocs, type SearchDoc } from "../src/search/index.js"; +import { + searchBlocks, + messageDocs, + type SearchDoc, +} from "../src/search/index.js"; import { createInitialState } from "../src/state.js"; import { topicFallback } from "../src/panel/topic.js"; import { truncateLargeToolOutputs } from "../src/truncate-tools.js"; import { buildRestoredContentPreview } from "../src/decompress.js"; import { defaultConfig } from "../src/config.js"; -import type { CompressionBlock, CompressionState, CoreMessage } from "../src/types.js"; +import type { + CompressionBlock, + CompressionState, + CoreMessage, +} from "../src/types.js"; const EMOJI = "\u{1F980}"; function assertNoUnpairedSurrogates(s: string): void { - for (let i = 0; i < s.length; i++) { - const c = s.charCodeAt(i); - if (c >= 0xd800 && c <= 0xdbff) { - const n = s.charCodeAt(i + 1); - assert.ok(n >= 0xdc00 && n <= 0xdfff, `lone high surrogate at ${i} in ${JSON.stringify(s.slice(Math.max(0, i - 4), i + 4))}`); - i++; - } else if (c >= 0xdc00 && c <= 0xdfff) { - assert.fail(`lone low surrogate at ${i} in ${JSON.stringify(s.slice(Math.max(0, i - 4), i + 4))}`); - } + for (let i = 0; i < s.length; i++) { + const c = s.charCodeAt(i); + if (c >= 0xd800 && c <= 0xdbff) { + const n = s.charCodeAt(i + 1); + assert.ok( + n >= 0xdc00 && n <= 0xdfff, + `lone high surrogate at ${i} in ${JSON.stringify(s.slice(Math.max(0, i - 4), i + 4))}`, + ); + i++; + } else if (c >= 0xdc00 && c <= 0xdfff) { + assert.fail( + `lone low surrogate at ${i} in ${JSON.stringify(s.slice(Math.max(0, i - 4), i + 4))}`, + ); } + } } test("clampPrefix: ascii and short inputs unchanged", () => { - assert.equal(clampPrefix("abcdef", 3), "abc"); - assert.equal(clampPrefix("abc", 5), "abc"); - assert.equal(clampPrefix("", 3), ""); + assert.equal(clampPrefix("abcdef", 3), "abc"); + assert.equal(clampPrefix("abc", 5), "abc"); + assert.equal(clampPrefix("", 3), ""); }); test("clampPrefix: pair straddling the cut drops the stray high half", () => { - // units: a b D83E DF80 — cutting at 3 would strand D83E - assert.equal(clampPrefix(`ab${EMOJI}`, 3), "ab"); - assert.equal(clampPrefix(`ab${EMOJI}`, 4), `ab${EMOJI}`); + // units: a b D83E DF80 — cutting at 3 would strand D83E + assert.equal(clampPrefix(`ab${EMOJI}`, 3), "ab"); + assert.equal(clampPrefix(`ab${EMOJI}`, 4), `ab${EMOJI}`); }); test("clampWindow: edges snap off surrogate pairs", () => { - const t = `ab${EMOJI}cd`; - // start at 3 points at DF80 (low half of the pair at 2-3) -> snap forward - assert.equal(clampWindow(t, 3, 6), "cd"); - // start at 2 lands exactly on the pair -> kept intact - assert.equal(clampWindow(t, 2, 6), `${EMOJI}cd`); - // end at 3 cuts between D83E/DF80 -> snap back - assert.equal(clampWindow(t, 0, 3), "ab"); - assert.equal(clampWindow(t, 0, 6), t); - assert.equal(clampWindow("abc", 5, 3), ""); + const t = `ab${EMOJI}cd`; + // start at 3 points at DF80 (low half of the pair at 2-3) -> snap forward + assert.equal(clampWindow(t, 3, 6), "cd"); + // start at 2 lands exactly on the pair -> kept intact + assert.equal(clampWindow(t, 2, 6), `${EMOJI}cd`); + // end at 3 cuts between D83E/DF80 -> snap back + assert.equal(clampWindow(t, 0, 3), "ab"); + assert.equal(clampWindow(t, 0, 6), t); + assert.equal(clampWindow("abc", 5, 3), ""); }); test("compactEntry stub: emoji straddling the 200-char clamp leaves no lone surrogate (#816)", () => { - const state = createInitialState(); - state.blocks.push({ - blockId: "b1", runId: "r0", tier: 1, - summary: "s", directMessageIds: [], effectiveMessageIds: [], directBlockIds: [], - createdAt: 1000, survivedCount: 0, generation: "young", active: true, - compressCallId: "call1", startRef: "m00001", endRef: "m00050", - }); - // 198 ascii + astral at units 198-199 + tail: old slice(0,199) ended on D83E - const summary = "a".repeat(198) + EMOJI + "tail"; - const call: CoreMessage = { - id: "mc1", role: "assistant", contentType: "tool-call", toolName: "compress", - toolCallId: "call1", text: JSON.stringify({ content: [{ startId: "m00001", endId: "m00050", summary }] }), - }; - const { messages } = hideConsumedCompressCalls(state, [call]); - const stub = (JSON.parse(messages[0]!.text ?? "") as { content: { summary: string }[] }).content[0]!.summary; - assert.ok(stub.length <= 200, `stub length ${stub.length}`); - assert.ok(stub.endsWith("\u2026")); - assertNoUnpairedSurrogates(stub); - assertNoUnpairedSurrogates(JSON.stringify(stub)); + const state = createInitialState(); + state.blocks.push({ + blockId: "b1", + runId: "r0", + tier: 1, + summary: "s", + directMessageIds: [], + effectiveMessageIds: [], + directBlockIds: [], + createdAt: 1000, + survivedCount: 0, + generation: "young", + active: true, + compressCallId: "call1", + startRef: "m00001", + endRef: "m00050", + }); + // 198 ascii + astral at units 198-199 + tail: old slice(0,199) ended on D83E + const summary = "a".repeat(198) + EMOJI + "tail"; + const call: CoreMessage = { + id: "mc1", + role: "assistant", + contentType: "tool-call", + toolName: "compress", + toolCallId: "call1", + text: JSON.stringify({ + content: [{ startId: "m00001", endId: "m00050", summary }], + }), + }; + const { messages } = hideConsumedCompressCalls(state, [call]); + const stub = ( + JSON.parse(messages[0]!.text ?? "") as { content: { summary: string }[] } + ).content[0]!.summary; + assert.ok(stub.length <= 200, `stub length ${stub.length}`); + assert.ok(stub.endsWith("\u2026")); + assertNoUnpairedSurrogates(stub); + assertNoUnpairedSurrogates(JSON.stringify(stub)); }); test("compactEntry stub: astral char fully inside the prefix survives intact", () => { - const state = createInitialState(); - state.blocks.push({ - blockId: "b1", runId: "r0", tier: 1, - summary: "s", directMessageIds: [], effectiveMessageIds: [], directBlockIds: [], - createdAt: 1000, survivedCount: 0, generation: "young", active: true, - compressCallId: "call1", startRef: "m00001", endId: "m00050", - }); - const summary = "a".repeat(197) + EMOJI + "b".repeat(10); - const call: CoreMessage = { - id: "mc1", role: "assistant", contentType: "tool-call", toolName: "compress", - toolCallId: "call1", text: JSON.stringify({ content: [{ startId: "m00001", endId: "m00050", summary }] }), - }; - const { messages } = hideConsumedCompressCalls(state, [call]); - const stub = (JSON.parse(messages[0]!.text ?? "") as { content: { summary: string }[] }).content[0]!.summary; - assert.equal(stub.length, 200); - assert.ok(stub.includes(EMOJI)); - assertNoUnpairedSurrogates(stub); + const state = createInitialState(); + state.blocks.push({ + blockId: "b1", + runId: "r0", + tier: 1, + summary: "s", + directMessageIds: [], + effectiveMessageIds: [], + directBlockIds: [], + createdAt: 1000, + survivedCount: 0, + generation: "young", + active: true, + compressCallId: "call1", + startRef: "m00001", + endId: "m00050", + }); + const summary = "a".repeat(197) + EMOJI + "b".repeat(10); + const call: CoreMessage = { + id: "mc1", + role: "assistant", + contentType: "tool-call", + toolName: "compress", + toolCallId: "call1", + text: JSON.stringify({ + content: [{ startId: "m00001", endId: "m00050", summary }], + }), + }; + const { messages } = hideConsumedCompressCalls(state, [call]); + const stub = ( + JSON.parse(messages[0]!.text ?? "") as { content: { summary: string }[] } + ).content[0]!.summary; + assert.equal(stub.length, 200); + assert.ok(stub.includes(EMOJI)); + assertNoUnpairedSurrogates(stub); }); test("search title: message text straddling the 60-unit title cut stays paired", () => { - // units 0-58 'h', 59-60 astral, then the query term: old slice(0,60) kept D83E alone - const text = "h".repeat(59) + EMOJI + " needle details"; - const docs = messageDocs([{ ref: "m00001", role: "user", text }]); - const r = searchBlocks(docs, "needle"); - assert.equal(r.length, 1); - assertNoUnpairedSurrogates(r[0]!.title); - assert.ok(r[0]!.title.startsWith("user: ")); + // units 0-58 'h', 59-60 astral, then the query term: old slice(0,60) kept D83E alone + const text = "h".repeat(59) + EMOJI + " needle details"; + const docs = messageDocs([{ ref: "m00001", role: "user", text }]); + const r = searchBlocks(docs, "needle"); + assert.equal(r.length, 1); + assertNoUnpairedSurrogates(r[0]!.title); + assert.ok(r[0]!.title.startsWith("user: ")); }); test("search preview: centered window splitting both surrogate edges stays paired", () => { - // layout: x*19 | EMOJI@19-20 | y*9 | needle@30-35 | w*23 | EMOJI@59-60 | v*20 - const text = "x".repeat(19) + EMOJI + "y".repeat(9) + "needle" + "w".repeat(23) + EMOJI + "v".repeat(20); - const docs: SearchDoc[] = [{ kind: "block", ref: "b1", text, title: "t", blockId: "b1", tier: 1 }]; - const r = searchBlocks(docs, "needle", { previewLength: 40 }); - assert.equal(r.length, 1); - const p = r[0]!.preview; - assert.ok(p.startsWith("\u2026"), "window has a head ellipsis"); - assert.ok(p.endsWith("\u2026"), "window has a tail ellipsis"); - assertNoUnpairedSurrogates(p); + // layout: x*19 | EMOJI@19-20 | y*9 | needle@30-35 | w*23 | EMOJI@59-60 | v*20 + const text = + "x".repeat(19) + + EMOJI + + "y".repeat(9) + + "needle" + + "w".repeat(23) + + EMOJI + + "v".repeat(20); + const docs: SearchDoc[] = [ + { kind: "block", ref: "b1", text, title: "t", blockId: "b1", tier: 1 }, + ]; + const r = searchBlocks(docs, "needle", { previewLength: 40 }); + assert.equal(r.length, 1); + const p = r[0]!.preview; + assert.ok(p.startsWith("\u2026"), "window has a head ellipsis"); + assert.ok(p.endsWith("\u2026"), "window has a tail ellipsis"); + assertNoUnpairedSurrogates(p); }); test("buildRecap: block summary straddling the 200-unit preview cut stays paired", () => { - const state: CompressionState = { - ...createInitialState(), - blocks: [ - { - blockId: "b1", runId: "r0", tier: 1, - summary: "a".repeat(198) + EMOJI + "tail", - directMessageIds: [], effectiveMessageIds: ["m1"], directBlockIds: [], - createdAt: 1000, survivedCount: 0, generation: "young", active: true, - }, - ], - }; - const recap = buildRecap(state); - assert.ok(recap.includes("..."), "long summary gets an ellipsis marker"); - assertNoUnpairedSurrogates(recap); + const state: CompressionState = { + ...createInitialState(), + blocks: [ + { + blockId: "b1", + runId: "r0", + tier: 1, + summary: "a".repeat(198) + EMOJI + "tail", + directMessageIds: [], + effectiveMessageIds: ["m1"], + directBlockIds: [], + createdAt: 1000, + survivedCount: 0, + generation: "young", + active: true, + }, + ], + }; + const recap = buildRecap(state); + assert.ok(recap.includes("..."), "long summary gets an ellipsis marker"); + assertNoUnpairedSurrogates(recap); }); test("deriveTopicFromSummary: heading straddling the 60-unit cut stays paired", () => { - const summary = `# ${"h".repeat(59)}${EMOJI} rest\nbody`; - const topic = deriveTopicFromSummary(summary); - assert.ok(topic !== undefined); - assert.ok(topic!.length <= 60); - assertNoUnpairedSurrogates(topic!); + const summary = `# ${"h".repeat(59)}${EMOJI} rest\nbody`; + const topic = deriveTopicFromSummary(summary); + assert.ok(topic !== undefined); + assert.ok(topic!.length <= 60); + assertNoUnpairedSurrogates(topic!); }); test("topicFallback: astral char straddling the 30-unit cut stays paired", () => { - // units 0-28 'h', 29-30 astral: old slice(0,30) kept D83E alone - const out = topicFallback(`h${"h".repeat(28)}${EMOJI}rest of the sentence`); - assert.ok(out.endsWith("\u2026")); - assert.ok(!out.includes(EMOJI)); - assertNoUnpairedSurrogates(out); + // units 0-28 'h', 29-30 astral: old slice(0,30) kept D83E alone + const out = topicFallback(`h${"h".repeat(28)}${EMOJI}rest of the sentence`); + assert.ok(out.endsWith("\u2026")); + assert.ok(!out.includes(EMOJI)); + assertNoUnpairedSurrogates(out); }); test("topicFallback: short topics pass through untouched", () => { - assert.equal(topicFallback(`ok ${EMOJI}`), `ok ${EMOJI}`); + assert.equal(topicFallback(`ok ${EMOJI}`), `ok ${EMOJI}`); }); test("truncateLargeToolOutputs: prefix/suffix cuts straddling pairs stay paired", () => { - // x*9 | EMOJI@9-10 | m*300 | EMOJI@311-312 | q*9 (len 322): - // the prefix cut at 10 splits the first pair; the suffix start at len-10=312 lands on a low half - const text = "x".repeat(9) + EMOJI + "m".repeat(300) + EMOJI + "q".repeat(9); - const messages: CoreMessage[] = [ - { id: "t1", role: "tool", contentType: "tool-result", toolName: "bash", toolCallId: "c1", text }, - { id: "u1", role: "user", contentType: "text", text: "go" }, - ]; - const config = defaultConfig(1000); - const r = truncateLargeToolOutputs(messages, config.modelContextLimit, config, (t) => Math.max(1, t.length), { - minOutputTokens: 1, - keepPrefixChars: 10, - keepSuffixChars: 10, - protectRecentMessages: 0, - }); - assert.equal(r.truncatedCount, 1); - const out = r.messages[0]!.text ?? ""; - assert.ok(out.includes("[truncated for context space]")); - assert.ok(out.startsWith("x".repeat(9))); - assert.ok(out.endsWith("q".repeat(9))); - assertNoUnpairedSurrogates(out); + // x*9 | EMOJI@9-10 | m*300 | EMOJI@311-312 | q*9 (len 322): + // the prefix cut at 10 splits the first pair; the suffix start at len-10=312 lands on a low half + const text = "x".repeat(9) + EMOJI + "m".repeat(300) + EMOJI + "q".repeat(9); + const messages: CoreMessage[] = [ + { + id: "t1", + role: "tool", + contentType: "tool-result", + toolName: "bash", + toolCallId: "c1", + text, + }, + { id: "u1", role: "user", contentType: "text", text: "go" }, + ]; + const config = defaultConfig(1000); + const r = truncateLargeToolOutputs( + messages, + config.modelContextLimit, + config, + (t) => Math.max(1, t.length), + { + minOutputTokens: 1, + keepPrefixChars: 10, + keepSuffixChars: 10, + protectRecentMessages: 0, + }, + ); + assert.equal(r.truncatedCount, 1); + const out = r.messages[0]!.text ?? ""; + assert.ok(out.includes("[truncated for context space]")); + assert.ok(out.startsWith("x".repeat(9))); + assert.ok(out.endsWith("q".repeat(9))); + assertNoUnpairedSurrogates(out); }); test("buildRestoredContentPreview: per-message 200-unit cut straddling a pair stays paired", () => { - const state = createInitialState(); - // z*199 | EMOJI@199-200 | tail: old slice(0,200) kept D83E alone - const messages: CoreMessage[] = [ - { id: "m1", role: "assistant", contentType: "text", text: "z".repeat(199) + EMOJI + "tail" }, - ]; - const { preview, restoredCount } = buildRestoredContentPreview(messages, new Set(["m1"]), state); - assert.equal(restoredCount, 1); - assert.ok(preview.startsWith("[assistant] z")); - assert.ok(preview.includes("...")); - assertNoUnpairedSurrogates(preview); + const state = createInitialState(); + // z*199 | EMOJI@199-200 | tail: old slice(0,200) kept D83E alone + const messages: CoreMessage[] = [ + { + id: "m1", + role: "assistant", + contentType: "text", + text: "z".repeat(199) + EMOJI + "tail", + }, + ]; + const { preview, restoredCount } = buildRestoredContentPreview( + messages, + new Set(["m1"]), + state, + ); + assert.equal(restoredCount, 1); + assert.ok(preview.startsWith("[assistant] z")); + assert.ok(preview.includes("...")); + assertNoUnpairedSurrogates(preview); }); test("parse diagnostics: rawPrefix 800-unit cut straddling a pair stays paired", () => { - // j*799 | EMOJI@799-800 | k*50: old slice(0,800) kept D83E alone - const raw = "j".repeat(799) + EMOJI + "k".repeat(50); - const r = parseCompressArgs(raw); - assert.equal(r.diagnostics.rawPrefix?.length, 799); - assertNoUnpairedSurrogates(r.diagnostics.rawPrefix ?? ""); + // j*799 | EMOJI@799-800 | k*50: old slice(0,800) kept D83E alone + const raw = "j".repeat(799) + EMOJI + "k".repeat(50); + const r = parseCompressArgs(raw); + assert.equal(r.diagnostics.rawPrefix?.length, 799); + assertNoUnpairedSurrogates(r.diagnostics.rawPrefix ?? ""); }); diff --git a/tests/validation.test.ts b/tests/validation.test.ts index 6b7a316..0928637 100644 --- a/tests/validation.test.ts +++ b/tests/validation.test.ts @@ -6,7 +6,11 @@ import { assignRefs } from "../src/refs.js"; import { defaultConfig } from "../src/config.js"; import type { CoreMessage } from "../src/types.js"; -function msg(id: string, text: string, role: CoreMessage["role"] = "user"): CoreMessage { +function msg( + id: string, + text: string, + role: CoreMessage["role"] = "user", +): CoreMessage { return { id, role, contentType: "text", text }; } @@ -20,16 +24,17 @@ function setupRefs(messages: CoreMessage[]) { } const longText = "x".repeat(6000); -const validSummary = "This is a meaningful summary that captures the key information of the compressed range including paths and decisions."; +const validSummary = + "This is a meaningful summary that captures the key information of the compressed range including paths and decisions."; test("validation: empty summary is rejected", () => { const core = createCore(); - const messages = [ - msg("a", longText), - msg("b", longText), - ]; + const messages = [msg("a", longText), msg("b", longText)]; const state = setupRefs(messages); - const config = defaultConfig(200000, { preserveRecentMessages: 0, preserveRecentTokens: 0 }); + const config = defaultConfig(200000, { + preserveRecentMessages: 0, + preserveRecentTokens: 0, + }); const result = core.applyCompression({ ranges: [{ startRef: "m00001", endRef: "m00002", summary: "" }], @@ -46,7 +51,10 @@ test("validation: whitespace-only summary is rejected", () => { const core = createCore(); const messages = [msg("a", longText), msg("b", longText)]; const state = setupRefs(messages); - const config = defaultConfig(200000, { preserveRecentMessages: 0, preserveRecentTokens: 0 }); + const config = defaultConfig(200000, { + preserveRecentMessages: 0, + preserveRecentTokens: 0, + }); const result = core.applyCompression({ ranges: [{ startRef: "m00001", endRef: "m00002", summary: " \n\t " }], @@ -63,8 +71,13 @@ test("validation: summary below minSummaryLength is rejected", () => { const messages = [msg("a", longText), msg("b", longText)]; const state = setupRefs(messages); const config = defaultConfig(200000, { - preserveRecentMessages: 0, preserveRecentTokens: 0, - compress: { minCompressRange: 0, maxSummaryLength: 0, minSummaryLength: 100 }, + preserveRecentMessages: 0, + preserveRecentTokens: 0, + compress: { + minCompressRange: 0, + maxSummaryLength: 0, + minSummaryLength: 100, + }, }); const result = core.applyCompression({ @@ -82,8 +95,13 @@ test("validation: summary above maxSummaryLength is rejected", () => { const messages = [msg("a", longText), msg("b", longText)]; const state = setupRefs(messages); const config = defaultConfig(200000, { - preserveRecentMessages: 0, preserveRecentTokens: 0, - compress: { minCompressRange: 0, maxSummaryLength: 50, minSummaryLength: 0 }, + preserveRecentMessages: 0, + preserveRecentTokens: 0, + compress: { + minCompressRange: 0, + maxSummaryLength: 50, + minSummaryLength: 0, + }, }); const result = core.applyCompression({ @@ -101,8 +119,13 @@ test("validation: range below minCompressRange is rejected", () => { const messages = [msg("a", "short"), msg("b", "text")]; const state = setupRefs(messages); const config = defaultConfig(200000, { - preserveRecentMessages: 0, preserveRecentTokens: 0, - compress: { minCompressRange: 5000, maxSummaryLength: 0, minSummaryLength: 0 }, + preserveRecentMessages: 0, + preserveRecentTokens: 0, + compress: { + minCompressRange: 5000, + maxSummaryLength: 0, + minSummaryLength: 0, + }, }); const result = core.applyCompression({ @@ -119,12 +142,21 @@ test("validation: batch aggregate — many small ranges pass if total >= minComp const core = createCore(); const chunk = "x".repeat(1200); const messages = [ - msg("a", chunk), msg("b", chunk), msg("c", chunk), msg("d", chunk), msg("e", chunk), + msg("a", chunk), + msg("b", chunk), + msg("c", chunk), + msg("d", chunk), + msg("e", chunk), ]; const state = setupRefs(messages); const config = defaultConfig(200000, { - preserveRecentMessages: 0, preserveRecentTokens: 0, - compress: { minCompressRange: 5000, maxSummaryLength: 0, minSummaryLength: 0 }, + preserveRecentMessages: 0, + preserveRecentTokens: 0, + compress: { + minCompressRange: 5000, + maxSummaryLength: 0, + minSummaryLength: 0, + }, }); // 5 separate ranges, each 1200 chars — individually below 5000 but total 6000 @@ -153,7 +185,10 @@ test("validation: range with zero compressible messages (covered by non-nested b msg("d", longText), ]; let state = setupRefs(messages); - const config = defaultConfig(200000, { preserveRecentMessages: 0, preserveRecentTokens: 0 }); + const config = defaultConfig(200000, { + preserveRecentMessages: 0, + preserveRecentTokens: 0, + }); const first = core.applyCompression({ ranges: [{ startRef: "m00001", endRef: "m00003", summary: validSummary }], @@ -180,7 +215,10 @@ test("validation: block-boundary compress with 0 direct msgs but consumed blocks const core = createCore(); const messages = [msg("a", longText), msg("b", longText), msg("c", longText)]; let state = setupRefs(messages); - const config = defaultConfig(200000, { preserveRecentMessages: 0, preserveRecentTokens: 0 }); + const config = defaultConfig(200000, { + preserveRecentMessages: 0, + preserveRecentTokens: 0, + }); const t1 = core.applyCompression({ ranges: [{ startRef: "m00001", endRef: "m00002", summary: validSummary }], @@ -206,7 +244,8 @@ test("validation: all checks disabled (0s) allows any input", () => { const messages = [msg("a", "x"), msg("b", "y")]; const state = setupRefs(messages); const config = defaultConfig(200000, { - preserveRecentMessages: 0, preserveRecentTokens: 0, + preserveRecentMessages: 0, + preserveRecentTokens: 0, compress: { minCompressRange: 0, maxSummaryLength: 0, minSummaryLength: 0 }, }); @@ -229,7 +268,10 @@ test("validation: errors are collected per-range in batch compress", () => { msg("d", longText), ]; const state = setupRefs(messages); - const config = defaultConfig(200000, { preserveRecentMessages: 0, preserveRecentTokens: 0 }); + const config = defaultConfig(200000, { + preserveRecentMessages: 0, + preserveRecentTokens: 0, + }); const result = core.applyCompression({ ranges: [ diff --git a/tests/viable.test.ts b/tests/viable.test.ts index 360fb74..787cfe5 100644 --- a/tests/viable.test.ts +++ b/tests/viable.test.ts @@ -10,7 +10,10 @@ test("viableRanges drops fragmented ranges below the floor", () => { { startRef: "m00200", endRef: "m00201", tokens: 4_700 }, ]; const kept = viableRanges(ranges); - assert.deepEqual(kept.map((r) => r.tokens), [1_000, 4_700]); + assert.deepEqual( + kept.map((r) => r.tokens), + [1_000, 4_700], + ); assert.equal(VIABLE_RANGE_MIN_TOKENS, 200); }); diff --git a/tests/wire-bili-message-roundtrip.test.ts b/tests/wire-bili-message-roundtrip.test.ts index 2998594..35bc213 100644 --- a/tests/wire-bili-message-roundtrip.test.ts +++ b/tests/wire-bili-message-roundtrip.test.ts @@ -2,94 +2,156 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { anthropicToCore, coreToAnthropic } from "../src/wire/anthropic.js"; import { openaiToCore, coreToOpenai } from "../src/wire/openai.js"; -import { responsesToCore, coreToResponses, patchResponsesInput } from "../src/wire/responses.js"; -import type { AnthropicBlock, AnthropicRequestBody } from "../src/wire/anthropic.js"; +import { + responsesToCore, + coreToResponses, + patchResponsesInput, +} from "../src/wire/responses.js"; +import type { + AnthropicBlock, + AnthropicRequestBody, +} from "../src/wire/anthropic.js"; import type { OpenAIRequestBody } from "../src/wire/openai.js"; -import type { ResponseInputItem, ResponsesRequestBody } from "../src/wire/responses.js"; +import type { + ResponseInputItem, + ResponsesRequestBody, +} from "../src/wire/responses.js"; -const IMG_DATA = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII="; +const IMG_DATA = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII="; const DATA_URL = `data:image/png;base64,${IMG_DATA}`; -function block(rebuilt: { content: string | AnthropicBlock[] }, i: number): Record { - return (rebuilt.content as unknown[])[i] as Record; +function block( + rebuilt: { content: string | AnthropicBlock[] }, + i: number, +): Record { + return (rebuilt.content as unknown[])[i] as Record; } // 1. Anthropic image block round-trips losslessly (source.base64 + media_type). test("anthropic: image block is restored verbatim via rawAnthropicBlock", () => { - const source = { type: "base64", media_type: "image/png", data: IMG_DATA }; - const body: AnthropicRequestBody = { - model: "claude", - max_tokens: 100, - messages: [{ role: "user", content: [{ type: "image", source }] }], - }; - const { msgs } = anthropicToCore(body); - assert.equal(msgs.length, 1); - assert.equal(msgs[0]?.contentType, "text"); - assert.equal(msgs[0]?.text, "[image]"); - assert.ok(msgs[0]?.rawAnthropicBlock, "sidecar rawAnthropicBlock stored"); - const rebuilt = coreToAnthropic(msgs); - assert.equal(rebuilt.length, 1); - const img = block(rebuilt[0]!, 0); - assert.equal(img.type, "image", "image block reconstructed (not a text placeholder)"); - assert.deepEqual(img.source, source, "source preserved verbatim (base64 + media_type)"); + const source = { type: "base64", media_type: "image/png", data: IMG_DATA }; + const body: AnthropicRequestBody = { + model: "claude", + max_tokens: 100, + messages: [{ role: "user", content: [{ type: "image", source }] }], + }; + const { msgs } = anthropicToCore(body); + assert.equal(msgs.length, 1); + assert.equal(msgs[0]?.contentType, "text"); + assert.equal(msgs[0]?.text, "[image]"); + assert.ok(msgs[0]?.rawAnthropicBlock, "sidecar rawAnthropicBlock stored"); + const rebuilt = coreToAnthropic(msgs); + assert.equal(rebuilt.length, 1); + const img = block(rebuilt[0]!, 0); + assert.equal( + img.type, + "image", + "image block reconstructed (not a text placeholder)", + ); + assert.deepEqual( + img.source, + source, + "source preserved verbatim (base64 + media_type)", + ); }); // 2. Anthropic thinking + signature round-trips (signature was previously dropped). test("anthropic: thinking signature is restored", () => { - const body: AnthropicRequestBody = { - model: "claude", - max_tokens: 100, - messages: [ - { role: "assistant", content: [{ type: "thinking", thinking: "let me consider", signature: "sig_EqMAC" }] }, + const body: AnthropicRequestBody = { + model: "claude", + max_tokens: 100, + messages: [ + { + role: "assistant", + content: [ + { + type: "thinking", + thinking: "let me consider", + signature: "sig_EqMAC", + }, ], - }; - const { msgs } = anthropicToCore(body); - assert.equal(msgs[0]?.contentType, "reasoning"); - assert.equal(msgs[0]?.thinkingSignature, "sig_EqMAC", "thinkingSignature sidecar stored"); - const rebuilt = coreToAnthropic(msgs); - const th = block(rebuilt[0]!, 0); - assert.equal(th.type, "thinking"); - assert.equal(th.thinking, "let me consider"); - assert.equal(th.signature, "sig_EqMAC", "signature reattached (Anthropic rejects thinking without it)"); + }, + ], + }; + const { msgs } = anthropicToCore(body); + assert.equal(msgs[0]?.contentType, "reasoning"); + assert.equal( + msgs[0]?.thinkingSignature, + "sig_EqMAC", + "thinkingSignature sidecar stored", + ); + const rebuilt = coreToAnthropic(msgs); + const th = block(rebuilt[0]!, 0); + assert.equal(th.type, "thinking"); + assert.equal(th.thinking, "let me consider"); + assert.equal( + th.signature, + "sig_EqMAC", + "signature reattached (Anthropic rejects thinking without it)", + ); }); // 3. Anthropic tool_result.is_error round-trips (was previously dropped). test("anthropic: tool_result.is_error is restored", () => { - const body: AnthropicRequestBody = { - model: "claude", - max_tokens: 100, - messages: [ - { role: "assistant", content: [{ type: "tool_use", id: "t1", name: "Bash", input: { cmd: "ls" } }] }, - { role: "user", content: [{ type: "tool_result", tool_use_id: "t1", content: "boom", is_error: true }] }, + const body: AnthropicRequestBody = { + model: "claude", + max_tokens: 100, + messages: [ + { + role: "assistant", + content: [ + { type: "tool_use", id: "t1", name: "Bash", input: { cmd: "ls" } }, ], - }; - const { msgs } = anthropicToCore(body); - const result = msgs.find((m) => m.contentType === "tool-result"); - assert.equal(result?.toolIsError, true, "toolIsError sidecar stored"); - const rebuilt = coreToAnthropic(msgs); - const userMsg = rebuilt.find((m) => m.role === "user")!; - const tr = block(userMsg, 0); - assert.equal(tr.type, "tool_result"); - assert.equal(tr.is_error, true, "is_error reconstructed"); + }, + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "t1", + content: "boom", + is_error: true, + }, + ], + }, + ], + }; + const { msgs } = anthropicToCore(body); + const result = msgs.find((m) => m.contentType === "tool-result"); + assert.equal(result?.toolIsError, true, "toolIsError sidecar stored"); + const rebuilt = coreToAnthropic(msgs); + const userMsg = rebuilt.find((m) => m.role === "user")!; + const tr = block(userMsg, 0); + assert.equal(tr.type, "tool_result"); + assert.equal(tr.is_error, true, "is_error reconstructed"); }); // 3b. Sanity: a non-error tool_result does NOT gain is_error. test("anthropic: tool_result without is_error stays clean", () => { - const body: AnthropicRequestBody = { - model: "claude", - max_tokens: 100, - messages: [ - { role: "assistant", content: [{ type: "tool_use", id: "t1", name: "Bash", input: { cmd: "ls" } }] }, - { role: "user", content: [{ type: "tool_result", tool_use_id: "t1", content: "ok" }] }, + const body: AnthropicRequestBody = { + model: "claude", + max_tokens: 100, + messages: [ + { + role: "assistant", + content: [ + { type: "tool_use", id: "t1", name: "Bash", input: { cmd: "ls" } }, ], - }; - const { msgs } = anthropicToCore(body); - const result = msgs.find((m) => m.contentType === "tool-result"); - assert.equal(result?.toolIsError, undefined); - const rebuilt = coreToAnthropic(msgs); - const userMsg = rebuilt.find((m) => m.role === "user")!; - const tr = block(userMsg, 0); - assert.equal(tr.is_error, undefined, "no spurious is_error"); + }, + { + role: "user", + content: [{ type: "tool_result", tool_use_id: "t1", content: "ok" }], + }, + ], + }; + const { msgs } = anthropicToCore(body); + const result = msgs.find((m) => m.contentType === "tool-result"); + assert.equal(result?.toolIsError, undefined); + const rebuilt = coreToAnthropic(msgs); + const userMsg = rebuilt.find((m) => m.role === "user")!; + const tr = block(userMsg, 0); + assert.equal(tr.is_error, undefined, "no spurious is_error"); }); // 4. OpenAI leading system/developer prefix is hoisted OUT of the fold space @@ -97,63 +159,82 @@ test("anthropic: tool_result without is_error stays clean", () => { // never enter the id/fingerprint space); originalRole still round-trips for // mid-conversation system traffic. test("openai: leading developer/system prefix is hoisted, not folded", () => { - const body: OpenAIRequestBody = { - messages: [ - { role: "developer", content: "you are a dev" }, - { role: "system", content: "sys" }, - { role: "user", content: "hi" }, - ], - }; - const { msgs, systemText } = openaiToCore(body); - assert.equal(systemText, "you are a dev\n\nsys", "prefix returned separately"); - assert.ok(msgs.every((m) => m.role !== "system"), "no system piece in the fold space"); - assert.equal(msgs[0]?.role, "user"); + const body: OpenAIRequestBody = { + messages: [ + { role: "developer", content: "you are a dev" }, + { role: "system", content: "sys" }, + { role: "user", content: "hi" }, + ], + }; + const { msgs, systemText } = openaiToCore(body); + assert.equal( + systemText, + "you are a dev\n\nsys", + "prefix returned separately", + ); + assert.ok( + msgs.every((m) => m.role !== "system"), + "no system piece in the fold space", + ); + assert.equal(msgs[0]?.role, "user"); }); // 4b. Mid-conversation system traffic keeps the role sidecar round-trip. test("openai: mid-conversation developer role is restored", () => { - const body: OpenAIRequestBody = { - messages: [ - { role: "user", content: "hi" }, - { role: "developer", content: "you are a dev" }, - ], - }; - const { msgs } = openaiToCore(body); - const dev = msgs.find((m) => m.role === "system")!; - assert.equal(dev.originalRole, "developer", "originalRole sidecar stored"); - const rebuilt = coreToOpenai(msgs); - assert.equal(rebuilt[1]?.role, "developer", "developer role reconstructed"); - assert.equal(rebuilt[1]?.content, "you are a dev"); + const body: OpenAIRequestBody = { + messages: [ + { role: "user", content: "hi" }, + { role: "developer", content: "you are a dev" }, + ], + }; + const { msgs } = openaiToCore(body); + const dev = msgs.find((m) => m.role === "system")!; + assert.equal(dev.originalRole, "developer", "originalRole sidecar stored"); + const rebuilt = coreToOpenai(msgs); + assert.equal(rebuilt[1]?.role, "developer", "developer role reconstructed"); + assert.equal(rebuilt[1]?.content, "you are a dev"); }); // 5. OpenAI image_url round-trips (image was previously dropped entirely). test("openai: user image_url content part is restored", () => { - const body: OpenAIRequestBody = { - messages: [ - { - role: "user", - content: [ - { type: "text", text: "what is this?" }, - { type: "image_url", image_url: { url: DATA_URL } }, - ], - }, + const body: OpenAIRequestBody = { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "what is this?" }, + { type: "image_url", image_url: { url: DATA_URL } }, ], - }; - const { msgs } = openaiToCore(body); - assert.ok(msgs[0]?.text?.startsWith("what is this?"), "text preserved"); - assert.equal(msgs[0]?.imageMediaType, "image/png"); - assert.equal(msgs[0]?.imageBase64, IMG_DATA); - assert.ok(msgs[0]?.rawOpenaiContent, "rawOpenaiContent sidecar stored"); - const rebuilt = coreToOpenai(msgs); - const u = rebuilt[0]!; - assert.equal(u.role, "user"); - assert.ok(Array.isArray(u.content), "content rebuilt as array (text + image)"); - const parts = u.content as unknown as { type: string; [k: string]: unknown }[]; - const text = parts.find((p) => p.type === "text"); - assert.ok(typeof text?.text === "string" && text.text.startsWith("what is this?")); - const img = parts.find((p) => p.type === "image_url"); - assert.ok(img, "image_url part reconstructed"); - assert.deepEqual((img as unknown as { image_url: { url: string } }).image_url.url, DATA_URL, "image data URL restored"); + }, + ], + }; + const { msgs } = openaiToCore(body); + assert.ok(msgs[0]?.text?.startsWith("what is this?"), "text preserved"); + assert.equal(msgs[0]?.imageMediaType, "image/png"); + assert.equal(msgs[0]?.imageBase64, IMG_DATA); + assert.ok(msgs[0]?.rawOpenaiContent, "rawOpenaiContent sidecar stored"); + const rebuilt = coreToOpenai(msgs); + const u = rebuilt[0]!; + assert.equal(u.role, "user"); + assert.ok( + Array.isArray(u.content), + "content rebuilt as array (text + image)", + ); + const parts = u.content as unknown as { + type: string; + [k: string]: unknown; + }[]; + const text = parts.find((p) => p.type === "text"); + assert.ok( + typeof text?.text === "string" && text.text.startsWith("what is this?"), + ); + const img = parts.find((p) => p.type === "image_url"); + assert.ok(img, "image_url part reconstructed"); + assert.deepEqual( + (img as unknown as { image_url: { url: string } }).image_url.url, + DATA_URL, + "image data URL restored", + ); }); // 5b. OpenAI multi-image user message preserves ALL images. firstImagePart @@ -161,49 +242,68 @@ test("openai: user image_url content part is restored", () => { // dropped on the coreToOpenai rebuild (and the omp wire-fold gate failed // open on the whole payload). All data-URL parts must round-trip in order. test("openai: user message with multiple image_url parts round-trips ALL images", () => { - const DATA_URL2 = `data:image/png;base64,${IMG_DATA}AAAA`; - const DATA_URL3 = `data:image/png;base64,${IMG_DATA}AAAAA`; - const body: OpenAIRequestBody = { - messages: [ - { - role: "user", - content: [ - { type: "text", text: "compare these?" }, - { type: "image_url", image_url: { url: DATA_URL } }, - { type: "image_url", image_url: { url: DATA_URL2 } }, - { type: "image_url", image_url: { url: DATA_URL3 } }, - ], - }, + const DATA_URL2 = `data:image/png;base64,${IMG_DATA}AAAA`; + const DATA_URL3 = `data:image/png;base64,${IMG_DATA}AAAAA`; + const body: OpenAIRequestBody = { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "compare these?" }, + { type: "image_url", image_url: { url: DATA_URL } }, + { type: "image_url", image_url: { url: DATA_URL2 } }, + { type: "image_url", image_url: { url: DATA_URL3 } }, ], - }; - const { msgs } = openaiToCore(body); - assert.ok(msgs[0]?.rawOpenaiContentParts, "multi-image sidecar stored (not singular rawOpenaiContent)"); - assert.equal((msgs[0]?.rawOpenaiContentParts ?? []).length, 3); - const rebuilt = coreToOpenai(msgs); - const u = rebuilt[0]!; - assert.ok(Array.isArray(u.content), "content rebuilt as array"); - const parts = u.content as unknown as { type: string; [k: string]: unknown }[]; - const imgs = parts.filter((p) => p.type === "image_url"); - assert.equal(imgs.length, 3, "all 3 image_url parts preserved"); - const urls = imgs.map((p) => (p as unknown as { image_url: { url: string } }).image_url.url); - assert.deepEqual(urls, [DATA_URL, DATA_URL2, DATA_URL3], "image order preserved"); - const text = parts.find((p) => p.type === "text"); - assert.ok(typeof text?.text === "string" && text.text.startsWith("compare these?"), "text preserved"); + }, + ], + }; + const { msgs } = openaiToCore(body); + assert.ok( + msgs[0]?.rawOpenaiContentParts, + "multi-image sidecar stored (not singular rawOpenaiContent)", + ); + assert.equal((msgs[0]?.rawOpenaiContentParts ?? []).length, 3); + const rebuilt = coreToOpenai(msgs); + const u = rebuilt[0]!; + assert.ok(Array.isArray(u.content), "content rebuilt as array"); + const parts = u.content as unknown as { + type: string; + [k: string]: unknown; + }[]; + const imgs = parts.filter((p) => p.type === "image_url"); + assert.equal(imgs.length, 3, "all 3 image_url parts preserved"); + const urls = imgs.map( + (p) => (p as unknown as { image_url: { url: string } }).image_url.url, + ); + assert.deepEqual( + urls, + [DATA_URL, DATA_URL2, DATA_URL3], + "image order preserved", + ); + const text = parts.find((p) => p.type === "text"); + assert.ok( + typeof text?.text === "string" && text.text.startsWith("compare these?"), + "text preserved", + ); }); // 5c. Malformed but JSON-valid image part (image_url:null) must degrade to a // non-image part instead of throwing — previously raised TypeError reading // null.url inside allImageParts, crashing the whole turn in proxy mode. test("openai: malformed image_url:null part is skipped without throwing", () => { - const body = { - messages: [ - { role: "user", content: [{ type: "image_url", image_url: null }] }, - ], - } as unknown as OpenAIRequestBody; - const { msgs } = openaiToCore(body); - assert.equal(msgs.length, 1); - assert.equal(msgs[0]?.imageMediaType, undefined, "null image_url degrades to non-image"); - assert.equal(msgs[0]?.imageBase64, undefined); + const body = { + messages: [ + { role: "user", content: [{ type: "image_url", image_url: null }] }, + ], + } as unknown as OpenAIRequestBody; + const { msgs } = openaiToCore(body); + assert.equal(msgs.length, 1); + assert.equal( + msgs[0]?.imageMediaType, + undefined, + "null image_url degrades to non-image", + ); + assert.equal(msgs[0]?.imageBase64, undefined); }); // 5d. A single REMOTE (non-data) URL image gets no base64 sidecar, so it must @@ -211,109 +311,162 @@ test("openai: malformed image_url:null part is skipped without throwing", () => // data: URLs, so a lone remote-URL image produced NO sidecar at all and // was silently dropped on the coreToOpenai rebuild (issues #187/#291). test("openai: single remote-URL image_url round-trips via rawOpenaiContent", () => { - const REMOTE_URL = "https://example.com/a.png"; - const body: OpenAIRequestBody = { - messages: [ - { - role: "user", - content: [ - { type: "text", text: "t" }, - { type: "image_url", image_url: { url: REMOTE_URL } }, - ], - }, + const REMOTE_URL = "https://example.com/a.png"; + const body: OpenAIRequestBody = { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "t" }, + { type: "image_url", image_url: { url: REMOTE_URL } }, ], - }; - const { msgs } = openaiToCore(body); - assert.ok(msgs[0]?.rawOpenaiContent, "rawOpenaiContent sidecar stored for a single remote-URL image"); - assert.equal(msgs[0]?.imageBase64, undefined, "no base64 split for non-data URLs"); - assert.equal(msgs[0]?.imageMediaType, undefined, "no mediaType split for non-data URLs"); - const rebuilt = coreToOpenai(msgs); - const u = rebuilt[0]!; - assert.ok(Array.isArray(u.content), "content rebuilt as array"); - const parts = u.content as unknown as { type: string; [k: string]: unknown }[]; - const img = parts.find((p) => p.type === "image_url") as unknown as { image_url: { url: string } }; - assert.equal(img?.image_url.url, REMOTE_URL, "remote-URL image restored verbatim"); - const text = parts.find((p) => p.type === "text"); - assert.ok(typeof text?.text === "string" && text.text.startsWith("t"), "text preserved"); + }, + ], + }; + const { msgs } = openaiToCore(body); + assert.ok( + msgs[0]?.rawOpenaiContent, + "rawOpenaiContent sidecar stored for a single remote-URL image", + ); + assert.equal( + msgs[0]?.imageBase64, + undefined, + "no base64 split for non-data URLs", + ); + assert.equal( + msgs[0]?.imageMediaType, + undefined, + "no mediaType split for non-data URLs", + ); + const rebuilt = coreToOpenai(msgs); + const u = rebuilt[0]!; + assert.ok(Array.isArray(u.content), "content rebuilt as array"); + const parts = u.content as unknown as { + type: string; + [k: string]: unknown; + }[]; + const img = parts.find((p) => p.type === "image_url") as unknown as { + image_url: { url: string }; + }; + assert.equal( + img?.image_url.url, + REMOTE_URL, + "remote-URL image restored verbatim", + ); + const text = parts.find((p) => p.type === "text"); + assert.ok( + typeof text?.text === "string" && text.text.startsWith("t"), + "text preserved", + ); }); // 5e. Image-only user message (no text part) with a remote URL: the message // text is empty, so survival depends entirely on the sidecar. test("openai: image-only remote-URL user message survives rebuild", () => { - const REMOTE_URL = "https://example.com/b.jpg"; - const body: OpenAIRequestBody = { - messages: [{ role: "user", content: [{ type: "image_url", image_url: { url: REMOTE_URL } }] }], - }; - const { msgs } = openaiToCore(body); - assert.equal(msgs[0]?.text, ""); - assert.ok(msgs[0]?.rawOpenaiContent, "sidecar stored despite empty text"); - const rebuilt = coreToOpenai(msgs); - const u = rebuilt[0]!; - assert.ok(Array.isArray(u.content), "content rebuilt as array (not an empty string)"); - const parts = u.content as unknown as { type: string; [k: string]: unknown }[]; - assert.equal(parts.length, 1, "exactly one part — no empty text part injected"); - const img = parts[0] as unknown as { type: string; image_url: { url: string } }; - assert.equal(img.type, "image_url"); - assert.equal(img.image_url.url, REMOTE_URL); + const REMOTE_URL = "https://example.com/b.jpg"; + const body: OpenAIRequestBody = { + messages: [ + { + role: "user", + content: [{ type: "image_url", image_url: { url: REMOTE_URL } }], + }, + ], + }; + const { msgs } = openaiToCore(body); + assert.equal(msgs[0]?.text, ""); + assert.ok(msgs[0]?.rawOpenaiContent, "sidecar stored despite empty text"); + const rebuilt = coreToOpenai(msgs); + const u = rebuilt[0]!; + assert.ok( + Array.isArray(u.content), + "content rebuilt as array (not an empty string)", + ); + const parts = u.content as unknown as { + type: string; + [k: string]: unknown; + }[]; + assert.equal( + parts.length, + 1, + "exactly one part — no empty text part injected", + ); + const img = parts[0] as unknown as { + type: string; + image_url: { url: string }; + }; + assert.equal(img.type, "image_url"); + assert.equal(img.image_url.url, REMOTE_URL); }); // 5f. Mixed data-URL + remote-URL multi-image message: allImageParts used to // keep only data-URL parts, so the remote one vanished from // rawOpenaiContentParts (wider gap than the single-image case). test("openai: mixed data-URL + remote-URL multi-image message preserves ALL images in order", () => { - const REMOTE_URL = "https://example.com/c.webp"; - const body: OpenAIRequestBody = { - messages: [ - { - role: "user", - content: [ - { type: "text", text: "both?" }, - { type: "image_url", image_url: { url: DATA_URL } }, - { type: "image_url", image_url: { url: REMOTE_URL } }, - ], - }, + const REMOTE_URL = "https://example.com/c.webp"; + const body: OpenAIRequestBody = { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "both?" }, + { type: "image_url", image_url: { url: DATA_URL } }, + { type: "image_url", image_url: { url: REMOTE_URL } }, ], - }; - const { msgs } = openaiToCore(body); - assert.ok(msgs[0]?.rawOpenaiContentParts, "multi-image sidecar stored"); - assert.equal((msgs[0]?.rawOpenaiContentParts ?? []).length, 2, "BOTH images collected (remote included)"); - const rebuilt = coreToOpenai(msgs); - const u = rebuilt[0]!; - const parts = u.content as unknown as { type: string; [k: string]: unknown }[]; - const imgs = parts.filter((p) => p.type === "image_url"); - assert.equal(imgs.length, 2, "all 2 image_url parts preserved"); - const urls = imgs.map((p) => (p as unknown as { image_url: { url: string } }).image_url.url); - assert.deepEqual(urls, [DATA_URL, REMOTE_URL], "order preserved"); + }, + ], + }; + const { msgs } = openaiToCore(body); + assert.ok(msgs[0]?.rawOpenaiContentParts, "multi-image sidecar stored"); + assert.equal( + (msgs[0]?.rawOpenaiContentParts ?? []).length, + 2, + "BOTH images collected (remote included)", + ); + const rebuilt = coreToOpenai(msgs); + const u = rebuilt[0]!; + const parts = u.content as unknown as { + type: string; + [k: string]: unknown; + }[]; + const imgs = parts.filter((p) => p.type === "image_url"); + assert.equal(imgs.length, 2, "all 2 image_url parts preserved"); + const urls = imgs.map( + (p) => (p as unknown as { image_url: { url: string } }).image_url.url, + ); + assert.deepEqual(urls, [DATA_URL, REMOTE_URL], "order preserved"); }); // 6. Responses API input_image round-trips (image was previously dropped). test("responses: user input_image is restored via rawResponsesItem", () => { - const body: ResponsesRequestBody = { - input: [ - { - type: "message", - role: "user", - content: [ - { type: "input_text", text: "see this" }, - { type: "input_image", image_url: DATA_URL }, - ], - }, + const body: ResponsesRequestBody = { + input: [ + { + type: "message", + role: "user", + content: [ + { type: "input_text", text: "see this" }, + { type: "input_image", image_url: DATA_URL }, ], - }; - const { msgs } = responsesToCore(body); - assert.ok(msgs[0]?.text?.startsWith("see this"), "text preserved"); - assert.equal(msgs[0]?.imageMediaType, "image/png"); - assert.equal(msgs[0]?.imageBase64, IMG_DATA); - assert.ok(msgs[0]?.rawResponsesItem, "rawResponsesItem sidecar stored"); - const rebuilt = coreToResponses(msgs); - assert.equal(rebuilt.length, 1); - const m = rebuilt[0] as { type: string; content: unknown }; - assert.equal(m.type, "message"); - assert.ok(Array.isArray(m.content), "message content rebuilt as array"); - const parts = m.content as unknown as { type: string; [k: string]: unknown }[]; - const img = parts.find((p) => p.type === "input_image"); - assert.ok(img, "input_image reconstructed"); - assert.equal(img?.image_url, DATA_URL, "image url restored"); + }, + ], + }; + const { msgs } = responsesToCore(body); + assert.ok(msgs[0]?.text?.startsWith("see this"), "text preserved"); + assert.equal(msgs[0]?.imageMediaType, "image/png"); + assert.equal(msgs[0]?.imageBase64, IMG_DATA); + assert.ok(msgs[0]?.rawResponsesItem, "rawResponsesItem sidecar stored"); + const rebuilt = coreToResponses(msgs); + assert.equal(rebuilt.length, 1); + const m = rebuilt[0] as { type: string; content: unknown }; + assert.equal(m.type, "message"); + assert.ok(Array.isArray(m.content), "message content rebuilt as array"); + const parts = m.content as unknown as { + type: string; + [k: string]: unknown; + }[]; + const img = parts.find((p) => p.type === "input_image"); + assert.ok(img, "input_image reconstructed"); + assert.equal(img?.image_url, DATA_URL, "image url restored"); }); // 6b. Image-ONLY user item (no input_text) previously produced NO core @@ -322,18 +475,40 @@ test("responses: user input_image is restored via rawResponsesItem", () => { // tracked with the "[image]" placeholder (anthropic codec precedent) and // re-emitted verbatim via rawResponsesItem while unmodified. test("responses: image-only user item is tracked and round-trips verbatim", () => { - const body: ResponsesRequestBody = { - input: [{ type: "message", role: "user", content: [{ type: "input_image", image_url: DATA_URL }] }], - }; - const { msgs } = responsesToCore(body); - assert.equal(msgs.length, 1, "image-only item produces exactly one core message"); - assert.equal(msgs[0]?.text, "[image]", "placeholder text used for identity + display"); - assert.ok(msgs[0]?.rawResponsesItem, "rawResponsesItem sidecar stored"); - assert.equal(msgs[0]?.imageMediaType, "image/png", "data-URL sidecar fields still populated"); - assert.equal(msgs[0]?.imageBase64, IMG_DATA); - const rebuilt = coreToResponses(msgs); - assert.equal(rebuilt.length, 1); - assert.deepEqual(rebuilt[0], body.input[0], "raw item re-emitted verbatim (images intact)"); + const body: ResponsesRequestBody = { + input: [ + { + type: "message", + role: "user", + content: [{ type: "input_image", image_url: DATA_URL }], + }, + ], + }; + const { msgs } = responsesToCore(body); + assert.equal( + msgs.length, + 1, + "image-only item produces exactly one core message", + ); + assert.equal( + msgs[0]?.text, + "[image]", + "placeholder text used for identity + display", + ); + assert.ok(msgs[0]?.rawResponsesItem, "rawResponsesItem sidecar stored"); + assert.equal( + msgs[0]?.imageMediaType, + "image/png", + "data-URL sidecar fields still populated", + ); + assert.equal(msgs[0]?.imageBase64, IMG_DATA); + const rebuilt = coreToResponses(msgs); + assert.equal(rebuilt.length, 1); + assert.deepEqual( + rebuilt[0], + body.input[0], + "raw item re-emitted verbatim (images intact)", + ); }); // 6c. The tracked image-only item must survive patchResponsesInput (the @@ -342,89 +517,164 @@ test("responses: image-only user item is tracked and round-trips verbatim", () = // was only kept by the layout fallback (untracked → invisible to the // compression pipeline, accumulating unbounded). test("responses: image-only item survives patchResponsesInput and prunes with its turn", () => { - const body: ResponsesRequestBody = { - input: [ - { type: "message", role: "user", content: [{ type: "input_image", image_url: DATA_URL }] }, - { type: "message", role: "assistant", content: "got it" }, - ], - }; - const proj = responsesToCore(body); - assert.equal(proj.msgs.length, 2, "both items tracked"); - const imgMsg = proj.msgs.find((m) => m.text === "[image]")!; - assert.ok(imgMsg, "image-only item has a tracked core id"); - const kept = patchResponsesInput(proj, proj.msgs); - assert.ok(JSON.stringify(kept).includes(DATA_URL), "image survives the patch rebuild"); - const pruned = patchResponsesInput(proj, proj.msgs.filter((m) => m.id !== imgMsg.id)); - assert.ok(!JSON.stringify(pruned).includes(DATA_URL), "pruning the id removes the image from the rebuild"); + const body: ResponsesRequestBody = { + input: [ + { + type: "message", + role: "user", + content: [{ type: "input_image", image_url: DATA_URL }], + }, + { type: "message", role: "assistant", content: "got it" }, + ], + }; + const proj = responsesToCore(body); + assert.equal(proj.msgs.length, 2, "both items tracked"); + const imgMsg = proj.msgs.find((m) => m.text === "[image]")!; + assert.ok(imgMsg, "image-only item has a tracked core id"); + const kept = patchResponsesInput(proj, proj.msgs); + assert.ok( + JSON.stringify(kept).includes(DATA_URL), + "image survives the patch rebuild", + ); + const pruned = patchResponsesInput( + proj, + proj.msgs.filter((m) => m.id !== imgMsg.id), + ); + assert.ok( + !JSON.stringify(pruned).includes(DATA_URL), + "pruning the id removes the image from the rebuild", + ); }); // 6d. Two identical image-only items must get distinct but STABLE ids across // turns (ClusterCounter disambiguation), so a later turn cannot collapse // or misalign them. test("responses: image-only item ids are stable and distinct across turns", () => { - const first: ResponsesRequestBody = { - input: [ - { type: "message", role: "user", content: [{ type: "input_image", image_url: DATA_URL }] }, - { type: "message", role: "assistant", content: "ok" }, - ], - }; - const second: ResponsesRequestBody = { - input: [ - ...first.input, - { type: "message", role: "user", content: [{ type: "input_image", image_url: DATA_URL }] }, - { type: "message", role: "assistant", content: "ok again" }, - ], - }; - const a = responsesToCore(first); - const b = responsesToCore(second); - const idsA = a.msgs.map((m) => m.id); - const idsB = b.msgs.map((m) => m.id); - assert.deepEqual(idsA.slice(0, 2), idsB.slice(0, 2), "earlier ids unchanged by later turns"); - const imgIds = b.msgs.filter((m) => m.text === "[image]").map((m) => m.id); - assert.equal(new Set(imgIds).size, 2, "identical image-only items get distinct ids"); + const first: ResponsesRequestBody = { + input: [ + { + type: "message", + role: "user", + content: [{ type: "input_image", image_url: DATA_URL }], + }, + { type: "message", role: "assistant", content: "ok" }, + ], + }; + const second: ResponsesRequestBody = { + input: [ + ...first.input, + { + type: "message", + role: "user", + content: [{ type: "input_image", image_url: DATA_URL }], + }, + { type: "message", role: "assistant", content: "ok again" }, + ], + }; + const a = responsesToCore(first); + const b = responsesToCore(second); + const idsA = a.msgs.map((m) => m.id); + const idsB = b.msgs.map((m) => m.id); + assert.deepEqual( + idsA.slice(0, 2), + idsB.slice(0, 2), + "earlier ids unchanged by later turns", + ); + const imgIds = b.msgs.filter((m) => m.text === "[image]").map((m) => m.id); + assert.equal( + new Set(imgIds).size, + 2, + "identical image-only items get distinct ids", + ); }); // 6e. MULTI-part image-only item: joining N−1 empty part texts yields a // truthy "\n", which is not real text — it must still take the "[image]" // placeholder (and keep matching canonicalUserText verbatim). test("responses: multi-input_image user item takes the [image] placeholder", () => { - const DATA_URL2 = `data:image/png;base64,${IMG_DATA}AAAA`; - const body: ResponsesRequestBody = { - input: [{ type: "message", role: "user", content: [{ type: "input_image", image_url: DATA_URL }, { type: "input_image", image_url: DATA_URL2 }] }], - }; - const { msgs } = responsesToCore(body); - assert.equal(msgs.length, 1); - assert.equal(msgs[0]?.text, "[image]", "no-text multi-image item uses the placeholder (not the \"\\n\" join artifact)"); - const rebuilt = coreToResponses(msgs); - assert.deepEqual(rebuilt[0], body.input[0], "raw item re-emitted verbatim"); + const DATA_URL2 = `data:image/png;base64,${IMG_DATA}AAAA`; + const body: ResponsesRequestBody = { + input: [ + { + type: "message", + role: "user", + content: [ + { type: "input_image", image_url: DATA_URL }, + { type: "input_image", image_url: DATA_URL2 }, + ], + }, + ], + }; + const { msgs } = responsesToCore(body); + assert.equal(msgs.length, 1); + assert.equal( + msgs[0]?.text, + "[image]", + 'no-text multi-image item uses the placeholder (not the "\\n" join artifact)', + ); + const rebuilt = coreToResponses(msgs); + assert.deepEqual(rebuilt[0], body.input[0], "raw item re-emitted verbatim"); }); // 6f. Whitespace-only text + image: the whitespace IS real text (a text part // exists), so it must be kept as-is — both responsesToCore and // canonicalUserText must agree on it for the verbatim round-trip to hold. test("responses: whitespace-only text plus image keeps its exact text", () => { - const body: ResponsesRequestBody = { - input: [{ type: "message", role: "user", content: [{ type: "input_text", text: " " }, { type: "input_image", image_url: DATA_URL }] }], - }; - const { msgs } = responsesToCore(body); - assert.equal(msgs.length, 1); - assert.notEqual(msgs[0]?.text, "[image]", "whitespace text counts as real text, not the placeholder"); - assert.ok(msgs[0]?.text?.startsWith(" "), "exact whitespace text preserved (kernel joins non-text parts with \\n)"); - const rebuilt = coreToResponses(msgs); - assert.deepEqual(rebuilt[0], body.input[0], "verbatim round-trip holds for truthy-blank text"); + const body: ResponsesRequestBody = { + input: [ + { + type: "message", + role: "user", + content: [ + { type: "input_text", text: " " }, + { type: "input_image", image_url: DATA_URL }, + ], + }, + ], + }; + const { msgs } = responsesToCore(body); + assert.equal(msgs.length, 1); + assert.notEqual( + msgs[0]?.text, + "[image]", + "whitespace text counts as real text, not the placeholder", + ); + assert.ok( + msgs[0]?.text?.startsWith(" "), + "exact whitespace text preserved (kernel joins non-text parts with \\n)", + ); + const rebuilt = coreToResponses(msgs); + assert.deepEqual( + rebuilt[0], + body.input[0], + "verbatim round-trip holds for truthy-blank text", + ); }); // 6g. Image part whose image_url is malformed (non-string): still tracked and // re-emitted verbatim; no base64 sidecar fields. test("responses: input_image with non-string image_url is tracked without sidecar fields", () => { - const badPart = { type: "input_image", image_url: { not: "a string" } } as unknown as { type: string; [k: string]: unknown }; - const body = { input: [{ type: "message", role: "user", content: [badPart] }] } as unknown as ResponsesRequestBody; - const { msgs } = responsesToCore(body); - assert.equal(msgs.length, 1, "tracked despite malformed url"); - assert.equal(msgs[0]?.text, "[image]"); - assert.equal(msgs[0]?.imageBase64, undefined, "no base64 split for non-string urls"); - const rebuilt = coreToResponses(msgs); - assert.deepEqual(rebuilt[0], body.input[0], "malformed item re-emitted verbatim"); + const badPart = { + type: "input_image", + image_url: { not: "a string" }, + } as unknown as { type: string; [k: string]: unknown }; + const body = { + input: [{ type: "message", role: "user", content: [badPart] }], + } as unknown as ResponsesRequestBody; + const { msgs } = responsesToCore(body); + assert.equal(msgs.length, 1, "tracked despite malformed url"); + assert.equal(msgs[0]?.text, "[image]"); + assert.equal( + msgs[0]?.imageBase64, + undefined, + "no base64 split for non-string urls", + ); + const rebuilt = coreToResponses(msgs); + assert.deepEqual( + rebuilt[0], + body.input[0], + "malformed item re-emitted verbatim", + ); }); // 6h. Empty-string input_text part + image: joining ["", ""] yields the @@ -434,18 +684,41 @@ test("responses: input_image with non-string image_url is tracked without sideca // upgrade; switching it to "[image]" would move its id for no functional // gain. Verbatim round-trip holds either way (#187/#291 review). test("responses: empty input_text plus image keeps its join-artifact text and stable id", () => { - const body: ResponsesRequestBody = { - input: [{ type: "message", role: "user", content: [{ type: "input_text", text: "" }, { type: "input_image", image_url: DATA_URL }] }], - }; - const { msgs } = responsesToCore(body); - assert.equal(msgs.length, 1); - assert.equal(msgs[0]?.text, "\n", "empty text part joins to the \"\\n\" artifact, not the placeholder"); - const pureImage: ResponsesRequestBody = { - input: [{ type: "message", role: "user", content: [{ type: "input_image", image_url: DATA_URL }] }], - }; - assert.notEqual(msgs[0]?.id, responsesToCore(pureImage).msgs[0]?.id, "distinct from the pure image-only placeholder id"); - const rebuilt = coreToResponses(msgs); - assert.deepEqual(rebuilt[0], body.input[0], "verbatim round-trip holds"); + const body: ResponsesRequestBody = { + input: [ + { + type: "message", + role: "user", + content: [ + { type: "input_text", text: "" }, + { type: "input_image", image_url: DATA_URL }, + ], + }, + ], + }; + const { msgs } = responsesToCore(body); + assert.equal(msgs.length, 1); + assert.equal( + msgs[0]?.text, + "\n", + 'empty text part joins to the "\\n" artifact, not the placeholder', + ); + const pureImage: ResponsesRequestBody = { + input: [ + { + type: "message", + role: "user", + content: [{ type: "input_image", image_url: DATA_URL }], + }, + ], + }; + assert.notEqual( + msgs[0]?.id, + responsesToCore(pureImage).msgs[0]?.id, + "distinct from the pure image-only placeholder id", + ); + const rebuilt = coreToResponses(msgs); + assert.deepEqual(rebuilt[0], body.input[0], "verbatim round-trip holds"); }); // 7. Responses reasoning is routed into the compression pipeline (NOT the @@ -453,59 +726,80 @@ test("responses: empty input_text plus image keeps its join-artifact text and st // item — including encrypted_content — round-trips verbatim via // rawResponsesItem while the turn is still live. test("responses: reasoning enters msgs[] as a tracked reasoning message (not preamble)", () => { - const body: ResponsesRequestBody = { - input: [ - { type: "reasoning", id: "rs_abc", summary: [{ type: "summary_text", text: "thinking" }], encrypted_content: "ENC_BLOB" }, - { type: "message", role: "user", content: "hi" }, - ], - }; - const { msgs, preamble } = responsesToCore(body); - assert.equal(preamble.length, 0, "reasoning is NOT in the opaque preamble"); - const r = msgs.find((m) => m.contentType === "reasoning"); - assert.ok(r, "reasoning entered msgs[] as contentType reasoning"); - assert.ok(r?.rawResponsesItem, "raw reasoning item carried in rawResponsesItem"); - const rebuilt = coreToResponses(msgs); - const out = rebuilt.find((i) => (i as { id?: string }).id === "rs_abc") as { type: string; encrypted_content?: string }; - assert.ok(out, "reasoning item rebuilt"); - assert.equal(out.type, "reasoning"); - assert.equal(out.encrypted_content, "ENC_BLOB", "encrypted_content preserved verbatim"); + const body: ResponsesRequestBody = { + input: [ + { + type: "reasoning", + id: "rs_abc", + summary: [{ type: "summary_text", text: "thinking" }], + encrypted_content: "ENC_BLOB", + }, + { type: "message", role: "user", content: "hi" }, + ], + }; + const { msgs, preamble } = responsesToCore(body); + assert.equal(preamble.length, 0, "reasoning is NOT in the opaque preamble"); + const r = msgs.find((m) => m.contentType === "reasoning"); + assert.ok(r, "reasoning entered msgs[] as contentType reasoning"); + assert.ok( + r?.rawResponsesItem, + "raw reasoning item carried in rawResponsesItem", + ); + const rebuilt = coreToResponses(msgs); + const out = rebuilt.find((i) => (i as { id?: string }).id === "rs_abc") as { + type: string; + encrypted_content?: string; + }; + assert.ok(out, "reasoning item rebuilt"); + assert.equal(out.type, "reasoning"); + assert.equal( + out.encrypted_content, + "ENC_BLOB", + "encrypted_content preserved verbatim", + ); }); // 8. additional_tools (and other opaque host directives) still go to the // preamble verbatim — only reasoning was promoted into compression. test("responses: additional_tools stays in the opaque preamble", () => { - const body: ResponsesRequestBody = { - input: [ - { type: "additional_tools", tools: [{ name: "exec" }] }, - { type: "reasoning", id: "rs_1" }, - { type: "message", role: "user", content: "hi" }, - ], - }; - const { msgs, preamble } = responsesToCore(body); - assert.equal(preamble.length, 1, "only additional_tools is opaque"); - assert.equal(preamble[0]?.type, "additional_tools"); - assert.ok(msgs.find((m) => m.contentType === "reasoning"), "reasoning went to msgs[], not preamble"); + const body: ResponsesRequestBody = { + input: [ + { type: "additional_tools", tools: [{ name: "exec" }] }, + { type: "reasoning", id: "rs_1" }, + { type: "message", role: "user", content: "hi" }, + ], + }; + const { msgs, preamble } = responsesToCore(body); + assert.equal(preamble.length, 1, "only additional_tools is opaque"); + assert.equal(preamble[0]?.type, "additional_tools"); + assert.ok( + msgs.find((m) => m.contentType === "reasoning"), + "reasoning went to msgs[], not preamble", + ); }); // 9. ACP_REASONING_KEEP=none drops reasoning entirely (escape hatch). test("responses: ACP_REASONING_KEEP=none drops all reasoning", () => { - const prev = process.env.ACP_REASONING_KEEP; - process.env.ACP_REASONING_KEEP = "none"; - try { - const body: ResponsesRequestBody = { - input: [ - { type: "reasoning", id: "rs_abc", encrypted_content: "ENC" }, - { type: "message", role: "user", content: "hi" }, - ], - }; - const { msgs, preamble, droppedReasoning } = responsesToCore(body); - assert.equal(preamble.length, 0); - assert.ok(!msgs.find((m) => m.contentType === "reasoning"), "no reasoning in msgs[]"); - assert.equal(droppedReasoning, 1, "droppedReasoning counted"); - } finally { - if (prev === undefined) delete process.env.ACP_REASONING_KEEP; - else process.env.ACP_REASONING_KEEP = prev; - } + const prev = process.env.ACP_REASONING_KEEP; + process.env.ACP_REASONING_KEEP = "none"; + try { + const body: ResponsesRequestBody = { + input: [ + { type: "reasoning", id: "rs_abc", encrypted_content: "ENC" }, + { type: "message", role: "user", content: "hi" }, + ], + }; + const { msgs, preamble, droppedReasoning } = responsesToCore(body); + assert.equal(preamble.length, 0); + assert.ok( + !msgs.find((m) => m.contentType === "reasoning"), + "no reasoning in msgs[]", + ); + assert.equal(droppedReasoning, 1, "droppedReasoning counted"); + } finally { + if (prev === undefined) delete process.env.ACP_REASONING_KEEP; + else process.env.ACP_REASONING_KEEP = prev; + } }); // 10. Reasoning VANISHES from the rebuilt input once its turn is covered by a @@ -513,75 +807,96 @@ test("responses: ACP_REASONING_KEEP=none drops all reasoning", () => { // drops covered message ids before coreToResponses runs; simulating that prune // here, the reasoning item must disappear while ordinary messages survive. test("responses: reasoning is dropped from output after its turn is compressed", () => { - const body: ResponsesRequestBody = { - input: [ - { type: "reasoning", id: "rs_abc", encrypted_content: "ENC" }, - { type: "message", role: "user", content: "hi" }, - { type: "message", role: "assistant", content: "hello" }, - ], - }; - const { msgs } = responsesToCore(body); - const reasoningId = msgs.find((m) => m.contentType === "reasoning")!.id; - const pruned = msgs.filter((m) => m.id !== reasoningId); - const rebuilt = coreToResponses(pruned); - const gone = rebuilt.find((i) => (i as { type?: string }).type === "reasoning"); - assert.equal(gone, undefined, "reasoning item disappears once its turn is compressed"); - assert.equal(rebuilt.length, 2, "user + assistant messages survive"); + const body: ResponsesRequestBody = { + input: [ + { type: "reasoning", id: "rs_abc", encrypted_content: "ENC" }, + { type: "message", role: "user", content: "hi" }, + { type: "message", role: "assistant", content: "hello" }, + ], + }; + const { msgs } = responsesToCore(body); + const reasoningId = msgs.find((m) => m.contentType === "reasoning")!.id; + const pruned = msgs.filter((m) => m.id !== reasoningId); + const rebuilt = coreToResponses(pruned); + const gone = rebuilt.find( + (i) => (i as { type?: string }).type === "reasoning", + ); + assert.equal( + gone, + undefined, + "reasoning item disappears once its turn is compressed", + ); + assert.equal(rebuilt.length, 2, "user + assistant messages survive"); }); // 11. Reasoning id is stable across turns — Codex re-sends the same reasoning // items every turn, so the same input item must yield the same BiliMessage id // or the kernel would accumulate phantom duplicates. test("responses: reasoning id is stable across turns (same item → same id)", () => { - const body = (): ResponsesRequestBody => ({ - input: [ - { type: "reasoning", id: "rs_abc", summary: [{ type: "summary_text", text: "t" }] }, - { type: "message", role: "user", content: "hi" }, - ], - }); - const a = responsesToCore(body()).msgs.find((m) => m.contentType === "reasoning")!.id; - const b = responsesToCore(body()).msgs.find((m) => m.contentType === "reasoning")!.id; - assert.equal(a, b, "same reasoning item yields the same message id across turns"); + const body = (): ResponsesRequestBody => ({ + input: [ + { + type: "reasoning", + id: "rs_abc", + summary: [{ type: "summary_text", text: "t" }], + }, + { type: "message", role: "user", content: "hi" }, + ], + }); + const a = responsesToCore(body()).msgs.find( + (m) => m.contentType === "reasoning", + )!.id; + const b = responsesToCore(body()).msgs.find( + (m) => m.contentType === "reasoning", + )!.id; + assert.equal( + a, + b, + "same reasoning item yields the same message id across turns", + ); }); // 12. Multiple reasoning items in one turn each get distinct ids and survive // round-trip in order. test("responses: multiple reasoning items keep distinct ids and order", () => { - const body: ResponsesRequestBody = { - input: [ - { type: "reasoning", id: "rs_1", encrypted_content: "A" }, - { type: "reasoning", id: "rs_2", encrypted_content: "B" }, - { type: "message", role: "user", content: "hi" }, - ], - }; - const { msgs } = responsesToCore(body); - const rs = msgs.filter((m) => m.contentType === "reasoning"); - assert.equal(rs.length, 2); - assert.notEqual(rs[0]!.id, rs[1]!.id, "distinct ids"); - const rebuilt = coreToResponses(msgs); - const ids = rebuilt - .filter((i) => (i as { type?: string }).type === "reasoning") - .map((i) => (i as { id?: string }).id); - assert.deepEqual(ids, ["rs_1", "rs_2"], "order + ids preserved on rebuild"); + const body: ResponsesRequestBody = { + input: [ + { type: "reasoning", id: "rs_1", encrypted_content: "A" }, + { type: "reasoning", id: "rs_2", encrypted_content: "B" }, + { type: "message", role: "user", content: "hi" }, + ], + }; + const { msgs } = responsesToCore(body); + const rs = msgs.filter((m) => m.contentType === "reasoning"); + assert.equal(rs.length, 2); + assert.notEqual(rs[0]!.id, rs[1]!.id, "distinct ids"); + const rebuilt = coreToResponses(msgs); + const ids = rebuilt + .filter((i) => (i as { type?: string }).type === "reasoning") + .map((i) => (i as { id?: string }).id); + assert.deepEqual(ids, ["rs_1", "rs_2"], "order + ids preserved on rebuild"); }); // 13. Reasoning without encrypted_content (older API shape) still round-trips // via rawResponsesItem. test("responses: reasoning without encrypted_content round-trips", () => { - const body: ResponsesRequestBody = { - input: [ - { type: "reasoning", id: "rs_x", summary: [{ type: "summary_text", text: "t" }] }, - { type: "message", role: "user", content: "hi" }, - ], - }; - const { msgs } = responsesToCore(body); - const rebuilt = coreToResponses(msgs); - const out = rebuilt.find((i) => (i as { id?: string }).id === "rs_x") as - | { type: string; encrypted_content?: string } - | undefined; - assert.ok(out, "reasoning without encrypted_content still rebuilt"); - assert.equal(out!.type, "reasoning"); - assert.equal(out!.encrypted_content, undefined); + const body: ResponsesRequestBody = { + input: [ + { + type: "reasoning", + id: "rs_x", + summary: [{ type: "summary_text", text: "t" }], + }, + { type: "message", role: "user", content: "hi" }, + ], + }; + const { msgs } = responsesToCore(body); + const rebuilt = coreToResponses(msgs); + const out = rebuilt.find((i) => (i as { id?: string }).id === "rs_x") as + { type: string; encrypted_content?: string } | undefined; + assert.ok(out, "reasoning without encrypted_content still rebuilt"); + assert.equal(out!.type, "reasoning"); + assert.equal(out!.encrypted_content, undefined); }); // 14. Prior-response ACTION items (computer_call, mcp_call, ...) are routed @@ -589,63 +904,92 @@ test("responses: reasoning without encrypted_content round-trips", () => { // accumulate unbounded every turn and break the prompt-cache prefix. They // round-trip verbatim via rawResponsesItem while their turn is live. test("responses: call items (computer_call/mcp_call) enter msgs[] not preamble", () => { - const body: ResponsesRequestBody = { - input: [ - { type: "additional_tools", tools: [] } as ResponseInputItem, - { type: "mcp_list_tools", server_label: "s" } as ResponseInputItem, - { type: "computer_call", id: "cc_1", action: { type: "screenshot" } } as ResponseInputItem, - { type: "mcp_call", id: "mc_1", name: "search", arguments: "{}" } as ResponseInputItem, - { type: "message", role: "user", content: "go" }, - ], - }; - const { msgs, preamble } = responsesToCore(body); - assert.deepEqual( - preamble.map((p) => p.type), - ["additional_tools", "mcp_list_tools"], - "only definitions stay in the preamble", - ); - const calls = msgs.filter((m) => m.contentType === "reasoning" && m.rawResponsesItem); - assert.equal(calls.length, 2, "both call items routed into msgs[]"); - const rebuilt = coreToResponses(msgs); - const types = rebuilt.map((i) => i.type); - assert.ok(types.includes("computer_call"), "computer_call round-trips verbatim"); - assert.ok(types.includes("mcp_call"), "mcp_call round-trips verbatim"); + const body: ResponsesRequestBody = { + input: [ + { type: "additional_tools", tools: [] } as ResponseInputItem, + { type: "mcp_list_tools", server_label: "s" } as ResponseInputItem, + { + type: "computer_call", + id: "cc_1", + action: { type: "screenshot" }, + } as ResponseInputItem, + { + type: "mcp_call", + id: "mc_1", + name: "search", + arguments: "{}", + } as ResponseInputItem, + { type: "message", role: "user", content: "go" }, + ], + }; + const { msgs, preamble } = responsesToCore(body); + assert.deepEqual( + preamble.map((p) => p.type), + ["additional_tools", "mcp_list_tools"], + "only definitions stay in the preamble", + ); + const calls = msgs.filter( + (m) => m.contentType === "reasoning" && m.rawResponsesItem, + ); + assert.equal(calls.length, 2, "both call items routed into msgs[]"); + const rebuilt = coreToResponses(msgs); + const types = rebuilt.map((i) => i.type); + assert.ok( + types.includes("computer_call"), + "computer_call round-trips verbatim", + ); + assert.ok(types.includes("mcp_call"), "mcp_call round-trips verbatim"); }); // 15. Call items are hidden once their turn is compressed (same prune // semantics as reasoning). test("responses: call items drop from output after their turn is compressed", () => { - const body: ResponsesRequestBody = { - input: [ - { type: "computer_call", id: "cc_1", action: { type: "screenshot" } } as ResponseInputItem, - { type: "message", role: "user", content: "hi" }, - ], - }; - const { msgs } = responsesToCore(body); - const callMsg = msgs.find((m) => m.contentType === "reasoning" && (m.rawResponsesItem as { type?: string }).type === "computer_call"); - assert.ok(callMsg, "computer_call entered msgs[]"); - const pruned = msgs.filter((m) => m.id !== callMsg!.id); - const rebuilt = coreToResponses(pruned); - assert.equal( - rebuilt.find((i) => i.type === "computer_call"), - undefined, - "computer_call disappears once its turn is compressed", - ); + const body: ResponsesRequestBody = { + input: [ + { + type: "computer_call", + id: "cc_1", + action: { type: "screenshot" }, + } as ResponseInputItem, + { type: "message", role: "user", content: "hi" }, + ], + }; + const { msgs } = responsesToCore(body); + const callMsg = msgs.find( + (m) => + m.contentType === "reasoning" && + (m.rawResponsesItem as { type?: string }).type === "computer_call", + ); + assert.ok(callMsg, "computer_call entered msgs[]"); + const pruned = msgs.filter((m) => m.id !== callMsg!.id); + const rebuilt = coreToResponses(pruned); + assert.equal( + rebuilt.find((i) => i.type === "computer_call"), + undefined, + "computer_call disappears once its turn is compressed", + ); }); // 16. A call item is given a distinct id namespace from reasoning, so the two // never collide even when present in the same turn. test("responses: call item and reasoning keep distinct ids in the same turn", () => { - const body: ResponsesRequestBody = { - input: [ - { type: "reasoning", id: "shared", encrypted_content: "E" }, - { type: "mcp_call", id: "shared", name: "n", arguments: "{}" } as ResponseInputItem, - ], - }; - const { msgs } = responsesToCore(body); - const ids = msgs.filter((m) => m.contentType === "reasoning").map((m) => m.id); - assert.equal(ids.length, 2); - assert.notEqual(ids[0], ids[1], "same raw id does not collide across kinds"); + const body: ResponsesRequestBody = { + input: [ + { type: "reasoning", id: "shared", encrypted_content: "E" }, + { + type: "mcp_call", + id: "shared", + name: "n", + arguments: "{}", + } as ResponseInputItem, + ], + }; + const { msgs } = responsesToCore(body); + const ids = msgs + .filter((m) => m.contentType === "reasoning") + .map((m) => m.id); + assert.equal(ids.length, 2); + assert.notEqual(ids[0], ids[1], "same raw id does not collide across kinds"); }); // 17. ACP_REASONING_KEEP=none drops chain-of-thought but MUST preserve call @@ -653,158 +997,242 @@ test("responses: call item and reasoning keep distinct ids in the same turn", () // bucket, not because they are reasoning. Dropping a computer_call would // corrupt the Responses conversation replay. test("responses: ACP_REASONING_KEEP=none drops reasoning but keeps call items", () => { - const prev = process.env.ACP_REASONING_KEEP; - process.env.ACP_REASONING_KEEP = "none"; - try { - const body: ResponsesRequestBody = { - input: [ - { type: "reasoning", id: "rs_1", encrypted_content: "E" }, - { type: "computer_call", id: "cc_1", action: { type: "screenshot" } } as ResponseInputItem, - { type: "mcp_call", id: "mc_1", name: "n", arguments: "{}" } as ResponseInputItem, - ], - }; - const { msgs } = responsesToCore(body); - const reasoning = msgs.filter((m) => (m.rawResponsesItem as { type?: string }).type === "reasoning"); - const calls = msgs.filter((m) => (m.rawResponsesItem as { type?: string }).type !== "reasoning" && m.contentType === "reasoning"); - assert.equal(reasoning.length, 0, "chain-of-thought is dropped under ACP_REASONING_KEEP=none"); - assert.equal(calls.length, 2, "computer_call + mcp_call survive (replay-critical)"); - } finally { - if (prev === undefined) delete process.env.ACP_REASONING_KEEP; - else process.env.ACP_REASONING_KEEP = prev; - } + const prev = process.env.ACP_REASONING_KEEP; + process.env.ACP_REASONING_KEEP = "none"; + try { + const body: ResponsesRequestBody = { + input: [ + { type: "reasoning", id: "rs_1", encrypted_content: "E" }, + { + type: "computer_call", + id: "cc_1", + action: { type: "screenshot" }, + } as ResponseInputItem, + { + type: "mcp_call", + id: "mc_1", + name: "n", + arguments: "{}", + } as ResponseInputItem, + ], + }; + const { msgs } = responsesToCore(body); + const reasoning = msgs.filter( + (m) => (m.rawResponsesItem as { type?: string }).type === "reasoning", + ); + const calls = msgs.filter( + (m) => + (m.rawResponsesItem as { type?: string }).type !== "reasoning" && + m.contentType === "reasoning", + ); + assert.equal( + reasoning.length, + 0, + "chain-of-thought is dropped under ACP_REASONING_KEEP=none", + ); + assert.equal( + calls.length, + 2, + "computer_call + mcp_call survive (replay-critical)", + ); + } finally { + if (prev === undefined) delete process.env.ACP_REASONING_KEEP; + else process.env.ACP_REASONING_KEEP = prev; + } }); // 10. OpenAI's EasyInputMessage shorthand legally omits `type` (omp and bare // /v1/responses clients send `{ role, content }`): those items must fold // exactly like their typed `message` form instead of falling into the preamble. test("responses: EasyInputMessage without type folds like a typed message", () => { - const body: ResponsesRequestBody = { - input: [ - { role: "user", content: "hello" }, - { role: "assistant", content: "hi there" }, - { type: "message", role: "user", content: "second" }, - ] as unknown as ResponseInputItem[], - }; - const { msgs, preamble, systemParts } = responsesToCore(body); - assert.equal(preamble.length, 0, "type-less messages are not opaque"); - assert.equal(msgs.length, 3); - assert.equal(msgs[0]?.role, "user"); - assert.equal(msgs[0]?.text, "hello"); - assert.equal(msgs[1]?.role, "assistant"); - assert.equal(msgs[1]?.text, "hi there"); - assert.equal(systemParts.length, 0); + const body: ResponsesRequestBody = { + input: [ + { role: "user", content: "hello" }, + { role: "assistant", content: "hi there" }, + { type: "message", role: "user", content: "second" }, + ] as unknown as ResponseInputItem[], + }; + const { msgs, preamble, systemParts } = responsesToCore(body); + assert.equal(preamble.length, 0, "type-less messages are not opaque"); + assert.equal(msgs.length, 3); + assert.equal(msgs[0]?.role, "user"); + assert.equal(msgs[0]?.text, "hello"); + assert.equal(msgs[1]?.role, "assistant"); + assert.equal(msgs[1]?.text, "hi there"); + assert.equal(systemParts.length, 0); }); // 11. Folded text splices back into the EasyInputMessage slot; re-emitted // items carry the canonical typed form (interchangeable on the wire). test("responses: EasyInputMessage round-trips through patchResponsesInput", () => { - const body: ResponsesRequestBody = { - input: [ - { role: "user", content: "fold me" }, - { role: "assistant", content: "ok" }, - { role: "user", content: "and me" }, - ] as unknown as ResponseInputItem[], - }; - const projection = responsesToCore(body); - const patched = projection.msgs.map((m, i) => (i === 0 ? { ...m, text: `[sum] ${m.text}` } : m)); - const rebuilt = patchResponsesInput(projection, patched); - assert.ok(Array.isArray(rebuilt)); - const first = rebuilt[0] as { type?: string; role?: string; content?: unknown }; - assert.equal(first.type, "message", "re-emitted in canonical typed form"); - assert.equal(first.role, "user"); - assert.equal(first.content, "[sum] fold me"); - assert.equal((rebuilt[1] as { content?: unknown }).content, "ok", "untouched slots keep content"); + const body: ResponsesRequestBody = { + input: [ + { role: "user", content: "fold me" }, + { role: "assistant", content: "ok" }, + { role: "user", content: "and me" }, + ] as unknown as ResponseInputItem[], + }; + const projection = responsesToCore(body); + const patched = projection.msgs.map((m, i) => + i === 0 ? { ...m, text: `[sum] ${m.text}` } : m, + ); + const rebuilt = patchResponsesInput(projection, patched); + assert.ok(Array.isArray(rebuilt)); + const first = rebuilt[0] as { + type?: string; + role?: string; + content?: unknown; + }; + assert.equal(first.type, "message", "re-emitted in canonical typed form"); + assert.equal(first.role, "user"); + assert.equal(first.content, "[sum] fold me"); + assert.equal( + (rebuilt[1] as { content?: unknown }).content, + "ok", + "untouched slots keep content", + ); }); // 12. Type-less system/developer shorthands follow the typed path into // systemParts (hosts re-inject them via injectResponsesDeveloperMessage). test("responses: EasyInputMessage system role joins systemParts", () => { - const body: ResponsesRequestBody = { - instructions: "be brief", - input: [{ role: "system", content: "extra rules" }] as unknown as ResponseInputItem[], - }; - const { msgs, systemParts } = responsesToCore(body); - assert.deepEqual(systemParts, ["be brief", "extra rules"]); - assert.equal(msgs.length, 0); + const body: ResponsesRequestBody = { + instructions: "be brief", + input: [ + { role: "system", content: "extra rules" }, + ] as unknown as ResponseInputItem[], + }; + const { msgs, systemParts } = responsesToCore(body); + assert.deepEqual(systemParts, ["be brief", "extra rules"]); + assert.equal(msgs.length, 0); }); // 13. Items with neither `type` nor a message `role` stay opaque. test("responses: type-less non-message items stay in the preamble", () => { - const body: ResponsesRequestBody = { - input: [{ foo: "bar" }] as unknown as ResponseInputItem[], - }; - const { msgs, preamble } = responsesToCore(body); - assert.equal(msgs.length, 0); - assert.equal(preamble.length, 1); - assert.deepEqual(preamble[0], { foo: "bar" }); + const body: ResponsesRequestBody = { + input: [{ foo: "bar" }] as unknown as ResponseInputItem[], + }; + const { msgs, preamble } = responsesToCore(body); + assert.equal(msgs.length, 0); + assert.equal(preamble.length, 1); + assert.deepEqual(preamble[0], { foo: "bar" }); }); // --- Anthropic tool_result embedded media (#366) --- -const TR_IMG_SOURCE = { type: "base64", media_type: "image/png", data: IMG_DATA }; +const TR_IMG_SOURCE = { + type: "base64", + media_type: "image/png", + data: IMG_DATA, +}; -function trBody(content: AnthropicToolResult["content"], extra: Record = {}): AnthropicRequestBody { - return { - model: "claude", - max_tokens: 100, - messages: [ - { role: "user", content: [{ type: "text", text: "run the tool" }] }, - { role: "assistant", content: [{ type: "tool_use", id: "t1", name: "screenshot", input: {} }] }, - { role: "user", content: [{ type: "tool_result", tool_use_id: "t1", content, ...extra }] }, +function trBody( + content: AnthropicToolResult["content"], + extra: Record = {}, +): AnthropicRequestBody { + return { + model: "claude", + max_tokens: 100, + messages: [ + { role: "user", content: [{ type: "text", text: "run the tool" }] }, + { + role: "assistant", + content: [ + { type: "tool_use", id: "t1", name: "screenshot", input: {} }, ], - }; + }, + { + role: "user", + content: [ + { type: "tool_result", tool_use_id: "t1", content, ...extra }, + ], + }, + ], + }; } -function rebuiltToolResult(msgs: ReturnType["msgs"]): Record { - const rebuilt = coreToAnthropic(msgs); - const userMsg = rebuilt.filter((m) => m.role === "user").pop()!; - return block(userMsg, 0); +function rebuiltToolResult( + msgs: ReturnType["msgs"], +): Record { + const rebuilt = coreToAnthropic(msgs); + const userMsg = rebuilt.filter((m) => m.role === "user").pop()!; + return block(userMsg, 0); } test("anthropic: tool_result [text, image] round-trips the image verbatim", () => { - const original = { type: "text" as const, text: "done" }; - const body = trBody([original, { type: "image", source: TR_IMG_SOURCE }]); - const { msgs } = anthropicToCore(body); - const res = msgs.find((m) => m.contentType === "tool-result")!; - assert.ok(res.rawAnthropicBlock, "sidecar carries the original block"); - // Text extraction (and thus the id seed) is unchanged by the fix. - assert.equal(res.text, "done\n"); - const tr = rebuiltToolResult(msgs); - assert.deepEqual(tr.content, [original, { type: "image", source: TR_IMG_SOURCE }], "content array re-emitted verbatim, original order"); + const original = { type: "text" as const, text: "done" }; + const body = trBody([original, { type: "image", source: TR_IMG_SOURCE }]); + const { msgs } = anthropicToCore(body); + const res = msgs.find((m) => m.contentType === "tool-result")!; + assert.ok(res.rawAnthropicBlock, "sidecar carries the original block"); + // Text extraction (and thus the id seed) is unchanged by the fix. + assert.equal(res.text, "done\n"); + const tr = rebuiltToolResult(msgs); + assert.deepEqual( + tr.content, + [original, { type: "image", source: TR_IMG_SOURCE }], + "content array re-emitted verbatim, original order", + ); }); test("anthropic: image-only tool_result survives (empty text)", () => { - const body = trBody([{ type: "image", source: TR_IMG_SOURCE }]); - const { msgs } = anthropicToCore(body); - const res = msgs.find((m) => m.contentType === "tool-result")!; - assert.equal(res.text, ""); - assert.ok(res.rawAnthropicBlock); - const tr = rebuiltToolResult(msgs); - assert.deepEqual(tr.content, [{ type: "image", source: TR_IMG_SOURCE }]); + const body = trBody([{ type: "image", source: TR_IMG_SOURCE }]); + const { msgs } = anthropicToCore(body); + const res = msgs.find((m) => m.contentType === "tool-result")!; + assert.equal(res.text, ""); + assert.ok(res.rawAnthropicBlock); + const tr = rebuiltToolResult(msgs); + assert.deepEqual(tr.content, [{ type: "image", source: TR_IMG_SOURCE }]); }); test("anthropic: tool_result is_error + cache_control survive via sidecar", () => { - const body = trBody( - [{ type: "text", text: "boom" }, { type: "image", source: TR_IMG_SOURCE }], - { is_error: true, cache_control: { type: "ephemeral" } }, - ); - const { msgs } = anthropicToCore(body); - const tr = rebuiltToolResult(msgs); - assert.equal(tr.is_error, true, "is_error reconstructed"); - assert.deepEqual(tr.cache_control, { type: "ephemeral" }, "cache_control reconstructed"); + const body = trBody( + [ + { type: "text", text: "boom" }, + { type: "image", source: TR_IMG_SOURCE }, + ], + { is_error: true, cache_control: { type: "ephemeral" } }, + ); + const { msgs } = anthropicToCore(body); + const tr = rebuiltToolResult(msgs); + assert.equal(tr.is_error, true, "is_error reconstructed"); + assert.deepEqual( + tr.cache_control, + { type: "ephemeral" }, + "cache_control reconstructed", + ); }); test("anthropic: string and text-only tool_results keep the legacy shape", () => { - const strBody = trBody("done"); - const strMsgs = anthropicToCore(strBody).msgs; - const strRes = strMsgs.find((m) => m.contentType === "tool-result")!; - assert.equal(strRes.rawAnthropicBlock, undefined, "string content gets no sidecar"); - assert.equal(rebuiltToolResult(strMsgs).content, "done", "string content stays a string"); - - const arrBody = trBody([{ type: "text", text: "a" }, { type: "text", text: "b" }]); - const arrMsgs = anthropicToCore(arrBody).msgs; - const arrRes = arrMsgs.find((m) => m.contentType === "tool-result")!; - assert.equal(arrRes.rawAnthropicBlock, undefined, "text-only array gets no sidecar"); - assert.equal(arrRes.text, "a\nb"); - assert.equal(rebuiltToolResult(arrMsgs).content, "a\nb", "text-only array keeps the joined-string rebuild"); + const strBody = trBody("done"); + const strMsgs = anthropicToCore(strBody).msgs; + const strRes = strMsgs.find((m) => m.contentType === "tool-result")!; + assert.equal( + strRes.rawAnthropicBlock, + undefined, + "string content gets no sidecar", + ); + assert.equal( + rebuiltToolResult(strMsgs).content, + "done", + "string content stays a string", + ); + + const arrBody = trBody([ + { type: "text", text: "a" }, + { type: "text", text: "b" }, + ]); + const arrMsgs = anthropicToCore(arrBody).msgs; + const arrRes = arrMsgs.find((m) => m.contentType === "tool-result")!; + assert.equal( + arrRes.rawAnthropicBlock, + undefined, + "text-only array gets no sidecar", + ); + assert.equal(arrRes.text, "a\nb"); + assert.equal( + rebuiltToolResult(arrMsgs).content, + "a\nb", + "text-only array keeps the joined-string rebuild", + ); }); diff --git a/tests/wire-conversation-signal.test.ts b/tests/wire-conversation-signal.test.ts index 8723689..aff74b5 100644 --- a/tests/wire-conversation-signal.test.ts +++ b/tests/wire-conversation-signal.test.ts @@ -1,7 +1,13 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { conversationSignalAnthropic, type AnthropicRequestBody } from "../src/wire/anthropic.js"; -import { conversationSignalOpenai, type OpenAIRequestBody } from "../src/wire/openai.js"; +import { + conversationSignalAnthropic, + type AnthropicRequestBody, +} from "../src/wire/anthropic.js"; +import { + conversationSignalOpenai, + type OpenAIRequestBody, +} from "../src/wire/openai.js"; import { hashId } from "../src/wire/util.js"; const DEFAULT = hashId("default"); @@ -11,44 +17,80 @@ const DEFAULT = hashId("default"); const malformedBodies: unknown[] = [{}, [1, 2], null, "str", 42]; test("conversationSignalAnthropic: bodies without a messages array fall back to the default fingerprint (#298)", () => { - for (const raw of malformedBodies) { - assert.doesNotThrow(() => conversationSignalAnthropic(raw as AnthropicRequestBody)); - assert.equal(conversationSignalAnthropic(raw as AnthropicRequestBody), DEFAULT); - } + for (const raw of malformedBodies) { + assert.doesNotThrow(() => + conversationSignalAnthropic(raw as AnthropicRequestBody), + ); + assert.equal( + conversationSignalAnthropic(raw as AnthropicRequestBody), + DEFAULT, + ); + } }); test("conversationSignalOpenai: bodies without a messages array fall back to the default fingerprint (#298)", () => { - for (const raw of malformedBodies) { - assert.doesNotThrow(() => conversationSignalOpenai(raw as OpenAIRequestBody)); - assert.equal(conversationSignalOpenai(raw as OpenAIRequestBody), DEFAULT); - } + for (const raw of malformedBodies) { + assert.doesNotThrow(() => + conversationSignalOpenai(raw as OpenAIRequestBody), + ); + assert.equal(conversationSignalOpenai(raw as OpenAIRequestBody), DEFAULT); + } }); test("conversationSignal*: header short-circuit wins even on a malformed body", () => { - const body = {} as unknown as AnthropicRequestBody & OpenAIRequestBody; - assert.equal(conversationSignalAnthropic(body, " s-1 "), "s-1"); - assert.equal(conversationSignalOpenai(body, " s-2 "), "s-2"); + const body = {} as unknown as AnthropicRequestBody & OpenAIRequestBody; + assert.equal(conversationSignalAnthropic(body, " s-1 "), "s-1"); + assert.equal(conversationSignalOpenai(body, " s-2 "), "s-2"); }); test("conversationSignal*: non-user and null message elements never crash and fall back to default (#298)", () => { - const anthropicBody = { messages: [null, { role: "assistant", content: "hi" }] } as unknown as AnthropicRequestBody; - assert.equal(conversationSignalAnthropic(anthropicBody), DEFAULT); - const openaiBody = { messages: [null, { role: "system", content: "sys" }] } as unknown as OpenAIRequestBody; - assert.equal(conversationSignalOpenai(openaiBody), DEFAULT); + const anthropicBody = { + messages: [null, { role: "assistant", content: "hi" }], + } as unknown as AnthropicRequestBody; + assert.equal(conversationSignalAnthropic(anthropicBody), DEFAULT); + const openaiBody = { + messages: [null, { role: "system", content: "sys" }], + } as unknown as OpenAIRequestBody; + assert.equal(conversationSignalOpenai(openaiBody), DEFAULT); }); test("conversationSignalAnthropic: first user message content still drives the fingerprint", () => { - const body = { messages: [{ role: "user", content: "hello" }, { role: "assistant", content: "x" }] } satisfies AnthropicRequestBody; - assert.equal(conversationSignalAnthropic(body), hashId(JSON.stringify("hello"))); - const other = { ...body, messages: [{ role: "user", content: "world" }] } satisfies AnthropicRequestBody; - assert.notEqual(conversationSignalAnthropic(other), conversationSignalAnthropic(body)); - assert.match(conversationSignalAnthropic(body), /^[0-9a-f]{16}$/); + const body = { + messages: [ + { role: "user", content: "hello" }, + { role: "assistant", content: "x" }, + ], + } satisfies AnthropicRequestBody; + assert.equal( + conversationSignalAnthropic(body), + hashId(JSON.stringify("hello")), + ); + const other = { + ...body, + messages: [{ role: "user", content: "world" }], + } satisfies AnthropicRequestBody; + assert.notEqual( + conversationSignalAnthropic(other), + conversationSignalAnthropic(body), + ); + assert.match(conversationSignalAnthropic(body), /^[0-9a-f]{16}$/); }); test("conversationSignalOpenai: first user message content still drives the fingerprint", () => { - const body = { messages: [{ role: "user", content: "hello" }, { role: "assistant", content: "x" }] } satisfies OpenAIRequestBody; - assert.equal(conversationSignalOpenai(body), hashId("hello")); - const other = { ...body, messages: [{ role: "user", content: "world" }] } satisfies OpenAIRequestBody; - assert.notEqual(conversationSignalOpenai(other), conversationSignalOpenai(body)); - assert.match(conversationSignalOpenai(body), /^[0-9a-f]{16}$/); + const body = { + messages: [ + { role: "user", content: "hello" }, + { role: "assistant", content: "x" }, + ], + } satisfies OpenAIRequestBody; + assert.equal(conversationSignalOpenai(body), hashId("hello")); + const other = { + ...body, + messages: [{ role: "user", content: "world" }], + } satisfies OpenAIRequestBody; + assert.notEqual( + conversationSignalOpenai(other), + conversationSignalOpenai(body), + ); + assert.match(conversationSignalOpenai(body), /^[0-9a-f]{16}$/); }); diff --git a/tests/wire-demoted-thinking.test.ts b/tests/wire-demoted-thinking.test.ts index acf7a26..ea251f3 100644 --- a/tests/wire-demoted-thinking.test.ts +++ b/tests/wire-demoted-thinking.test.ts @@ -2,176 +2,243 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { splitDemotedThinking } from "../src/wire/demoted-thinking.js"; -import { coreToOpenai, openaiToCore, type OpenAIRequestBody } from "../src/wire/openai.js"; -import { responsesToCore, type ResponsesRequestBody } from "../src/wire/responses.js"; +import { + coreToOpenai, + openaiToCore, + type OpenAIRequestBody, +} from "../src/wire/openai.js"; +import { + responsesToCore, + type ResponsesRequestBody, +} from "../src/wire/responses.js"; // --- splitDemotedThinking unit ------------------------------------------------- test("splitDemotedThinking reverses each dialect form byte-exactly", () => { - assert.deepStrictEqual(splitDemotedThinking("\nA\n\n\n\n两个 PR："), { - reasoning: "A", - text: "\n\n两个 PR：", - }); - assert.deepStrictEqual(splitDemotedThinking("\nB\n\ntail"), { - reasoning: "B", - text: "tail", - }); - assert.deepStrictEqual(splitDemotedThinking("```thinking\nC\n```\ntail"), { - reasoning: "C", - text: "tail", - }); + assert.deepStrictEqual( + splitDemotedThinking("\nA\n\n\n\n两个 PR："), + { + reasoning: "A", + text: "\n\n两个 PR：", + }, + ); + assert.deepStrictEqual( + splitDemotedThinking("\nB\n\ntail"), + { + reasoning: "B", + text: "tail", + }, + ); + assert.deepStrictEqual(splitDemotedThinking("```thinking\nC\n```\ntail"), { + reasoning: "C", + text: "tail", + }); }); test("splitDemotedThinking handles stacked blocks and glue", () => { - assert.deepStrictEqual(splitDemotedThinking("\nA\n\n\nB\n\n\ntext"), { - reasoning: "A\nB", - text: "\ntext", - }); - // thinking-only turn: glue consumed, text empty - assert.deepStrictEqual(splitDemotedThinking("\nA\n"), { reasoning: "A", text: "" }); + assert.deepStrictEqual( + splitDemotedThinking("\nA\n\n\nB\n\n\ntext"), + { + reasoning: "A\nB", + text: "\ntext", + }, + ); + // thinking-only turn: glue consumed, text empty + assert.deepStrictEqual(splitDemotedThinking("\nA\n"), { + reasoning: "A", + text: "", + }); }); test("splitDemotedThinking refuses malformed / non-tag content", () => { - assert.strictEqual(splitDemotedThinking(""), null); - assert.strictEqual(splitDemotedThinking("plain answer"), null); - assert.strictEqual(splitDemotedThinking("pre \nA\n"), null); // tag must open at offset 0 - assert.strictEqual(splitDemotedThinking("\nA\n"), null); // unterminated - assert.strictEqual(splitDemotedThinking("x"), null); // renderer always breaks after the open tag - assert.strictEqual(splitDemotedThinking("\n"), null); // empty block + assert.strictEqual(splitDemotedThinking(""), null); + assert.strictEqual(splitDemotedThinking("plain answer"), null); + assert.strictEqual(splitDemotedThinking("pre \nA\n"), null); // tag must open at offset 0 + assert.strictEqual(splitDemotedThinking("\nA\n"), null); // unterminated + assert.strictEqual(splitDemotedThinking("x"), null); // renderer always breaks after the open tag + assert.strictEqual(splitDemotedThinking("\n"), null); // empty block }); // --- openaiToCore invariance ---------------------------------------------------- -const INLINE_TURN = "\nThe user asks.\n\n\n\n两个 PR：内核与代理。"; +const INLINE_TURN = + "\nThe user asks.\n\n\n\n两个 PR：内核与代理。"; const FIELD_TEXT = "\n\n两个 PR：内核与代理。"; const FIELD_REASONING = "The user asks."; const inlineBody: OpenAIRequestBody = { - model: "glm-x", - messages: [ - { role: "user", content: "q" }, - { role: "assistant", content: INLINE_TURN }, - { role: "user", content: "r" }, - ], + model: "glm-x", + messages: [ + { role: "user", content: "q" }, + { role: "assistant", content: INLINE_TURN }, + { role: "user", content: "r" }, + ], }; const fieldBody: OpenAIRequestBody = { - model: "glm-x", - messages: [ - { role: "user", content: "q" }, - { role: "assistant", content: FIELD_TEXT, reasoning_content: FIELD_REASONING }, - { role: "user", content: "r" }, - ], + model: "glm-x", + messages: [ + { role: "user", content: "q" }, + { + role: "assistant", + content: FIELD_TEXT, + reasoning_content: FIELD_REASONING, + }, + { role: "user", content: "r" }, + ], }; test("openaiToCore: inline and reasoning_content field share one identity space", () => { - const inline = openaiToCore(inlineBody).msgs; - const field = openaiToCore(fieldBody).msgs; - assert.strictEqual(inline.length, 4); - // Same ids, same order, same contentTypes — one fingerprint space. - assert.deepStrictEqual( - inline.map((m) => ({ id: m.id, role: m.role, contentType: m.contentType, text: m.text })), - field.map((m) => ({ id: m.id, role: m.role, contentType: m.contentType, text: m.text })), - ); - const reasoning = inline[1]; - assert.strictEqual(reasoning.contentType, "reasoning"); - assert.strictEqual(reasoning.text, FIELD_REASONING); - assert.strictEqual(inline[2].text, FIELD_TEXT); + const inline = openaiToCore(inlineBody).msgs; + const field = openaiToCore(fieldBody).msgs; + assert.strictEqual(inline.length, 4); + // Same ids, same order, same contentTypes — one fingerprint space. + assert.deepStrictEqual( + inline.map((m) => ({ + id: m.id, + role: m.role, + contentType: m.contentType, + text: m.text, + })), + field.map((m) => ({ + id: m.id, + role: m.role, + contentType: m.contentType, + text: m.text, + })), + ); + const reasoning = inline[1]; + assert.strictEqual(reasoning.contentType, "reasoning"); + assert.strictEqual(reasoning.text, FIELD_REASONING); + assert.strictEqual(inline[2].text, FIELD_TEXT); }); test("openaiToCore: reasoning_content field wins; tagged content is not double-split", () => { - const body: OpenAIRequestBody = { - model: "glm-x", - messages: [{ role: "assistant", content: "\nA\ntext", reasoning_content: "R" }], - }; - const msgs = openaiToCore(body).msgs; - assert.strictEqual(msgs.length, 2); - assert.strictEqual(msgs[0].text, "R"); - assert.strictEqual(msgs[1].text, "\nA\ntext"); + const body: OpenAIRequestBody = { + model: "glm-x", + messages: [ + { + role: "assistant", + content: "\nA\ntext", + reasoning_content: "R", + }, + ], + }; + const msgs = openaiToCore(body).msgs; + assert.strictEqual(msgs.length, 2); + assert.strictEqual(msgs[0].text, "R"); + assert.strictEqual(msgs[1].text, "\nA\ntext"); }); test("openaiToCore: thinking-only turn yields reasoning with no empty text piece", () => { - const msgs = openaiToCore({ - model: "glm-x", - messages: [{ role: "assistant", content: "\nonly\n" }], - }).msgs; - assert.deepStrictEqual( - msgs.map((m) => m.contentType), - ["reasoning"], - ); + const msgs = openaiToCore({ + model: "glm-x", + messages: [{ role: "assistant", content: "\nonly\n" }], + }).msgs; + assert.deepStrictEqual( + msgs.map((m) => m.contentType), + ["reasoning"], + ); }); test("openaiToCore: user content starting with a tag is never split", () => { - const msgs = openaiToCore({ - model: "glm-x", - messages: [{ role: "user", content: "\nA\nq" }], - }).msgs; - assert.deepStrictEqual( - msgs.map((m) => m.contentType), - ["text"], - ); - assert.strictEqual(msgs[0].text, "\nA\nq"); + const msgs = openaiToCore({ + model: "glm-x", + messages: [{ role: "user", content: "\nA\nq" }], + }).msgs; + assert.deepStrictEqual( + msgs.map((m) => m.contentType), + ["text"], + ); + assert.strictEqual(msgs[0].text, "\nA\nq"); }); test("coreToOpenai round-trip normalizes inline form to the field form", () => { - const wire = coreToOpenai(openaiToCore(inlineBody).msgs); - const assistant = wire.find((m) => m.role === "assistant"); - assert.ok(assistant); - assert.strictEqual(assistant.reasoning_content, FIELD_REASONING); - assert.strictEqual(assistant.content, FIELD_TEXT); + const wire = coreToOpenai(openaiToCore(inlineBody).msgs); + const assistant = wire.find((m) => m.role === "assistant"); + assert.ok(assistant); + assert.strictEqual(assistant.reasoning_content, FIELD_REASONING); + assert.strictEqual(assistant.content, FIELD_TEXT); }); // --- responsesToCore invariance ------------------------------------------------- const inlineResponsesBody = (): ResponsesRequestBody => ({ - model: "resp-x", - input: [ - { type: "message", role: "user", content: "q" }, - { type: "message", role: "assistant", content: [{ type: "output_text", text: INLINE_TURN }] }, - { type: "message", role: "user", content: "r" }, - ] as ResponsesRequestBody["input"], + model: "resp-x", + input: [ + { type: "message", role: "user", content: "q" }, + { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: INLINE_TURN }], + }, + { type: "message", role: "user", content: "r" }, + ] as ResponsesRequestBody["input"], }); const itemResponsesBody = (): ResponsesRequestBody => ({ - model: "resp-x", - input: [ - { type: "message", role: "user", content: "q" }, - { type: "reasoning", summary: [{ type: "summary_text", text: FIELD_REASONING }] }, - { type: "message", role: "assistant", content: [{ type: "output_text", text: FIELD_TEXT }] }, - { type: "message", role: "user", content: "r" }, - ] as ResponsesRequestBody["input"], + model: "resp-x", + input: [ + { type: "message", role: "user", content: "q" }, + { + type: "reasoning", + summary: [{ type: "summary_text", text: FIELD_REASONING }], + }, + { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: FIELD_TEXT }], + }, + { type: "message", role: "user", content: "r" }, + ] as ResponsesRequestBody["input"], }); test("responsesToCore: inline message and reasoning item share one identity space", () => { - const inline = responsesToCore(inlineResponsesBody()).msgs; - const item = responsesToCore(itemResponsesBody()).msgs; - assert.strictEqual(inline.length, 4); - assert.strictEqual(item.length, 4); - assert.deepStrictEqual( - inline.map((m) => ({ id: m.id, role: m.role, contentType: m.contentType, text: m.text })), - item.map((m) => ({ id: m.id, role: m.role, contentType: m.contentType, text: m.text })), - ); + const inline = responsesToCore(inlineResponsesBody()).msgs; + const item = responsesToCore(itemResponsesBody()).msgs; + assert.strictEqual(inline.length, 4); + assert.strictEqual(item.length, 4); + assert.deepStrictEqual( + inline.map((m) => ({ + id: m.id, + role: m.role, + contentType: m.contentType, + text: m.text, + })), + item.map((m) => ({ + id: m.id, + role: m.role, + contentType: m.contentType, + text: m.text, + })), + ); }); test("responsesToCore: thinking-only message yields reasoning with no empty text piece", () => { - const proj = responsesToCore({ - model: "resp-x", - input: [ - { type: "message", role: "assistant", content: [{ type: "output_text", text: "\nonly\n" }] }, - ] as ResponsesRequestBody["input"], - }); - assert.deepStrictEqual( - proj.msgs.map((m) => m.contentType), - ["reasoning"], - ); + const proj = responsesToCore({ + model: "resp-x", + input: [ + { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "\nonly\n" }], + }, + ] as ResponsesRequestBody["input"], + }); + assert.deepStrictEqual( + proj.msgs.map((m) => m.contentType), + ["reasoning"], + ); }); test("responsesToCore: user content starting with a tag is never split", () => { - const proj = responsesToCore({ - model: "resp-x", - input: [{ type: "message", role: "user", content: "\nA\nq" }] as ResponsesRequestBody["input"], - }); - assert.deepStrictEqual( - proj.msgs.map((m) => m.contentType), - ["text"], - ); - assert.strictEqual(proj.msgs[0].text, "\nA\nq"); + const proj = responsesToCore({ + model: "resp-x", + input: [ + { type: "message", role: "user", content: "\nA\nq" }, + ] as ResponsesRequestBody["input"], + }); + assert.deepStrictEqual( + proj.msgs.map((m) => m.contentType), + ["text"], + ); + assert.strictEqual(proj.msgs[0].text, "\nA\nq"); }); diff --git a/tests/wire-formats.test.ts b/tests/wire-formats.test.ts index 0a6bab3..5136be4 100644 --- a/tests/wire-formats.test.ts +++ b/tests/wire-formats.test.ts @@ -7,7 +7,10 @@ import { } from "../src/wire/formats.js"; test("WIRE_FORMATS lists exactly the codecs shipped by this package", () => { - assert.deepEqual([...WIRE_FORMATS], ["anthropic", "openai", "responses", "google"]); + assert.deepEqual( + [...WIRE_FORMATS], + ["anthropic", "openai", "responses", "google"], + ); for (const f of WIRE_FORMATS) { assert.equal(isWireFormat(f), true); } diff --git a/tests/wire-mirror.test.ts b/tests/wire-mirror.test.ts index 1383c95..de86229 100644 --- a/tests/wire-mirror.test.ts +++ b/tests/wire-mirror.test.ts @@ -2,258 +2,351 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { - mirrorAnthropicMessages, - mirrorAnthropicToCore, - mirrorOpenaiMessages, - mirrorOpenaiToCore, - mirrorResponsesInput, - mirrorResponsesToCore, - type MirrorMessage, + mirrorAnthropicMessages, + mirrorAnthropicToCore, + mirrorOpenaiMessages, + mirrorOpenaiToCore, + mirrorResponsesInput, + mirrorResponsesToCore, + type MirrorMessage, } from "../src/wire/mirror.js"; import { openaiToCore, type OpenAIRequestBody } from "../src/wire/openai.js"; import { anthropicToCore } from "../src/wire/anthropic.js"; -import { responsesToCore, type ResponsesRequestBody } from "../src/wire/responses.js"; +import { + responsesToCore, + type ResponsesRequestBody, +} from "../src/wire/responses.js"; -const ids = (msgs: Array<{ id: string; contentType: string }>) => msgs.map((m) => `${m.contentType}:${m.id}`); +const ids = (msgs: Array<{ id: string; contentType: string }>) => + msgs.map((m) => `${m.contentType}:${m.id}`); const view: MirrorMessage[] = [ - { role: "user", blocks: [{ type: "text", text: "Q" }] }, - { - role: "assistant", - blocks: [ - { type: "thinking", thinking: "each." }, - { type: "text", text: "\n\n两个 PR：" }, - ], - }, - { - role: "assistant", - blocks: [ - { type: "thinking", thinking: "plan" }, - { type: "toolCall", id: "call_1", name: "compress", arguments: { ranges: ["m00001..m00002"] } }, - ], - }, - { role: "toolResult", toolCallId: "call_1", blocks: [{ type: "text", text: "done" }] }, - { role: "meta", text: "note" }, + { role: "user", blocks: [{ type: "text", text: "Q" }] }, + { + role: "assistant", + blocks: [ + { type: "thinking", thinking: "each." }, + { type: "text", text: "\n\n两个 PR：" }, + ], + }, + { + role: "assistant", + blocks: [ + { type: "thinking", thinking: "plan" }, + { + type: "toolCall", + id: "call_1", + name: "compress", + arguments: { ranges: ["m00001..m00002"] }, + }, + ], + }, + { + role: "toolResult", + toolCallId: "call_1", + blocks: [{ type: "text", text: "done" }], + }, + { role: "meta", text: "note" }, ]; // --- openai mirror --------------------------------------------------------- test("mirrorOpenaiMessages emits the completions wire layout", () => { - const messages = mirrorOpenaiMessages(view, "SYS"); - assert.deepEqual(messages[0], { role: "system", content: "SYS" }); - assert.deepEqual(messages[1], { role: "user", content: "Q" }); - assert.deepEqual(messages[2], { role: "assistant", content: "\n\n两个 PR：", reasoning_content: "each." }); - assert.deepEqual(messages[3], { - role: "assistant", - content: "", - reasoning_content: "plan", - tool_calls: [ - { - id: "call_1", - type: "function", - function: { name: "compress", arguments: '{"ranges":["m00001..m00002"]}' }, - }, - ], - }); - assert.deepEqual(messages[4], { role: "tool", tool_call_id: "call_1", content: "done" }); - assert.deepEqual(messages[5], { role: "developer", content: "note" }); - assert.equal(messages.length, 6); + const messages = mirrorOpenaiMessages(view, "SYS"); + assert.deepEqual(messages[0], { role: "system", content: "SYS" }); + assert.deepEqual(messages[1], { role: "user", content: "Q" }); + assert.deepEqual(messages[2], { + role: "assistant", + content: "\n\n两个 PR：", + reasoning_content: "each.", + }); + assert.deepEqual(messages[3], { + role: "assistant", + content: "", + reasoning_content: "plan", + tool_calls: [ + { + id: "call_1", + type: "function", + function: { + name: "compress", + arguments: '{"ranges":["m00001..m00002"]}', + }, + }, + ], + }); + assert.deepEqual(messages[4], { + role: "tool", + tool_call_id: "call_1", + content: "done", + }); + assert.deepEqual(messages[5], { role: "developer", content: "note" }); + assert.equal(messages.length, 6); }); test("mirrorOpenaiMessages drops empty turns and keeps whitespace-only text", () => { - const out = mirrorOpenaiMessages( - [ - { role: "user", blocks: [{ type: "text", text: "" }] }, // empty → dropped - { role: "assistant", blocks: [] }, // nothing at all → dropped - { role: "assistant", blocks: [{ type: "text", text: " " }] }, // whitespace-only → kept - { role: "meta", text: "" }, // empty meta → dropped - ], - "SYS", - ); - assert.deepEqual(out, [ - { role: "system", content: "SYS" }, - { role: "assistant", content: " " }, - ]); + const out = mirrorOpenaiMessages( + [ + { role: "user", blocks: [{ type: "text", text: "" }] }, // empty → dropped + { role: "assistant", blocks: [] }, // nothing at all → dropped + { role: "assistant", blocks: [{ type: "text", text: " " }] }, // whitespace-only → kept + { role: "meta", text: "" }, // empty meta → dropped + ], + "SYS", + ); + assert.deepEqual(out, [ + { role: "system", content: "SYS" }, + { role: "assistant", content: " " }, + ]); }); test("openai mirror ids match the live reasoning_content wire (issue #103)", () => { - const live: OpenAIRequestBody = { - model: "glm-x", - messages: [ - { role: "system", content: "SYS" }, - { role: "user", content: "Q" }, - { role: "assistant", content: "\n\n两个 PR：", reasoning_content: "each." }, - { - role: "assistant", - content: "", - reasoning_content: "plan", - tool_calls: [ - { - id: "call_1", - type: "function", - function: { name: "compress", arguments: '{"ranges":["m00001..m00002"]}' }, - }, - ], + const live: OpenAIRequestBody = { + model: "glm-x", + messages: [ + { role: "system", content: "SYS" }, + { role: "user", content: "Q" }, + { + role: "assistant", + content: "\n\n两个 PR：", + reasoning_content: "each.", + }, + { + role: "assistant", + content: "", + reasoning_content: "plan", + tool_calls: [ + { + id: "call_1", + type: "function", + function: { + name: "compress", + arguments: '{"ranges":["m00001..m00002"]}', }, - { role: "tool", tool_call_id: "call_1", content: "done" }, - { role: "developer", content: "note" }, + }, ], - }; - assert.deepEqual(ids(mirrorOpenaiToCore(view, "SYS")), ids(openaiToCore(live).msgs)); + }, + { role: "tool", tool_call_id: "call_1", content: "done" }, + { role: "developer", content: "note" }, + ], + }; + assert.deepEqual( + ids(mirrorOpenaiToCore(view, "SYS")), + ids(openaiToCore(live).msgs), + ); }); test("openai mirror ids match the live inline- wire (issue #64 demoted)", () => { - // A host that demotes thinking INLINE (glm/qwen/deepseek dialect via the - // pi host encoder) puts the same turn on the wire as one text blob; the - // kernel codec normalizes it into the same identity space as the mirror. - const live: OpenAIRequestBody = { - model: "glm-x", - messages: [ - { role: "system", content: "SYS" }, - { role: "user", content: "Q" }, - { role: "assistant", content: "\neach.\n\n\n\n两个 PR：" }, - { - role: "assistant", - content: "\nplan\n\n", - tool_calls: [ - { - id: "call_1", - type: "function", - function: { name: "compress", arguments: '{"ranges":["m00001..m00002"]}' }, - }, - ], + // A host that demotes thinking INLINE (glm/qwen/deepseek dialect via the + // pi host encoder) puts the same turn on the wire as one text blob; the + // kernel codec normalizes it into the same identity space as the mirror. + const live: OpenAIRequestBody = { + model: "glm-x", + messages: [ + { role: "system", content: "SYS" }, + { role: "user", content: "Q" }, + { role: "assistant", content: "\neach.\n\n\n\n两个 PR：" }, + { + role: "assistant", + content: "\nplan\n\n", + tool_calls: [ + { + id: "call_1", + type: "function", + function: { + name: "compress", + arguments: '{"ranges":["m00001..m00002"]}', }, - { role: "tool", tool_call_id: "call_1", content: "done" }, - { role: "developer", content: "note" }, + }, ], - }; - assert.deepEqual(ids(mirrorOpenaiToCore(view, "SYS")), ids(openaiToCore(live).msgs)); + }, + { role: "tool", tool_call_id: "call_1", content: "done" }, + { role: "developer", content: "note" }, + ], + }; + assert.deepEqual( + ids(mirrorOpenaiToCore(view, "SYS")), + ids(openaiToCore(live).msgs), + ); }); // --- anthropic mirror ------------------------------------------------------ test("mirrorAnthropicMessages emits the messages wire layout", () => { - const out = mirrorAnthropicMessages([ - { role: "user", blocks: [{ type: "text", text: "Q" }] }, - { - role: "assistant", - blocks: [ - { type: "thinking", thinking: "sig'd", signature: "sig-1" }, - { type: "thinking", thinking: "unsigned" }, - { type: "text", text: " " }, // whitespace-only → dropped - { type: "text", text: "\n\n答案" }, - { type: "toolCall", id: "tu_1", name: "compress", arguments: { a: 1 } }, - ], - }, - { role: "toolResult", toolCallId: "tu_1", blocks: [{ type: "text", text: "done" }] }, - { role: "meta", text: "note" }, - ]); - assert.deepEqual(out, [ - { role: "user", content: [{ type: "text", text: "Q" }] }, - { - role: "assistant", - content: [ - { type: "thinking", thinking: "sig'd", signature: "sig-1" }, - { type: "thinking", thinking: "unsigned" }, - { type: "text", text: "\n\n答案" }, - { type: "tool_use", id: "tu_1", name: "compress", input: { a: 1 } }, - ], - }, - { role: "user", content: [{ type: "tool_result", tool_use_id: "tu_1", content: "done" }] }, - { role: "user", content: [{ type: "text", text: "note" }] }, - ]); - // unsigned thinking carries no signature field at all - assert.equal("signature" in (out[1] as { content: Array> }).content[1], false); + const out = mirrorAnthropicMessages([ + { role: "user", blocks: [{ type: "text", text: "Q" }] }, + { + role: "assistant", + blocks: [ + { type: "thinking", thinking: "sig'd", signature: "sig-1" }, + { type: "thinking", thinking: "unsigned" }, + { type: "text", text: " " }, // whitespace-only → dropped + { type: "text", text: "\n\n答案" }, + { type: "toolCall", id: "tu_1", name: "compress", arguments: { a: 1 } }, + ], + }, + { + role: "toolResult", + toolCallId: "tu_1", + blocks: [{ type: "text", text: "done" }], + }, + { role: "meta", text: "note" }, + ]); + assert.deepEqual(out, [ + { role: "user", content: [{ type: "text", text: "Q" }] }, + { + role: "assistant", + content: [ + { type: "thinking", thinking: "sig'd", signature: "sig-1" }, + { type: "thinking", thinking: "unsigned" }, + { type: "text", text: "\n\n答案" }, + { type: "tool_use", id: "tu_1", name: "compress", input: { a: 1 } }, + ], + }, + { + role: "user", + content: [{ type: "tool_result", tool_use_id: "tu_1", content: "done" }], + }, + { role: "user", content: [{ type: "text", text: "note" }] }, + ]); + // unsigned thinking carries no signature field at all + assert.equal( + "signature" in + (out[1] as { content: Array> }).content[1], + false, + ); }); test("anthropic mirror folds into reasoning pieces (issue #103)", () => { - const msgs = mirrorAnthropicToCore([ - { role: "user", blocks: [{ type: "text", text: "Q" }] }, - { - role: "assistant", - blocks: [ - { type: "thinking", thinking: "think A", signature: "sig" }, - { type: "text", text: "answer" }, - ], - }, - ]); - assert.deepEqual( - msgs.map((m) => m.contentType), - ["text", "reasoning", "text"], - ); - const live = anthropicToCore({ - model: "claude-x", - messages: [ - { role: "user", content: [{ type: "text", text: "Q" }] }, - { - role: "assistant", - content: [ - { type: "thinking", thinking: "think A", signature: "sig" }, - { type: "text", text: "answer" }, - ], - }, + const msgs = mirrorAnthropicToCore([ + { role: "user", blocks: [{ type: "text", text: "Q" }] }, + { + role: "assistant", + blocks: [ + { type: "thinking", thinking: "think A", signature: "sig" }, + { type: "text", text: "answer" }, + ], + }, + ]); + assert.deepEqual( + msgs.map((m) => m.contentType), + ["text", "reasoning", "text"], + ); + const live = anthropicToCore({ + model: "claude-x", + messages: [ + { role: "user", content: [{ type: "text", text: "Q" }] }, + { + role: "assistant", + content: [ + { type: "thinking", thinking: "think A", signature: "sig" }, + { type: "text", text: "answer" }, ], - } as Parameters[0]); - assert.deepEqual(ids(msgs), ids(live.msgs)); + }, + ], + } as Parameters[0]); + assert.deepEqual(ids(msgs), ids(live.msgs)); }); // --- responses mirror ------------------------------------------------------ test("mirrorResponsesInput emits items in content order", () => { - const out = mirrorResponsesInput([ - { role: "user", blocks: [{ type: "text", text: "Q" }] }, + const out = mirrorResponsesInput([ + { role: "user", blocks: [{ type: "text", text: "Q" }] }, + { + role: "assistant", + blocks: [ + { type: "thinking", thinking: "plan" }, + { type: "text", text: " " }, // whitespace-only → dropped + { type: "text", text: "answer" }, { - role: "assistant", - blocks: [ - { type: "thinking", thinking: "plan" }, - { type: "text", text: " " }, // whitespace-only → dropped - { type: "text", text: "answer" }, - { type: "toolCall", id: "call_1", name: "compress", arguments: { a: 1 } }, - ], + type: "toolCall", + id: "call_1", + name: "compress", + arguments: { a: 1 }, }, - { role: "toolResult", toolCallId: "call_1", blocks: [{ type: "text", text: "done" }] }, - { role: "meta", text: "note" }, - ]); - assert.deepEqual(out, [ - { type: "message", role: "user", content: [{ type: "input_text", text: "Q" }] }, - { type: "reasoning", summary: [{ type: "summary_text", text: "plan" }] }, - { type: "message", role: "assistant", content: [{ type: "output_text", text: "answer" }] }, - { type: "function_call", call_id: "call_1", name: "compress", arguments: '{"a":1}' }, - { type: "function_call_output", call_id: "call_1", output: "done" }, - { type: "message", role: "user", content: [{ type: "input_text", text: "note" }] }, - ]); + ], + }, + { + role: "toolResult", + toolCallId: "call_1", + blocks: [{ type: "text", text: "done" }], + }, + { role: "meta", text: "note" }, + ]); + assert.deepEqual(out, [ + { + type: "message", + role: "user", + content: [{ type: "input_text", text: "Q" }], + }, + { type: "reasoning", summary: [{ type: "summary_text", text: "plan" }] }, + { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "answer" }], + }, + { + type: "function_call", + call_id: "call_1", + name: "compress", + arguments: '{"a":1}', + }, + { type: "function_call_output", call_id: "call_1", output: "done" }, + { + type: "message", + role: "user", + content: [{ type: "input_text", text: "note" }], + }, + ]); }); test("responses mirror ids match the live reasoning-item wire (issue #64 responses variant)", () => { - const live: ResponsesRequestBody = { - model: "qwen-x", - instructions: "SYS", - input: [ - { type: "message", role: "user", content: [{ type: "input_text", text: "Q" }] }, - { type: "reasoning", summary: [{ type: "summary_text", text: "plan" }] }, - { type: "message", role: "assistant", content: [{ type: "output_text", text: "answer" }] }, - ], - }; - const view2: MirrorMessage[] = [ - { role: "user", blocks: [{ type: "text", text: "Q" }] }, - { - role: "assistant", - blocks: [ - { type: "thinking", thinking: "plan" }, - { type: "text", text: "answer" }, - ], - }, - ]; - assert.deepEqual(ids(mirrorResponsesToCore(view2, "SYS")), ids(responsesToCore(live).msgs)); + const live: ResponsesRequestBody = { + model: "qwen-x", + instructions: "SYS", + input: [ + { + type: "message", + role: "user", + content: [{ type: "input_text", text: "Q" }], + }, + { type: "reasoning", summary: [{ type: "summary_text", text: "plan" }] }, + { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "answer" }], + }, + ], + }; + const view2: MirrorMessage[] = [ + { role: "user", blocks: [{ type: "text", text: "Q" }] }, + { + role: "assistant", + blocks: [ + { type: "thinking", thinking: "plan" }, + { type: "text", text: "answer" }, + ], + }, + ]; + assert.deepEqual( + ids(mirrorResponsesToCore(view2, "SYS")), + ids(responsesToCore(live).msgs), + ); }); test("responses mirror keeps thinking-only turns as reasoning items", () => { - const msgs = mirrorResponsesToCore( - [{ role: "assistant", blocks: [{ type: "thinking", thinking: "only think" }] }], - "SYS", - ); - assert.deepEqual( - msgs.map((m) => m.contentType), - ["reasoning"], - ); + const msgs = mirrorResponsesToCore( + [ + { + role: "assistant", + blocks: [{ type: "thinking", thinking: "only think" }], + }, + ], + "SYS", + ); + assert.deepEqual( + msgs.map((m) => m.contentType), + ["reasoning"], + ); }); diff --git a/tests/wire-proxy-message-id.test.ts b/tests/wire-proxy-message-id.test.ts index edeb341..22e1018 100644 --- a/tests/wire-proxy-message-id.test.ts +++ b/tests/wire-proxy-message-id.test.ts @@ -3,131 +3,153 @@ import assert from "node:assert/strict"; import { anthropicToCore } from "../src/wire/anthropic.js"; import { openaiToCore } from "../src/wire/openai.js"; import { responsesToCore } from "../src/wire/responses.js"; -import type { AnthropicMessage, AnthropicRequestBody } from "../src/wire/anthropic.js"; +import type { + AnthropicMessage, + AnthropicRequestBody, +} from "../src/wire/anthropic.js"; import type { OpenAIRequestBody } from "../src/wire/openai.js"; import type { ResponsesRequestBody } from "../src/wire/responses.js"; // --- Anthropic --- function anthropicBody(...messages: AnthropicMessage[]): AnthropicRequestBody { - return { model: "claude", max_tokens: 100, messages }; + return { model: "claude", max_tokens: 100, messages }; } test("fingerprint ids are stable: identical content across two conversions yields the same id", () => { - const body = anthropicBody( - { role: "user", content: "hello" }, - { role: "assistant", content: "hi" }, - { role: "user", content: "do something" }, + const body = anthropicBody( + { role: "user", content: "hello" }, + { role: "assistant", content: "hi" }, + { role: "user", content: "do something" }, + ); + const a = anthropicToCore(body).msgs; + const b = anthropicToCore(body).msgs; + assert.equal(a.length, b.length); + for (let i = 0; i < a.length; i++) { + assert.equal( + a[i].id, + b[i].id, + `id stable across conversions at index ${i}`, ); - const a = anthropicToCore(body).msgs; - const b = anthropicToCore(body).msgs; - assert.equal(a.length, b.length); - for (let i = 0; i < a.length; i++) { - assert.equal(a[i].id, b[i].id, `id stable across conversions at index ${i}`); - assert.ok(a[i].id.startsWith("h_"), "fingerprint prefix"); - } + assert.ok(a[i].id.startsWith("h_"), "fingerprint prefix"); + } }); test("fingerprint ids survive client reordering (reordering keeps ids bound to content, not position)", () => { - // A realistic case: another plugin summarized away a middle message, and - // the client reorders the rest. With raw-${idx} the ids would shift and - // downstream compression effectiveMessageIds would point at the wrong - // messages. With content fingerprints each message keeps its own id. - const bodyA = anthropicBody( - { role: "user", content: "alpha" }, - { role: "assistant", content: "beta" }, - { role: "user", content: "gamma" }, - { role: "assistant", content: "delta" }, - { role: "user", content: "epsilon" }, + // A realistic case: another plugin summarized away a middle message, and + // the client reorders the rest. With raw-${idx} the ids would shift and + // downstream compression effectiveMessageIds would point at the wrong + // messages. With content fingerprints each message keeps its own id. + const bodyA = anthropicBody( + { role: "user", content: "alpha" }, + { role: "assistant", content: "beta" }, + { role: "user", content: "gamma" }, + { role: "assistant", content: "delta" }, + { role: "user", content: "epsilon" }, + ); + // Drop "beta" and "delta", reorder epsilon before gamma. + const bodyB = anthropicBody( + { role: "user", content: "alpha" }, + { role: "user", content: "epsilon" }, + { role: "user", content: "gamma" }, + ); + const idsA = new Map(anthropicToCore(bodyA).msgs.map((m) => [m.text, m.id])); + const msgsB = anthropicToCore(bodyB).msgs; + for (const m of msgsB) { + assert.equal( + m.id, + idsA.get(m.text), + `id for "${m.text}" follows the content, not the position`, ); - // Drop "beta" and "delta", reorder epsilon before gamma. - const bodyB = anthropicBody( - { role: "user", content: "alpha" }, - { role: "user", content: "epsilon" }, - { role: "user", content: "gamma" }, - ); - const idsA = new Map(anthropicToCore(bodyA).msgs.map((m) => [m.text, m.id])); - const msgsB = anthropicToCore(bodyB).msgs; - for (const m of msgsB) { - assert.equal(m.id, idsA.get(m.text), `id for "${m.text}" follows the content, not the position`); - } + } }); test("duplicate messages get distinct cluster ids (not collapsed onto one id)", () => { - // Two identical "ok" turns must not share an id — otherwise a compression - // `covered` set keyed on that id would swallow BOTH (losing recent content). - const body = anthropicBody( - { role: "user", content: "ok" }, - { role: "user", content: "do work" }, - { role: "user", content: "ok" }, - ); - const msgs = anthropicToCore(body).msgs; - const ids = msgs.filter((m) => m.text === "ok").map((m) => m.id); - assert.equal(ids.length, 2, "both ok messages present"); - assert.notEqual(ids[0], ids[1], "duplicates get distinct cluster suffixes"); - // The first is the bare hash, the second is the hash with _1. - assert.ok(!/_\d+$/.test(ids[0]), "first occurrence has no cluster suffix"); - assert.ok(/_1$/.test(ids[1]), "second occurrence gets _1 suffix"); + // Two identical "ok" turns must not share an id — otherwise a compression + // `covered` set keyed on that id would swallow BOTH (losing recent content). + const body = anthropicBody( + { role: "user", content: "ok" }, + { role: "user", content: "do work" }, + { role: "user", content: "ok" }, + ); + const msgs = anthropicToCore(body).msgs; + const ids = msgs.filter((m) => m.text === "ok").map((m) => m.id); + assert.equal(ids.length, 2, "both ok messages present"); + assert.notEqual(ids[0], ids[1], "duplicates get distinct cluster suffixes"); + // The first is the bare hash, the second is the hash with _1. + assert.ok(!/_\d+$/.test(ids[0]), "first occurrence has no cluster suffix"); + assert.ok(/_1$/.test(ids[1]), "second occurrence gets _1 suffix"); }); // --- OpenAI Chat --- test("openaiToCore: fingerprint ids distinct from mNNNNN and stable", () => { - const body: OpenAIRequestBody = { - model: "gpt", - messages: [ - { role: "user", content: "ping" }, - { role: "assistant", content: "pong" }, - ], - }; - const ids = openaiToCore(body).msgs.map((m) => m.id); - for (const id of ids) { - assert.ok(id.startsWith("h_"), `id ${id} has h_ prefix`); - assert.ok(!/^m\d+$/.test(id), "does not collide with kernel mNNNNN"); - } - // Stable across conversions. - const ids2 = openaiToCore(body).msgs.map((m) => m.id); - assert.deepEqual(ids, ids2, "stable across conversions"); + const body: OpenAIRequestBody = { + model: "gpt", + messages: [ + { role: "user", content: "ping" }, + { role: "assistant", content: "pong" }, + ], + }; + const ids = openaiToCore(body).msgs.map((m) => m.id); + for (const id of ids) { + assert.ok(id.startsWith("h_"), `id ${id} has h_ prefix`); + assert.ok(!/^m\d+$/.test(id), "does not collide with kernel mNNNNN"); + } + // Stable across conversions. + const ids2 = openaiToCore(body).msgs.map((m) => m.id); + assert.deepEqual(ids, ids2, "stable across conversions"); }); // --- Responses --- test("responsesToCore: fingerprint ids stable + duplicates distinct", () => { - const body: ResponsesRequestBody = { - model: "gpt", - input: [ - { type: "message", role: "user", content: "hello" }, - { type: "message", role: "user", content: "hello" }, - { type: "message", role: "assistant", content: "world" }, - ], - }; - const msgs = responsesToCore(body).msgs; - assert.equal(msgs.length, 3); - const helloIds = msgs.filter((m) => m.text === "hello").map((m) => m.id); - assert.notEqual(helloIds[0], helloIds[1], "duplicate hellos distinct"); - assert.ok(helloIds[0].startsWith("h_")); - assert.ok(helloIds[1].endsWith("_1")); - // Stable. - const ids2 = responsesToCore(body).msgs.map((m) => m.id); - assert.deepEqual(msgs.map((m) => m.id), ids2); + const body: ResponsesRequestBody = { + model: "gpt", + input: [ + { type: "message", role: "user", content: "hello" }, + { type: "message", role: "user", content: "hello" }, + { type: "message", role: "assistant", content: "world" }, + ], + }; + const msgs = responsesToCore(body).msgs; + assert.equal(msgs.length, 3); + const helloIds = msgs.filter((m) => m.text === "hello").map((m) => m.id); + assert.notEqual(helloIds[0], helloIds[1], "duplicate hellos distinct"); + assert.ok(helloIds[0].startsWith("h_")); + assert.ok(helloIds[1].endsWith("_1")); + // Stable. + const ids2 = responsesToCore(body).msgs.map((m) => m.id); + assert.deepEqual( + msgs.map((m) => m.id), + ids2, + ); }); test("responsesToCore: tool call + result ids bind to call_id, not position", () => { - // Same function name/args in two distinct calls must differ via call_id. - const body: ResponsesRequestBody = { - model: "gpt", - input: [ - { type: "message", role: "user", content: "run it twice" }, - { type: "function_call", name: "exec", call_id: "c1", arguments: "{}" }, - { type: "function_call_output", call_id: "c1", output: "done1" }, - { type: "function_call", name: "exec", call_id: "c2", arguments: "{}" }, - { type: "function_call_output", call_id: "c2", output: "done2" }, - ], - }; - const msgs = responsesToCore(body).msgs; - const callIds = msgs.filter((m) => m.contentType === "tool-call").map((m) => m.id); - assert.equal(callIds.length, 2); - assert.notEqual(callIds[0], callIds[1], "two same-name same-args calls differ via call_id"); - const resultIds = msgs.filter((m) => m.contentType === "tool-result").map((m) => m.id); - assert.notEqual(resultIds[0], resultIds[1], "two results differ via call_id"); + // Same function name/args in two distinct calls must differ via call_id. + const body: ResponsesRequestBody = { + model: "gpt", + input: [ + { type: "message", role: "user", content: "run it twice" }, + { type: "function_call", name: "exec", call_id: "c1", arguments: "{}" }, + { type: "function_call_output", call_id: "c1", output: "done1" }, + { type: "function_call", name: "exec", call_id: "c2", arguments: "{}" }, + { type: "function_call_output", call_id: "c2", output: "done2" }, + ], + }; + const msgs = responsesToCore(body).msgs; + const callIds = msgs + .filter((m) => m.contentType === "tool-call") + .map((m) => m.id); + assert.equal(callIds.length, 2); + assert.notEqual( + callIds[0], + callIds[1], + "two same-name same-args calls differ via call_id", + ); + const resultIds = msgs + .filter((m) => m.contentType === "tool-result") + .map((m) => m.id); + assert.notEqual(resultIds[0], resultIds[1], "two results differ via call_id"); }); diff --git a/tests/wire-reasoning-echo.test.ts b/tests/wire-reasoning-echo.test.ts index 82a0c0d..57d9ed1 100644 --- a/tests/wire-reasoning-echo.test.ts +++ b/tests/wire-reasoning-echo.test.ts @@ -1,7 +1,11 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { coreToOpenai, openaiToCore, type OpenAIRequestBody } from "../src/wire/openai.js"; +import { + coreToOpenai, + openaiToCore, + type OpenAIRequestBody, +} from "../src/wire/openai.js"; import { createCore } from "../src/compress.js"; import { createInitialState } from "../src/state.js"; import { defaultConfig } from "../src/config.js"; @@ -14,154 +18,213 @@ import { prune } from "../src/prune.js"; // that lost the key while accepting a blank one, so the core round-trip must // not turn "blank" into "absent". -const BASH_CALL = { id: "call_bash", type: "function", function: { name: "bash", arguments: '{"cmd":"ls"}' } } as const; -const GREP_CALL = { id: "call_grep", type: "function", function: { name: "grep", arguments: '{"q":"x"}' } } as const; +const BASH_CALL = { + id: "call_bash", + type: "function", + function: { name: "bash", arguments: '{"cmd":"ls"}' }, +} as const; +const GREP_CALL = { + id: "call_grep", + type: "function", + function: { name: "grep", arguments: '{"q":"x"}' }, +} as const; -const roundTrip = (messages: OpenAIRequestBody["messages"]) => coreToOpenai(openaiToCore({ model: "deepseek-v4-flash", messages }).msgs); +const roundTrip = (messages: OpenAIRequestBody["messages"]) => + coreToOpenai(openaiToCore({ model: "deepseek-v4-flash", messages }).msgs); test("a blank reasoning_content field survives the round-trip", () => { - const wire = roundTrip([ - { role: "assistant", content: ".", reasoning_content: "", tool_calls: [BASH_CALL] }, - { role: "tool", tool_call_id: BASH_CALL.id, content: "ok" }, - ]); - const assistant = wire[0]!; - assert.equal(assistant.role, "assistant"); - assert.ok("reasoning_content" in assistant, "the key must be re-emitted"); - assert.strictEqual(assistant.reasoning_content, ""); - assert.deepStrictEqual( - assistant.tool_calls?.map((call) => call.id), - [BASH_CALL.id], - ); + const wire = roundTrip([ + { + role: "assistant", + content: ".", + reasoning_content: "", + tool_calls: [BASH_CALL], + }, + { role: "tool", tool_call_id: BASH_CALL.id, content: "ok" }, + ]); + const assistant = wire[0]!; + assert.equal(assistant.role, "assistant"); + assert.ok("reasoning_content" in assistant, "the key must be re-emitted"); + assert.strictEqual(assistant.reasoning_content, ""); + assert.deepStrictEqual( + assistant.tool_calls?.map((call) => call.id), + [BASH_CALL.id], + ); }); test("an absent reasoning_content field is not invented", () => { - const wire = roundTrip([{ role: "assistant", content: ".", tool_calls: [BASH_CALL] }, { role: "tool", tool_call_id: BASH_CALL.id, content: "ok" }]); - assert.ok(!("reasoning_content" in wire[0]!), "no field, no echo"); + const wire = roundTrip([ + { role: "assistant", content: ".", tool_calls: [BASH_CALL] }, + { role: "tool", tool_call_id: BASH_CALL.id, content: "ok" }, + ]); + assert.ok(!("reasoning_content" in wire[0]!), "no field, no echo"); }); test("an inline thinking block still wins over a blank field", () => { - const wire = roundTrip([{ role: "assistant", content: "\nreasoned about it\n\n\n\nanswer", reasoning_content: "" }]); - assert.strictEqual(wire[0]!.reasoning_content, "reasoned about it"); - assert.strictEqual(wire[0]!.content, "\n\nanswer"); + const wire = roundTrip([ + { + role: "assistant", + content: "\nreasoned about it\n\n\n\nanswer", + reasoning_content: "", + }, + ]); + assert.strictEqual(wire[0]!.reasoning_content, "reasoned about it"); + assert.strictEqual(wire[0]!.content, "\n\nanswer"); }); test("a text-only assistant turn keeps its blank field", () => { - const wire = roundTrip([ - { role: "assistant", content: "hello", reasoning_content: "" }, - { role: "user", content: "again" }, - ]); - assert.strictEqual(wire[0]!.reasoning_content, ""); - assert.strictEqual(wire[0]!.content, "hello"); + const wire = roundTrip([ + { role: "assistant", content: "hello", reasoning_content: "" }, + { role: "user", content: "again" }, + ]); + assert.strictEqual(wire[0]!.reasoning_content, ""); + assert.strictEqual(wire[0]!.content, "hello"); }); test("a blank field does not leak onto a turn that never carried it", () => { - const wire = roundTrip([ - { role: "assistant", content: ".", reasoning_content: "", tool_calls: [BASH_CALL] }, - { role: "tool", tool_call_id: BASH_CALL.id, content: "ok" }, - { role: "assistant", content: "done" }, - ]); - assert.strictEqual(wire[0]!.reasoning_content, ""); - assert.ok(!("reasoning_content" in wire[2]!), "the second turn keeps the key absent"); + const wire = roundTrip([ + { + role: "assistant", + content: ".", + reasoning_content: "", + tool_calls: [BASH_CALL], + }, + { role: "tool", tool_call_id: BASH_CALL.id, content: "ok" }, + { role: "assistant", content: "done" }, + ]); + assert.strictEqual(wire[0]!.reasoning_content, ""); + assert.ok( + !("reasoning_content" in wire[2]!), + "the second turn keeps the key absent", + ); }); test("the presence marker does not enter core identity", () => { - const blank = openaiToCore({ - model: "deepseek-v4-flash", - messages: [ - { role: "assistant", content: ".", reasoning_content: "", tool_calls: [BASH_CALL] }, - { role: "tool", tool_call_id: BASH_CALL.id, content: "ok" }, - ], - }).msgs; - const absent = openaiToCore({ - model: "deepseek-v4-flash", - messages: [ - { role: "assistant", content: ".", tool_calls: [BASH_CALL] }, - { role: "tool", tool_call_id: BASH_CALL.id, content: "ok" }, - ], - }).msgs; - assert.deepStrictEqual( - blank.map((m) => m.id), - absent.map((m) => m.id), - ); + const blank = openaiToCore({ + model: "deepseek-v4-flash", + messages: [ + { + role: "assistant", + content: ".", + reasoning_content: "", + tool_calls: [BASH_CALL], + }, + { role: "tool", tool_call_id: BASH_CALL.id, content: "ok" }, + ], + }).msgs; + const absent = openaiToCore({ + model: "deepseek-v4-flash", + messages: [ + { role: "assistant", content: ".", tool_calls: [BASH_CALL] }, + { role: "tool", tool_call_id: BASH_CALL.id, content: "ok" }, + ], + }).msgs; + assert.deepStrictEqual( + blank.map((m) => m.id), + absent.map((m) => m.id), + ); }); test("parallel calls keep their call order and their results keep theirs", () => { - const wire = roundTrip([ - { role: "assistant", content: ".", reasoning_content: "", tool_calls: [BASH_CALL, GREP_CALL] }, - { role: "tool", tool_call_id: GREP_CALL.id, content: "grep out" }, - { role: "tool", tool_call_id: BASH_CALL.id, content: "bash out" }, - { role: "user", content: "next" }, - ]); - assert.strictEqual(wire[0]!.reasoning_content, ""); - assert.deepStrictEqual( - wire[0]!.tool_calls?.map((call) => call.id), - [BASH_CALL.id, GREP_CALL.id], - ); - assert.deepStrictEqual( - wire.slice(1, 3).map((m) => m.tool_call_id), - [GREP_CALL.id, BASH_CALL.id], - ); + const wire = roundTrip([ + { + role: "assistant", + content: ".", + reasoning_content: "", + tool_calls: [BASH_CALL, GREP_CALL], + }, + { role: "tool", tool_call_id: GREP_CALL.id, content: "grep out" }, + { role: "tool", tool_call_id: BASH_CALL.id, content: "bash out" }, + { role: "user", content: "next" }, + ]); + assert.strictEqual(wire[0]!.reasoning_content, ""); + assert.deepStrictEqual( + wire[0]!.tool_calls?.map((call) => call.id), + [BASH_CALL.id, GREP_CALL.id], + ); + assert.deepStrictEqual( + wire.slice(1, 3).map((m) => m.tool_call_id), + [GREP_CALL.id, BASH_CALL.id], + ); }); test("a blank echo survives the full pipeline and a history fold", () => { - // #289 post-fold shape: summary ahead, open blank-echo turn followed only by its result. - const core = createCore(); - const state = createInitialState(); - const base = defaultConfig(200000); - const config = { - ...base, - compress: { ...base.compress, minCompressRange: 0, minSummaryLength: 0 }, - }; - const body: OpenAIRequestBody = { - model: "deepseek-v4-flash", - messages: [ - { role: "user", content: "step one" }, - { role: "assistant", content: ".", reasoning_content: "", tool_calls: [BASH_CALL] }, - { role: "tool", tool_call_id: BASH_CALL.id, content: "done one" }, - { role: "user", content: "step two" }, - { role: "assistant", content: "ok two" }, - { role: "user", content: "step three" }, - { role: "assistant", content: "", reasoning_content: "", tool_calls: [GREP_CALL] }, - { role: "tool", tool_call_id: GREP_CALL.id, content: "done three" }, - ], - }; - - const turn = core.processTurn({ - messages: openaiToCore(body).msgs, - state, - config, - tokenCount: 4000, - renderTags: "none", - }); - - const tailCall = turn.messages.find( - (m) => m.contentType === "tool-call" && m.toolCallId === GREP_CALL.id, - ); - assert.ok(tailCall?.reasoningPresent, "marker must survive the pipeline"); - - const anchor = turn.messages.find((m) => m.role === "user" && m.text === "step three"); - assert.ok(anchor, "anchor user message present"); - const anchorRef = Number(turn.state.messageRefs.byRaw[anchor!.id]!.slice(1)); - const applied = core.applyCompression({ - ranges: [ - { - startRef: "m00001", - endRef: `m${String(anchorRef - 1).padStart(5, "0")}`, - summary: "steps one and two recap", - }, - ], - messages: turn.messages, - state: turn.state, - config, - }); - assert.equal(applied.result.errors.length, 0); - - const rebuilt = coreToOpenai(prune(turn.messages, applied.state)); - assert.ok(rebuilt.some((m) => m.role === "system"), "fold injected a summary message"); - const tailWire = rebuilt.find( - (m) => m.role === "assistant" && Array.isArray(m.tool_calls), - ); - assert.ok(tailWire, "tail assistant turn still present"); - assert.ok("reasoning_content" in tailWire!, "blank key must survive the fold rebuild"); - assert.strictEqual(tailWire!.reasoning_content, ""); + // #289 post-fold shape: summary ahead, open blank-echo turn followed only by its result. + const core = createCore(); + const state = createInitialState(); + const base = defaultConfig(200000); + const config = { + ...base, + compress: { ...base.compress, minCompressRange: 0, minSummaryLength: 0 }, + }; + const body: OpenAIRequestBody = { + model: "deepseek-v4-flash", + messages: [ + { role: "user", content: "step one" }, + { + role: "assistant", + content: ".", + reasoning_content: "", + tool_calls: [BASH_CALL], + }, + { role: "tool", tool_call_id: BASH_CALL.id, content: "done one" }, + { role: "user", content: "step two" }, + { role: "assistant", content: "ok two" }, + { role: "user", content: "step three" }, + { + role: "assistant", + content: "", + reasoning_content: "", + tool_calls: [GREP_CALL], + }, + { role: "tool", tool_call_id: GREP_CALL.id, content: "done three" }, + ], + }; + + const turn = core.processTurn({ + messages: openaiToCore(body).msgs, + state, + config, + tokenCount: 4000, + renderTags: "none", + }); + + const tailCall = turn.messages.find( + (m) => m.contentType === "tool-call" && m.toolCallId === GREP_CALL.id, + ); + assert.ok(tailCall?.reasoningPresent, "marker must survive the pipeline"); + + const anchor = turn.messages.find( + (m) => m.role === "user" && m.text === "step three", + ); + assert.ok(anchor, "anchor user message present"); + const anchorRef = Number(turn.state.messageRefs.byRaw[anchor!.id]!.slice(1)); + const applied = core.applyCompression({ + ranges: [ + { + startRef: "m00001", + endRef: `m${String(anchorRef - 1).padStart(5, "0")}`, + summary: "steps one and two recap", + }, + ], + messages: turn.messages, + state: turn.state, + config, + }); + assert.equal(applied.result.errors.length, 0); + + const rebuilt = coreToOpenai(prune(turn.messages, applied.state)); + assert.ok( + rebuilt.some((m) => m.role === "system"), + "fold injected a summary message", + ); + const tailWire = rebuilt.find( + (m) => m.role === "assistant" && Array.isArray(m.tool_calls), + ); + assert.ok(tailWire, "tail assistant turn still present"); + assert.ok( + "reasoning_content" in tailWire!, + "blank key must survive the fold rebuild", + ); + assert.strictEqual(tailWire!.reasoning_content, ""); }); diff --git a/tests/wire-schema-top-level.test.ts b/tests/wire-schema-top-level.test.ts index 7ebaabe..6bc1187 100644 --- a/tests/wire-schema-top-level.test.ts +++ b/tests/wire-schema-top-level.test.ts @@ -33,16 +33,31 @@ const COMBINATORS = ["oneOf", "allOf", "anyOf", "not"] as const; type ToolLike = { name?: string }; -function schemaOf(tool: ToolLike & Record, shape: string): Record { - if (shape === "anthropic") return tool.input_schema as Record; +function schemaOf( + tool: ToolLike & Record, + shape: string, +): Record { + if (shape === "anthropic") + return tool.input_schema as Record; if (shape === "google") return tool.parameters as Record; - const fn = tool.function as { parameters?: Record } | undefined; - return (tool.parameters as Record | undefined) ?? fn?.parameters ?? {}; + const fn = tool.function as + { parameters?: Record } | undefined; + return ( + (tool.parameters as Record | undefined) ?? + fn?.parameters ?? + {} + ); } -function assertPlainTopLevel(tool: ToolLike & Record, shape: string): void { +function assertPlainTopLevel( + tool: ToolLike & Record, + shape: string, +): void { const schema = schemaOf(tool, shape); - assert.ok(schema && typeof schema === "object", `${tool.name}: ${shape} schema missing`); + assert.ok( + schema && typeof schema === "object", + `${tool.name}: ${shape} schema missing`, + ); for (const key of COMBINATORS) { assert.ok( schema[key] === undefined, @@ -51,7 +66,10 @@ function assertPlainTopLevel(tool: ToolLike & Record, shape: st } } -function walk(list: readonly (ToolLike & Record)[], shape: string): void { +function walk( + list: readonly (ToolLike & Record)[], + shape: string, +): void { for (const tool of list) assertPlainTopLevel(tool, shape); } @@ -65,15 +83,27 @@ test("anthropic input_schema: no top-level combinators on any tool", () => { test("openai function.parameters: no top-level combinators on any tool", () => { walk(ACP_TOOLS_OPENAI, "openai"); assertPlainTopLevel(ABSORB_TOOL_OPENAI as Record, "openai"); - assertPlainTopLevel(IMAGE_FULL_TOOL_OPENAI as Record, "openai"); - assertPlainTopLevel(RETRIEVE_TOOL_OPENAI as Record, "openai"); + assertPlainTopLevel( + IMAGE_FULL_TOOL_OPENAI as Record, + "openai", + ); + assertPlainTopLevel( + RETRIEVE_TOOL_OPENAI as Record, + "openai", + ); }); test("responses flat parameters: no top-level combinators on any tool", () => { walk(ACP_TOOLS_RESPONSES, "responses"); walk(ACP_READONLY_TOOLS_RESPONSES, "responses"); - assertPlainTopLevel(IMAGE_FULL_TOOL_RESPONSES as Record, "responses"); - assertPlainTopLevel(RETRIEVE_TOOL_RESPONSES as Record, "responses"); + assertPlainTopLevel( + IMAGE_FULL_TOOL_RESPONSES as Record, + "responses", + ); + assertPlainTopLevel( + RETRIEVE_TOOL_RESPONSES as Record, + "responses", + ); }); test("google functionDeclarations parameters: no top-level combinators on any tool", () => { @@ -83,12 +113,27 @@ test("google functionDeclarations parameters: no top-level combinators on any to test("content alternatives use scalar types and self-contained object branches", () => { const params = ACP_TOOLS_OPENAI[0].function.parameters as { - properties: { content: { anyOf: { type: string; items?: { anyOf: { type: string; properties?: object; required?: string[] }[] } }[] } }; + properties: { + content: { + anyOf: { + type: string; + items?: { + anyOf: { type: string; properties?: object; required?: string[] }[]; + }; + }[]; + }; + }; }; const alternatives = params.properties.content.anyOf; - assert.deepEqual(alternatives.map((entry) => entry.type), ["array", "string"]); + assert.deepEqual( + alternatives.map((entry) => entry.type), + ["array", "string"], + ); const entries = alternatives[0].items!.anyOf; - assert.deepEqual(entries.map((entry) => entry.type), ["string", "object", "object"]); + assert.deepEqual( + entries.map((entry) => entry.type), + ["string", "object", "object"], + ); for (const entry of entries.slice(1)) { assert.ok(entry.properties); for (const key of entry.required!) assert.ok(key in entry.properties!); diff --git a/tests/wire-subagent-namespace.test.ts b/tests/wire-subagent-namespace.test.ts index 6e0937b..e46ba02 100644 --- a/tests/wire-subagent-namespace.test.ts +++ b/tests/wire-subagent-namespace.test.ts @@ -3,49 +3,79 @@ import assert from "node:assert/strict"; import { subagentNamespace } from "../src/wire/responses.js"; test("subagentNamespace: first-seen instructions anchor the identity; identical instructions reuse it", () => { - const id = "ns-unit-a"; - assert.equal(subagentNamespace(id, "You are Codex, the main agent."), id); - assert.equal(subagentNamespace(id, "You are Codex, the main agent."), id); + const id = "ns-unit-a"; + assert.equal(subagentNamespace(id, "You are Codex, the main agent."), id); + assert.equal(subagentNamespace(id, "You are Codex, the main agent."), id); }); test("subagentNamespace: differing instructions map to a stable separate |sub: namespace", () => { - const id = "ns-unit-b"; - assert.equal(subagentNamespace(id, "main prompt"), id); - const sub = subagentNamespace(id, "guardian prompt"); - assert.match(sub, /^ns-unit-b\|sub:[0-9a-f]{16}$/); - assert.equal(subagentNamespace(id, "guardian prompt"), sub, "same subagent instructions reuse the sub namespace"); - assert.notEqual(subagentNamespace(id, "reviewer prompt"), sub, "a different subagent gets its own namespace"); - assert.equal(subagentNamespace(id, "main prompt"), id, "main conversation keeps the anchored namespace"); + const id = "ns-unit-b"; + assert.equal(subagentNamespace(id, "main prompt"), id); + const sub = subagentNamespace(id, "guardian prompt"); + assert.match(sub, /^ns-unit-b\|sub:[0-9a-f]{16}$/); + assert.equal( + subagentNamespace(id, "guardian prompt"), + sub, + "same subagent instructions reuse the sub namespace", + ); + assert.notEqual( + subagentNamespace(id, "reviewer prompt"), + sub, + "a different subagent gets its own namespace", + ); + assert.equal( + subagentNamespace(id, "main prompt"), + id, + "main conversation keeps the anchored namespace", + ); }); test("subagentNamespace: absent or empty instructions never anchor or split", () => { - const id = "ns-unit-c"; - assert.equal(subagentNamespace(id, undefined), id); - assert.equal(subagentNamespace(id, " "), id); - assert.equal(subagentNamespace(id, "real prompt"), id); + const id = "ns-unit-c"; + assert.equal(subagentNamespace(id, undefined), id); + assert.equal(subagentNamespace(id, " "), id); + assert.equal(subagentNamespace(id, "real prompt"), id); }); import { createSubagentNamespaces } from "../src/wire/responses.js"; test("createSubagentNamespaces: instances own their anchor state independently", () => { - const a = createSubagentNamespaces(); - const b = createSubagentNamespaces(); - assert.equal(a.namespaceFor("ns-iso", "main prompt"), "ns-iso"); - assert.equal(b.namespaceFor("ns-iso", "other prompt"), "ns-iso", "b anchors independently — no shared module state"); - const subA = a.namespaceFor("ns-iso", "guardian prompt"); - assert.match(subA, /^ns-iso\|sub:[0-9a-f]{16}$/); - assert.equal(a.namespaceFor("ns-iso", "main prompt"), "ns-iso", "a keeps its own anchor"); - assert.equal(a.namespaceFor("ns-iso", "guardian prompt"), subA, "stable within the instance"); + const a = createSubagentNamespaces(); + const b = createSubagentNamespaces(); + assert.equal(a.namespaceFor("ns-iso", "main prompt"), "ns-iso"); + assert.equal( + b.namespaceFor("ns-iso", "other prompt"), + "ns-iso", + "b anchors independently — no shared module state", + ); + const subA = a.namespaceFor("ns-iso", "guardian prompt"); + assert.match(subA, /^ns-iso\|sub:[0-9a-f]{16}$/); + assert.equal( + a.namespaceFor("ns-iso", "main prompt"), + "ns-iso", + "a keeps its own anchor", + ); + assert.equal( + a.namespaceFor("ns-iso", "guardian prompt"), + subA, + "stable within the instance", + ); }); test("createSubagentNamespaces: fresh instance re-anchors (restart semantics documented)", () => { - const first = createSubagentNamespaces(); - assert.equal(first.namespaceFor("ns-restart", "main prompt"), "ns-restart"); - // Simulated restart: new instance, subagent request arrives FIRST — it - // becomes the anchor, so the main prompt lands in a |sub: namespace. - // This is the documented first-seen tradeoff; hosts needing - // restart-stable namespaces must persist their own instance state. - const second = createSubagentNamespaces(); - assert.equal(second.namespaceFor("ns-restart", "guardian prompt"), "ns-restart"); - assert.match(second.namespaceFor("ns-restart", "main prompt"), /^ns-restart\|sub:[0-9a-f]{16}$/); + const first = createSubagentNamespaces(); + assert.equal(first.namespaceFor("ns-restart", "main prompt"), "ns-restart"); + // Simulated restart: new instance, subagent request arrives FIRST — it + // becomes the anchor, so the main prompt lands in a |sub: namespace. + // This is the documented first-seen tradeoff; hosts needing + // restart-stable namespaces must persist their own instance state. + const second = createSubagentNamespaces(); + assert.equal( + second.namespaceFor("ns-restart", "guardian prompt"), + "ns-restart", + ); + assert.match( + second.namespaceFor("ns-restart", "main prompt"), + /^ns-restart\|sub:[0-9a-f]{16}$/, + ); }); diff --git a/tests/wire-tool-result-toolname.test.ts b/tests/wire-tool-result-toolname.test.ts index 7c9ac95..eafbf28 100644 --- a/tests/wire-tool-result-toolname.test.ts +++ b/tests/wire-tool-result-toolname.test.ts @@ -1,8 +1,14 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { anthropicToCore, type AnthropicRequestBody } from "../src/wire/anthropic.js"; +import { + anthropicToCore, + type AnthropicRequestBody, +} from "../src/wire/anthropic.js"; import { openaiToCore, type OpenAIRequestBody } from "../src/wire/openai.js"; -import { responsesToCore, type ResponsesRequestBody } from "../src/wire/responses.js"; +import { + responsesToCore, + type ResponsesRequestBody, +} from "../src/wire/responses.js"; import { deriveMessageId } from "../src/wire/message-id.js"; import { applyAbsorb, isAbsorbCandidate } from "../src/absorb.js"; import { isMessageProtected, isNeverPreserveRecent } from "../src/protected.js"; @@ -43,45 +49,95 @@ function anthropicBody(): AnthropicRequestBody { model: "claude-x", messages: [ { role: "user", content: "run the build" }, - { role: "assistant", content: [{ type: "tool_use", id: "tu_1", name: "bash", input: {} }] }, - { role: "user", content: [{ type: "tool_result", tool_use_id: "tu_1", content: ANTHROPIC_TEXT }] }, - { role: "assistant", content: [{ type: "tool_use", id: "tu_2", name: "compress", input: {} }] }, - { role: "user", content: [{ type: "tool_result", tool_use_id: "tu_2", content: "ok" }] }, - { role: "user", content: [{ type: "tool_result", tool_use_id: "tu_orphan", content: "orphaned" }] }, + { + role: "assistant", + content: [{ type: "tool_use", id: "tu_1", name: "bash", input: {} }], + }, + { + role: "user", + content: [ + { type: "tool_result", tool_use_id: "tu_1", content: ANTHROPIC_TEXT }, + ], + }, + { + role: "assistant", + content: [ + { type: "tool_use", id: "tu_2", name: "compress", input: {} }, + ], + }, + { + role: "user", + content: [{ type: "tool_result", tool_use_id: "tu_2", content: "ok" }], + }, + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "tu_orphan", + content: "orphaned", + }, + ], + }, ], }; } test("anthropicToCore backfills toolName onto paired tool_results", () => { const { msgs } = anthropicToCore(anthropicBody()); - const bashResult = msgs.find((m) => m.contentType === "tool-result" && m.toolCallId === "tu_1"); - const compressResult = msgs.find((m) => m.contentType === "tool-result" && m.toolCallId === "tu_2"); - const orphan = msgs.find((m) => m.contentType === "tool-result" && m.toolCallId === "tu_orphan"); + const bashResult = msgs.find( + (m) => m.contentType === "tool-result" && m.toolCallId === "tu_1", + ); + const compressResult = msgs.find( + (m) => m.contentType === "tool-result" && m.toolCallId === "tu_2", + ); + const orphan = msgs.find( + (m) => m.contentType === "tool-result" && m.toolCallId === "tu_orphan", + ); assert.equal(bashResult?.toolName, "bash"); assert.equal(compressResult?.toolName, "compress"); - assert.ok(orphan && !("toolName" in orphan), "unpaired result must not gain a toolName"); + assert.ok( + orphan && !("toolName" in orphan), + "unpaired result must not gain a toolName", + ); }); test("anthropicToCore result ids stay stable under the toolName backfill", () => { const { msgs } = anthropicToCore(anthropicBody()); - const bashResult = msgs.find((m) => m.contentType === "tool-result" && m.toolCallId === "tu_1")!; + const bashResult = msgs.find( + (m) => m.contentType === "tool-result" && m.toolCallId === "tu_1", + )!; assert.equal( bashResult.id, - deriveMessageId("tool", "tool-result", ANTHROPIC_TEXT, { toolCallId: "tu_1" }), + deriveMessageId("tool", "tool-result", ANTHROPIC_TEXT, { + toolCallId: "tu_1", + }), ); }); test("excludeTools guard fires on wire-projected anthropic results (issue #213)", () => { const { msgs } = anthropicToCore(anthropicBody()); - const bashResult = msgs.find((m) => m.contentType === "tool-result" && m.toolCallId === "tu_1")!; - assert.equal(isAbsorbCandidate(bashResult, absorbConfig({ excludeTools: ["bash"] })), false); - assert.equal(isAbsorbCandidate(bashResult, absorbConfig({ excludeTools: [] })), true); + const bashResult = msgs.find( + (m) => m.contentType === "tool-result" && m.toolCallId === "tu_1", + )!; + assert.equal( + isAbsorbCandidate(bashResult, absorbConfig({ excludeTools: ["bash"] })), + false, + ); + assert.equal( + isAbsorbCandidate(bashResult, absorbConfig({ excludeTools: [] })), + true, + ); }); test("isNeverPreserveRecent and isMessageProtected fire on wire-projected results", () => { const { msgs } = anthropicToCore(anthropicBody()); - const bashResult = msgs.find((m) => m.contentType === "tool-result" && m.toolCallId === "tu_1")!; - const compressResult = msgs.find((m) => m.contentType === "tool-result" && m.toolCallId === "tu_2")!; + const bashResult = msgs.find( + (m) => m.contentType === "tool-result" && m.toolCallId === "tu_1", + )!; + const compressResult = msgs.find( + (m) => m.contentType === "tool-result" && m.toolCallId === "tu_2", + )!; assert.equal(isNeverPreserveRecent(bashResult), true); assert.equal(isMessageProtected(compressResult, defaultConfig(200000)), true); }); @@ -90,7 +146,9 @@ test("applyAbsorb rejects absorbing an ACP-managed tool result projected from wi const { msgs } = anthropicToCore(anthropicBody()); const coreMsgs = msgs as CoreMessage[]; const state = refsFor(coreMsgs); - const compressResult = coreMsgs.find((m) => m.contentType === "tool-result" && m.toolCallId === "tu_2")!; + const compressResult = coreMsgs.find( + (m) => m.contentType === "tool-result" && m.toolCallId === "tu_2", + )!; const ref = refForRaw(state.messageRefs, compressResult.id)!; const outcome = applyAbsorb({ ref, @@ -100,7 +158,10 @@ test("applyAbsorb rejects absorbing an ACP-managed tool result projected from wi config: defaultConfig(200000), }); assert.equal(outcome.ok, false); - assert.match(outcome.resultText, /absorb failed: .* is an ACP-managed tool result/); + assert.match( + outcome.resultText, + /absorb failed: .* is an ACP-managed tool result/, + ); }); test("applyAbsorb rejects absorbing a protectedTools-matched result projected from wire", () => { @@ -108,15 +169,31 @@ test("applyAbsorb rejects absorbing a protectedTools-matched result projected fr model: "claude-x", messages: [ { role: "user", content: "secret" }, - { role: "assistant", content: [{ type: "tool_use", id: "tu_s", name: "vault_read", input: {} }] }, - { role: "user", content: [{ type: "tool_result", tool_use_id: "tu_s", content: "s".repeat(480) }] }, + { + role: "assistant", + content: [ + { type: "tool_use", id: "tu_s", name: "vault_read", input: {} }, + ], + }, + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "tu_s", + content: "s".repeat(480), + }, + ], + }, ], }; const { msgs } = anthropicToCore(body); const coreMsgs = msgs as CoreMessage[]; const state = refsFor(coreMsgs); const config = { ...defaultConfig(200000), protectedTools: ["vault_*"] }; - const result = coreMsgs.find((m) => m.contentType === "tool-result" && m.toolCallId === "tu_s")!; + const result = coreMsgs.find( + (m) => m.contentType === "tool-result" && m.toolCallId === "tu_s", + )!; const ref = refForRaw(state.messageRefs, result.id)!; const outcome = applyAbsorb({ ref, @@ -139,15 +216,32 @@ function openaiBody(): OpenAIRequestBody { { role: "assistant", content: null, - tool_calls: [{ id: "call_1", type: "function", function: { name: "bash", arguments: "{}" } }], + tool_calls: [ + { + id: "call_1", + type: "function", + function: { name: "bash", arguments: "{}" }, + }, + ], }, { role: "tool", tool_call_id: "call_1", content: "y".repeat(480) }, { role: "assistant", content: null, - tool_calls: [{ id: "call_2", type: "function", function: { name: "read", arguments: "{}" } }], + tool_calls: [ + { + id: "call_2", + type: "function", + function: { name: "read", arguments: "{}" }, + }, + ], + }, + { + role: "tool", + tool_call_id: "call_2", + name: "legacy_name", + content: "z", }, - { role: "tool", tool_call_id: "call_2", name: "legacy_name", content: "z" }, { role: "tool", tool_call_id: "call_orphan", content: "w" }, ], }; @@ -155,19 +249,37 @@ function openaiBody(): OpenAIRequestBody { test("openaiToCore backfills toolName onto paired role:tool results", () => { const { msgs } = openaiToCore(openaiBody()); - const bashResult = msgs.find((m) => m.contentType === "tool-result" && m.toolCallId === "call_1"); - const legacyResult = msgs.find((m) => m.contentType === "tool-result" && m.toolCallId === "call_2"); - const orphan = msgs.find((m) => m.contentType === "tool-result" && m.toolCallId === "call_orphan"); + const bashResult = msgs.find( + (m) => m.contentType === "tool-result" && m.toolCallId === "call_1", + ); + const legacyResult = msgs.find( + (m) => m.contentType === "tool-result" && m.toolCallId === "call_2", + ); + const orphan = msgs.find( + (m) => m.contentType === "tool-result" && m.toolCallId === "call_orphan", + ); assert.equal(bashResult?.toolName, "bash"); - assert.equal(legacyResult?.toolName, "legacy_name", "explicit message.name wins over the call map"); + assert.equal( + legacyResult?.toolName, + "legacy_name", + "explicit message.name wins over the call map", + ); assert.ok(orphan && !("toolName" in orphan)); }); test("excludeTools guard fires on wire-projected openai results", () => { const { msgs } = openaiToCore(openaiBody()); - const bashResult = msgs.find((m) => m.contentType === "tool-result" && m.toolCallId === "call_1")!; - assert.equal(isAbsorbCandidate(bashResult, absorbConfig({ excludeTools: ["bash"] })), false); - assert.equal(isAbsorbCandidate(bashResult, absorbConfig({ excludeTools: [] })), true); + const bashResult = msgs.find( + (m) => m.contentType === "tool-result" && m.toolCallId === "call_1", + )!; + assert.equal( + isAbsorbCandidate(bashResult, absorbConfig({ excludeTools: ["bash"] })), + false, + ); + assert.equal( + isAbsorbCandidate(bashResult, absorbConfig({ excludeTools: [] })), + true, + ); }); // --- Responses --- @@ -178,8 +290,17 @@ function responsesBody(): ResponsesRequestBody { input: [ { type: "message", role: "user", content: "run" }, { type: "function_call", call_id: "fc_1", name: "bash", arguments: "{}" }, - { type: "function_call_output", call_id: "fc_1", output: "z".repeat(480) }, - { type: "custom_tool_call", call_id: "ctc_1", name: "my_tool", input: "{}" }, + { + type: "function_call_output", + call_id: "fc_1", + output: "z".repeat(480), + }, + { + type: "custom_tool_call", + call_id: "ctc_1", + name: "my_tool", + input: "{}", + }, { type: "custom_tool_call_output", call_id: "ctc_1", output: "out" }, { type: "custom_tool_call_output", call_id: "ctc_orphan", output: "o" }, ], @@ -188,9 +309,15 @@ function responsesBody(): ResponsesRequestBody { test("responsesToCore backfills toolName onto paired call outputs", () => { const { msgs } = responsesToCore(responsesBody()); - const fnResult = msgs.find((m) => m.contentType === "tool-result" && m.toolCallId === "fc_1"); - const customResult = msgs.find((m) => m.contentType === "tool-result" && m.toolCallId === "ctc_1"); - const orphan = msgs.find((m) => m.contentType === "tool-result" && m.toolCallId === "ctc_orphan"); + const fnResult = msgs.find( + (m) => m.contentType === "tool-result" && m.toolCallId === "fc_1", + ); + const customResult = msgs.find( + (m) => m.contentType === "tool-result" && m.toolCallId === "ctc_1", + ); + const orphan = msgs.find( + (m) => m.contentType === "tool-result" && m.toolCallId === "ctc_orphan", + ); assert.equal(fnResult?.toolName, "bash"); assert.equal(customResult?.toolName, "my_tool"); assert.ok(orphan && !("toolName" in orphan)); @@ -198,16 +325,28 @@ test("responsesToCore backfills toolName onto paired call outputs", () => { test("responsesToCore output ids stay stable under the toolName backfill", () => { const { msgs } = responsesToCore(responsesBody()); - const fnResult = msgs.find((m) => m.contentType === "tool-result" && m.toolCallId === "fc_1")!; + const fnResult = msgs.find( + (m) => m.contentType === "tool-result" && m.toolCallId === "fc_1", + )!; assert.equal( fnResult.id, - deriveMessageId("tool", "tool-result", "z".repeat(480), { toolCallId: "fc_1" }), + deriveMessageId("tool", "tool-result", "z".repeat(480), { + toolCallId: "fc_1", + }), ); }); test("excludeTools guard fires on wire-projected responses results", () => { const { msgs } = responsesToCore(responsesBody()); - const fnResult = msgs.find((m) => m.contentType === "tool-result" && m.toolCallId === "fc_1")!; - assert.equal(isAbsorbCandidate(fnResult, absorbConfig({ excludeTools: ["bash"] })), false); - assert.equal(isAbsorbCandidate(fnResult, absorbConfig({ excludeTools: [] })), true); + const fnResult = msgs.find( + (m) => m.contentType === "tool-result" && m.toolCallId === "fc_1", + )!; + assert.equal( + isAbsorbCandidate(fnResult, absorbConfig({ excludeTools: ["bash"] })), + false, + ); + assert.equal( + isAbsorbCandidate(fnResult, absorbConfig({ excludeTools: [] })), + true, + ); }); diff --git a/tsconfig.json b/tsconfig.json index 8629a6a..c8958c3 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,25 +1,25 @@ { - "compilerOptions": { - "target": "ES2022", - "lib": ["ES2022", "ES2024.Promise"], - "module": "ESNext", - "moduleResolution": "Bundler", - "strict": true, - "esModuleInterop": true, - "skipLibCheck": true, - "forceConsistentCasingInFileNames": true, - "resolveJsonModule": true, - "declaration": true, - "declarationMap": true, - "sourceMap": true, - "outDir": "./dist", - "rootDir": "./src", - "noUnusedLocals": true, - "noUnusedParameters": true, - "noFallthroughCasesInSwitch": true, - "noUncheckedIndexedAccess": true, - "verbatimModuleSyntax": true - }, - "include": ["src/**/*.ts"], - "exclude": ["node_modules", "dist", "tests"] + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2022", "ES2024.Promise"], + "module": "ESNext", + "moduleResolution": "Bundler", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "outDir": "./dist", + "rootDir": "./src", + "noUnusedLocals": true, + "noUnusedParameters": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedIndexedAccess": true, + "verbatimModuleSyntax": true + }, + "include": ["src/**/*.ts"], + "exclude": ["node_modules", "dist", "tests"] } diff --git a/tsup.config.ts b/tsup.config.ts index 3726c11..a4eceea 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -1,10 +1,15 @@ import { defineConfig } from "tsup"; export default defineConfig({ - entry: ["src/index.ts", "src/wire/index.ts", "src/panel/index.ts", "src/persist/index.ts"], - format: ["esm"], - dts: false, - clean: false, - sourcemap: true, - target: "es2022", + entry: [ + "src/index.ts", + "src/wire/index.ts", + "src/panel/index.ts", + "src/persist/index.ts", + ], + format: ["esm"], + dts: false, + clean: false, + sourcemap: true, + target: "es2022", });