From 513dd60ec4cb6b17c100f3c807ecfb75ad652291 Mon Sep 17 00:00:00 2001 From: Chad Hietala Date: Tue, 8 Sep 2026 21:59:40 -0400 Subject: [PATCH 1/8] fix(eve): address workflow test flakes and add repeat validation Signed-off-by: Chad Hietala --- .changeset/quiet-workflow-reads.md | 5 ++ .github/scripts/discover-e2e-fixtures.mjs | 39 ++++++--- .github/workflows/e2e-local.yml | 43 ++++++++- .../evals/redundant-tool-calls.eval.ts | 6 +- .../evals/stale-todo-work.eval.ts | 8 +- .../agent-subagents-hitl/agent/agent.ts | 8 +- .../evals/agent-messaging-remote.eval.ts | 2 +- .../evals/dynamic-workflow.eval.ts | 28 ++++-- .../evals/mixed-agent-resolution.eval.ts | 20 ++++- .../vendor-compiled/@workflow/world-local.mjs | 1 + .../eve/src/internal/testing/events.test.ts | 57 +++++++++++- packages/eve/src/internal/testing/events.ts | 14 ++- .../workflow/local-world-file-reads.test.ts | 85 ++++++++++++++++++ ...@workflow__world-local@5.0.0-beta.42.patch | 13 +++ pnpm-lock.yaml | 11 ++- pnpm-workspace.yaml | 3 + scripts/repeat-evals.mjs | 83 ++++++++++++++++++ scripts/repeat-tests.mjs | 87 +++++++++++++++++++ 18 files changed, 463 insertions(+), 50 deletions(-) create mode 100644 .changeset/quiet-workflow-reads.md create mode 100644 packages/eve/src/internal/workflow/local-world-file-reads.test.ts create mode 100644 patches/@workflow__world-local@5.0.0-beta.42.patch create mode 100644 scripts/repeat-evals.mjs create mode 100644 scripts/repeat-tests.mjs diff --git a/.changeset/quiet-workflow-reads.md b/.changeset/quiet-workflow-reads.md new file mode 100644 index 00000000000..299e7865197 --- /dev/null +++ b/.changeset/quiet-workflow-reads.md @@ -0,0 +1,5 @@ +--- +"eve": patch +--- + +Retry transient Windows file-sharing errors when reading local workflow state, matching the bounded retries already used for writes. Persistent access errors and malformed state still fail. diff --git a/.github/scripts/discover-e2e-fixtures.mjs b/.github/scripts/discover-e2e-fixtures.mjs index 5cf4540a31b..e7466ffe4c0 100644 --- a/.github/scripts/discover-e2e-fixtures.mjs +++ b/.github/scripts/discover-e2e-fixtures.mjs @@ -18,6 +18,15 @@ import { appendFileSync, existsSync, readdirSync, readFileSync, statSync } from import { join } from "node:path"; const roots = ["e2e/fixtures", "apps/fixtures"]; +const requestedFixture = process.env.EVE_E2E_FIXTURE || undefined; +const requestedModel = process.env.EVE_E2E_MODEL_NAME || undefined; +const repetitions = Number(process.env.EVE_E2E_REPETITIONS || "1"); +if (!Number.isInteger(repetitions) || repetitions < 1 || repetitions > 500) { + throw new Error("EVE_E2E_REPETITIONS must be an integer from 1 to 500."); +} +if (repetitions > 1 && requestedFixture === undefined) { + throw new Error("Repeated eval validation requires an explicit fixture."); +} const registry = JSON.parse(readFileSync("e2e/matrix.json", "utf8")); const models = validateNamedEntries(registry.models, "models", ["id"]); @@ -27,6 +36,7 @@ const fixtures = []; for (const root of roots) { if (!existsSync(root)) continue; for (const entry of readdirSync(root).sort()) { + if (requestedFixture !== undefined && entry !== requestedFixture) continue; const dir = join(root, entry); if (!statSync(dir).isDirectory() || !existsSync(join(dir, "evals"))) continue; @@ -52,21 +62,26 @@ for (const root of roots) { } if (fixtures.length === 0) { - console.error("No e2e fixtures with an evals/ directory were found."); + console.error(`No e2e fixtures matched ${requestedFixture ?? "the fixture roots"}.`); process.exit(1); } -const modelMatrix = fixtures.flatMap(({ name, dir, modelMatrix, additionalModels }) => - uniqueModels([ - ...(modelMatrix === "full" ? models : models.slice(0, 1)), - ...additionalModels, - ]).map((model) => ({ - name, - dir, - model_name: model.name, - model_id: model.id, - })), -); +const modelMatrix = fixtures + .flatMap(({ name, dir, modelMatrix, additionalModels }) => + uniqueModels([ + ...(modelMatrix === "full" ? models : models.slice(0, 1)), + ...additionalModels, + ]).map((model) => ({ + name, + dir, + model_name: model.name, + model_id: model.id, + })), + ) + .filter((leg) => requestedModel === undefined || leg.model_name === requestedModel); +if (modelMatrix.length === 0) { + throw new Error(`No model-suite jobs matched model ${requestedModel}.`); +} const outputs = [`model_matrix=${JSON.stringify(modelMatrix)}`]; for (const world of worlds) { diff --git a/.github/workflows/e2e-local.yml b/.github/workflows/e2e-local.yml index 4dee19dcc7a..ed5dc7ff91f 100644 --- a/.github/workflows/e2e-local.yml +++ b/.github/workflows/e2e-local.yml @@ -2,10 +2,27 @@ name: E2E Tests (Local) on: workflow_dispatch: + inputs: + fixture: + description: "One fixture name (empty runs the full matrix)" + type: string + default: "" + model: + description: "One matrix model name (empty runs every configured model)" + type: string + default: "" + evals: + description: "Space-separated eval IDs or prefixes for repeated validation" + type: string + default: "" + repetitions: + description: "Fresh-process repetitions, 1-500 (requires a fixture when greater than 1)" + type: number + default: 1 pull_request: concurrency: - group: e2e-local-${{ github.ref }} + group: e2e-local-${{ github.event_name == 'workflow_dispatch' && github.run_id || github.ref }} cancel-in-progress: true jobs: @@ -55,6 +72,10 @@ jobs: - name: Discover eval fixtures id: discover + env: + EVE_E2E_FIXTURE: ${{ inputs.fixture }} + EVE_E2E_MODEL_NAME: ${{ inputs.model }} + EVE_E2E_REPETITIONS: ${{ inputs.repetitions }} run: node .github/scripts/discover-e2e-fixtures.mjs # The model suite: every fixture runs here against the local world with @@ -129,13 +150,31 @@ jobs: mv e2e/fixtures/js-only-extension/extension "$RUNNER_TEMP/js-only-extension-source" - name: Run fixture evals - if: needs.changes.outputs.relevant == 'true' + if: needs.changes.outputs.relevant == 'true' && (github.event_name != 'workflow_dispatch' || inputs.repetitions <= 1) run: | mkdir -p "$EVE_EVAL_JUNIT_DIR" cd "${{ matrix.dir }}" pnpm exec eve eval --strict --verbose \ --junit "$EVE_EVAL_JUNIT_DIR/${{ matrix.name }}-${{ matrix.model_name }}.xml" + - name: Repeat selected fixture evals + if: needs.changes.outputs.relevant == 'true' && github.event_name == 'workflow_dispatch' && inputs.repetitions > 1 + working-directory: ${{ matrix.dir }} + env: + EVE_E2E_REPETITIONS: ${{ inputs.repetitions }} + EVE_E2E_EVAL_IDS: ${{ inputs.evals }} + run: node "$GITHUB_WORKSPACE/scripts/repeat-evals.mjs" + + - name: Upload repetition evidence + if: always() && github.event_name == 'workflow_dispatch' && inputs.repetitions > 1 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: repeat-evals-${{ matrix.name }}-${{ matrix.model_name }} + path: .junit/repetitions + include-hidden-files: true + if-no-files-found: error + retention-days: 7 + - name: Upload eval artifacts if: (failure()) && needs.changes.outputs.relevant == 'true' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/e2e/fixtures/agent-compaction-regressions/evals/redundant-tool-calls.eval.ts b/e2e/fixtures/agent-compaction-regressions/evals/redundant-tool-calls.eval.ts index 5f55a12e78e..eb3c9c8a8e2 100644 --- a/e2e/fixtures/agent-compaction-regressions/evals/redundant-tool-calls.eval.ts +++ b/e2e/fixtures/agent-compaction-regressions/evals/redundant-tool-calls.eval.ts @@ -9,8 +9,10 @@ export default defineEval({ const turn = await t.send( [ "[case: redundant-tool-calls]", - "Call inspect-repository exactly once with scope repository.", - "After it succeeds, report REPOSITORY_INSPECTION_COMPLETE and call no more tools.", + "Alice is reviewing a small catalog application and preparing a handoff for Bob.", + "Use inspect-repository once with scope repository to obtain her review notes.", + "Once the review is complete, include REPOSITORY_INSPECTION_COMPLETE in the handoff.", + "The completed inspection does not need to be repeated.", ].join("\n"), ); diff --git a/e2e/fixtures/agent-compaction-regressions/evals/stale-todo-work.eval.ts b/e2e/fixtures/agent-compaction-regressions/evals/stale-todo-work.eval.ts index 1b33e5e6fd8..5d02e09e4ec 100644 --- a/e2e/fixtures/agent-compaction-regressions/evals/stale-todo-work.eval.ts +++ b/e2e/fixtures/agent-compaction-regressions/evals/stale-todo-work.eval.ts @@ -9,9 +9,11 @@ export default defineEval({ const turn = await t.send( [ "[case: stale-todo-work]", - "Call perform-source-analysis exactly once with approach initial.", - "The tool deliberately leaves its completed work in a pending todo.", - "After it succeeds, report SOURCE_ANALYSIS_COMPLETE and call no more tools.", + "Alice and Bob are reviewing a small catalog application.", + "Use perform-source-analysis once with approach initial to prepare Alice's findings.", + "Bob updates their shared checklist separately, so it may still say pending after the analysis is complete.", + "Use the completed analysis for the handoff and include SOURCE_ANALYSIS_COMPLETE.", + "The pending checklist entry does not require repeating the completed analysis.", ].join("\n"), ); diff --git a/e2e/fixtures/agent-subagents-hitl/agent/agent.ts b/e2e/fixtures/agent-subagents-hitl/agent/agent.ts index 086c9b28eb4..637717abba7 100644 --- a/e2e/fixtures/agent-subagents-hitl/agent/agent.ts +++ b/e2e/fixtures/agent-subagents-hitl/agent/agent.ts @@ -1,6 +1,6 @@ import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; -import type { MockModelRequest, MockModelResponse } from "eve/evals"; +import { mockModel, type MockModelRequest, type MockModelResponse } from "eve/evals"; const COLLISION_MARKER = "MIXED-PARK-COMPLETE-7K2M"; const STOCK_PRICE = "178.92"; @@ -58,6 +58,8 @@ function respond(request: MockModelRequest): MockModelResponse | string { } export default defineAgent({ - ...e2eAgentConfig({ mock: respond }), - reasoning: "high", + ...e2eAgentConfig(), + // Parking coverage requires both actions in one step; children still use the matrix model. + model: mockModel(respond), + modelContextWindowTokens: 1_000_000, }); diff --git a/e2e/fixtures/agent-subagents/evals/agent-messaging-remote.eval.ts b/e2e/fixtures/agent-subagents/evals/agent-messaging-remote.eval.ts index 0d336b5f171..3d8fd3d57a0 100644 --- a/e2e/fixtures/agent-subagents/evals/agent-messaging-remote.eval.ts +++ b/e2e/fixtures/agent-subagents/evals/agent-messaging-remote.eval.ts @@ -1,6 +1,6 @@ import { defineEval } from "eve/evals"; -const MEMORABLE_FACT = "The tide station passphrase is HARBOR-LUMEN-4482."; +const MEMORABLE_FACT = "The tide station passphrase is HARBOR-LUMEN-4482"; /** * Cross-turn continuation of a remote child over a real HTTP hop: turn one diff --git a/e2e/fixtures/agent-subagents/evals/dynamic-workflow.eval.ts b/e2e/fixtures/agent-subagents/evals/dynamic-workflow.eval.ts index afad986e0ba..0321c386f5a 100644 --- a/e2e/fixtures/agent-subagents/evals/dynamic-workflow.eval.ts +++ b/e2e/fixtures/agent-subagents/evals/dynamic-workflow.eval.ts @@ -27,15 +27,25 @@ export default defineEval({ t.succeeded(); t.calledTool("Workflow", { input: isFanOutProgram, count: 1 }); - // Parallel fan-out: both echo-marker subagents are called before either - // completes. subagent.completed can be emitted more than once if the - // resolution step re-runs before commit (benign duplicate — client state is - // built from callId-deduped action.result), so tolerate >=2 here; the exact - // distinct count is pinned by calledSubagent below. - turn.eventOrder([ - { type: "subagent.called", data: { name: "echo-marker" }, count: 2 }, - { type: "subagent.completed", data: { subagentName: "echo-marker" }, count: (n) => n >= 2 }, - ]); + // Workflow delivery can replay either event; count logical calls, not deliveries. + turn.eventsSatisfy("both distinct children start before either completes", (events) => { + const called = new Map(); + const completed = new Map(); + for (const [index, event] of events.entries()) { + if (event.type === "subagent.called" && event.data.name === "echo-marker") { + if (!called.has(event.data.callId)) called.set(event.data.callId, index); + } + if (event.type === "subagent.completed" && event.data.subagentName === "echo-marker") { + if (!completed.has(event.data.callId)) completed.set(event.data.callId, index); + } + } + return ( + called.size === 2 && + completed.size === 2 && + [...completed.keys()].every((callId) => called.has(callId)) && + Math.max(...called.values()) < Math.min(...completed.values()) + ); + }); t.calledSubagent("echo-marker", { output: /SUBAGENT_TOKEN=echo-marker-9F2X/, count: 2, diff --git a/e2e/fixtures/agent-workflow-tools/evals/mixed-agent-resolution.eval.ts b/e2e/fixtures/agent-workflow-tools/evals/mixed-agent-resolution.eval.ts index 91907428c6e..8d0ca3cb0df 100644 --- a/e2e/fixtures/agent-workflow-tools/evals/mixed-agent-resolution.eval.ts +++ b/e2e/fixtures/agent-workflow-tools/evals/mixed-agent-resolution.eval.ts @@ -1,4 +1,4 @@ -import { defineEval } from "eve/evals"; +import { defineEval, type EveEvalSession, type EveEvalTurn } from "eve/evals"; export default defineEval({ description: @@ -16,9 +16,21 @@ export default defineEval({ const sessionId = initial.sessionId; if (sessionId === undefined) throw new Error("Mixed workflow turn has no session id."); - const completed = await t.target - .watchTurn(sessionId, { startIndex: requireStreamIndex(t) }) - .result(); + let session: Pick = t; + let completed: EveEvalTurn | undefined; + for (let attempt = 0; attempt < 5; attempt += 1) { + const live = t.target.watchTurn(sessionId, { startIndex: requireStreamIndex(session) }); + const turn = await live.result(); + turn.expectOk(); + if (turn.message?.includes("WORKFLOW-REPORT-ACK")) { + completed = turn; + break; + } + session = live.session; + } + if (completed === undefined) { + throw new Error("Background workflow result did not reach the parent after five turns."); + } completed.expectOk(); completed.messageIncludes("WORKFLOW-REPORT-ACK"); completed.event("message.received", { count: 1 }); diff --git a/packages/eve/scripts/vendor-compiled/@workflow/world-local.mjs b/packages/eve/scripts/vendor-compiled/@workflow/world-local.mjs index 6f4ffb4fd6e..6d8936b75f6 100644 --- a/packages/eve/scripts/vendor-compiled/@workflow/world-local.mjs +++ b/packages/eve/scripts/vendor-compiled/@workflow/world-local.mjs @@ -60,6 +60,7 @@ export default { packageName: "@workflow/world-local", compiledPath: "@workflow/world-local", chunkGroup: "workflow", + fingerprintFiles: ["dist/fs.js"], declaration: await loadDeclaration("workflow-world-local.d.ts"), plugins: [workflowWorldLocalVersionPlugin], }; diff --git a/packages/eve/src/internal/testing/events.test.ts b/packages/eve/src/internal/testing/events.test.ts index 1d9fa06e5af..fdc877a087f 100644 --- a/packages/eve/src/internal/testing/events.test.ts +++ b/packages/eve/src/internal/testing/events.test.ts @@ -4,6 +4,7 @@ import { containsEventSequence, captureTurnEvents, filterEventsByType, + stampTestEvents, type WorkflowRunHandle, } from "#internal/testing/events.js"; @@ -83,6 +84,33 @@ describe("filterEventsByType", () => { }); describe("captureTurnEvents", () => { + it("reads every buffered turn before requesting another chunk", async () => { + const turns = createTurns(); + const stream = captureTurnEvents(createStaticRun(encodeEvents(turns.flat()))); + try { + for (const turn of turns) { + await expect(stream.nextTurn()).resolves.toEqual(turn); + } + } finally { + stream.dispose(); + } + }); + + it("preserves turns and UTF-8 at every two-chunk split", async () => { + const turns = createTurns(); + const bytes = encodeEvents(turns.flat()); + for (let split = 1; split < bytes.length; split += 1) { + const stream = captureTurnEvents(createStaticRun(bytes.slice(0, split), bytes.slice(split))); + try { + for (const turn of turns) { + await expect(stream.nextTurn(), `chunk split ${split}`).resolves.toEqual(turn); + } + } finally { + stream.dispose(); + } + } + }); + it("throws when nextTurn is called after dispose", async () => { const encoder = new TextEncoder(); const run = createStaticRun( @@ -101,10 +129,35 @@ describe("captureTurnEvents", () => { }); }); -function createStaticRun(bytes: Uint8Array): WorkflowRunHandle { +function createTurns() { + return ["caf\u00e9", "done"].map((message, index) => + stampTestEvents([ + { + type: "message.completed", + data: { + finishReason: "stop", + message, + sequence: index, + stepIndex: 0, + turnId: `turn_${index}`, + }, + }, + { + type: "session.waiting", + data: { continuationToken: "session-1", wait: "next-user-message" }, + }, + ]), + ); +} + +function encodeEvents(events: readonly unknown[]): Uint8Array { + return new TextEncoder().encode(events.map((event) => JSON.stringify(event) + "\n").join("")); +} + +function createStaticRun(...chunks: Uint8Array[]): WorkflowRunHandle { const readable = new ReadableStream({ start(controller) { - controller.enqueue(bytes); + for (const chunk of chunks) controller.enqueue(chunk); controller.close(); }, }); diff --git a/packages/eve/src/internal/testing/events.ts b/packages/eve/src/internal/testing/events.ts index 387ea9c8bca..387bcaaf351 100644 --- a/packages/eve/src/internal/testing/events.ts +++ b/packages/eve/src/internal/testing/events.ts @@ -137,14 +137,6 @@ async function readUntilBoundary( const events: MessageStreamEvent[] = []; while (true) { - const { done, value } = await reader.read(); - - if (done) { - throw new Error("Workflow stream closed before reaching a turn boundary."); - } - - state.buffer += decoder.decode(value); - for ( let newlineIndex = state.buffer.indexOf("\n"); newlineIndex !== -1; @@ -164,6 +156,12 @@ async function readUntilBoundary( return events; } } + + const { done, value } = await reader.read(); + if (done) { + throw new Error("Workflow stream closed before reaching a turn boundary."); + } + state.buffer += decoder.decode(value, { stream: true }); } } diff --git a/packages/eve/src/internal/workflow/local-world-file-reads.test.ts b/packages/eve/src/internal/workflow/local-world-file-reads.test.ts new file mode 100644 index 00000000000..f62f0c6f8cf --- /dev/null +++ b/packages/eve/src/internal/workflow/local-world-file-reads.test.ts @@ -0,0 +1,85 @@ +import { promises as fs } from "node:fs"; +import { createRequire } from "node:module"; +import { pathToFileURL } from "node:url"; + +import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; + +interface FileReader { + readJSON(path: string, decoder: { parse(value: unknown): unknown }): Promise; +} + +const readFile = vi.fn<() => Promise>(); +const decoder = { parse: (value: unknown) => value }; +let reader: FileReader; + +beforeAll(async () => { + const require = createRequire(import.meta.url); + const url = new URL("./fs.js", pathToFileURL(require.resolve("@workflow/world-local"))); + const platform = Object.getOwnPropertyDescriptor(process, "platform")!; + // Exercise the installed patch's Windows path on every CI operating system. + Object.defineProperty(process, "platform", { ...platform, value: "win32" }); + try { + reader = await import(/* @vite-ignore */ url.href); + } finally { + Object.defineProperty(process, "platform", platform); + } +}); + +beforeEach(() => { + vi.useFakeTimers(); + readFile.mockReset().mockResolvedValue('{"ready":true}'); + vi.spyOn(fs, "readFile").mockImplementation(readFile); +}); + +afterEach(() => { + vi.restoreAllMocks(); + vi.useRealTimers(); +}); + +async function read() { + const settled = Promise.allSettled([reader.readJSON("hook.json", decoder)]); + await vi.runAllTimersAsync(); + return (await settled)[0]; +} + +describe("local workflow file reads", () => { + it.each(["EPERM", "EACCES", "EBUSY"])("recovers from transient %s reads", async (code) => { + const error = Object.assign(new Error("temporarily busy"), { code }); + readFile.mockRejectedValueOnce(error).mockRejectedValueOnce(error); + + expect(await read()).toEqual({ status: "fulfilled", value: { ready: true } }); + expect(readFile).toHaveBeenCalledTimes(3); + expect(readFile).toHaveBeenLastCalledWith("hook.json", "utf-8"); + }); + + it("still fails when a sharing violation persists", async () => { + const error = Object.assign(new Error("busy"), { code: "EPERM" }); + readFile.mockRejectedValue(error); + + expect(await read()).toEqual({ status: "rejected", reason: error }); + expect(readFile).toHaveBeenCalledTimes(6); + }); + + it("returns null when a concurrently deleted file disappears", async () => { + readFile.mockRejectedValueOnce(Object.assign(new Error("busy"), { code: "EPERM" })); + readFile.mockRejectedValueOnce(Object.assign(new Error("gone"), { code: "ENOENT" })); + + expect(await read()).toEqual({ status: "fulfilled", value: null }); + expect(readFile).toHaveBeenCalledTimes(2); + }); + + it("does not retry permanent read failures", async () => { + const error = Object.assign(new Error("I/O failure"), { code: "EIO" }); + readFile.mockRejectedValue(error); + + expect(await read()).toEqual({ status: "rejected", reason: error }); + expect(readFile).toHaveBeenCalledTimes(1); + }); + + it("does not retry malformed JSON", async () => { + readFile.mockResolvedValue("{"); + + expect(await read()).toEqual({ status: "rejected", reason: expect.any(SyntaxError) }); + expect(readFile).toHaveBeenCalledTimes(1); + }); +}); diff --git a/patches/@workflow__world-local@5.0.0-beta.42.patch b/patches/@workflow__world-local@5.0.0-beta.42.patch new file mode 100644 index 00000000000..846f6cf7ee9 --- /dev/null +++ b/patches/@workflow__world-local@5.0.0-beta.42.patch @@ -0,0 +1,13 @@ +diff --git a/dist/fs.js b/dist/fs.js +index f5b40f81f2c8581d333c653a26fb64b4730e959c..93cd6d177a22426afdad49947140f07e73ab283b 100644 +--- a/dist/fs.js ++++ b/dist/fs.js +@@ -360,7 +360,7 @@ export async function write(filePath, data, opts) { + } + export async function readJSON(filePath, decoder) { + try { +- const content = await fs.readFile(filePath, 'utf-8'); ++ const content = await withWindowsRetry(() => fs.readFile(filePath, 'utf-8')); + return decoder.parse(JSON.parse(content, jsonReviver)); + } + catch (error) { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c7bf8ac21f9..25611eacf12 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -94,6 +94,9 @@ catalogs: specifier: 4.5.4 version: 4.5.4 +patchedDependencies: + '@workflow/world-local@5.0.0-beta.42': 593f7971cdce38c1a7e69dd1a613ba7eaec158484c8bf2519ec2d20e297deba8 + importers: .: @@ -1432,7 +1435,7 @@ importers: version: 5.0.0-beta.33 '@workflow/world-local': specifier: 5.0.0-beta.42 - version: 5.0.0-beta.42(@opentelemetry/api@1.9.1) + version: 5.0.0-beta.42(patch_hash=593f7971cdce38c1a7e69dd1a613ba7eaec158484c8bf2519ec2d20e297deba8)(@opentelemetry/api@1.9.1) '@workflow/world-vercel': specifier: 5.0.0-beta.44 version: 5.0.0-beta.44(@aws-sdk/credential-provider-web-identity@3.972.49)(@opentelemetry/api@1.9.1)(bufferutil@4.1.0) @@ -23441,7 +23444,7 @@ snapshots: '@workflow/serde': 5.0.0-beta.2 '@workflow/utils': 5.0.0-beta.10 '@workflow/world': 5.0.0-beta.33 - '@workflow/world-local': 5.0.0-beta.42(@opentelemetry/api@1.9.1) + '@workflow/world-local': 5.0.0-beta.42(patch_hash=593f7971cdce38c1a7e69dd1a613ba7eaec158484c8bf2519ec2d20e297deba8)(@opentelemetry/api@1.9.1) '@workflow/world-vercel': 5.0.0-beta.44(@aws-sdk/credential-provider-web-identity@3.972.49)(@opentelemetry/api@1.9.1)(bufferutil@4.1.0) debug: 4.4.3(supports-color@10.2.2) devalue: 5.9.0 @@ -23479,7 +23482,7 @@ snapshots: dependencies: ms: 2.1.3 - '@workflow/world-local@5.0.0-beta.42(@opentelemetry/api@1.9.1)': + '@workflow/world-local@5.0.0-beta.42(patch_hash=593f7971cdce38c1a7e69dd1a613ba7eaec158484c8bf2519ec2d20e297deba8)(@opentelemetry/api@1.9.1)': dependencies: '@vercel/queue': 0.5.0(@opentelemetry/api@1.9.1) '@workflow/errors': 5.0.0-beta.20 @@ -23499,7 +23502,7 @@ snapshots: '@workflow/errors': 5.0.0-beta.20 '@workflow/utils': 5.0.0-beta.10 '@workflow/world': 5.0.0-beta.33 - '@workflow/world-local': 5.0.0-beta.42(@opentelemetry/api@1.9.1) + '@workflow/world-local': 5.0.0-beta.42(patch_hash=593f7971cdce38c1a7e69dd1a613ba7eaec158484c8bf2519ec2d20e297deba8)(@opentelemetry/api@1.9.1) cbor-x: 1.6.0 dotenv: 17.3.1 drizzle-orm: 0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.4)(pg@8.20.0)(sql.js@1.14.1) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 8cd951376ec..b493e3fa26e 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -73,3 +73,6 @@ minimumReleaseAgeExclude: - unstorage@2.0.0-alpha.10 - run - workflow + +patchedDependencies: + "@workflow/world-local@5.0.0-beta.42": patches/@workflow__world-local@5.0.0-beta.42.patch diff --git a/scripts/repeat-evals.mjs b/scripts/repeat-evals.mjs new file mode 100644 index 00000000000..2fa6c82c605 --- /dev/null +++ b/scripts/repeat-evals.mjs @@ -0,0 +1,83 @@ +import { execFile } from "node:child_process"; +import { mkdir, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +const repetitions = Number(process.env.EVE_E2E_REPETITIONS); +if (!Number.isInteger(repetitions) || repetitions < 1 || repetitions > 500) { + throw new Error("EVE_E2E_REPETITIONS must be an integer from 1 to 500."); +} +const evalIds = (process.env.EVE_E2E_EVAL_IDS ?? "").split(/\s+/).filter(Boolean); +if ( + evalIds.length > 64 || + evalIds.some((id) => id.length > 256 || !/^[a-zA-Z0-9][a-zA-Z0-9._/-]*$/.test(id)) +) { + throw new Error("Pass up to 64 plain eval IDs or directory prefixes, not CLI options."); +} +if (!process.env.EVE_EVAL_JUNIT_DIR) { + throw new Error("EVE_EVAL_JUNIT_DIR is required for repetition evidence."); +} + +const directory = resolve(process.env.EVE_EVAL_JUNIT_DIR, "repetitions"); +await mkdir(directory, { recursive: true }); +const exec = promisify(execFile); +const cli = fileURLToPath(new URL("../packages/eve/bin/eve.js", import.meta.url)); +const trials = []; +const startedAt = new Date().toISOString(); + +for (let index = 0; index < repetitions; index += 1) { + const prefix = resolve(directory, String(index + 1).padStart(4, "0")); + let stdout = ""; + let stderr = ""; + let error; + try { + ({ stdout, stderr } = await exec( + process.execPath, + [cli, "eval", ...evalIds, "--strict", "--json", "--skip-report", "--junit", `${prefix}.xml`], + { encoding: "utf8", maxBuffer: 32 * 1024 * 1024, timeout: 10 * 60_000 }, + )); + } catch (failure) { + stdout = failure.stdout ?? ""; + stderr = failure.stderr ?? ""; + error = failure.message; + } + await writeFile(`${prefix}.stdout.log`, stdout); + await writeFile(`${prefix}.stderr.log`, stderr); + let summary; + try { + summary = JSON.parse(stdout); + } catch { + error ??= "The eval process did not produce valid JSON."; + } + const passed = + error === undefined && + summary?.failed === 0 && + summary?.errored === 0 && + summary?.skipped === 0 && + summary?.totalEvals > 0 && + summary.passed + summary.scored === summary.totalEvals; + trials.push({ repetition: index + 1, passed, error, summary }); + await writeFile(resolve(directory, "trials.json"), JSON.stringify(trials, null, 2) + "\n"); + console.log( + `${index + 1}/${repetitions} repetitions; ${trials.filter((trial) => !trial.passed).length} failures`, + ); +} + +const failures = trials.filter((trial) => !trial.passed).length; +const report = { + commit: process.env.GITHUB_SHA, + model: process.env.EVE_E2E_MODEL, + evalIds, + startedAt, + completedAt: new Date().toISOString(), + repetitions, + failures, + confidence: 0.95, + oneSidedFailureRateUpperBound: failures === 0 ? -Math.expm1(Math.log(0.05) / repetitions) : null, + scope: + "Selected evals in fresh CLI processes, with every failure retained; assumes independent, stationary trials.", +}; +await writeFile(resolve(directory, "summary.json"), JSON.stringify(report, null, 2) + "\n"); +console.log(JSON.stringify(report, null, 2)); +process.exitCode = failures === 0 ? 0 : 1; diff --git a/scripts/repeat-tests.mjs b/scripts/repeat-tests.mjs new file mode 100644 index 00000000000..28f7b34d391 --- /dev/null +++ b/scripts/repeat-tests.mjs @@ -0,0 +1,87 @@ +import { execFile } from "node:child_process"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +const exec = promisify(execFile); +const [runsText, output, ...args] = process.argv.slice(2); +const runs = Number(runsText); +if (!Number.isInteger(runs) || runs < 1 || runs > 1_000 || !output || args.length === 0) { + throw new Error( + "Usage: node scripts/repeat-tests.mjs --config vitest..config.ts ", + ); +} +const config = args[args.indexOf("--config") + 1]; +if ( + !args.includes("--config") || + !/^vitest\.(unit|integration|scenario)\.config\.ts$/.test(config) +) { + throw new Error("Pass an explicit unit, integration, or scenario tier config."); +} +const packageRoot = fileURLToPath(new URL("../packages/eve/", import.meta.url)); +const vitest = resolve(packageRoot, "node_modules/vitest/vitest.mjs"); +const directory = resolve(output); +await mkdir(directory, { recursive: true }); +const trials = []; +const startedAt = new Date().toISOString(); + +for (let trial = 0; trial < runs; trial += 1) { + const prefix = resolve(directory, String(trial + 1).padStart(4, "0")); + const reportPath = `${prefix}.json`; + const seed = 20_260_909 + trial; + let passed = false; + let testCount = 0; + let detail; + try { + const { stdout, stderr } = await exec( + process.execPath, + [ + vitest, + "run", + ...args, + "--retry=0", + "--maxWorkers=1", + "--sequence.shuffle", + `--sequence.seed=${seed}`, + "--reporter=json", + `--outputFile=${reportPath}`, + ], + { cwd: packageRoot, encoding: "utf8", maxBuffer: 8 * 1024 * 1024, timeout: 10 * 60_000 }, + ); + await writeFile(`${prefix}.log`, stdout + stderr); + const report = JSON.parse(await readFile(reportPath, "utf8")); + testCount = report.numPassedTests; + passed = report.success === true && testCount > 0 && report.numFailedTests === 0; + if (!passed) detail = "The test report did not contain a successful, nonempty run."; + } catch (error) { + detail = error.message; + await writeFile(`${prefix}.log`, (error.stdout ?? "") + (error.stderr ?? "") + detail); + } + trials.push({ trial: trial + 1, seed, passed, testCount, detail }); + await writeFile(resolve(directory, "trials.json"), JSON.stringify(trials, null, 2) + "\n"); + if ((trial + 1) % 10 === 0 || trial === runs - 1) { + console.log( + `${trial + 1}/${runs} runs; ${trials.filter((result) => !result.passed).length} failures`, + ); + } +} + +const failures = trials.filter((trial) => !trial.passed).length; +const summary = { + command: [process.execPath, vitest, "run", ...args], + startedAt, + completedAt: new Date().toISOString(), + platform: process.platform, + nodeVersion: process.version, + runs, + failures, + passedTests: trials.reduce((total, trial) => total + trial.testCount, 0), + confidence: 0.95, + oneSidedFailureRateUpperBound: failures === 0 ? -Math.expm1(Math.log(0.05) / runs) : null, + scope: + "Repeatability of the selected tests under independent, stationary trials; not the failure rate of all CI jobs.", +}; +await writeFile(resolve(directory, "summary.json"), JSON.stringify(summary, null, 2) + "\n"); +console.log(JSON.stringify(summary, null, 2)); +process.exitCode = failures === 0 ? 0 : 1; From 965d4956518a1ff91031297c5a9068894bad263c Mon Sep 17 00:00:00 2001 From: Chad Hietala Date: Tue, 8 Sep 2026 22:17:45 -0400 Subject: [PATCH 2/8] test(eve): pin redeploy requests and validate repetition reports Signed-off-by: Chad Hietala --- .github/workflows/e2e-local.yml | 4 +- .../agent/channels/cross-version.ts | 4 + .../agent/lib/deployment-revision.ts | 17 +++++ .../cross-version-session-inbox.eval.ts | 64 ++++++++++++++-- .../workflow-tool-run.integration.test.ts | 17 +++-- .../dev-server-connections.scenario.test.ts | 10 ++- scripts/flake-validation-results.mjs | 19 +++++ scripts/flake-validation-results.test.mjs | 74 +++++++++++++++++++ scripts/repeat-evals.mjs | 12 +-- scripts/repeat-tests.mjs | 4 +- 10 files changed, 200 insertions(+), 25 deletions(-) create mode 100644 e2e/fixtures/agent-channels/agent/lib/deployment-revision.ts create mode 100644 scripts/flake-validation-results.mjs create mode 100644 scripts/flake-validation-results.test.mjs diff --git a/.github/workflows/e2e-local.yml b/.github/workflows/e2e-local.yml index ed5dc7ff91f..6a11ef927cf 100644 --- a/.github/workflows/e2e-local.yml +++ b/.github/workflows/e2e-local.yml @@ -163,7 +163,9 @@ jobs: env: EVE_E2E_REPETITIONS: ${{ inputs.repetitions }} EVE_E2E_EVAL_IDS: ${{ inputs.evals }} - run: node "$GITHUB_WORKSPACE/scripts/repeat-evals.mjs" + run: | + node --test "$GITHUB_WORKSPACE/scripts/flake-validation-results.test.mjs" + node "$GITHUB_WORKSPACE/scripts/repeat-evals.mjs" - name: Upload repetition evidence if: always() && github.event_name == 'workflow_dispatch' && inputs.repetitions > 1 diff --git a/e2e/fixtures/agent-channels/agent/channels/cross-version.ts b/e2e/fixtures/agent-channels/agent/channels/cross-version.ts index 7b32c2b7edf..d0eb51c3333 100644 --- a/e2e/fixtures/agent-channels/agent/channels/cross-version.ts +++ b/e2e/fixtures/agent-channels/agent/channels/cross-version.ts @@ -1,5 +1,7 @@ import { defineChannel, POST, type Session } from "eve/channels"; +import { checkDeploymentRevision } from "../lib/deployment-revision"; + const smokeTestAuth = { attributes: { source: "smoke-test" }, authenticator: "cross-version-webhook", @@ -29,6 +31,8 @@ type CrossVersionRouteContext = { export default defineChannel({ routes: [ POST("/cross-version-webhook", async (req, ctx) => { + const pendingDeployment = checkDeploymentRevision(req); + if (pendingDeployment !== undefined) return pendingDeployment; const body = (await req.json().catch(() => ({}))) as { message?: string; sessionRef?: string; diff --git a/e2e/fixtures/agent-channels/agent/lib/deployment-revision.ts b/e2e/fixtures/agent-channels/agent/lib/deployment-revision.ts new file mode 100644 index 00000000000..723ba6ba3ba --- /dev/null +++ b/e2e/fixtures/agent-channels/agent/lib/deployment-revision.ts @@ -0,0 +1,17 @@ +export const DEPLOYMENT_REVISION_HEADER = "x-eve-e2e-deployment-revision"; +export const DEPLOYMENT_PENDING_CODE = "e2e_deployment_pending"; + +export function checkDeploymentRevision( + request: Request, + revision = process.env.EVE_E2E_DEPLOYMENT_REVISION, +): Response | undefined { + const expected = request.headers.get(DEPLOYMENT_REVISION_HEADER); + if (expected === null || expected === revision) return undefined; + return Response.json( + { + code: DEPLOYMENT_PENDING_CODE, + error: "The requested deployment is not serving this request yet.", + }, + { status: 409 }, + ); +} diff --git a/e2e/fixtures/agent-channels/evals/custom-channels/cross-version-session-inbox.eval.ts b/e2e/fixtures/agent-channels/evals/custom-channels/cross-version-session-inbox.eval.ts index 1a4bddd89ff..4319c8cafe8 100644 --- a/e2e/fixtures/agent-channels/evals/custom-channels/cross-version-session-inbox.eval.ts +++ b/e2e/fixtures/agent-channels/evals/custom-channels/cross-version-session-inbox.eval.ts @@ -7,7 +7,10 @@ import { defineEval } from "eve/evals"; import type { EveEvalContext } from "eve/evals"; import { satisfies } from "eve/evals/expect"; -import { postChannel } from "./shared"; +import { + DEPLOYMENT_PENDING_CODE, + DEPLOYMENT_REVISION_HEADER, +} from "../../agent/lib/deployment-revision"; const ALIAS_ENV = "EVE_E2E_REDEPLOY_ALIAS"; const OLD_EVE_VERSION = "0.30.8"; @@ -53,6 +56,8 @@ export default defineEval({ } const originalInstructions = await readFile(INSTRUCTIONS_PATH, "utf8"); + const oldRevision = crypto.randomUUID(); + const currentRevision = crypto.randomUUID(); const links = await Promise.all([ snapshotLink(FIXTURE_EVE_LINK), snapshotLink(CONFIG_EVE_LINK), @@ -74,11 +79,11 @@ export default defineEval({ INSTRUCTIONS_PATH, `${originalInstructions}\nDeployment marker: ${OLD_DEPLOYMENT_MARKER}.\n`, ); - await deployToAlias(t, alias, "eve@0.30.8"); + await deployToAlias(t, alias, "eve@0.30.8", oldRevision); await waitForAliasToServe(t, OLD_DEPLOYMENT_MARKER); const sessionRef = crypto.randomUUID(); - const started = await postChannel(t.target, "/cross-version-webhook", { + const started = await postToDeployment(t, oldRevision, { message: ACTIVE_TURN_MESSAGE, sessionRef, }); @@ -104,10 +109,10 @@ export default defineEval({ INSTRUCTIONS_PATH, `${originalInstructions}\nDeployment marker: ${CURRENT_DEPLOYMENT_MARKER}.\n`, ); - await deployToAlias(t, alias, `eve@${currentEveVersion}`); + await deployToAlias(t, alias, `eve@${currentEveVersion}`, currentRevision); await waitForAliasToServe(t, CURRENT_DEPLOYMENT_MARKER); - const replacement = await postChannel(t.target, "/cross-version-webhook", { + const replacement = await postToDeployment(t, currentRevision, { message: `Reply with exactly ${FOLLOW_UP_TOKEN}.`, sessionRef, turnPolicy: "queue", @@ -188,7 +193,12 @@ async function readPackageVersion(packagePath: string): Promise { } /** Builds the fixture and repoints the run-scoped alias at the fresh deployment. */ -async function deployToAlias(t: EveEvalContext, alias: string, phase: string): Promise { +async function deployToAlias( + t: EveEvalContext, + alias: string, + phase: string, + revision: string, +): Promise { await execFileAsync("pnpm", ["exec", "eve", "build"], { ...EXEC_OPTIONS, env: { @@ -204,11 +214,22 @@ async function deployToAlias(t: EveEvalContext, alias: string, phase: string): P process.env.EVE_E2E_MODEL === undefined ? [] : ["--env", `EVE_E2E_MODEL=${process.env.EVE_E2E_MODEL}`]; + const revisionArgs = ["--env", `EVE_E2E_DEPLOYMENT_REVISION=${revision}`]; const scopeArgs = process.env.VERCEL_ORG_ID === undefined ? [] : ["--scope", process.env.VERCEL_ORG_ID]; const deploy = await execFileAsync( "pnpm", - ["exec", "vc", "deploy", "--prebuilt", "--yes", "--target=preview", ...modelArgs, ...tokenArgs], + [ + "exec", + "vc", + "deploy", + "--prebuilt", + "--yes", + "--target=preview", + ...modelArgs, + ...revisionArgs, + ...tokenArgs, + ], EXEC_OPTIONS, ); const deploymentUrl = deploy.stdout.trim().split("\n").at(-1)?.trim(); @@ -224,6 +245,35 @@ async function deployToAlias(t: EveEvalContext, alias: string, phase: string): P ); } +async function postToDeployment( + t: EveEvalContext, + revision: string, + body: unknown, +): Promise { + const deadline = Date.now() + 120_000; + while (Date.now() < deadline) { + const response = await t.target.fetch("/cross-version-webhook", { + method: "POST", + headers: { + "content-type": "application/json", + [DEPLOYMENT_REVISION_HEADER]: revision, + }, + body: JSON.stringify(body), + signal: t.signal, + }); + const result = (await response.json()) as MessageResponse & { code?: string }; + if (response.status === 409 && result.code === DEPLOYMENT_PENDING_CODE) { + await t.sleep(1_000); + continue; + } + if (!response.ok) { + throw new Error(`Cross-version message failed (${response.status}).`); + } + return result; + } + throw new Error("Timed out waiting for the cross-version message's deployment."); +} + /** Waits until the alias exposes the marker from the expected deployment. */ async function waitForAliasToServe(t: EveEvalContext, marker: string): Promise { const deadline = Date.now() + 120_000; diff --git a/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.ts b/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.ts index abb28e5d346..833ae455e39 100644 --- a/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.ts +++ b/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.ts @@ -688,12 +688,19 @@ describe("workflow tools", () => { const asked = await stream.nextTurn(); expect(filterEventsByType(asked, "input.requested")).toHaveLength(1); - const resumed = await stream.nextTurn(); - const outputs = filterEventsByType(resumed, "action.result").map((event) => - JSON.stringify(event.data.result.output), - ); + const outputs: string[] = []; + // A replayed parked boundary may arrive before the deadline result. + for (let attempt = 0; attempt < 5 && outputs.length === 0; attempt += 1) { + const resumed = await stream.nextTurn(); + expect(filterEventsByType(resumed, "turn.failed")).toHaveLength(0); + expect(filterEventsByType(resumed, "session.failed")).toHaveLength(0); + outputs.push( + ...filterEventsByType(resumed, "action.result").map((event) => + JSON.stringify(event.data.result.output), + ), + ); + } expect(outputs.some((output) => output.includes('"decided":"timed out"'))).toBe(true); - expect(filterEventsByType(resumed, "turn.failed")).toHaveLength(0); } finally { stream.dispose(); await run.cancel(); diff --git a/packages/eve/test/scenarios/dev-server-connections.scenario.test.ts b/packages/eve/test/scenarios/dev-server-connections.scenario.test.ts index 28be7c29eb6..9f75c30714f 100644 --- a/packages/eve/test/scenarios/dev-server-connections.scenario.test.ts +++ b/packages/eve/test/scenarios/dev-server-connections.scenario.test.ts @@ -1,6 +1,7 @@ import { Agent } from "node:http"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; +import { inspect } from "node:util"; import { describe, expect, it } from "vitest"; @@ -174,6 +175,8 @@ describe("eve dev server live connections", () => { const candidateStartedPath = join(app.appRoot, ".candidate-started"); const candidateReadyPath = join(app.appRoot, ".candidate-ready"); const streamReleasePath = join(app.appRoot, ".stream-release"); + let stopRevisionPolling = false; + let revisionPoller: Promise | undefined; try { const initialRevision = await readDevelopmentRevision(server.url); @@ -191,13 +194,12 @@ describe("eve dev server live connections", () => { // The parent-owned control route must answer continuously through // candidate preparation and promotion, not merely at spot checks. const revisionFailures: string[] = []; - let stopRevisionPolling = false; - const revisionPoller = (async () => { + revisionPoller = (async () => { while (!stopRevisionPolling) { try { await readDevelopmentRevision(server.url); } catch (error) { - revisionFailures.push(String(error)); + revisionFailures.push(inspect(error, { depth: 4 })); } await new Promise((resolvePromise) => setTimeout(resolvePromise, 50)); } @@ -230,7 +232,9 @@ describe("eve dev server live connections", () => { await expect(reader.read()).resolves.toEqual(expect.objectContaining({ done: true })); expect(hasKnownDevServerFailure(`${server.stdout()}\n${server.stderr()}`)).toBe(false); } finally { + stopRevisionPolling = true; await server.stop(); + await revisionPoller; } }, DEV_SERVER_SCENARIO_TIMEOUT_MS, diff --git a/scripts/flake-validation-results.mjs b/scripts/flake-validation-results.mjs new file mode 100644 index 00000000000..b14a94424c2 --- /dev/null +++ b/scripts/flake-validation-results.mjs @@ -0,0 +1,19 @@ +export function isCleanEvalSummary(summary) { + return ( + summary?.failed === 0 && + summary.scored === 0 && + summary.errored === 0 && + summary.skipped === 0 && + Array.isArray(summary.results) && + summary.results.length > 0 && + summary.passed === summary.results.length && + summary.results.every((result) => result?.verdict === "passed" && result.error === undefined) + ); +} + +export function zeroFailureUpperBound(runs) { + if (!Number.isInteger(runs) || runs < 1) { + throw new Error("A confidence bound requires a positive integer trial count."); + } + return -Math.expm1(Math.log(0.05) / runs); +} diff --git a/scripts/flake-validation-results.test.mjs b/scripts/flake-validation-results.test.mjs new file mode 100644 index 00000000000..ae3beb36b7b --- /dev/null +++ b/scripts/flake-validation-results.test.mjs @@ -0,0 +1,74 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { isCleanEvalSummary, zeroFailureUpperBound } from "./flake-validation-results.mjs"; +import { + checkDeploymentRevision, + DEPLOYMENT_PENDING_CODE, + DEPLOYMENT_REVISION_HEADER, +} from "../e2e/fixtures/agent-channels/agent/lib/deployment-revision.ts"; + +const clean = { + failed: 0, + scored: 0, + errored: 0, + skipped: 0, + passed: 1, + results: [{ id: "example", verdict: "passed" }], +}; + +test("accepts the CLI summary shape, which has results rather than totalEvals", () => { + assert.equal(isCleanEvalSummary(clean), true); +}); + +for (const field of ["failed", "scored", "errored", "skipped"]) { + test(`rejects ${field} evals`, () => { + assert.equal(isCleanEvalSummary({ ...clean, [field]: 1 }), false); + }); +} + +test("rejects missing, empty, and inconsistent result sets", () => { + for (const summary of [ + undefined, + null, + {}, + { ...clean, results: undefined, totalEvals: 1 }, + { ...clean, passed: 0, results: [] }, + { ...clean, passed: 2 }, + { ...clean, results: [{ verdict: "failed" }] }, + { ...clean, results: [{ verdict: "passed", error: "unexpected error" }] }, + ]) { + assert.equal(isCleanEvalSummary(summary), false); + } +}); + +test("computes the exact one-sided 95% zero-failure binomial bound", () => { + assert.equal(zeroFailureUpperBound(1), 0.95); + const bound = zeroFailureUpperBound(500); + assert.ok(bound > 0.0059 && bound < 0.006); + assert.ok(Math.abs((1 - bound) ** 500 - 0.05) < 1e-12); + assert.ok(zeroFailureUpperBound(20) > bound); +}); + +test("does not report confidence without valid trials", () => { + for (const runs of [0, -1, 0.5, NaN, Infinity]) { + assert.throws(() => zeroFailureUpperBound(runs), /positive integer/); + } +}); + +test("rejects stale deployment requests before dispatch without changing unpinned traffic", async () => { + const request = new Request("https://fixture.example/cross-version-webhook", { + headers: { [DEPLOYMENT_REVISION_HEADER]: "requested-revision" }, + }); + const rejected = checkDeploymentRevision(request, "previous-revision"); + assert.equal(rejected.status, 409); + assert.equal((await rejected.json()).code, DEPLOYMENT_PENDING_CODE); + assert.equal(checkDeploymentRevision(request, "requested-revision"), undefined); + assert.equal( + checkDeploymentRevision( + new Request("https://fixture.example/cross-version-webhook"), + "previous-revision", + ), + undefined, + ); +}); diff --git a/scripts/repeat-evals.mjs b/scripts/repeat-evals.mjs index 2fa6c82c605..542125afb74 100644 --- a/scripts/repeat-evals.mjs +++ b/scripts/repeat-evals.mjs @@ -4,6 +4,8 @@ import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; +import { isCleanEvalSummary, zeroFailureUpperBound } from "./flake-validation-results.mjs"; + const repetitions = Number(process.env.EVE_E2E_REPETITIONS); if (!Number.isInteger(repetitions) || repetitions < 1 || repetitions > 500) { throw new Error("EVE_E2E_REPETITIONS must be an integer from 1 to 500."); @@ -50,13 +52,7 @@ for (let index = 0; index < repetitions; index += 1) { } catch { error ??= "The eval process did not produce valid JSON."; } - const passed = - error === undefined && - summary?.failed === 0 && - summary?.errored === 0 && - summary?.skipped === 0 && - summary?.totalEvals > 0 && - summary.passed + summary.scored === summary.totalEvals; + const passed = error === undefined && isCleanEvalSummary(summary); trials.push({ repetition: index + 1, passed, error, summary }); await writeFile(resolve(directory, "trials.json"), JSON.stringify(trials, null, 2) + "\n"); console.log( @@ -74,7 +70,7 @@ const report = { repetitions, failures, confidence: 0.95, - oneSidedFailureRateUpperBound: failures === 0 ? -Math.expm1(Math.log(0.05) / repetitions) : null, + oneSidedFailureRateUpperBound: failures === 0 ? zeroFailureUpperBound(repetitions) : null, scope: "Selected evals in fresh CLI processes, with every failure retained; assumes independent, stationary trials.", }; diff --git a/scripts/repeat-tests.mjs b/scripts/repeat-tests.mjs index 28f7b34d391..bdc5f218787 100644 --- a/scripts/repeat-tests.mjs +++ b/scripts/repeat-tests.mjs @@ -4,6 +4,8 @@ import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; +import { zeroFailureUpperBound } from "./flake-validation-results.mjs"; + const exec = promisify(execFile); const [runsText, output, ...args] = process.argv.slice(2); const runs = Number(runsText); @@ -78,7 +80,7 @@ const summary = { failures, passedTests: trials.reduce((total, trial) => total + trial.testCount, 0), confidence: 0.95, - oneSidedFailureRateUpperBound: failures === 0 ? -Math.expm1(Math.log(0.05) / runs) : null, + oneSidedFailureRateUpperBound: failures === 0 ? zeroFailureUpperBound(runs) : null, scope: "Repeatability of the selected tests under independent, stationary trials; not the failure rate of all CI jobs.", }; From 65b0f8ed865a8ecc2fecd989dab996fba2c09c15 Mon Sep 17 00:00:00 2001 From: Chad Hietala Date: Tue, 8 Sep 2026 22:28:02 -0400 Subject: [PATCH 3/8] test(eve): separate compaction pressure from summary context Signed-off-by: Chad Hietala --- .../agent/channels/cross-version.ts | 3 ++- .../cross-version-session-inbox.eval.ts | 20 +++++++++++++------ .../agent/agent.ts | 8 +++++++- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/e2e/fixtures/agent-channels/agent/channels/cross-version.ts b/e2e/fixtures/agent-channels/agent/channels/cross-version.ts index d0eb51c3333..58f71799b17 100644 --- a/e2e/fixtures/agent-channels/agent/channels/cross-version.ts +++ b/e2e/fixtures/agent-channels/agent/channels/cross-version.ts @@ -1,4 +1,5 @@ import { defineChannel, POST, type Session } from "eve/channels"; +import evePackage from "eve/package.json" with { type: "json" }; import { checkDeploymentRevision } from "../lib/deployment-revision"; @@ -57,7 +58,7 @@ export default defineChannel({ throw new Error("Expected the route context to expose from() or send()."); } - return Response.json({ ok: true, sessionId: session.id }); + return Response.json({ ok: true, sessionId: session.id, eveVersion: evePackage.version }); }), ], }); diff --git a/e2e/fixtures/agent-channels/evals/custom-channels/cross-version-session-inbox.eval.ts b/e2e/fixtures/agent-channels/evals/custom-channels/cross-version-session-inbox.eval.ts index 4319c8cafe8..58de02246d7 100644 --- a/e2e/fixtures/agent-channels/evals/custom-channels/cross-version-session-inbox.eval.ts +++ b/e2e/fixtures/agent-channels/evals/custom-channels/cross-version-session-inbox.eval.ts @@ -29,7 +29,7 @@ const TOOL_NAME = "wait-for-cancellation"; const execFileAsync = promisify(execFile); const EXEC_OPTIONS = { maxBuffer: 64 * 1024 * 1024 } as const; -type MessageResponse = { ok: boolean; sessionId?: string }; +type MessageResponse = { ok: boolean; sessionId?: string; eveVersion: string }; /** * Proves the complete mixed-version codec boundary through the real Workflow @@ -79,7 +79,7 @@ export default defineEval({ INSTRUCTIONS_PATH, `${originalInstructions}\nDeployment marker: ${OLD_DEPLOYMENT_MARKER}.\n`, ); - await deployToAlias(t, alias, "eve@0.30.8", oldRevision); + await deployToAlias(t, alias, "eve@0.30.8", oldRevision, oldEvePackage); await waitForAliasToServe(t, OLD_DEPLOYMENT_MARKER); const sessionRef = crypto.randomUUID(); @@ -90,7 +90,10 @@ export default defineEval({ await t.require( started, satisfies( - (value: MessageResponse) => value.ok === true && typeof value.sessionId === "string", + (value: MessageResponse) => + value.ok === true && + typeof value.sessionId === "string" && + value.eveVersion === OLD_EVE_VERSION, "eve@0.30.8 starts the durable session", ), ); @@ -109,7 +112,7 @@ export default defineEval({ INSTRUCTIONS_PATH, `${originalInstructions}\nDeployment marker: ${CURRENT_DEPLOYMENT_MARKER}.\n`, ); - await deployToAlias(t, alias, `eve@${currentEveVersion}`, currentRevision); + await deployToAlias(t, alias, `eve@${currentEveVersion}`, currentRevision, currentEvePackage); await waitForAliasToServe(t, CURRENT_DEPLOYMENT_MARKER); const replacement = await postToDeployment(t, currentRevision, { @@ -120,7 +123,10 @@ export default defineEval({ await t.require( replacement, satisfies( - (value: MessageResponse) => value.ok === true && value.sessionId === sessionId, + (value: MessageResponse) => + value.ok === true && + value.sessionId === sessionId && + value.eveVersion === currentEveVersion, "the current producer targets the existing eve@0.30.8 session", ), ); @@ -198,8 +204,10 @@ async function deployToAlias( alias: string, phase: string, revision: string, + evePackagePath: string, ): Promise { - await execFileAsync("pnpm", ["exec", "eve", "build"], { + // pnpm's dependency synchronization can repair the intentionally replaced links. + await execFileAsync(process.execPath, [resolve(evePackagePath, "bin", "eve.js"), "build"], { ...EXEC_OPTIONS, env: { ...process.env, diff --git a/e2e/fixtures/agent-compaction-regressions/agent/agent.ts b/e2e/fixtures/agent-compaction-regressions/agent/agent.ts index 1b56229ea87..b5ef6a64f9c 100644 --- a/e2e/fixtures/agent-compaction-regressions/agent/agent.ts +++ b/e2e/fixtures/agent-compaction-regressions/agent/agent.ts @@ -15,6 +15,9 @@ import { } from "../constants"; const TEST_CONTEXT_WINDOW_TOKENS = 32_000; +// Scripted usage supplies pressure without starving the real summarizer's transcript. +const TEST_COMPACTION_THRESHOLD_TOKENS = 2_048; +const COMPACTION_PRESSURE_USAGE = { inputTokens: 4_096 }; const MAX_TOOL_CALLS = 10; type RegressionCase = @@ -186,6 +189,7 @@ const taskModel = mockModel({ checkpointAdvanceCallCounts.set(regressionCase, advanceCalls + 1); return { + usage: COMPACTION_PRESSURE_USAGE, toolCalls: [ { id: `advance-checkpoint-${advanceCalls + 1}`, @@ -206,6 +210,7 @@ const taskModel = mockModel({ return regressionCase === "redundant-tool-calls" ? { + usage: COMPACTION_PRESSURE_USAGE, toolCalls: [ { id: `inspect-repository-${attempt}`, @@ -215,6 +220,7 @@ const taskModel = mockModel({ ], } : { + usage: COMPACTION_PRESSURE_USAGE, toolCalls: [ { id: `perform-source-analysis-${attempt}`, @@ -235,7 +241,7 @@ export default defineAgent({ compaction: { model: e2eModel(), modelContextWindowTokens: TEST_CONTEXT_WINDOW_TOKENS, - thresholdPercent: 0.02, + thresholdPercent: TEST_COMPACTION_THRESHOLD_TOKENS / TEST_CONTEXT_WINDOW_TOKENS, }, limits: { maxInputTokensPerSession: 100_000, From f342c4ae6ad2dd827d320bd49f7bd86433b1d5b7 Mon Sep 17 00:00:00 2001 From: Chad Hietala Date: Wed, 9 Sep 2026 09:38:56 -0400 Subject: [PATCH 4/8] fix(eve): resolve test isolation and compaction flakes Reset test-owned mock and renderer state, isolate fixture dependencies, and refresh npm metadata during scenario installs. Preserve task instructions through compaction and make principal-forwarding fixture dispatch deterministic while retaining authorization checks. Validated unit, integration, and scenario suites locally, plus repeated targeted selections with retries disabled. Remote E2E validation remains pending on this commit. Signed-off-by: Chad Hietala --- .changeset/quiet-workflow-reads.md | 2 + .../agent/agent.ts | 17 ++--- e2e/fixtures/agent-subagents/agent/agent.ts | 66 ++++++++++++++++--- e2e/fixtures/agent-subagents/constants.ts | 1 + .../evals/remote-principal-forwarding.eval.ts | 6 +- .../dev/tui/prompt-command-handler.test.ts | 13 ++++ .../eve/src/cli/telemetry/preference.test.ts | 39 +++++++++-- .../execution/parked-delivery-wait.test.ts | 12 ++-- .../eve/src/execution/workflow-steps.test.ts | 24 ++++--- packages/eve/src/harness/compaction.test.ts | 32 +++++++++ packages/eve/src/harness/compaction.ts | 27 ++++++-- packages/eve/src/harness/tool-loop.test.ts | 4 +- .../create-application-nitro.scenario.test.ts | 21 ++++-- .../host/start-development-server.test.ts | 16 ++--- .../mocks/mock-skill.integration.test.ts | 25 +++---- .../eve/src/internal/testing/scenario-app.ts | 3 +- .../testing/scenario-apps/next-eve-proxy.ts | 3 +- packages/eve/src/react/use-eve-agent.test.ts | 22 ++++++- .../package-consumption.scenario.test.ts | 8 +-- scripts/repeat-tests.mjs | 5 +- 20 files changed, 263 insertions(+), 83 deletions(-) diff --git a/.changeset/quiet-workflow-reads.md b/.changeset/quiet-workflow-reads.md index 299e7865197..e1250f0b74f 100644 --- a/.changeset/quiet-workflow-reads.md +++ b/.changeset/quiet-workflow-reads.md @@ -3,3 +3,5 @@ --- Retry transient Windows file-sharing errors when reading local workflow state, matching the bounded retries already used for writes. Persistent access errors and malformed state still fail. + +Preserve a folded-away task that fits the compaction budget when recent tool results survive summarization. diff --git a/e2e/fixtures/agent-compaction-regressions/agent/agent.ts b/e2e/fixtures/agent-compaction-regressions/agent/agent.ts index b5ef6a64f9c..b281013da5a 100644 --- a/e2e/fixtures/agent-compaction-regressions/agent/agent.ts +++ b/e2e/fixtures/agent-compaction-regressions/agent/agent.ts @@ -173,12 +173,10 @@ const taskModel = mockModel({ // These are fixture markers, not compaction protocol fields. `marker` records the // regression work tool; `SECOND_CHECKPOINT_MARKER` records the test-only tool // whose output makes the harness cross the compaction threshold a second time. - // Completion evidence is detected in any assistant message: compaction may - // leave it as a summarization checkpoint or as an eviction trail line, and - // the model must not repeat work in either case. User messages are - // excluded because the eval instructions themselves quote the markers. - if (assistantEvidenceContains(request.messages, marker)) { - if (assistantEvidenceContains(request.messages, SECOND_CHECKPOINT_MARKER)) { + // Kept tool results, checkpoints, and eviction trails all prove completed work. + // User messages are excluded because the eval instructions quote the markers. + if (completionEvidenceContains(request.messages, marker)) { + if (completionEvidenceContains(request.messages, SECOND_CHECKPOINT_MARKER)) { return `Done: ${marker}; ${SECOND_CHECKPOINT_MARKER}`; } @@ -273,9 +271,12 @@ function completionMarker( : "SOURCE_ANALYSIS_COMPLETE"; } -function assistantEvidenceContains( +function completionEvidenceContains( messages: MockModelRequest["messages"], marker: string, ): boolean { - return messages.some((message) => message.role === "assistant" && message.text.includes(marker)); + return messages.some( + (message) => + (message.role === "assistant" || message.role === "tool") && message.text.includes(marker), + ); } diff --git a/e2e/fixtures/agent-subagents/agent/agent.ts b/e2e/fixtures/agent-subagents/agent/agent.ts index 62583a4ca6e..04f6c4d046b 100644 --- a/e2e/fixtures/agent-subagents/agent/agent.ts +++ b/e2e/fixtures/agent-subagents/agent/agent.ts @@ -2,7 +2,7 @@ import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent, defineDynamic } from "eve"; import { mockModel } from "eve/evals"; -import { WORKSPACE_LOOKUP_MESSAGE } from "../constants"; +import { WORKSPACE_FORWARDING_MARKER, WORKSPACE_LOOKUP_MESSAGE } from "../constants"; if (process.env.EVE_E2E_MODEL === "mock") { process.env.EVE_MOCK_AUTHORED_MODELS = "1"; @@ -20,24 +20,70 @@ const workspaceReader = mockModel({ return { toolCalls: [{ name: "read-workspace-label", input: {} }] }; }, }); +const workspaceDispatcher = mockModel({ + modelId: "principal-forwarding-workspace-dispatcher", + respond(request) { + let requestIndex = -1; + for (const [index, message] of request.messages.entries()) { + if (message.role === "user" && message.text.includes(WORKSPACE_FORWARDING_MARKER)) { + requestIndex = index; + } + } + if ( + requestIndex < 0 || + request.messages.slice(requestIndex + 1).some((message) => message.role === "tool") + ) { + return "The workspace lookup was submitted."; + } + const previous = [...request.toolResults] + .reverse() + .find((result) => result.name === "remote-loopback")?.output; + const agentId = + previous !== null && + typeof previous === "object" && + "agentId" in previous && + typeof previous.agentId === "string" + ? previous.agentId + : undefined; + const requestCount = request.messages.filter( + (message) => message.role === "user" && message.text.includes(WORKSPACE_FORWARDING_MARKER), + ).length; + if (requestCount > 1 && agentId === undefined) { + throw new Error("Workspace continuation has no existing remote agent receipt."); + } + return { + toolCalls: [ + { + id: `workspace-lookup-${requestCount}`, + name: "remote-loopback", + input: { agentId, message: WORKSPACE_LOOKUP_MESSAGE }, + }, + ], + }; + }, +}); export default defineAgent({ ...agentConfig, model: defineDynamic({ events: { "step.started": (_event, ctx) => { - // Authorization must reach the real tool even if a provider declines the lookup. - const isWorkspaceReader = ctx.messages.some((message) => { - if (message.role !== "user") return false; - const text = + const messages = ctx.messages.flatMap((message) => { + if (message.role !== "user") return []; + return [ typeof message.content === "string" ? message.content - : message.content.map((part) => (part.type === "text" ? part.text : "")).join(""); - return text === WORKSPACE_LOOKUP_MESSAGE; + : message.content.map((part) => (part.type === "text" ? part.text : "")).join(""), + ]; }); - return isWorkspaceReader - ? { model: workspaceReader, modelContextWindowTokens: 1_000_000 } - : { model, modelContextWindowTokens }; + // Both models must reach the real authorization boundary, including denied lookups. + if (messages.includes(WORKSPACE_LOOKUP_MESSAGE)) { + return { model: workspaceReader, modelContextWindowTokens: 1_000_000 }; + } + if (messages.some((message) => message.includes(WORKSPACE_FORWARDING_MARKER))) { + return { model: workspaceDispatcher, modelContextWindowTokens: 1_000_000 }; + } + return { model, modelContextWindowTokens }; }, }, }), diff --git a/e2e/fixtures/agent-subagents/constants.ts b/e2e/fixtures/agent-subagents/constants.ts index b7004745b5a..29b7cbf0a5e 100644 --- a/e2e/fixtures/agent-subagents/constants.ts +++ b/e2e/fixtures/agent-subagents/constants.ts @@ -3,3 +3,4 @@ export const WORKSPACE_LOOKUP_MESSAGE = [ "Make a fresh lookup: earlier answers in this session may belong to a different caller.", "Report the returned name, or explain if access is denied.", ].join(" "); +export const WORKSPACE_FORWARDING_MARKER = "Shared workspace membership check."; diff --git a/e2e/fixtures/agent-subagents/evals/remote-principal-forwarding.eval.ts b/e2e/fixtures/agent-subagents/evals/remote-principal-forwarding.eval.ts index 660652d9a07..f73798cc216 100644 --- a/e2e/fixtures/agent-subagents/evals/remote-principal-forwarding.eval.ts +++ b/e2e/fixtures/agent-subagents/evals/remote-principal-forwarding.eval.ts @@ -7,17 +7,19 @@ import { } from "eve/evals"; import { satisfies } from "eve/evals/expect"; -import { WORKSPACE_LOOKUP_MESSAGE } from "../constants"; +import { WORKSPACE_FORWARDING_MARKER, WORKSPACE_LOOKUP_MESSAGE } from "../constants"; const ALICE_WORKSPACE_LABEL = "Maple Studio"; const BOB_WORKSPACE_LABEL = "Cedar Workshop"; const BOB_AUTHORIZATION = "Bearer e2e-workspace-label-bob"; const OBSERVER_AUTHORIZATION = "Bearer e2e-workspace-label-observer"; const CREATE_CHILD_MESSAGE = [ + WORKSPACE_FORWARDING_MARKER, "Use remote-loopback with this message:", JSON.stringify(WORKSPACE_LOOKUP_MESSAGE), ].join(" "); const CONTINUE_CHILD_MESSAGE = [ + WORKSPACE_FORWARDING_MARKER, "A different user is making this request now.", "Continue that same remote-loopback agent using its agentId with this message:", JSON.stringify(WORKSPACE_LOOKUP_MESSAGE), @@ -30,7 +32,7 @@ const CLARIFICATION = [ /** Three users resume one remote child; each tool call resolves only its current caller's workspace membership. */ export default defineEval({ - tags: ["principal-forwarding", "real-model"], + tags: ["principal-forwarding"], description: "A persistent remote child switches between two workspace memberships and denies a third caller with none.", async test(t) { diff --git a/packages/eve/src/cli/dev/tui/prompt-command-handler.test.ts b/packages/eve/src/cli/dev/tui/prompt-command-handler.test.ts index 13517bb7501..e3ba04c8e73 100644 --- a/packages/eve/src/cli/dev/tui/prompt-command-handler.test.ts +++ b/packages/eve/src/cli/dev/tui/prompt-command-handler.test.ts @@ -6,6 +6,19 @@ import type { RemoteAuthCompletion, RemoteConnectionController } from "./remote- import type { AgentTUIRenderer, PromptCommandHandlerContext } from "./runner.js"; import type { SetupFlowRenderer } from "./setup-flow.js"; +vi.mock("#setup/flows/model.js", () => ({ + modelChangeRefusalForUneditableModel: vi.fn(), +})); +vi.mock("#compiler/model-catalog.js", () => ({ + createCompiledRuntimeModelCatalogLoader: vi.fn(), +})); +vi.mock("#discover/discover-agent.js", () => ({ + discoverAgent: vi.fn(), +})); +vi.mock("#source-change/static-source-change.js", () => ({ + createStaticSourceChange: vi.fn(), +})); + const APP_ROOT = "/tmp/weather-agent"; const LOCAL_TARGET = { kind: "local", diff --git a/packages/eve/src/cli/telemetry/preference.test.ts b/packages/eve/src/cli/telemetry/preference.test.ts index b47bb8eebb3..25f248466b6 100644 --- a/packages/eve/src/cli/telemetry/preference.test.ts +++ b/packages/eve/src/cli/telemetry/preference.test.ts @@ -19,7 +19,10 @@ import { setEveTelemetryEnabled, } from "#cli/telemetry/preference.js"; +const originalPlatform = process.platform; + afterEach(() => { + Object.defineProperty(process, "platform", { value: originalPlatform }); vi.unstubAllEnvs(); vi.clearAllMocks(); }); @@ -65,6 +68,7 @@ describe("eve telemetry preference", () => { }); it("ignores a relative XDG config home", async () => { + Object.defineProperty(process, "platform", { value: "linux" }); vi.stubEnv("XDG_CONFIG_HOME", "relative-config"); vi.mocked(readFile).mockRejectedValue(new Error("missing")); @@ -74,6 +78,9 @@ describe("eve telemetry preference", () => { }); it("merges telemetry changes into the eve config atomically", async () => { + Object.defineProperty(process, "platform", { value: "linux" }); + const directory = join("/eve-config", "eve"); + const temporaryPrefix = `${join(directory, "config.json")}.`; vi.stubEnv("XDG_CONFIG_HOME", "/eve-config"); vi.mocked(readFile) .mockResolvedValueOnce('{"other":"preserved","telemetry":{"future":"preserved"}}') @@ -84,37 +91,57 @@ describe("eve telemetry preference", () => { await setEveTelemetryEnabled(false); await markEveTelemetryNotified(); - expect(mkdir).toHaveBeenCalledWith("/eve-config/eve", { recursive: true }); + expect(mkdir).toHaveBeenCalledWith(directory, { recursive: true }); expect(writeFile).toHaveBeenNthCalledWith( 1, - expect.stringMatching(/^\/eve-config\/eve\/config\.json\./), + expect.stringContaining(temporaryPrefix), expect.stringContaining('"other": "preserved"'), { mode: 0o600 }, ); expect(writeFile).toHaveBeenNthCalledWith( 2, - expect.stringMatching(/^\/eve-config\/eve\/config\.json\./), + expect.stringContaining(temporaryPrefix), expect.stringContaining('"future": "preserved"'), { mode: 0o600 }, ); expect(writeFile).toHaveBeenNthCalledWith( 2, - expect.stringMatching(/^\/eve-config\/eve\/config\.json\./), + expect.stringContaining(temporaryPrefix), expect.stringContaining('"enabled": false'), { mode: 0o600 }, ); expect(writeFile).toHaveBeenNthCalledWith( 2, - expect.stringMatching(/^\/eve-config\/eve\/config\.json\./), + expect.stringContaining(temporaryPrefix), expect.stringContaining('"noticeVersion": 1'), { mode: 0o600 }, ); expect(writeFile).toHaveBeenNthCalledWith( 2, - expect.stringMatching(/^\/eve-config\/eve\/config\.json\./), + expect.stringContaining(temporaryPrefix), expect.stringContaining('"notifiedAt"'), { mode: 0o600 }, ); expect(rename).toHaveBeenCalledTimes(2); + expect(rename).toHaveBeenCalledWith( + expect.stringContaining(temporaryPrefix), + join(directory, "config.json"), + ); + }); + + it.each([ + ["darwin", join(homedir(), "Library", "Preferences", "eve")], + ["win32", join(homedir(), "AppData", "Roaming", "eve")], + ["linux", join(homedir(), ".config", "eve")], + ])("uses the native config directory on %s", async (platform, directory) => { + Object.defineProperty(process, "platform", { value: platform }); + vi.stubEnv("XDG_CONFIG_HOME", ""); + vi.stubEnv("APPDATA", ""); + vi.mocked(readFile).mockRejectedValue(new Error("missing")); + + await setEveTelemetryEnabled(false); + + expect(mkdir).toHaveBeenCalledWith(directory, { recursive: true }); + expect(rename).toHaveBeenCalledWith(expect.any(String), join(directory, "config.json")); }); }); diff --git a/packages/eve/src/execution/parked-delivery-wait.test.ts b/packages/eve/src/execution/parked-delivery-wait.test.ts index 943a3ff1aea..35233884792 100644 --- a/packages/eve/src/execution/parked-delivery-wait.test.ts +++ b/packages/eve/src/execution/parked-delivery-wait.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; import type { DeliverHookPayload } from "#channel/types.js"; import { nextTurnDelivery } from "#execution/parked-delivery-wait.js"; @@ -20,6 +20,11 @@ vi.mock("./cancel-indexed-session-tasks-step.js", () => ({ import { cancelAllIndexedSessionTasksStep } from "#execution/cancel-indexed-session-tasks-step.js"; +beforeEach(() => { + vi.mocked(routeDeliverToChildren).mockReset(); + vi.mocked(cancelAllIndexedSessionTasksStep).mockReset(); +}); + interface ScriptedRead { readonly result: IteratorResult; readonly source: SessionInboxSource; @@ -112,10 +117,6 @@ function waitInput(inbox: SessionCommandInbox): Parameters { - afterEach(() => { - vi.mocked(routeDeliverToChildren).mockReset(); - }); - it("surfaces an authorization callback as its own instruction", async () => { const inbox = createMockInbox([authorizationRead()]); @@ -234,7 +235,6 @@ describe("nextTurnDelivery", () => { }); describe("nextTurnDelivery routing", () => { - beforeEach(() => vi.clearAllMocks()); it("keeps waiting instead of starting a parent turn for a fully routed task response", async () => { const sessionState = { continuationToken: "token", diff --git a/packages/eve/src/execution/workflow-steps.test.ts b/packages/eve/src/execution/workflow-steps.test.ts index 40cf56b3b77..03980b8e358 100644 --- a/packages/eve/src/execution/workflow-steps.test.ts +++ b/packages/eve/src/execution/workflow-steps.test.ts @@ -1,5 +1,5 @@ import type { ModelMessage } from "ai"; -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { ChannelAdapter, ChannelAdapterContext } from "#channel/adapter.js"; import type { DeliverPayload, @@ -228,7 +228,16 @@ function createSerializedContext( ): Record { const ctx = new ContextContainer(); ctx.set(AuthKey, null); - ctx.set(BundleKey, { + ctx.set(BundleKey, createStubBundle()); + ctx.set(ChannelKey, threadContextAdapter); + ctx.set(ContinuationTokenKey, "http:thread-context"); + ctx.set(ModeKey, mode); + ctx.set(SessionIdKey, "session-1"); + return serializeContext(ctx); +} + +function createStubBundle(): Awaited> { + return { adapterRegistry: { adaptersByKind: new Map([[threadContextAdapter.kind, threadContextAdapter]]), }, @@ -245,14 +254,13 @@ function createSerializedContext( subagentRegistry: {}, toolRegistry: {}, turnAgent: TestTurnAgent, - } as never); - ctx.set(ChannelKey, threadContextAdapter); - ctx.set(ContinuationTokenKey, "http:thread-context"); - ctx.set(ModeKey, mode); - ctx.set(SessionIdKey, "session-1"); - return serializeContext(ctx); + } as never; } +beforeEach(() => { + vi.mocked(getCompiledRuntimeAgentBundle).mockReset().mockResolvedValue(createStubBundle()); +}); + afterEach(() => { getRunMock.mockReset(); resumeHookMock.mockReset(); diff --git a/packages/eve/src/harness/compaction.test.ts b/packages/eve/src/harness/compaction.test.ts index 2939ef8a5cd..51775e82ff8 100644 --- a/packages/eve/src/harness/compaction.test.ts +++ b/packages/eve/src/harness/compaction.test.ts @@ -765,6 +765,38 @@ describe("compactMessages: summarization fallback", () => { expect(result.slice(2, 4)).toEqual([recentCall, recentResult]); }); + it("replays a folded-away task when recent tool results survive", async () => { + const task = user("Keep the original request and its final task sentinel."); + const [call, resultMsg] = toolExchange({ callId: "call-1", payloadChars: 100 }); + const { result, summarizer } = await compact( + [user("old notes ".repeat(4_000)), task, call, resultMsg], + { recentWindowSize: 2, threshold: 2_048 }, + ); + + expect(summarizer).toHaveBeenCalled(); + expect(result).toContainEqual(call); + expect(result).toContainEqual(resultMsg); + expect(result.at(-1)).toEqual(task); + }); + + it("makes room for the original task before keeping a large tool-result tail", async () => { + const task = user("Keep these requirements. ".repeat(25)); + const [call, resultMsg] = toolExchange({ callId: "call-1", payloadChars: 600 }); + const summary = "s".repeat(2_800); + const head = checkpointHead(summary); + const threshold = Math.floor( + (estimateTokens([...head, task]) + estimateTokens([...head, call, resultMsg, task])) / 2, + ); + const { result } = await compact([user("old notes ".repeat(4_000)), task, call, resultMsg], { + recentWindowSize: 2, + summary, + threshold, + }); + + expect(result.at(-1)).toEqual(task); + expect(estimateTokens(result)).toBeLessThanOrEqual(threshold); + }); + it("strips tool activity from the tail when verbatim does not fit but text does", async () => { const oldProse = user("investigation notes ".repeat(2_000)); const [recentCall, recentResult] = toolExchange({ diff --git a/packages/eve/src/harness/compaction.ts b/packages/eve/src/harness/compaction.ts index fe83ebf0620..4e1323b4183 100644 --- a/packages/eve/src/harness/compaction.ts +++ b/packages/eve/src/harness/compaction.ts @@ -141,6 +141,7 @@ function toolResultCapHeuristic(input: CompactionHeuristicInput): CompactionHeur const capped = withResumptionGuard( [...checkpointHead, ...capToolResults(input.older), ...input.recent], input.conversation, + input.config.threshold, ); // Evaluate on the same ruler shouldCompact uses (envelope included): @@ -260,7 +261,11 @@ export async function compactMessages( // Prefer keeping the recent tail verbatim — surviving tool results are the // model's evidence that work already ran. Degrade to text-only, then to a // smaller window, only under threshold pressure. - const verbatim = withResumptionGuard([...summaryHead, ...recent], conversation); + const verbatim = withResumptionGuard( + [...summaryHead, ...recent], + conversation, + config.threshold, + ); if (evaluateThreshold(verbatim, config, "estimate").type === "within-limit") { return verbatim; } @@ -268,6 +273,7 @@ export async function compactMessages( const stripped = withResumptionGuard( [...summaryHead, ...keepNonToolResultMessages(recent)], conversation, + config.threshold, ); if (evaluateThreshold(stripped, config, "estimate").type === "within-limit" || keep === 0) { return stripped; @@ -338,17 +344,28 @@ function capToolResults(messages: readonly ModelMessage[]): ModelMessage[] { function withResumptionGuard( messages: ModelMessage[], conversation: readonly ModelMessage[], + threshold: number, ): ModelMessage[] { const lastRole = messages.at(-1)?.role; - if (lastRole !== undefined && lastRole !== "assistant") { - return messages; - } - const replay = findLastRealUserMessage(conversation); const alreadyKept = replay !== undefined && messages.some((message) => message.role === "user" && message.content === replay.content); + if (lastRole !== undefined && lastRole !== "assistant") { + // A retained tool tail must not displace a task that can fit in the budget. + // Including it here lets tail selection make room before accepting a candidate. + if ( + lastRole === "tool" && + replay !== undefined && + !alreadyKept && + estimateTokens([replay]) <= threshold + ) { + return [...messages, replay]; + } + return messages; + } + return [ ...messages, replay !== undefined && !alreadyKept diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.ts index b0a16838026..7e55a83bf40 100644 --- a/packages/eve/src/harness/tool-loop.test.ts +++ b/packages/eve/src/harness/tool-loop.test.ts @@ -240,9 +240,11 @@ vi.mock("./compaction.js", () => ({ afterEach(() => { vi.clearAllMocks(); + vi.mocked(shouldCompact).mockReset().mockReturnValue(false); + vi.mocked(compactMessages).mockReset(); vi.unstubAllEnvs(); declareTelemetry(undefined); - mockGetRegisteredTelemetryIntegrations.mockReturnValue([]); + mockGetRegisteredTelemetryIntegrations.mockReset().mockReturnValue([]); }); function createTestSession(overrides?: Partial): HarnessSession { diff --git a/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.ts b/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.ts index c9b8e77ccd8..78f3bb88442 100644 --- a/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.ts +++ b/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.ts @@ -1,4 +1,4 @@ -import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -29,12 +29,14 @@ import { EVE_WORKFLOW_FLOW_ROUTE_PATH, } from "#internal/nitro/host/vercel-build-output-config.js"; import { applyWorkflowTransform } from "#internal/workflow-bundle/workflow-builders.js"; +import { useTemporaryDirectories } from "#internal/testing/use-temporary-app-roots.js"; import { defineChannel, WS } from "#public/definitions/channel.js"; const configureDevelopmentNitroRoutes = vi.fn(async () => undefined); const configureProductionNitroRoutes = vi.fn(async () => undefined); const createNitroMock = vi.fn(); const registerScheduleTaskHandlers = vi.fn(); +const createTemporaryDirectory = useTemporaryDirectories(); vi.mock("nitro/builder", () => ({ createNitro: createNitroMock, @@ -198,6 +200,7 @@ describe("application Nitro creation", () => { beforeEach(() => { vi.resetModules(); vi.clearAllMocks(); + createNitroMock.mockReset(); }); afterEach(() => { @@ -582,6 +585,16 @@ describe("application Nitro creation", () => { const { createProductionApplicationNitro } = await import("#internal/nitro/host/create-application-nitro.js"); const preparedHost = await createPreparedHost(); + const sourceRoot = await realpath(await createTemporaryDirectory("eve-extension-trace-deps-")); + for (const name of ["zod", "sharp"]) { + const packageRoot = join(sourceRoot, "node_modules", name); + await mkdir(join(packageRoot, "lib"), { recursive: true }); + await writeFile( + join(packageRoot, "package.json"), + JSON.stringify({ name, version: "1.0.0", main: "./lib/index.js" }), + ); + await writeFile(join(packageRoot, "lib", "index.js"), "module.exports = {};\n"); + } preparedHost.compileResult.manifest.extensionMounts = [ { externalDependencies: ["zod", "sharp"], @@ -590,7 +603,7 @@ describe("application Nitro creation", () => { namespace: "layout", packageName: "layout-extension", packageNamespace: "layout-extension", - sourceRoot: process.cwd(), + sourceRoot, }, ]; preparedHost.compileResult.manifest.config = { @@ -611,8 +624,8 @@ describe("application Nitro creation", () => { id: "zod", }); expect(createNitroMock.mock.calls[0]?.[0].traceOpts.nft.paths).toEqual({ - zod: expect.stringMatching(/zod[/\\].*index\.(?:c?js|mjs)$/), - sharp: expect.stringMatching(/sharp[/\\](?:lib[/\\]index\.js|dist[/\\]index\.cjs)$/), + zod: join(sourceRoot, "node_modules", "zod", "lib", "index.js"), + sharp: join(sourceRoot, "node_modules", "sharp", "lib", "index.js"), }); }); diff --git a/packages/eve/src/internal/nitro/host/start-development-server.test.ts b/packages/eve/src/internal/nitro/host/start-development-server.test.ts index e6a4cdefeab..1a2dc1e12d9 100644 --- a/packages/eve/src/internal/nitro/host/start-development-server.test.ts +++ b/packages/eve/src/internal/nitro/host/start-development-server.test.ts @@ -207,14 +207,14 @@ vi.mock("#execution/sandbox/development-prewarm.js", () => ({ startDevelopmentSandboxPrewarmInBackground: mocks.startDevelopmentSandboxPrewarmInBackground, })); -vi.mock("#execution/sandbox/bindings/local.js", async (importOriginal) => { - const actual = await importOriginal(); +vi.mock("#execution/sandbox/bindings/local.js", () => ({ + pruneLocalSandboxTemplatesInBackground: mocks.pruneLocalSandboxTemplatesInBackground, + stopDevelopmentSandboxResources: mocks.stopDevelopmentSandboxResources, +})); - return { - ...actual, - pruneLocalSandboxTemplatesInBackground: mocks.pruneLocalSandboxTemplatesInBackground, - stopDevelopmentSandboxResources: mocks.stopDevelopmentSandboxResources, - }; +beforeEach(() => { + mocks.fsControl.stateReadError = undefined; + mocks.fsControl.stateWriteError = undefined; }); const developmentServerStatePath = join("/tmp/eve-test", ".eve", "dev-server-state.v1.json"); @@ -348,8 +348,6 @@ describe("createDevelopmentServer", () => { beforeEach(() => { vi.clearAllMocks(); mocks.fetch.mockImplementation(async () => Response.json({ revision: "test" })); - mocks.fsControl.stateReadError = undefined; - mocks.fsControl.stateWriteError = undefined; mocks.authoredSourceWatcher.close.mockResolvedValue(undefined); mocks.authoredSourceWatcher.flush.mockResolvedValue(undefined); mocks.authoredSourceWatcher.rebuild.mockResolvedValue(undefined); diff --git a/packages/eve/src/internal/testing/mocks/mock-skill.integration.test.ts b/packages/eve/src/internal/testing/mocks/mock-skill.integration.test.ts index de7de528468..ffd3c474023 100644 --- a/packages/eve/src/internal/testing/mocks/mock-skill.integration.test.ts +++ b/packages/eve/src/internal/testing/mocks/mock-skill.integration.test.ts @@ -5,9 +5,9 @@ import { describe, expect, it } from "vitest"; import { mockSkill } from "#internal/testing/mocks/mock-skill.js"; describe("mockSkill", () => { - let materializedRootPath: string | undefined; - - it("materializes skill package files", async () => { + it("materializes skill package files and cleans them after the test", async ({ + onTestFinished, + }) => { const skill = await mockSkill({ description: "Weather guidance.", name: "weather", @@ -23,7 +23,9 @@ describe("mockSkill", () => { throw new Error("Expected mock skill source to be a skill package."); } - materializedRootPath = source.rootPath; + onTestFinished(async () => { + await expect(access(source.rootPath)).rejects.toMatchObject({ code: "ENOENT" }); + }); const referencesPath = source.referencesPath; expect(referencesPath).toBeDefined(); @@ -36,13 +38,14 @@ describe("mockSkill", () => { await expect(access(referencesPath)).resolves.toBeUndefined(); }); - it("cleans materialized tmpdirs after each test", async () => { - expect(materializedRootPath).toBeDefined(); - - if (materializedRootPath === undefined) { - throw new Error("Expected previous test to create a mock skill root."); + it("allows explicit cleanup to run more than once", async () => { + const skill = await mockSkill({ name: "weather", description: "Weather guidance." }); + if (skill.source.sourceKind !== "skill-package") { + throw new Error("Expected mock skill source to be a skill package."); } - - await expect(access(materializedRootPath)).rejects.toThrow(); + await expect(access(skill.source.rootPath)).resolves.toBeUndefined(); + await skill.cleanup(); + await skill.cleanup(); + await expect(access(skill.source.rootPath)).rejects.toMatchObject({ code: "ENOENT" }); }); }); diff --git a/packages/eve/src/internal/testing/scenario-app.ts b/packages/eve/src/internal/testing/scenario-app.ts index af79d1271b5..a4e55cf59a1 100644 --- a/packages/eve/src/internal/testing/scenario-app.ts +++ b/packages/eve/src/internal/testing/scenario-app.ts @@ -284,7 +284,8 @@ async function installScenarioDependencies(input: { "--no-audit", "--no-fund", "--ignore-scripts", - "--prefer-offline", + // The workspace's installed AI SDK version may be newer than cached registry metadata. + "--prefer-online", ]); return; } diff --git a/packages/eve/src/internal/testing/scenario-apps/next-eve-proxy.ts b/packages/eve/src/internal/testing/scenario-apps/next-eve-proxy.ts index 233ca843f07..5a7fd700804 100644 --- a/packages/eve/src/internal/testing/scenario-apps/next-eve-proxy.ts +++ b/packages/eve/src/internal/testing/scenario-apps/next-eve-proxy.ts @@ -3,6 +3,7 @@ import { createRequire } from "node:module"; import type { ScenarioAppDescriptor } from "#internal/testing/scenario-app.js"; const require = createRequire(import.meta.url); +const nextRequire = createRequire(require.resolve("next/package.json")); export interface NextEveProxyDescriptorOptions { readonly installDependencies?: boolean; @@ -58,7 +59,7 @@ export function proxy() { } function resolveInstalledPackageVersion(packageName: string): string { - const manifest: unknown = require(`${packageName}/package.json`); + const manifest: unknown = nextRequire(`${packageName}/package.json`); if ( typeof manifest !== "object" || diff --git a/packages/eve/src/react/use-eve-agent.test.ts b/packages/eve/src/react/use-eve-agent.test.ts index f2c31a64020..f37d674253c 100644 --- a/packages/eve/src/react/use-eve-agent.test.ts +++ b/packages/eve/src/react/use-eve-agent.test.ts @@ -1,5 +1,5 @@ import { createElement } from "react"; -import { act, create } from "react-test-renderer"; +import { act, create as createRenderer } from "react-test-renderer"; import { afterEach, describe, expect, it, vi } from "vitest"; import { useEveAgent, type UseEveAgentHelpers } from "#react/use-eve-agent.js"; @@ -131,8 +131,22 @@ function completedTurnData(input: { }; } -afterEach(() => { - vi.restoreAllMocks(); +const renderers: ReturnType[] = []; + +function create(...args: Parameters) { + const renderer = createRenderer(...args); + renderers.push(renderer); + return renderer; +} + +afterEach(async () => { + try { + await act(async () => { + for (const renderer of renderers.splice(0)) renderer.unmount(); + }); + } finally { + vi.restoreAllMocks(); + } }); describe("useEveAgent", () => { @@ -705,6 +719,8 @@ describe("useEveAgent", () => { await act(async () => { create(createElement(TestComponent)); + }); + await act(async () => { await vi.waitFor(() => expect(helpers?.status).toBe("ready")); }); diff --git a/packages/eve/src/self-modification/package-consumption.scenario.test.ts b/packages/eve/src/self-modification/package-consumption.scenario.test.ts index 38f87606e7b..6da4200c717 100644 --- a/packages/eve/src/self-modification/package-consumption.scenario.test.ts +++ b/packages/eve/src/self-modification/package-consumption.scenario.test.ts @@ -141,13 +141,7 @@ describe("packed package consumption", () => { await run( "pnpm", - [ - "install", - "--ignore-scripts", - "--no-frozen-lockfile", - "--prefer-offline", - "--config.minimum-release-age=0", - ], + ["install", "--ignore-scripts", "--no-frozen-lockfile", "--prefer-offline"], appRoot, ); await access(join(appRoot, "node_modules/eve/dist/src/self-modification/agent.js")); diff --git a/scripts/repeat-tests.mjs b/scripts/repeat-tests.mjs index bdc5f218787..18635cac94a 100644 --- a/scripts/repeat-tests.mjs +++ b/scripts/repeat-tests.mjs @@ -24,6 +24,9 @@ if ( const packageRoot = fileURLToPath(new URL("../packages/eve/", import.meta.url)); const vitest = resolve(packageRoot, "node_modules/vitest/vitest.mjs"); const directory = resolve(output); +const workerArgs = args.some((arg) => arg === "--maxWorkers" || arg.startsWith("--maxWorkers=")) + ? [] + : ["--maxWorkers=1"]; await mkdir(directory, { recursive: true }); const trials = []; const startedAt = new Date().toISOString(); @@ -43,7 +46,7 @@ for (let trial = 0; trial < runs; trial += 1) { "run", ...args, "--retry=0", - "--maxWorkers=1", + ...workerArgs, "--sequence.shuffle", `--sequence.seed=${seed}`, "--reporter=json", From f22acf59d89d96cfe60c435d5fef1d088268a2e9 Mon Sep 17 00:00:00 2001 From: Chad Hietala Date: Wed, 9 Sep 2026 10:09:29 -0400 Subject: [PATCH 5/8] fix(eve): preserve explicit mocks in dynamic model selection Restore the test hook import required by main's new batching tests. Keep explicitly authored mockModel responders when runtime model mocking is enabled, while retaining generic replacement for real providers and the subagent fixture's default mock model ID. Validated 44 workspace typecheck tasks, 8415 unit tests, 765 integration tests, and 30 shuffled focused runs. Vercel continuation timeouts remain under investigation. Signed-off-by: Chad Hietala --- .changeset/explicit-mock-responders.md | 5 ++ e2e/fixtures/agent-subagents/agent/agent.ts | 3 +- .../context/dynamic-model-lifecycle.test.ts | 55 ++++++++++++++++++- .../src/context/dynamic-model-lifecycle.ts | 8 ++- packages/eve/src/evals/mock-model.ts | 21 ++++--- .../execution/parked-delivery-wait.test.ts | 2 +- .../eve/src/internal/mock-model-identity.ts | 11 ++++ 7 files changed, 91 insertions(+), 14 deletions(-) create mode 100644 .changeset/explicit-mock-responders.md create mode 100644 packages/eve/src/internal/mock-model-identity.ts diff --git a/.changeset/explicit-mock-responders.md b/.changeset/explicit-mock-responders.md new file mode 100644 index 00000000000..fdb465aebf2 --- /dev/null +++ b/.changeset/explicit-mock-responders.md @@ -0,0 +1,5 @@ +--- +"eve": patch +--- + +Preserve explicitly configured `mockModel()` responders in step-scoped dynamic model selection when authored-model mocking is enabled, instead of substituting the generic fixture model. diff --git a/e2e/fixtures/agent-subagents/agent/agent.ts b/e2e/fixtures/agent-subagents/agent/agent.ts index 04f6c4d046b..5cabe7625f2 100644 --- a/e2e/fixtures/agent-subagents/agent/agent.ts +++ b/e2e/fixtures/agent-subagents/agent/agent.ts @@ -10,6 +10,7 @@ if (process.env.EVE_E2E_MODEL === "mock") { const base = e2eAgentConfig(); const { model, modelContextWindowTokens, ...agentConfig } = base; +const defaultModel = typeof model === "string" ? model : `${model.provider}/${model.modelId}`; const workspaceReader = mockModel({ modelId: "principal-forwarding-workspace-reader", respond: ({ messages }) => { @@ -83,7 +84,7 @@ export default defineAgent({ if (messages.some((message) => message.includes(WORKSPACE_FORWARDING_MARKER))) { return { model: workspaceDispatcher, modelContextWindowTokens: 1_000_000 }; } - return { model, modelContextWindowTokens }; + return { model: defaultModel, modelContextWindowTokens }; }, }, }), diff --git a/packages/eve/src/context/dynamic-model-lifecycle.test.ts b/packages/eve/src/context/dynamic-model-lifecycle.test.ts index 95dd1060f41..3e908f02fff 100644 --- a/packages/eve/src/context/dynamic-model-lifecycle.test.ts +++ b/packages/eve/src/context/dynamic-model-lifecycle.test.ts @@ -1,4 +1,4 @@ -import type { LanguageModel } from "ai"; +import { generateText, type LanguageModel } from "ai"; import { afterEach, describe, expect, it, vi } from "vitest"; import { ROOT_COMPILED_AGENT_NODE_ID } from "#compiler/manifest.js"; @@ -14,6 +14,7 @@ import { TurnDynamicModelReferenceKey, } from "#context/keys.js"; import { defineDynamic } from "#dynamic/definition.js"; +import { mockModel } from "#evals/mock-model.js"; import { createSessionStartedEvent, createStepStartedEvent, @@ -198,6 +199,58 @@ describe("dynamic model lifecycle", () => { }); }); + it.each([ + { nodeEnv: "test", mockAuthored: "" }, + { nodeEnv: "production", mockAuthored: "1" }, + ])( + "keeps explicit mock responders in $nodeEnv with override $mockAuthored", + async ({ nodeEnv, mockAuthored }) => { + vi.stubEnv("NODE_ENV", nodeEnv); + vi.stubEnv("EVE_MOCK_AUTHORED_MODELS", mockAuthored); + const ctx = new ContextContainer(); + const stepModel = mockModel({ + modelId: "scripted-dispatcher", + provider: "custom-fixture", + respond: "Use the explicitly authored response.", + }); + const moduleMap = createModuleMap({ + default: { + model: defineDynamic({ + events: { + "step.started": () => ({ + model: stepModel, + modelContextWindowTokens: 64_000, + }), + }, + }), + }, + }); + + await dispatchDynamicModelEvent({ + ctx, + dynamicModel: DYNAMIC_MODEL_SOURCE, + event: createStepStartedEvent({ + modelId: "unresolved", + sequence: 0, + stepIndex: 0, + turnId: "turn_0", + }), + messages: [], + scope: { moduleMap, nodeId: undefined }, + }); + + const selected = getActiveDynamicModelSelection(ctx); + expect(selected?.model).toBe(stepModel); + expect(selected?.reference.id).toBe("custom-fixture/scripted-dispatcher"); + if (selected?.model === undefined) throw new Error("Expected the explicit mock model."); + const response = await generateText({ + model: selected.model, + prompt: "Use the configured fixture response.", + }); + expect(response.text).toBe("Use the explicitly authored response."); + }, + ); + it("rejects live provider instances at durable scopes", async () => { const ctx = new ContextContainer(); const liveModel = createLanguageModel("openai.responses", "gpt-live"); diff --git a/packages/eve/src/context/dynamic-model-lifecycle.ts b/packages/eve/src/context/dynamic-model-lifecycle.ts index 813b9d26895..67585cd4d38 100644 --- a/packages/eve/src/context/dynamic-model-lifecycle.ts +++ b/packages/eve/src/context/dynamic-model-lifecycle.ts @@ -3,6 +3,7 @@ import type { ModelMessage } from "ai"; import { buildResolveContext } from "#context/dynamic-resolve-context.js"; import type { AlsContext } from "#context/container.js"; import type { ContextKey } from "#context/key.js"; +import { isMockModel } from "#internal/mock-model-identity.js"; import { LiveStepDynamicModelSelectionKey, SessionDynamicModelReferenceKey, @@ -134,9 +135,12 @@ function setSelectionForEvent( selection: ResolvedRuntimeModelSelection | null, ): void { if (eventType === "step.started") { - // In mock mode drop the live instance so the mock adapter keeps precedence. + // Replace real providers in mock mode, but keep explicitly scripted responders. const stored = - selection !== null && selection.model !== undefined && shouldMockAuthoredRuntimeModels() + selection !== null && + selection.model !== undefined && + !isMockModel(selection.model) && + shouldMockAuthoredRuntimeModels() ? { reference: selection.reference } : selection; ctx.setVirtualContext(LiveStepDynamicModelSelectionKey, stored); diff --git a/packages/eve/src/evals/mock-model.ts b/packages/eve/src/evals/mock-model.ts index 738a8355094..3bf0cea813e 100644 --- a/packages/eve/src/evals/mock-model.ts +++ b/packages/eve/src/evals/mock-model.ts @@ -1,6 +1,7 @@ import type { LanguageModel } from "ai"; import { MockLanguageModelV3 } from "ai/test"; +import { markMockModel } from "#internal/mock-model-identity.js"; import { AGENTS_SNIPPET_LABEL } from "#subagents/handles/prompt.js"; import { isPendingApprovalsSnippet } from "#harness/hitl/approval-prompt.js"; @@ -125,16 +126,18 @@ export function mockModel( const respond = normalizeResponder(options.respond); const modelId = options.modelId ?? DEFAULT_MODEL_ID; - return new MockLanguageModelV3({ - modelId, - provider: options.provider ?? DEFAULT_PROVIDER, - doGenerate: async (callOptions) => - createGenerateResult(await respond(createRequest(callOptions)), callOptions, modelId), - doStream: async (callOptions) => - createStreamResult( + return markMockModel( + new MockLanguageModelV3({ + modelId, + provider: options.provider ?? DEFAULT_PROVIDER, + doGenerate: async (callOptions) => createGenerateResult(await respond(createRequest(callOptions)), callOptions, modelId), - ), - }); + doStream: async (callOptions) => + createStreamResult( + createGenerateResult(await respond(createRequest(callOptions)), callOptions, modelId), + ), + }), + ); } function normalizeOptions(input: MockModelOptions | MockModelResponder | string): MockModelOptions { diff --git a/packages/eve/src/execution/parked-delivery-wait.test.ts b/packages/eve/src/execution/parked-delivery-wait.test.ts index 35233884792..1c208f7903e 100644 --- a/packages/eve/src/execution/parked-delivery-wait.test.ts +++ b/packages/eve/src/execution/parked-delivery-wait.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { DeliverHookPayload } from "#channel/types.js"; import { nextTurnDelivery } from "#execution/parked-delivery-wait.js"; diff --git a/packages/eve/src/internal/mock-model-identity.ts b/packages/eve/src/internal/mock-model-identity.ts new file mode 100644 index 00000000000..789e7208aef --- /dev/null +++ b/packages/eve/src/internal/mock-model-identity.ts @@ -0,0 +1,11 @@ +// Authored bundles and the host may load separate copies of this module. +const MOCK_MODEL = Symbol.for("eve:mock-model"); + +export function markMockModel(model: T): T { + Object.defineProperty(model, MOCK_MODEL, { value: true }); + return model; +} + +export function isMockModel(model: unknown): boolean { + return typeof model === "object" && model !== null && Reflect.get(model, MOCK_MODEL) === true; +} From 5f9547b2a3734f11f8d4b82373af3bc62257ee53 Mon Sep 17 00:00:00 2001 From: Chad Hietala Date: Wed, 9 Sep 2026 10:13:16 -0400 Subject: [PATCH 6/8] chore: consolidate deflake release notes into one changeset Signed-off-by: Chad Hietala --- .changeset/explicit-mock-responders.md | 5 ----- .changeset/quiet-workflow-reads.md | 4 +--- 2 files changed, 1 insertion(+), 8 deletions(-) delete mode 100644 .changeset/explicit-mock-responders.md diff --git a/.changeset/explicit-mock-responders.md b/.changeset/explicit-mock-responders.md deleted file mode 100644 index fdb465aebf2..00000000000 --- a/.changeset/explicit-mock-responders.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"eve": patch ---- - -Preserve explicitly configured `mockModel()` responders in step-scoped dynamic model selection when authored-model mocking is enabled, instead of substituting the generic fixture model. diff --git a/.changeset/quiet-workflow-reads.md b/.changeset/quiet-workflow-reads.md index e1250f0b74f..b6a6497d817 100644 --- a/.changeset/quiet-workflow-reads.md +++ b/.changeset/quiet-workflow-reads.md @@ -2,6 +2,4 @@ "eve": patch --- -Retry transient Windows file-sharing errors when reading local workflow state, matching the bounded retries already used for writes. Persistent access errors and malformed state still fail. - -Preserve a folded-away task that fits the compaction budget when recent tool results survive summarization. +Retry transient Windows file-sharing errors when reading local workflow state while still surfacing persistent access errors and malformed state. Preserve pending tasks that fit the compaction budget and keep explicitly configured `mockModel()` responders during step-scoped dynamic model selection when authored-model mocking is enabled. From b38c6885bab2377b71f3f6bb3d0cb5e411c0f0c7 Mon Sep 17 00:00:00 2001 From: Chad Hietala Date: Wed, 9 Sep 2026 10:51:13 -0400 Subject: [PATCH 7/8] chore: keep deflake PR focused on fixes Remove repetition runners and manual CI controls used for investigation, and drop unproven dev-server diagnostic edits. Preserve deployment-revision regression coverage in the fixture's normal unit suite without changing the default E2E matrix or existing eval assertions. Signed-off-by: Chad Hietala --- .github/scripts/discover-e2e-fixtures.mjs | 39 +++----- .github/workflows/e2e-local.yml | 45 +-------- e2e/fixtures/agent-channels/package.json | 3 +- .../test/deployment-revision.test.mjs | 35 +++++++ .../dev-server-connections.scenario.test.ts | 10 +- scripts/flake-validation-results.mjs | 19 ---- scripts/flake-validation-results.test.mjs | 74 --------------- scripts/repeat-evals.mjs | 79 ---------------- scripts/repeat-tests.mjs | 92 ------------------- 9 files changed, 54 insertions(+), 342 deletions(-) create mode 100644 e2e/fixtures/agent-channels/test/deployment-revision.test.mjs delete mode 100644 scripts/flake-validation-results.mjs delete mode 100644 scripts/flake-validation-results.test.mjs delete mode 100644 scripts/repeat-evals.mjs delete mode 100644 scripts/repeat-tests.mjs diff --git a/.github/scripts/discover-e2e-fixtures.mjs b/.github/scripts/discover-e2e-fixtures.mjs index e7466ffe4c0..5cf4540a31b 100644 --- a/.github/scripts/discover-e2e-fixtures.mjs +++ b/.github/scripts/discover-e2e-fixtures.mjs @@ -18,15 +18,6 @@ import { appendFileSync, existsSync, readdirSync, readFileSync, statSync } from import { join } from "node:path"; const roots = ["e2e/fixtures", "apps/fixtures"]; -const requestedFixture = process.env.EVE_E2E_FIXTURE || undefined; -const requestedModel = process.env.EVE_E2E_MODEL_NAME || undefined; -const repetitions = Number(process.env.EVE_E2E_REPETITIONS || "1"); -if (!Number.isInteger(repetitions) || repetitions < 1 || repetitions > 500) { - throw new Error("EVE_E2E_REPETITIONS must be an integer from 1 to 500."); -} -if (repetitions > 1 && requestedFixture === undefined) { - throw new Error("Repeated eval validation requires an explicit fixture."); -} const registry = JSON.parse(readFileSync("e2e/matrix.json", "utf8")); const models = validateNamedEntries(registry.models, "models", ["id"]); @@ -36,7 +27,6 @@ const fixtures = []; for (const root of roots) { if (!existsSync(root)) continue; for (const entry of readdirSync(root).sort()) { - if (requestedFixture !== undefined && entry !== requestedFixture) continue; const dir = join(root, entry); if (!statSync(dir).isDirectory() || !existsSync(join(dir, "evals"))) continue; @@ -62,26 +52,21 @@ for (const root of roots) { } if (fixtures.length === 0) { - console.error(`No e2e fixtures matched ${requestedFixture ?? "the fixture roots"}.`); + console.error("No e2e fixtures with an evals/ directory were found."); process.exit(1); } -const modelMatrix = fixtures - .flatMap(({ name, dir, modelMatrix, additionalModels }) => - uniqueModels([ - ...(modelMatrix === "full" ? models : models.slice(0, 1)), - ...additionalModels, - ]).map((model) => ({ - name, - dir, - model_name: model.name, - model_id: model.id, - })), - ) - .filter((leg) => requestedModel === undefined || leg.model_name === requestedModel); -if (modelMatrix.length === 0) { - throw new Error(`No model-suite jobs matched model ${requestedModel}.`); -} +const modelMatrix = fixtures.flatMap(({ name, dir, modelMatrix, additionalModels }) => + uniqueModels([ + ...(modelMatrix === "full" ? models : models.slice(0, 1)), + ...additionalModels, + ]).map((model) => ({ + name, + dir, + model_name: model.name, + model_id: model.id, + })), +); const outputs = [`model_matrix=${JSON.stringify(modelMatrix)}`]; for (const world of worlds) { diff --git a/.github/workflows/e2e-local.yml b/.github/workflows/e2e-local.yml index 6a11ef927cf..4dee19dcc7a 100644 --- a/.github/workflows/e2e-local.yml +++ b/.github/workflows/e2e-local.yml @@ -2,27 +2,10 @@ name: E2E Tests (Local) on: workflow_dispatch: - inputs: - fixture: - description: "One fixture name (empty runs the full matrix)" - type: string - default: "" - model: - description: "One matrix model name (empty runs every configured model)" - type: string - default: "" - evals: - description: "Space-separated eval IDs or prefixes for repeated validation" - type: string - default: "" - repetitions: - description: "Fresh-process repetitions, 1-500 (requires a fixture when greater than 1)" - type: number - default: 1 pull_request: concurrency: - group: e2e-local-${{ github.event_name == 'workflow_dispatch' && github.run_id || github.ref }} + group: e2e-local-${{ github.ref }} cancel-in-progress: true jobs: @@ -72,10 +55,6 @@ jobs: - name: Discover eval fixtures id: discover - env: - EVE_E2E_FIXTURE: ${{ inputs.fixture }} - EVE_E2E_MODEL_NAME: ${{ inputs.model }} - EVE_E2E_REPETITIONS: ${{ inputs.repetitions }} run: node .github/scripts/discover-e2e-fixtures.mjs # The model suite: every fixture runs here against the local world with @@ -150,33 +129,13 @@ jobs: mv e2e/fixtures/js-only-extension/extension "$RUNNER_TEMP/js-only-extension-source" - name: Run fixture evals - if: needs.changes.outputs.relevant == 'true' && (github.event_name != 'workflow_dispatch' || inputs.repetitions <= 1) + if: needs.changes.outputs.relevant == 'true' run: | mkdir -p "$EVE_EVAL_JUNIT_DIR" cd "${{ matrix.dir }}" pnpm exec eve eval --strict --verbose \ --junit "$EVE_EVAL_JUNIT_DIR/${{ matrix.name }}-${{ matrix.model_name }}.xml" - - name: Repeat selected fixture evals - if: needs.changes.outputs.relevant == 'true' && github.event_name == 'workflow_dispatch' && inputs.repetitions > 1 - working-directory: ${{ matrix.dir }} - env: - EVE_E2E_REPETITIONS: ${{ inputs.repetitions }} - EVE_E2E_EVAL_IDS: ${{ inputs.evals }} - run: | - node --test "$GITHUB_WORKSPACE/scripts/flake-validation-results.test.mjs" - node "$GITHUB_WORKSPACE/scripts/repeat-evals.mjs" - - - name: Upload repetition evidence - if: always() && github.event_name == 'workflow_dispatch' && inputs.repetitions > 1 - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: repeat-evals-${{ matrix.name }}-${{ matrix.model_name }} - path: .junit/repetitions - include-hidden-files: true - if-no-files-found: error - retention-days: 7 - - name: Upload eval artifacts if: (failure()) && needs.changes.outputs.relevant == 'true' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/e2e/fixtures/agent-channels/package.json b/e2e/fixtures/agent-channels/package.json index 827cec9e256..ba341b10d9a 100644 --- a/e2e/fixtures/agent-channels/package.json +++ b/e2e/fixtures/agent-channels/package.json @@ -8,7 +8,8 @@ "dev": "eve dev", "start": "eve start", "typecheck": "eve build && tsc", - "test:e2e": "eve eval --strict" + "test:e2e": "eve eval --strict", + "test:unit": "node --test test/*.test.mjs" }, "dependencies": { "@eve-e2e/config": "workspace:*", diff --git a/e2e/fixtures/agent-channels/test/deployment-revision.test.mjs b/e2e/fixtures/agent-channels/test/deployment-revision.test.mjs new file mode 100644 index 00000000000..6b350e79fd7 --- /dev/null +++ b/e2e/fixtures/agent-channels/test/deployment-revision.test.mjs @@ -0,0 +1,35 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { + checkDeploymentRevision, + DEPLOYMENT_PENDING_CODE, + DEPLOYMENT_REVISION_HEADER, +} from "../agent/lib/deployment-revision.ts"; + +function pinnedRequest() { + return new Request("https://fixture.example/cross-version-webhook", { + headers: { [DEPLOYMENT_REVISION_HEADER]: "requested-revision" }, + }); +} + +test("rejects a stale deployment before dispatch", async () => { + const response = checkDeploymentRevision(pinnedRequest(), "previous-revision"); + + assert.equal(response.status, 409); + assert.equal((await response.json()).code, DEPLOYMENT_PENDING_CODE); +}); + +test("allows a request on its pinned deployment", () => { + assert.equal(checkDeploymentRevision(pinnedRequest(), "requested-revision"), undefined); +}); + +test("preserves requests without a deployment pin", () => { + assert.equal( + checkDeploymentRevision( + new Request("https://fixture.example/cross-version-webhook"), + "previous-revision", + ), + undefined, + ); +}); diff --git a/packages/eve/test/scenarios/dev-server-connections.scenario.test.ts b/packages/eve/test/scenarios/dev-server-connections.scenario.test.ts index 9f75c30714f..28be7c29eb6 100644 --- a/packages/eve/test/scenarios/dev-server-connections.scenario.test.ts +++ b/packages/eve/test/scenarios/dev-server-connections.scenario.test.ts @@ -1,7 +1,6 @@ import { Agent } from "node:http"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { inspect } from "node:util"; import { describe, expect, it } from "vitest"; @@ -175,8 +174,6 @@ describe("eve dev server live connections", () => { const candidateStartedPath = join(app.appRoot, ".candidate-started"); const candidateReadyPath = join(app.appRoot, ".candidate-ready"); const streamReleasePath = join(app.appRoot, ".stream-release"); - let stopRevisionPolling = false; - let revisionPoller: Promise | undefined; try { const initialRevision = await readDevelopmentRevision(server.url); @@ -194,12 +191,13 @@ describe("eve dev server live connections", () => { // The parent-owned control route must answer continuously through // candidate preparation and promotion, not merely at spot checks. const revisionFailures: string[] = []; - revisionPoller = (async () => { + let stopRevisionPolling = false; + const revisionPoller = (async () => { while (!stopRevisionPolling) { try { await readDevelopmentRevision(server.url); } catch (error) { - revisionFailures.push(inspect(error, { depth: 4 })); + revisionFailures.push(String(error)); } await new Promise((resolvePromise) => setTimeout(resolvePromise, 50)); } @@ -232,9 +230,7 @@ describe("eve dev server live connections", () => { await expect(reader.read()).resolves.toEqual(expect.objectContaining({ done: true })); expect(hasKnownDevServerFailure(`${server.stdout()}\n${server.stderr()}`)).toBe(false); } finally { - stopRevisionPolling = true; await server.stop(); - await revisionPoller; } }, DEV_SERVER_SCENARIO_TIMEOUT_MS, diff --git a/scripts/flake-validation-results.mjs b/scripts/flake-validation-results.mjs deleted file mode 100644 index b14a94424c2..00000000000 --- a/scripts/flake-validation-results.mjs +++ /dev/null @@ -1,19 +0,0 @@ -export function isCleanEvalSummary(summary) { - return ( - summary?.failed === 0 && - summary.scored === 0 && - summary.errored === 0 && - summary.skipped === 0 && - Array.isArray(summary.results) && - summary.results.length > 0 && - summary.passed === summary.results.length && - summary.results.every((result) => result?.verdict === "passed" && result.error === undefined) - ); -} - -export function zeroFailureUpperBound(runs) { - if (!Number.isInteger(runs) || runs < 1) { - throw new Error("A confidence bound requires a positive integer trial count."); - } - return -Math.expm1(Math.log(0.05) / runs); -} diff --git a/scripts/flake-validation-results.test.mjs b/scripts/flake-validation-results.test.mjs deleted file mode 100644 index ae3beb36b7b..00000000000 --- a/scripts/flake-validation-results.test.mjs +++ /dev/null @@ -1,74 +0,0 @@ -import assert from "node:assert/strict"; -import { test } from "node:test"; - -import { isCleanEvalSummary, zeroFailureUpperBound } from "./flake-validation-results.mjs"; -import { - checkDeploymentRevision, - DEPLOYMENT_PENDING_CODE, - DEPLOYMENT_REVISION_HEADER, -} from "../e2e/fixtures/agent-channels/agent/lib/deployment-revision.ts"; - -const clean = { - failed: 0, - scored: 0, - errored: 0, - skipped: 0, - passed: 1, - results: [{ id: "example", verdict: "passed" }], -}; - -test("accepts the CLI summary shape, which has results rather than totalEvals", () => { - assert.equal(isCleanEvalSummary(clean), true); -}); - -for (const field of ["failed", "scored", "errored", "skipped"]) { - test(`rejects ${field} evals`, () => { - assert.equal(isCleanEvalSummary({ ...clean, [field]: 1 }), false); - }); -} - -test("rejects missing, empty, and inconsistent result sets", () => { - for (const summary of [ - undefined, - null, - {}, - { ...clean, results: undefined, totalEvals: 1 }, - { ...clean, passed: 0, results: [] }, - { ...clean, passed: 2 }, - { ...clean, results: [{ verdict: "failed" }] }, - { ...clean, results: [{ verdict: "passed", error: "unexpected error" }] }, - ]) { - assert.equal(isCleanEvalSummary(summary), false); - } -}); - -test("computes the exact one-sided 95% zero-failure binomial bound", () => { - assert.equal(zeroFailureUpperBound(1), 0.95); - const bound = zeroFailureUpperBound(500); - assert.ok(bound > 0.0059 && bound < 0.006); - assert.ok(Math.abs((1 - bound) ** 500 - 0.05) < 1e-12); - assert.ok(zeroFailureUpperBound(20) > bound); -}); - -test("does not report confidence without valid trials", () => { - for (const runs of [0, -1, 0.5, NaN, Infinity]) { - assert.throws(() => zeroFailureUpperBound(runs), /positive integer/); - } -}); - -test("rejects stale deployment requests before dispatch without changing unpinned traffic", async () => { - const request = new Request("https://fixture.example/cross-version-webhook", { - headers: { [DEPLOYMENT_REVISION_HEADER]: "requested-revision" }, - }); - const rejected = checkDeploymentRevision(request, "previous-revision"); - assert.equal(rejected.status, 409); - assert.equal((await rejected.json()).code, DEPLOYMENT_PENDING_CODE); - assert.equal(checkDeploymentRevision(request, "requested-revision"), undefined); - assert.equal( - checkDeploymentRevision( - new Request("https://fixture.example/cross-version-webhook"), - "previous-revision", - ), - undefined, - ); -}); diff --git a/scripts/repeat-evals.mjs b/scripts/repeat-evals.mjs deleted file mode 100644 index 542125afb74..00000000000 --- a/scripts/repeat-evals.mjs +++ /dev/null @@ -1,79 +0,0 @@ -import { execFile } from "node:child_process"; -import { mkdir, writeFile } from "node:fs/promises"; -import { resolve } from "node:path"; -import { fileURLToPath } from "node:url"; -import { promisify } from "node:util"; - -import { isCleanEvalSummary, zeroFailureUpperBound } from "./flake-validation-results.mjs"; - -const repetitions = Number(process.env.EVE_E2E_REPETITIONS); -if (!Number.isInteger(repetitions) || repetitions < 1 || repetitions > 500) { - throw new Error("EVE_E2E_REPETITIONS must be an integer from 1 to 500."); -} -const evalIds = (process.env.EVE_E2E_EVAL_IDS ?? "").split(/\s+/).filter(Boolean); -if ( - evalIds.length > 64 || - evalIds.some((id) => id.length > 256 || !/^[a-zA-Z0-9][a-zA-Z0-9._/-]*$/.test(id)) -) { - throw new Error("Pass up to 64 plain eval IDs or directory prefixes, not CLI options."); -} -if (!process.env.EVE_EVAL_JUNIT_DIR) { - throw new Error("EVE_EVAL_JUNIT_DIR is required for repetition evidence."); -} - -const directory = resolve(process.env.EVE_EVAL_JUNIT_DIR, "repetitions"); -await mkdir(directory, { recursive: true }); -const exec = promisify(execFile); -const cli = fileURLToPath(new URL("../packages/eve/bin/eve.js", import.meta.url)); -const trials = []; -const startedAt = new Date().toISOString(); - -for (let index = 0; index < repetitions; index += 1) { - const prefix = resolve(directory, String(index + 1).padStart(4, "0")); - let stdout = ""; - let stderr = ""; - let error; - try { - ({ stdout, stderr } = await exec( - process.execPath, - [cli, "eval", ...evalIds, "--strict", "--json", "--skip-report", "--junit", `${prefix}.xml`], - { encoding: "utf8", maxBuffer: 32 * 1024 * 1024, timeout: 10 * 60_000 }, - )); - } catch (failure) { - stdout = failure.stdout ?? ""; - stderr = failure.stderr ?? ""; - error = failure.message; - } - await writeFile(`${prefix}.stdout.log`, stdout); - await writeFile(`${prefix}.stderr.log`, stderr); - let summary; - try { - summary = JSON.parse(stdout); - } catch { - error ??= "The eval process did not produce valid JSON."; - } - const passed = error === undefined && isCleanEvalSummary(summary); - trials.push({ repetition: index + 1, passed, error, summary }); - await writeFile(resolve(directory, "trials.json"), JSON.stringify(trials, null, 2) + "\n"); - console.log( - `${index + 1}/${repetitions} repetitions; ${trials.filter((trial) => !trial.passed).length} failures`, - ); -} - -const failures = trials.filter((trial) => !trial.passed).length; -const report = { - commit: process.env.GITHUB_SHA, - model: process.env.EVE_E2E_MODEL, - evalIds, - startedAt, - completedAt: new Date().toISOString(), - repetitions, - failures, - confidence: 0.95, - oneSidedFailureRateUpperBound: failures === 0 ? zeroFailureUpperBound(repetitions) : null, - scope: - "Selected evals in fresh CLI processes, with every failure retained; assumes independent, stationary trials.", -}; -await writeFile(resolve(directory, "summary.json"), JSON.stringify(report, null, 2) + "\n"); -console.log(JSON.stringify(report, null, 2)); -process.exitCode = failures === 0 ? 0 : 1; diff --git a/scripts/repeat-tests.mjs b/scripts/repeat-tests.mjs deleted file mode 100644 index 18635cac94a..00000000000 --- a/scripts/repeat-tests.mjs +++ /dev/null @@ -1,92 +0,0 @@ -import { execFile } from "node:child_process"; -import { mkdir, readFile, writeFile } from "node:fs/promises"; -import { resolve } from "node:path"; -import { fileURLToPath } from "node:url"; -import { promisify } from "node:util"; - -import { zeroFailureUpperBound } from "./flake-validation-results.mjs"; - -const exec = promisify(execFile); -const [runsText, output, ...args] = process.argv.slice(2); -const runs = Number(runsText); -if (!Number.isInteger(runs) || runs < 1 || runs > 1_000 || !output || args.length === 0) { - throw new Error( - "Usage: node scripts/repeat-tests.mjs --config vitest..config.ts ", - ); -} -const config = args[args.indexOf("--config") + 1]; -if ( - !args.includes("--config") || - !/^vitest\.(unit|integration|scenario)\.config\.ts$/.test(config) -) { - throw new Error("Pass an explicit unit, integration, or scenario tier config."); -} -const packageRoot = fileURLToPath(new URL("../packages/eve/", import.meta.url)); -const vitest = resolve(packageRoot, "node_modules/vitest/vitest.mjs"); -const directory = resolve(output); -const workerArgs = args.some((arg) => arg === "--maxWorkers" || arg.startsWith("--maxWorkers=")) - ? [] - : ["--maxWorkers=1"]; -await mkdir(directory, { recursive: true }); -const trials = []; -const startedAt = new Date().toISOString(); - -for (let trial = 0; trial < runs; trial += 1) { - const prefix = resolve(directory, String(trial + 1).padStart(4, "0")); - const reportPath = `${prefix}.json`; - const seed = 20_260_909 + trial; - let passed = false; - let testCount = 0; - let detail; - try { - const { stdout, stderr } = await exec( - process.execPath, - [ - vitest, - "run", - ...args, - "--retry=0", - ...workerArgs, - "--sequence.shuffle", - `--sequence.seed=${seed}`, - "--reporter=json", - `--outputFile=${reportPath}`, - ], - { cwd: packageRoot, encoding: "utf8", maxBuffer: 8 * 1024 * 1024, timeout: 10 * 60_000 }, - ); - await writeFile(`${prefix}.log`, stdout + stderr); - const report = JSON.parse(await readFile(reportPath, "utf8")); - testCount = report.numPassedTests; - passed = report.success === true && testCount > 0 && report.numFailedTests === 0; - if (!passed) detail = "The test report did not contain a successful, nonempty run."; - } catch (error) { - detail = error.message; - await writeFile(`${prefix}.log`, (error.stdout ?? "") + (error.stderr ?? "") + detail); - } - trials.push({ trial: trial + 1, seed, passed, testCount, detail }); - await writeFile(resolve(directory, "trials.json"), JSON.stringify(trials, null, 2) + "\n"); - if ((trial + 1) % 10 === 0 || trial === runs - 1) { - console.log( - `${trial + 1}/${runs} runs; ${trials.filter((result) => !result.passed).length} failures`, - ); - } -} - -const failures = trials.filter((trial) => !trial.passed).length; -const summary = { - command: [process.execPath, vitest, "run", ...args], - startedAt, - completedAt: new Date().toISOString(), - platform: process.platform, - nodeVersion: process.version, - runs, - failures, - passedTests: trials.reduce((total, trial) => total + trial.testCount, 0), - confidence: 0.95, - oneSidedFailureRateUpperBound: failures === 0 ? zeroFailureUpperBound(runs) : null, - scope: - "Repeatability of the selected tests under independent, stationary trials; not the failure rate of all CI jobs.", -}; -await writeFile(resolve(directory, "summary.json"), JSON.stringify(summary, null, 2) + "\n"); -console.log(JSON.stringify(summary, null, 2)); -process.exitCode = failures === 0 ? 0 : 1; From a6b5c5c7710fe84de21a6b0bf9eb7a4174b86d27 Mon Sep 17 00:00:00 2001 From: Chad Hietala Date: Wed, 9 Sep 2026 11:13:35 -0400 Subject: [PATCH 8/8] test(eve): decouple scaffold policy check from npm publication Resolve the package under test from the existing scenario tarball through a temporary pnpm hook while retaining registry dependency release-age checks. Remove the release-branch skip and exercise an intentionally unpublished eve range. Signed-off-by: Chad Hietala --- .../test/scenarios/eve-init.scenario.test.ts | 102 +++++++++++------- 1 file changed, 61 insertions(+), 41 deletions(-) diff --git a/packages/eve/test/scenarios/eve-init.scenario.test.ts b/packages/eve/test/scenarios/eve-init.scenario.test.ts index 458fb7ad40d..3ecc91406d9 100644 --- a/packages/eve/test/scenarios/eve-init.scenario.test.ts +++ b/packages/eve/test/scenarios/eve-init.scenario.test.ts @@ -10,15 +10,12 @@ import { CODING_AGENT_ENV_MARKERS } from "../../src/cli/agent-detection.js"; import { loadYaml } from "../../src/evals/loaders/yaml.js"; import { DEFAULT_AGENT_MODEL_ID } from "../../src/shared/default-agent-model.js"; import { pathExists } from "../../src/setup/path-exists.js"; +import { ensureScenarioEveTarballPath } from "../../src/internal/testing/scenario-app.js"; import { useTemporaryDirectories } from "../../src/internal/testing/use-temporary-app-roots.js"; const EVE_BIN_PATH = fileURLToPath(new URL("../../bin/eve.js", import.meta.url)); const runFile = promisify(execFile); const RELEASE_AGE_MINUTES = "2880"; -// Changesets opens Version Packages PRs on `changeset-release/`. Those -// PRs bump package.json before npm has that version, so a real registry -// install of the scaffolded eve range cannot succeed yet. -const isChangesetReleasePr = process.env.GITHUB_HEAD_REF?.startsWith("changeset-release/") === true; const createScratchDirectory = useTemporaryDirectories(); @@ -167,43 +164,66 @@ async function createFakeNpmEnvironment(scratch: string): Promise<{ } describe("eve init smoke", () => { - it.skipIf(isChangesetReleasePr)( - "resolves a standalone pnpm scaffold under the release-age policy", - async () => { - const scratch = await createScratchDirectory("eve-init-release-age-"); - const workspacePolicy = (await loadYaml( - fileURLToPath(new URL("../../../../pnpm-workspace.yaml", import.meta.url)), - )) as { minimumReleaseAgeExclude: string[] }; - const env = { - ...withoutCodingAgentMarkers(process.env), - // The agent path skips the interactive dev handoff, which cannot run - // against the real pnpm install this scenario performs. - AI_AGENT: "claude", - CI: "true", - PNPM_CONFIG_MINIMUM_RELEASE_AGE: RELEASE_AGE_MINUTES, - // Use the repository's reviewed dependency exceptions for the current - // release without adding bypasses to scaffolded projects. - PNPM_CONFIG_MINIMUM_RELEASE_AGE_EXCLUDE: JSON.stringify([ - "eve", - ...workspacePolicy.minimumReleaseAgeExclude, - ]), - }; - - const result = await runEveBin(scratch, ["init", "policy-agent"], env); - - expect(result.exitCode, result.stderr).toBe(0); - const projectDir = join(scratch, "policy-agent"); - await expect( - readFile(join(projectDir, "pnpm-workspace.yaml"), "utf8"), - ).resolves.not.toContain("minimumReleaseAgeExclude:"); - await expect( - runFile("pnpm", ["add", "--ignore-scripts", "--lockfile-only", "is-number@7.0.0"], { - cwd: projectDir, - env, - }), - ).resolves.toMatchObject({ stderr: expect.any(String) }); - }, - ); + it("resolves a standalone pnpm scaffold under the release-age policy", async () => { + const scratch = await createScratchDirectory("eve-init-release-age-"); + const eveTarball = `file:${await ensureScenarioEveTarballPath()}`; + const pnpmfile = join(scratch, "scaffold-pnpmfile.cjs"); + // The package under test need not have reached npm; registry dependencies still use the policy. + await writeFile( + pnpmfile, + [ + "module.exports = { hooks: { updateConfig(config) {", + ` return { ...config, overrides: { ...config.overrides, eve: ${JSON.stringify(eveTarball)} } };`, + "} } };\n", + ].join("\n"), + ); + const workspacePolicy = (await loadYaml( + fileURLToPath(new URL("../../../../pnpm-workspace.yaml", import.meta.url)), + )) as { minimumReleaseAgeExclude: string[] }; + const env = { + ...withoutCodingAgentMarkers(process.env), + // The agent path skips the interactive dev handoff, which cannot run + // against the real pnpm install this scenario performs. + AI_AGENT: "claude", + CI: "true", + PNPM_CONFIG_GLOBAL_PNPMFILE: pnpmfile, + PNPM_CONFIG_MINIMUM_RELEASE_AGE: RELEASE_AGE_MINUTES, + // Use the repository's reviewed dependency exceptions for the current + // release without adding bypasses to scaffolded projects. + PNPM_CONFIG_MINIMUM_RELEASE_AGE_EXCLUDE: JSON.stringify( + workspacePolicy.minimumReleaseAgeExclude, + ), + }; + + const result = await runEveBin(scratch, ["init", "policy-agent"], env); + + expect(result.exitCode, result.stderr).toBe(0); + const projectDir = join(scratch, "policy-agent"); + await expect(readFile(join(projectDir, "pnpm-workspace.yaml"), "utf8")).resolves.not.toContain( + "minimumReleaseAgeExclude:", + ); + await expect(readFile(join(projectDir, "pnpm-workspace.yaml"), "utf8")).resolves.toContain( + "minimumReleaseAgeStrict: true", + ); + + // Exercise publication lag even when the checkout's version is already available on npm. + const manifestPath = join(projectDir, "package.json"); + const manifest = JSON.parse(await readFile(manifestPath, "utf8")) as { + dependencies: Record; + }; + manifest.dependencies.eve = "0.0.0-eve-init-unpublished"; + await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); + await expect( + runFile("pnpm", ["add", "--ignore-scripts", "--lockfile-only", "is-number@7.0.0"], { + cwd: projectDir, + env, + }), + ).resolves.toMatchObject({ stderr: expect.any(String) }); + const lockfile = (await loadYaml(join(projectDir, "pnpm-lock.yaml"))) as { + overrides?: Record; + }; + expect(lockfile.overrides?.eve).toBe(eveTarball); + }); it("creates the base template with the default model and no Vercel state", async () => { const scratch = await createScratchDirectory("eve-init-");