fix(eve): deflake workflow and fixture tests

.changeset/quiet-workflow-reads.md

@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Retry transient Windows file-sharing errors when reading local workflow state while still surfacing persistent access errors and malformed state. Preserve pending tasks that fit the compaction budget and keep explicitly configured `mockModel()` responders during step-scoped dynamic model selection when authored-model mocking is enabled.

e2e/fixtures/agent-channels/agent/channels/cross-version.ts

@@ -1,4 +1,7 @@
 import { defineChannel, POST, type Session } from "eve/channels";
+import evePackage from "eve/package.json" with { type: "json" };
+
+import { checkDeploymentRevision } from "../lib/deployment-revision";
 
 const smokeTestAuth = {
   attributes: { source: "smoke-test" },
@@ -29,6 +32,8 @@ type CrossVersionRouteContext = {
 export default defineChannel({
   routes: [
     POST("/cross-version-webhook", async (req, ctx) => {
+      const pendingDeployment = checkDeploymentRevision(req);
+      if (pendingDeployment !== undefined) return pendingDeployment;
       const body = (await req.json().catch(() => ({}))) as {
         message?: string;
         sessionRef?: string;
@@ -53,7 +58,7 @@ export default defineChannel({
         throw new Error("Expected the route context to expose from() or send().");
       }
 
-      return Response.json({ ok: true, sessionId: session.id });
+      return Response.json({ ok: true, sessionId: session.id, eveVersion: evePackage.version });
     }),
   ],
 });

e2e/fixtures/agent-channels/agent/lib/deployment-revision.ts

@@ -0,0 +1,17 @@
+export const DEPLOYMENT_REVISION_HEADER = "x-eve-e2e-deployment-revision";
+export const DEPLOYMENT_PENDING_CODE = "e2e_deployment_pending";
+
+export function checkDeploymentRevision(
+  request: Request,
+  revision = process.env.EVE_E2E_DEPLOYMENT_REVISION,
+): Response | undefined {
+  const expected = request.headers.get(DEPLOYMENT_REVISION_HEADER);
+  if (expected === null || expected === revision) return undefined;
+  return Response.json(
+    {
+      code: DEPLOYMENT_PENDING_CODE,
+      error: "The requested deployment is not serving this request yet.",
+    },
+    { status: 409 },
+  );
+}

e2e/fixtures/agent-channels/evals/custom-channels/cross-version-session-inbox.eval.ts

@@ -7,7 +7,10 @@ import { defineEval } from "eve/evals";
 import type { EveEvalContext } from "eve/evals";
 import { satisfies } from "eve/evals/expect";
 
-import { postChannel } from "./shared";
+import {
+  DEPLOYMENT_PENDING_CODE,
+  DEPLOYMENT_REVISION_HEADER,
+} from "../../agent/lib/deployment-revision";
 
 const ALIAS_ENV = "EVE_E2E_REDEPLOY_ALIAS";
 const OLD_EVE_VERSION = "0.30.8";
@@ -26,7 +29,7 @@ const TOOL_NAME = "wait-for-cancellation";
 const execFileAsync = promisify(execFile);
 const EXEC_OPTIONS = { maxBuffer: 64 * 1024 * 1024 } as const;
 
-type MessageResponse = { ok: boolean; sessionId?: string };
+type MessageResponse = { ok: boolean; sessionId?: string; eveVersion: string };
 
 /**
  * Proves the complete mixed-version codec boundary through the real Workflow
@@ -53,6 +56,8 @@ export default defineEval({
     }
 
     const originalInstructions = await readFile(INSTRUCTIONS_PATH, "utf8");
+    const oldRevision = crypto.randomUUID();
+    const currentRevision = crypto.randomUUID();
     const links = await Promise.all([
       snapshotLink(FIXTURE_EVE_LINK),
       snapshotLink(CONFIG_EVE_LINK),
@@ -74,18 +79,21 @@ export default defineEval({
         INSTRUCTIONS_PATH,
         `${originalInstructions}\nDeployment marker: ${OLD_DEPLOYMENT_MARKER}.\n`,
       );
-      await deployToAlias(t, alias, "eve@0.30.8");
+      await deployToAlias(t, alias, "eve@0.30.8", oldRevision, oldEvePackage);
       await waitForAliasToServe(t, OLD_DEPLOYMENT_MARKER);
 
       const sessionRef = crypto.randomUUID();
-      const started = await postChannel<MessageResponse>(t.target, "/cross-version-webhook", {
+      const started = await postToDeployment(t, oldRevision, {
         message: ACTIVE_TURN_MESSAGE,
         sessionRef,
       });
       await t.require(
         started,
         satisfies(
-          (value: MessageResponse) => value.ok === true && typeof value.sessionId === "string",
+          (value: MessageResponse) =>
+            value.ok === true &&
+            typeof value.sessionId === "string" &&
+            value.eveVersion === OLD_EVE_VERSION,
           "eve@0.30.8 starts the durable session",
         ),
       );
@@ -104,18 +112,21 @@ export default defineEval({
         INSTRUCTIONS_PATH,
         `${originalInstructions}\nDeployment marker: ${CURRENT_DEPLOYMENT_MARKER}.\n`,
       );
-      await deployToAlias(t, alias, `eve@${currentEveVersion}`);
+      await deployToAlias(t, alias, `eve@${currentEveVersion}`, currentRevision, currentEvePackage);
       await waitForAliasToServe(t, CURRENT_DEPLOYMENT_MARKER);
 
-      const replacement = await postChannel<MessageResponse>(t.target, "/cross-version-webhook", {
+      const replacement = await postToDeployment(t, currentRevision, {
         message: `Reply with exactly ${FOLLOW_UP_TOKEN}.`,
         sessionRef,
         turnPolicy: "queue",
       });
       await t.require(
         replacement,
         satisfies(
-          (value: MessageResponse) => value.ok === true && value.sessionId === sessionId,
+          (value: MessageResponse) =>
+            value.ok === true &&
+            value.sessionId === sessionId &&
+            value.eveVersion === currentEveVersion,
           "the current producer targets the existing eve@0.30.8 session",
         ),
       );
@@ -188,8 +199,15 @@ async function readPackageVersion(packagePath: string): Promise<string> {
 }
 
 /** Builds the fixture and repoints the run-scoped alias at the fresh deployment. */
-async function deployToAlias(t: EveEvalContext, alias: string, phase: string): Promise<void> {
-  await execFileAsync("pnpm", ["exec", "eve", "build"], {
+async function deployToAlias(
+  t: EveEvalContext,
+  alias: string,
+  phase: string,
+  revision: string,
+  evePackagePath: string,
+): Promise<void> {
+  // pnpm's dependency synchronization can repair the intentionally replaced links.
+  await execFileAsync(process.execPath, [resolve(evePackagePath, "bin", "eve.js"), "build"], {
     ...EXEC_OPTIONS,
     env: {
       ...process.env,
@@ -204,11 +222,22 @@ async function deployToAlias(t: EveEvalContext, alias: string, phase: string): P
     process.env.EVE_E2E_MODEL === undefined
       ? []
       : ["--env", `EVE_E2E_MODEL=${process.env.EVE_E2E_MODEL}`];
+  const revisionArgs = ["--env", `EVE_E2E_DEPLOYMENT_REVISION=${revision}`];
   const scopeArgs =
     process.env.VERCEL_ORG_ID === undefined ? [] : ["--scope", process.env.VERCEL_ORG_ID];
   const deploy = await execFileAsync(
     "pnpm",
-    ["exec", "vc", "deploy", "--prebuilt", "--yes", "--target=preview", ...modelArgs, ...tokenArgs],
+    [
+      "exec",
+      "vc",
+      "deploy",
+      "--prebuilt",
+      "--yes",
+      "--target=preview",
+      ...modelArgs,
+      ...revisionArgs,
+      ...tokenArgs,
+    ],
     EXEC_OPTIONS,
   );
   const deploymentUrl = deploy.stdout.trim().split("\n").at(-1)?.trim();
@@ -224,6 +253,35 @@ async function deployToAlias(t: EveEvalContext, alias: string, phase: string): P
   );
 }
 
+async function postToDeployment(
+  t: EveEvalContext,
+  revision: string,
+  body: unknown,
+): Promise<MessageResponse> {
+  const deadline = Date.now() + 120_000;
+  while (Date.now() < deadline) {
+    const response = await t.target.fetch("/cross-version-webhook", {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        [DEPLOYMENT_REVISION_HEADER]: revision,
+      },
+      body: JSON.stringify(body),
+      signal: t.signal,
+    });
+    const result = (await response.json()) as MessageResponse & { code?: string };
+    if (response.status === 409 && result.code === DEPLOYMENT_PENDING_CODE) {
+      await t.sleep(1_000);
+      continue;
+    }
+    if (!response.ok) {
+      throw new Error(`Cross-version message failed (${response.status}).`);
+    }
+    return result;
+  }
+  throw new Error("Timed out waiting for the cross-version message's deployment.");
+}
+
 /** Waits until the alias exposes the marker from the expected deployment. */
 async function waitForAliasToServe(t: EveEvalContext, marker: string): Promise<void> {
   const deadline = Date.now() + 120_000;

e2e/fixtures/agent-channels/package.json

@@ -8,7 +8,8 @@
     "dev": "eve dev",
     "start": "eve start",
     "typecheck": "eve build && tsc",
-    "test:e2e": "eve eval --strict"
+    "test:e2e": "eve eval --strict",
+    "test:unit": "node --test test/*.test.mjs"
   },
   "dependencies": {
     "@eve-e2e/config": "workspace:*",

e2e/fixtures/agent-channels/test/deployment-revision.test.mjs

@@ -0,0 +1,35 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import {
+  checkDeploymentRevision,
+  DEPLOYMENT_PENDING_CODE,
+  DEPLOYMENT_REVISION_HEADER,
+} from "../agent/lib/deployment-revision.ts";
+
+function pinnedRequest() {
+  return new Request("https://fixture.example/cross-version-webhook", {
+    headers: { [DEPLOYMENT_REVISION_HEADER]: "requested-revision" },
+  });
+}
+
+test("rejects a stale deployment before dispatch", async () => {
+  const response = checkDeploymentRevision(pinnedRequest(), "previous-revision");
+
+  assert.equal(response.status, 409);
+  assert.equal((await response.json()).code, DEPLOYMENT_PENDING_CODE);
+});
+
+test("allows a request on its pinned deployment", () => {
+  assert.equal(checkDeploymentRevision(pinnedRequest(), "requested-revision"), undefined);
+});
+
+test("preserves requests without a deployment pin", () => {
+  assert.equal(
+    checkDeploymentRevision(
+      new Request("https://fixture.example/cross-version-webhook"),
+      "previous-revision",
+    ),
+    undefined,
+  );
+});

e2e/fixtures/agent-compaction-regressions/agent/agent.ts

@@ -15,6 +15,9 @@ import {
 } from "../constants";
 
 const TEST_CONTEXT_WINDOW_TOKENS = 32_000;
+// Scripted usage supplies pressure without starving the real summarizer's transcript.
+const TEST_COMPACTION_THRESHOLD_TOKENS = 2_048;
+const COMPACTION_PRESSURE_USAGE = { inputTokens: 4_096 };
 const MAX_TOOL_CALLS = 10;
 
 type RegressionCase =
@@ -170,12 +173,10 @@ const taskModel = mockModel({
     // These are fixture markers, not compaction protocol fields. `marker` records the
     // regression work tool; `SECOND_CHECKPOINT_MARKER` records the test-only tool
     // whose output makes the harness cross the compaction threshold a second time.
-    // Completion evidence is detected in any assistant message: compaction may
-    // leave it as a summarization checkpoint or as an eviction trail line, and
-    // the model must not repeat work in either case. User messages are
-    // excluded because the eval instructions themselves quote the markers.
-    if (assistantEvidenceContains(request.messages, marker)) {
-      if (assistantEvidenceContains(request.messages, SECOND_CHECKPOINT_MARKER)) {
+    // Kept tool results, checkpoints, and eviction trails all prove completed work.
+    // User messages are excluded because the eval instructions quote the markers.
+    if (completionEvidenceContains(request.messages, marker)) {
+      if (completionEvidenceContains(request.messages, SECOND_CHECKPOINT_MARKER)) {
         return `Done: ${marker}; ${SECOND_CHECKPOINT_MARKER}`;
       }
 
@@ -186,6 +187,7 @@ const taskModel = mockModel({
 
       checkpointAdvanceCallCounts.set(regressionCase, advanceCalls + 1);
       return {
+        usage: COMPACTION_PRESSURE_USAGE,
         toolCalls: [
           {
             id: `advance-checkpoint-${advanceCalls + 1}`,
@@ -206,6 +208,7 @@ const taskModel = mockModel({
 
     return regressionCase === "redundant-tool-calls"
       ? {
+          usage: COMPACTION_PRESSURE_USAGE,
           toolCalls: [
             {
               id: `inspect-repository-${attempt}`,
@@ -215,6 +218,7 @@ const taskModel = mockModel({
           ],
         }
       : {
+          usage: COMPACTION_PRESSURE_USAGE,
           toolCalls: [
             {
               id: `perform-source-analysis-${attempt}`,
@@ -235,7 +239,7 @@ export default defineAgent({
   compaction: {
     model: e2eModel(),
     modelContextWindowTokens: TEST_CONTEXT_WINDOW_TOKENS,
-    thresholdPercent: 0.02,
+    thresholdPercent: TEST_COMPACTION_THRESHOLD_TOKENS / TEST_CONTEXT_WINDOW_TOKENS,
   },
   limits: {
     maxInputTokensPerSession: 100_000,
@@ -267,9 +271,12 @@ function completionMarker(
     : "SOURCE_ANALYSIS_COMPLETE";
 }
 
-function assistantEvidenceContains(
+function completionEvidenceContains(
   messages: MockModelRequest["messages"],
   marker: string,
 ): boolean {
-  return messages.some((message) => message.role === "assistant" && message.text.includes(marker));
+  return messages.some(
+    (message) =>
+      (message.role === "assistant" || message.role === "tool") && message.text.includes(marker),
+  );
 }

e2e/fixtures/agent-compaction-regressions/evals/redundant-tool-calls.eval.ts

@@ -9,8 +9,10 @@ export default defineEval({
     const turn = await t.send(
       [
         "[case: redundant-tool-calls]",
-        "Call inspect-repository exactly once with scope repository.",
-        "After it succeeds, report REPOSITORY_INSPECTION_COMPLETE and call no more tools.",
+        "Alice is reviewing a small catalog application and preparing a handoff for Bob.",
+        "Use inspect-repository once with scope repository to obtain her review notes.",
+        "Once the review is complete, include REPOSITORY_INSPECTION_COMPLETE in the handoff.",
+        "The completed inspection does not need to be repeated.",
       ].join("\n"),
     );
 

e2e/fixtures/agent-compaction-regressions/evals/stale-todo-work.eval.ts

@@ -9,9 +9,11 @@ export default defineEval({
     const turn = await t.send(
       [
         "[case: stale-todo-work]",
-        "Call perform-source-analysis exactly once with approach initial.",
-        "The tool deliberately leaves its completed work in a pending todo.",
-        "After it succeeds, report SOURCE_ANALYSIS_COMPLETE and call no more tools.",
+        "Alice and Bob are reviewing a small catalog application.",
+        "Use perform-source-analysis once with approach initial to prepare Alice's findings.",
+        "Bob updates their shared checklist separately, so it may still say pending after the analysis is complete.",
+        "Use the completed analysis for the handoff and include SOURCE_ANALYSIS_COMPLETE.",
+        "The pending checklist entry does not require repeating the completed analysis.",
       ].join("\n"),
     );
 

e2e/fixtures/agent-subagents-hitl/agent/agent.ts

@@ -1,6 +1,6 @@
 import { e2eAgentConfig } from "@eve-e2e/config";
 import { defineAgent } from "eve";
-import type { MockModelRequest, MockModelResponse } from "eve/evals";
+import { mockModel, type MockModelRequest, type MockModelResponse } from "eve/evals";
 
 const COLLISION_MARKER = "MIXED-PARK-COMPLETE-7K2M";
 const STOCK_PRICE = "178.92";
@@ -58,6 +58,8 @@ function respond(request: MockModelRequest): MockModelResponse | string {
 }
 
 export default defineAgent({
-  ...e2eAgentConfig({ mock: respond }),
-  reasoning: "high",
+  ...e2eAgentConfig(),
+  // Parking coverage requires both actions in one step; children still use the matrix model.
+  model: mockModel(respond),
+  modelContextWindowTokens: 1_000_000,
 });