Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/bump-workflow-beta-42.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"eve": patch
---

Bump `@workflow/*` packages to the latest 5.0.0 betas (`core` beta.42, `errors` beta.17, `world` beta.27, `world-local` beta.36, `world-vercel` beta.38, `world-postgres` beta.34). The development Workflow world now advertises spec version 6 (slot-numbered event ids), matching the updated local world.
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import { equals, satisfies } from "eve/evals/expect";
export default defineEval({
tags: ["real-model"],
description:
"Messages sent during an active turn produce one limit prompt, and approve resets the budget.",
"Messages queued during an active turn preserve one limit prompt, and approve resets the budget.",
timeoutMs: 90_000,
async test(t) {
const active = await t.start(
Expand Down Expand Up @@ -45,38 +45,31 @@ export default defineEval({
if (activeState === undefined) {
throw new Error("The active eval session did not expose client state.");
}
// Both messages were queued behind the active turn, so eve delivers them
// coalesced into one follow-up turn while the limit prompt stays pending.
const queuedSession = t.target.watchTurn(active.sessionId, {
startIndex: activeState.streamIndex,
});
const queuedA = await queuedSession.result();
queuedA.event("message.received", { count: 1 });
queuedA.notEvent("input.requested");
queuedA.eventsSatisfy("delivers message A while preserving the pending prompt", (events) => {
const received = events.find((event) => event.type === "message.received");
return received !== undefined && received.data.message.includes("Queued message A");
});
const queuedState = queuedSession.session.state;
if (queuedState === undefined) {
throw new Error("The attached eval session did not expose client state.");
}
const queuedBSession = t.target.watchTurn(active.sessionId, {
startIndex: queuedState.streamIndex,
});
const queuedB = await queuedBSession.result();
queuedB.event("message.received", { count: 1 });
queuedB.notEvent("input.requested");
queuedB.eventsSatisfy("delivers message B while preserving the pending prompt", (events) => {
const received = events.find((event) => event.type === "message.received");
return received !== undefined && received.data.message.includes("Queued message B");
});
const queued = await queuedSession.result();
queued.event("message.received", { count: 1 });
queued.notEvent("input.requested");
queued.eventsSatisfy(
"coalesces messages A and B in order while preserving the pending prompt",
(events) => {
const received = events.find((event) => event.type === "message.received");
if (received === undefined) return false;
const messageAIndex = received.data.message.indexOf("Queued message A");
const messageBIndex = received.data.message.indexOf("Queued message B");
return messageAIndex !== -1 && messageBIndex > messageAIndex;
},
);
t.check(
[...prompted.events, ...queuedA.events, ...queuedB.events].filter(
(event) => event.type === "input.requested",
).length,
[...prompted.events, ...queued.events].filter((event) => event.type === "input.requested")
.length,
equals(1),
);

const resumed = await queuedBSession.session.respond([
const resumed = await queuedSession.session.respond([
{
optionId: "continue",
requestId: request.requestId,
Expand All @@ -87,18 +80,18 @@ export default defineEval({
resumed.notEvent("input.requested");
resumed.messageIncludes("approved");
t.check(resumed.sessionId, equals(active.sessionId));
t.check(queuedBSession.session.sessionId, equals(active.sessionId));
t.check(queuedSession.session.sessionId, equals(active.sessionId));

const nextPrompt = await queuedBSession.session.send(
const nextPrompt = await queuedSession.session.send(
'Post-approval probe: reply with exactly "same session".',
);
nextPrompt.event("input.requested", { count: 1 });
nextPrompt.notEvent("message.completed");
t.check(nextPrompt.sessionId, equals(active.sessionId));
t.check(nextPrompt.status, equals("waiting"));
t.check(queuedBSession.session.sessionId, equals(active.sessionId));
t.check(queuedSession.session.sessionId, equals(active.sessionId));

const nextRequest = queuedBSession.session.requireInputRequest({
const nextRequest = queuedSession.session.requireInputRequest({
display: "confirmation",
optionIds: ["continue", "stop"],
toolName: "session_limit_continuation",
Expand Down
10 changes: 5 additions & 5 deletions packages/eve/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -354,13 +354,13 @@
"@vercel/sandbox": "catalog:",
"@vercel/sandbox-drives": "catalog:",
"@vercel/sdk": "1.28.8",
"@workflow/core": "5.0.0-beta.41",
"@workflow/errors": "5.0.0-beta.16",
"@workflow/core": "5.0.0-beta.42",
"@workflow/errors": "5.0.0-beta.17",
"@workflow/serde": "5.0.0-beta.2",
"@workflow/utils": "5.0.0-beta.8",
"@workflow/world": "5.0.0-beta.26",
"@workflow/world-local": "5.0.0-beta.35",
"@workflow/world-vercel": "5.0.0-beta.37",
"@workflow/world": "5.0.0-beta.27",
"@workflow/world-local": "5.0.0-beta.36",
"@workflow/world-vercel": "5.0.0-beta.38",
"ai": "catalog:",
"autoevals": "0.0.132",
"chat": "4.34.0",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ async function call<T>(
export function createDevelopmentWorkflowWorld(): World {
const forwarded = buildForwardedOperations();
const world = {
specVersion: 5 as SpecVersion,
specVersion: 6 as SpecVersion,
async getDeploymentId() {
// Inside a pinned delivery, steps and child runs must record the
// delivery's generation — not whatever is active — so replay after a
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ describe("parent development Workflow World", () => {
workflowName: turnWorkflowReference.workflowId,
},
eventType: "run_created",
specVersion: 5,
specVersion: 6,
},
]);
const runId = readCreatedRunId(created);
Expand All @@ -108,7 +108,7 @@ describe("parent development Workflow World", () => {
runInput: {
deploymentId: "generation-a",
input: new Uint8Array(),
specVersion: 5,
specVersion: 6,
workflowName: workflowEntryReference.workflowId,
},
}),
Expand Down Expand Up @@ -142,7 +142,7 @@ describe("parent development Workflow World", () => {
workflowName: turnWorkflowReference.workflowId,
},
eventType: "run_created",
specVersion: 5,
specVersion: 6,
},
]);
await first.close();
Expand Down Expand Up @@ -187,7 +187,7 @@ describe("parent development Workflow World", () => {
workflowName: turnWorkflowReference.workflowId,
},
eventType: "run_created",
specVersion: 5,
specVersion: 6,
},
]);
const runId = readCreatedRunId(created);
Expand Down Expand Up @@ -236,7 +236,7 @@ describe("parent development Workflow World", () => {
workflowName: turnWorkflowReference.workflowId,
},
eventType: "run_created",
specVersion: 5,
specVersion: 6,
},
]);
const runId = readCreatedRunId(created);
Expand Down
6 changes: 3 additions & 3 deletions packages/eve/src/internal/workflow/validate-world.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ describe("validateWorkflowWorld", () => {
validateWorkflowWorld({
world: {
events: {},
specVersion: 5,
specVersion: 6,
},
}),
).toThrow("Configured Workflow world factory did not return a valid World.");
Expand All @@ -27,7 +27,7 @@ describe("validateWorkflowWorld", () => {
validateWorkflowWorld({
world: {
createQueueHandler: vi.fn(),
specVersion: 5,
specVersion: 6,
},
}),
).toThrow("Configured Workflow world factory did not return a valid World.");
Expand All @@ -49,6 +49,6 @@ function createMockWorld() {
return {
createQueueHandler: vi.fn(),
events: {},
specVersion: 5,
specVersion: 6,
};
}
Loading
Loading