[core] Single-orchestrator runtime (spec 9)

.changeset/quickjs-inert-after-terminal.md

@@ -0,0 +1,5 @@
+---
+"@workflow/core": patch
+---
+
+With `WORKFLOW_VM=quickjs`, replay ignores events for a step, wait or hook after its terminal event, as the default engine does.

.changeset/single-orchestrator-runtime.md

@@ -0,0 +1,16 @@
+---
+"@workflow/core": minor
+"@workflow/world": minor
+"@workflow/errors": patch
+"@workflow/world-vercel": minor
+"@workflow/world-local": minor
+"@workflow/world-postgres": minor
+"@workflow/world-testing": patch
+"@workflow/vitest": patch
+---
+
+Add opt-in spec 9 runs, driven by a single orchestrator per run: set `WORKFLOW_SPEC_VERSION=9`. New runs stay on spec 8 by default.
+
+- Without the setting, new runs are spec 8 and behave as before.
+- To opt in, first upgrade every deployment and process that calls `start()` or shares the World to this release, then set `WORKFLOW_SPEC_VERSION=9` on all of them. An older release fails writes to spec 9 runs (reads still work, nothing is corrupted), so do not roll back below this release while spec 9 runs are in flight.
+- Custom Worlds: `mintedSpecVersion()` keeps creating spec 8 runs. A World that implements the in-band writer fence declares `capabilities.inBandFence` and passes `{ inBandFence: true }` to create spec 9 runs.

.changeset/world-local-queue-slot-per-delivery.md

@@ -0,0 +1,5 @@
+---
+"@workflow/world-local": patch
+---
+
+Hold a queue concurrency slot for each delivery, not across the sleeps and backoffs between deliveries.

.changeset/world-local-retry-backoff.md

@@ -0,0 +1,5 @@
+---
+"@workflow/world-local": patch
+---
+
+Back off retries of a queue message whose deliveries keep failing after 32 attempts, as Vercel Queues does.

.github/scripts/vercel-e2e-matrix.js

@@ -107,6 +107,11 @@ const HTTP_TRANSPORT_APPS = [
   'hono',
 ];
 
+// Apps whose dedicated deployment opts in to spec 9 runs
+// (WORKFLOW_SPEC_VERSION=9). Every other Vercel lane runs the default, spec 8.
+// Both VMs, since each engine has its own spec 9 orchestrator.
+const SPEC9_APPS = ['nextjs-turbopack'];
+
 function project(name) {
   const found = PROJECTS.find((candidate) => candidate.name === name);
   if (!found) {
@@ -140,6 +145,12 @@ function buildMatrices() {
         app: appEntry(project(name)),
       })),
     },
+    'vercel-spec9-matrix': {
+      include: SPEC9_APPS.flatMap((name) => {
+        const app = project(name);
+        return (app.vms ?? VMS).map((vm) => ({ app: appEntry(app), vm }));
+      }),
+    },
   };
 }
 

.github/scripts/vercel-e2e-matrix.test.js

@@ -50,6 +50,13 @@ test('the transport lanes run their fixed app subsets', () => {
   ]);
 });
 
+test('the spec 9 lane runs its app on both VMs', () => {
+  assert.deepEqual(names(matrices['vercel-spec9-matrix']), [
+    'nextjs-turbopack/node',
+    'nextjs-turbopack/quickjs',
+  ]);
+});
+
 test('matrix entries carry no lane-internal fields', () => {
   for (const { app } of matrices['vercel-prod-matrix'].include) {
     assert.deepEqual(Object.keys(app).sort(), [

.github/workflows/event-log-race-repro.yml

@@ -100,8 +100,15 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/stable' }}
 
 jobs:
+  # Two cells on Vercel. `default` races the standard Git preview, whose new
+  # runs take the default spec version (8). `spec9` races a dedicated preview
+  # deployed with WORKFLOW_SPEC_VERSION=9, the single orchestrator, which is
+  # opt-in and so has no other race coverage. The variable is set on that
+  # deployment AND on the runner: a cross-deployment start() stamps the lower
+  # of the two versions, and the harness fails if a run comes back at another
+  # spec version. Both cells gate.
   event-log-race-repro:
-    name: Event Log Race Repro
+    name: Event Log Race Repro${{ matrix.spec == 'spec9' && ' (spec 9)' || '' }}
     runs-on: ubuntu-latest
     # Sized for the harness' default scale: its own test timeout is
     # `budget_ms + run_timeout_ms + 60s` (~17 min at the defaults), and the rest is
@@ -110,6 +117,10 @@ jobs:
     # the job before the summary is written.
     timeout-minutes: 25
     if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'event-log-race-repro') }}
+    strategy:
+      fail-fast: false
+      matrix:
+        spec: [default, spec9]
     # No comment-writing scopes: the PR comment for every lane is written by
     # `event-log-race-repro-comment` below.
     permissions:
@@ -135,12 +146,48 @@ jobs:
         run: pnpm turbo run build --filter='@workflow/cli'
 
       - name: Resolve Vercel deployment
-        id: deployment
+        id: git-deployment
+        if: ${{ matrix.spec == 'default' }}
         uses: ./.github/actions/resolve-vercel-deployment
         with:
           project-id: "prj_yjkM7UdHliv8bfxZ1sMJQf1pMpdi"
           token: ${{ secrets.VERCEL_LABS_TOKEN }}
 
+      # Same dedicated-deployment pattern as `e2e-vercel-spec9` in tests.yml.
+      - name: Deploy a dedicated spec 9 preview
+        id: spec9-deployment
+        if: ${{ matrix.spec == 'spec9' }}
+        uses: ./.github/actions/deploy-vercel-preview
+        with:
+          project-id: "prj_yjkM7UdHliv8bfxZ1sMJQf1pMpdi"
+          team-id: "team_nO2mCG4W8IxPIeKoSsqwAxxB"
+          token: ${{ secrets.VERCEL_LABS_TOKEN }}
+          env: |
+            WORKFLOW_SPEC_VERSION=9
+
+      - name: Select deployment
+        id: deployment
+        env:
+          GIT_URL: ${{ steps.git-deployment.outputs.deployment-url }}
+          GIT_ID: ${{ steps.git-deployment.outputs.deployment-id }}
+          GIT_ENVIRONMENT: ${{ steps.git-deployment.outputs.environment }}
+          SPEC9_URL: ${{ steps.spec9-deployment.outputs.deployment-url }}
+          SPEC9_ID: ${{ steps.spec9-deployment.outputs.deployment-id }}
+        run: |
+          if [ "${{ matrix.spec }}" = spec9 ]; then
+            {
+              echo "deployment-url=$SPEC9_URL"
+              echo "deployment-id=$SPEC9_ID"
+              echo "environment=preview"
+            } >> "$GITHUB_OUTPUT"
+          else
+            {
+              echo "deployment-url=$GIT_URL"
+              echo "deployment-id=$GIT_ID"
+              echo "environment=$GIT_ENVIRONMENT"
+            } >> "$GITHUB_OUTPUT"
+          fi
+
       - name: Run event log race repro
         id: repro
         continue-on-error: true
@@ -150,6 +197,7 @@ jobs:
           DEPLOYMENT_URL: ${{ steps.deployment.outputs.deployment-url }}
           VERCEL_DEPLOYMENT_ID: ${{ steps.deployment.outputs.deployment-id }}
           APP_NAME: nextjs-turbopack
+          WORKFLOW_SPEC_VERSION: ${{ matrix.spec == 'spec9' && '9' || '' }}
           WORKFLOW_VERCEL_ENV: ${{ steps.deployment.outputs.environment }}
           WORKFLOW_VERCEL_AUTH_TOKEN: ${{ secrets.VERCEL_LABS_TOKEN }}
           WORKFLOW_VERCEL_TEAM: "team_nO2mCG4W8IxPIeKoSsqwAxxB"
@@ -193,7 +241,7 @@ jobs:
         run: |
           node .github/scripts/render-event-log-race-repro-results.js \
             event-log-race-repro-results.json \
-            --label vercel \
+            --label "${{ matrix.spec == 'spec9' && 'vercel-spec9' || 'vercel' }}" \
             --run-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
             --run-attempt "${{ github.run_attempt }}" \
             --timestamp "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \
@@ -203,7 +251,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: event-log-race-repro-results
+          name: event-log-race-repro-results${{ matrix.spec == 'spec9' && '-vercel-spec9' || '' }}
           path: |
             event-log-race-repro-results.json
             event-log-race-repro-summary.md
@@ -383,10 +431,11 @@ jobs:
             previous_comment_args=(--previous-comment event-log-race-repro-previous-comment.md)
           fi
 
-          # Lane order here is the row order in the comment: the gating lane
-          # first, then the two report-only worlds.
+          # Lane order here is the row order in the comment: the two gating
+          # Vercel lanes first, then the two report-only worlds.
           node .github/scripts/render-event-log-race-repro-results.js \
             --lane vercel=lane-results/event-log-race-repro-results/event-log-race-repro-results.json \
+            --lane vercel-spec9=lane-results/event-log-race-repro-results-vercel-spec9/event-log-race-repro-results.json \
             --lane local=lane-results/event-log-race-repro-results-world-local/event-log-race-repro-results.json \
             --lane postgres=lane-results/event-log-race-repro-results-world-postgres/event-log-race-repro-results.json \
             --run-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \

.github/workflows/tests.yml

@@ -30,6 +30,7 @@ jobs:
       vercel-prod-matrix: ${{ steps.vercel-matrix.outputs.vercel-prod-matrix }}
       vercel-ws-transport-matrix: ${{ steps.vercel-matrix.outputs.vercel-ws-transport-matrix }}
       vercel-http-transport-matrix: ${{ steps.vercel-matrix.outputs.vercel-http-transport-matrix }}
+      vercel-spec9-matrix: ${{ steps.vercel-matrix.outputs.vercel-spec9-matrix }}
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v4
@@ -649,6 +650,97 @@ jobs:
           token: ${{ secrets.VERCEL_LABS_TOKEN }}
           e2e-start-ms: ${{ steps.e2eStart.outputs.ms }}
 
+  e2e-vercel-spec9:
+    # Spec 9 runs (the single orchestrator) are opt-in with
+    # WORKFLOW_SPEC_VERSION=9, so every other Vercel lane runs the default,
+    # spec 8. This lane deploys its own preview with the variable set and runs
+    # the full suite against it on both VMs.
+    #
+    # The variable is set on the deployment AND on the test runner: a
+    # cross-deployment start() stamps the lower of the target's and the
+    # caller's version, so with only the deployment env set this lane would
+    # silently test spec 8. The e2e test "stamps the spec version the lane
+    # opts in to" fails if either side stops seeing it.
+    #
+    # Same dedicated-deployment pattern as e2e-vercel-ws-transport above.
+    name: E2E Vercel Spec 9 Test (${{ matrix.app.name }}, ${{ matrix.vm }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    needs: ci-scope
+    if: ${{ needs.ci-scope.outputs.fast-path != 'true' }}
+    concurrency:
+      group: e2e-vercel-spec9-${{ matrix.app.name }}-${{ matrix.vm }}-${{ needs.ci-scope.outputs.e2e-vercel-slot }}
+      queue: max
+    permissions:
+      id-token: write
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.ci-scope.outputs.vercel-spec9-matrix) }}
+    env:
+      TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
+      TURBO_TEAM: ${{ vars.TURBO_TEAM }}
+      WORKFLOW_PUBLIC_MANIFEST: '1'
+      VERCEL_TEAM_ID: ${{ needs.ci-scope.outputs.vercel-team-id }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+
+      - name: Setup environment
+        uses: ./.github/actions/setup-workflow-dev
+        with:
+          build-packages: 'false'
+
+      - name: Build CLI
+        run: pnpm turbo run build --filter='@workflow/cli'
+
+      - name: Deploy a dedicated spec 9 preview
+        id: deployment
+        uses: ./.github/actions/deploy-vercel-preview
+        with:
+          project-id: ${{ matrix.app.project-id }}
+          team-id: ${{ env.VERCEL_TEAM_ID }}
+          token: ${{ secrets.VERCEL_LABS_TOKEN }}
+          env: |
+            WORKFLOW_SPEC_VERSION=9
+
+      - name: Record E2E start time
+        id: e2eStart
+        run: echo "ms=$(($(date +%s) * 1000))" >> "$GITHUB_OUTPUT"
+
+      - name: Run E2E Tests
+        run: pnpm run test:e2e --reporter=verbose --reporter=json --reporter=./packages/core/e2e/github-reporter.ts "--outputFile=e2e-vercel-spec9-$APP_NAME-$WORKFLOW_VM.json"
+        env:
+          NODE_OPTIONS: "--enable-source-maps"
+          DEPLOYMENT_URL: ${{ steps.deployment.outputs.deployment-url }}
+          VERCEL_DEPLOYMENT_ID: ${{ steps.deployment.outputs.deployment-id }}
+          APP_NAME: ${{ matrix.app.name }}
+          WORKFLOW_VM: ${{ matrix.vm }}
+          WORKFLOW_SPEC_VERSION: '9'
+          WORKFLOW_VERCEL_ENV: "preview"
+          WORKFLOW_VERCEL_AUTH_TOKEN: ${{ secrets.VERCEL_LABS_TOKEN }}
+          WORKFLOW_VERCEL_TEAM: ${{ env.VERCEL_TEAM_ID }}
+          WORKFLOW_VERCEL_PROJECT: ${{ matrix.app.project-id }}
+          WORKFLOW_VERCEL_PROJECT_SLUG: ${{ matrix.app.project-slug }}
+          VERCEL_WORKFLOW_SERVER_URL: ${{ github.ref != 'refs/heads/main' && secrets.VERCEL_WORKFLOW_SERVER_URL || '' }}
+
+      - name: Report E2E results
+        if: always()
+        uses: ./.github/actions/report-vercel-e2e
+        with:
+          results-name: vercel-spec9-${{ matrix.app.name }}-${{ matrix.vm }}
+          app: ${{ matrix.app.name }}
+          summary-title: E2E Vercel Spec 9 (${{ matrix.app.name }}, ${{ matrix.vm }})
+          capture-runtime-logs: ${{ job.status == 'failure' }}
+          deployment-id: ${{ steps.deployment.outputs.deployment-id }}
+          environment: preview
+          project-id: ${{ matrix.app.project-id }}
+          team-id: ${{ env.VERCEL_TEAM_ID }}
+          token: ${{ secrets.VERCEL_LABS_TOKEN }}
+          e2e-start-ms: ${{ steps.e2eStart.outputs.ms }}
+
   e2e-vercel-http-transport:
     # The counterpart to e2e-vercel-ws-transport. With
     # WORKFLOW_EVENTS_TRANSPORT defaulting to `ws`, e2e-vercel-prod — which
@@ -1461,7 +1553,7 @@ jobs:
   summary:
     name: E2E Summary
     runs-on: ubuntu-latest
-    needs: [ci-scope, e2e-vercel-prod, e2e-vercel-multi-region, e2e-vercel-ws-transport, e2e-vercel-http-transport, e2e-local-dev, e2e-local-prod, e2e-local-postgres, e2e-python, e2e-windows]
+    needs: [ci-scope, e2e-vercel-prod, e2e-vercel-multi-region, e2e-vercel-ws-transport, e2e-vercel-http-transport, e2e-vercel-spec9, e2e-local-dev, e2e-local-prod, e2e-local-postgres, e2e-python, e2e-windows]
     if: always() && !cancelled() && needs.ci-scope.outputs.fast-path != 'true'
     timeout-minutes: 10
 
@@ -1497,7 +1589,7 @@ jobs:
   e2e-required-check:
     name: E2E Required Check
     runs-on: ubuntu-latest
-    needs: [ci-scope, unit, e2e-package-build, e2e-vercel-prod, e2e-vercel-ws-transport, e2e-vercel-http-transport, e2e-local-dev, e2e-local-prod, e2e-local-postgres, e2e-windows]
+    needs: [ci-scope, unit, e2e-package-build, e2e-vercel-prod, e2e-vercel-ws-transport, e2e-vercel-http-transport, e2e-vercel-spec9, e2e-local-dev, e2e-local-prod, e2e-local-postgres, e2e-windows]
     if: always()
     timeout-minutes: 5
 
@@ -1509,6 +1601,7 @@ jobs:
           VERCEL_STATUS: ${{ needs.e2e-vercel-prod.result }}
           VERCEL_WS_STATUS: ${{ needs.e2e-vercel-ws-transport.result }}
           VERCEL_HTTP_STATUS: ${{ needs.e2e-vercel-http-transport.result }}
+          VERCEL_SPEC9_STATUS: ${{ needs.e2e-vercel-spec9.result }}
           LOCAL_DEV_STATUS: ${{ needs.e2e-local-dev.result }}
           LOCAL_PROD_STATUS: ${{ needs.e2e-local-prod.result }}
           POSTGRES_STATUS: ${{ needs.e2e-local-postgres.result }}
@@ -1554,6 +1647,7 @@ jobs:
             [[ "$VERCEL_STATUS" == "success" ]] || FAILED_JOBS+=("e2e-vercel-prod ($VERCEL_STATUS)")
             [[ "$VERCEL_WS_STATUS" == "success" ]] || FAILED_JOBS+=("e2e-vercel-ws-transport ($VERCEL_WS_STATUS)")
             [[ "$VERCEL_HTTP_STATUS" == "success" ]] || FAILED_JOBS+=("e2e-vercel-http-transport ($VERCEL_HTTP_STATUS)")
+            [[ "$VERCEL_SPEC9_STATUS" == "success" ]] || FAILED_JOBS+=("e2e-vercel-spec9 ($VERCEL_SPEC9_STATUS)")
             [[ "$LOCAL_DEV_STATUS" == "success" ]] || FAILED_JOBS+=("e2e-local-dev ($LOCAL_DEV_STATUS)")
             [[ "$LOCAL_PROD_STATUS" == "success" ]] || FAILED_JOBS+=("e2e-local-prod ($LOCAL_PROD_STATUS)")
             [[ "$POSTGRES_STATUS" == "success" ]] || FAILED_JOBS+=("e2e-local-postgres ($POSTGRES_STATUS)")

docs/content/docs/v5/api-reference/workflow-api/get-run.mdx

@@ -199,6 +199,10 @@ export async function POST(req: Request) {
 
 The options object is optional: `await run.cancel()` cancels the run without recording a reason.
 
+## Spec 9 (experimental)
+
+On runs of the experimental spec 9 runtime, `wakeUp()` asks the run's orchestrator to complete the sleeps instead of completing them itself, `stoppedCount` counts the sleeps it asked to complete, and the request is best-effort while the run is active. See [`run.wakeUp()` on spec 9 runs](/docs/changelog/spec-9#runwakeup-and-waitforsleep).
+
 ## Related functions
 
 - [`start()`](/docs/api-reference/workflow-api/start): Start a new workflow and get its run ID.