fix(runtime): preserve completed backends through bounded finalization

.ultrafuzz/prompts/strategies/invariants/invariant-testing-campaign.md

@@ -143,6 +143,15 @@ Use this configured invariant testing fuzzer timeout:
      relationships are contextual requirements beyond JSON Schema.
 
 3. Run the backend without path collisions.
+   - When the runtime supplies a **Checkpointed backend finalization** context,
+     wrap the complete supervised command below in its `ultrafuzz backend run`
+     command. That runner durably saves raw backend evidence on successful exit
+     and admits a bounded artifact-finalization grace window. Do not wrap the
+     preliminary smoke or claim completion from heartbeat/log activity. If that
+     context reports an existing completed checkpoint, reuse its restored raw
+     evidence and finalize artifacts without rerunning the expensive campaign.
+     Copy the runner's captured stdout/stderr into the declared log/evidence
+     paths; do not mistake an empty redirected wrapper stdout for backend logs.
    - Start the long campaign from this template, substituting the resolved
      worker count and the repository's own contract, config, and corpus
      conventions:
@@ -196,8 +205,12 @@ Use this configured invariant testing fuzzer timeout:
    - Choose the schema-defined execution variant that exactly matches whether
      the backend started, how it terminated, and whether its results are
      consumable. Record actual lifecycle timestamps, exit evidence, and failure
-     evidence; the finish time is the record finalization time even when the
-     backend never started.
+     evidence. For a backend that started, `execution.finished_at` and
+     `end_timestamp` are its actual exit time, not the later artifact-writing
+     time. Preserve that recorded exit time when resuming finalization from a
+     checkpoint; do not change the backend deadlines or invent a later backend
+     run. When the backend never started, use the time its unavailable/blocked
+     outcome was established.
    - Copy every plan-owned backend identity, version, execution worker/deadline
      setting, campaign command, and operational path exactly from
      `campaign-plan.json`. Bind the executed campaign command to the plan's

docs/reference/configuration.md

@@ -30,6 +30,7 @@ forge_rayon_threads = 1
 friction_log_enabled = false
 workspace_mode = "git-worktree"
 default_timeout_seconds = 3600
+finalization_grace_ms = 300000
 workflow_deadline_seconds = 86400
 controller_lease_seconds = 30
 refresh_prompts_on_resume = true
@@ -113,21 +114,22 @@ empty path components, and dot components fail validation.
 
 ## Run
 
-| Key                         | Type    | Meaning                                                                     |
-| --------------------------- | ------- | --------------------------------------------------------------------------- |
-| `output_dir`                | string  | Project-local run output directory. Defaults to `.ultrafuzz/runs`.          |
-| `max_parallel_agents`       | integer | Positive workflow submission concurrency default.                           |
-| `max_dynamic_nodes`         | integer | Positive run-wide safety limit for runtime-generated topology nodes.        |
-| `keep_workspaces`           | boolean | Keep every task worktree instead of deleting the disposable ones.           |
-| `forge_guard_enabled`       | boolean | Prepend a run-scoped Forge resource-limit wrapper to worker `PATH`.         |
-| `forge_vmem_limit_kb`       | integer | Forge virtual-memory ceiling in KiB. Defaults to 12 GiB.                    |
-| `forge_rayon_threads`       | integer | Default Forge Rayon worker count when the caller does not already set one.  |
-| `friction_log_enabled`      | boolean | Let agents record tooling roadblocks in a run-local friction log.           |
-| `workspace_mode`            | string  | Must be `git-worktree`.                                                     |
-| `default_timeout_seconds`   | integer | Default node timeout in seconds. The generated default is 3,600 (one hour). |
-| `workflow_deadline_seconds` | integer | Workflow wall-time limit, checked only when a command syncs (see below).    |
-| `controller_lease_seconds`  | integer | Lost-controller threshold used by the scoped renewable recovery supervisor. |
-| `refresh_prompts_on_resume` | boolean | Re-render unfinished tasks' prompts on `resume`. Defaults to `true`.        |
+| Key                         | Type    | Meaning                                                                                                                                      |
+| --------------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------- |
+| `output_dir`                | string  | Project-local run output directory. Defaults to `.ultrafuzz/runs`.                                                                           |
+| `max_parallel_agents`       | integer | Positive workflow submission concurrency default.                                                                                            |
+| `max_dynamic_nodes`         | integer | Positive run-wide safety limit for runtime-generated topology nodes.                                                                         |
+| `keep_workspaces`           | boolean | Keep every task worktree instead of deleting the disposable ones.                                                                            |
+| `forge_guard_enabled`       | boolean | Prepend a run-scoped Forge resource-limit wrapper to worker `PATH`.                                                                          |
+| `forge_vmem_limit_kb`       | integer | Forge virtual-memory ceiling in KiB. Defaults to 12 GiB.                                                                                     |
+| `forge_rayon_threads`       | integer | Default Forge Rayon worker count when the caller does not already set one.                                                                   |
+| `friction_log_enabled`      | boolean | Let agents record tooling roadblocks in a run-local friction log.                                                                            |
+| `workspace_mode`            | string  | Must be `git-worktree`.                                                                                                                      |
+| `default_timeout_seconds`   | integer | Default node timeout in seconds. The generated default is 3,600 (one hour).                                                                  |
+| `finalization_grace_ms`     | integer | Additional bounded finalization time after a checkpointed invariant backend succeeds (default 300,000 ms; 0 disables; maximum 3,600,000 ms). |
+| `workflow_deadline_seconds` | integer | Workflow wall-time limit, checked only when a command syncs (see below).                                                                     |
+| `controller_lease_seconds`  | integer | Lost-controller threshold used by the scoped renewable recovery supervisor.                                                                  |
+| `refresh_prompts_on_resume` | boolean | Re-render unfinished tasks' prompts on `resume`. Defaults to `true`.                                                                         |
 
 Other workspace modes are outside the product contract.
 
@@ -156,6 +158,48 @@ resume.
 `default_timeout_seconds` and `controller_lease_seconds` accept up to 86,400
 (one day).
 
+`finalization_grace_ms` applies to invariant campaign tasks that use the
+runtime-supplied `ultrafuzz backend run` wrapper. The original execution deadline
+still stops unfinished or failed backends. An exit code of zero admits a fixed
+finalization deadline: the later of the original deadline and backend completion
+plus the configured grace, never later than execution deadline plus grace.
+Heartbeats and provider retries do not extend it. Operator cancellation and the
+parent workflow's limits remain authoritative. Set the value to zero to retain
+the original single-deadline behavior; historical sealed configs without this
+field also keep that behavior.
+
+Completion is timestamped when the backend leader exits, not when inherited
+stdout/stderr pipes close. The supervisor kills remaining process-group members
+and drains logs for at most 250 ms; grace may begin during that drain, but a
+checkpoint is reusable only after its logs and evidence are sealed.
+
+The runtime supervisor saves bounded raw logs (16 MiB per stream), backend evidence (up to
+512 MiB and 10,000 entries in the complete checkpoint), the preliminary campaign plan,
+and a validated workspace patch under the run's `backend-checkpoints` directory
+before returning. Reuse is bound to the task, source revision, prompt and
+authenticated dependency markers, so changed inputs require fresh execution.
+Only an actual runtime-owned backend launch and successful exit admits grace;
+task-writable policy or receipt files cannot declare completion. The wrapper is
+an IPC client: its exit, disconnect, or SIGKILL cancels the runtime-owned process
+group, as do the original execution deadline and operator cancellation.
+Receipts are authenticated with a process-private key and bind the complete
+checkpoint inventory and SHA-256 contents, including logs and workspace patches.
+Missing, changed, extra, symlinked or hard-linked evidence fails closed before
+retry restoration. Writes use descriptor-anchored traversal and atomic staging,
+without following destination links; this feature requires Linux `/proc/self/fd`
+support. Set grace to zero on other hosts to keep single-deadline execution.
+These are unverified inputs, not findings or successful producer outputs.
+If finalization times out, an unchanged task retry in the same runtime process restores that evidence and
+gets only the configured finalization window, without rerunning fuzzing. Normal
+artifact validation still has to succeed. After a full runtime-process restart,
+the receipt key is intentionally unavailable: automatic reuse fails closed with
+an authentication diagnostic, preserving raw evidence for diagnosis instead of
+trusting writable disk state. `ultrafuzz node` reports the phase,
+both deadlines, checkpoint availability, and last observed agent activity;
+timeout errors include those fields and elapsed time. If that optional status
+file is unreadable or invalid, `node` and `node --watch` warn while still showing
+the independently validated node, attempt, and tool evidence.
+
 `workflow_deadline_seconds` is not a guaranteed wall-clock limit. Ultrafuzz
 records `workflow_deadline_at` in run state when the run is created, and again
 from each `replay`, `fork`, or `resume` that starts a controller (not one that

eslint.config.js

@@ -135,6 +135,7 @@ export default tseslint.config(
           "COMPILED_TASKS",
           "DYNAMIC_GROUPS",
           "FRICTION_LOG",
+          "FINALIZATION_GRACE_MS",
           "MAX_DYNAMIC_NODES",
           "RUN_ID_LITERAL",
           "RUN_ROOT_RELATIVE",

packages/cli/schema/cli-result.schema.json

@@ -1157,7 +1157,29 @@
             }
           }
         },
-        "tool_details_included": { "type": "boolean" }
+        "tool_details_included": { "type": "boolean" },
+        "backend_finalization": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "phase",
+            "started_at_ms",
+            "execution_deadline_ms",
+            "finalization_deadline_ms",
+            "finalization_grace_ms",
+            "last_activity_at_ms",
+            "checkpoint_available"
+          ],
+          "properties": {
+            "phase": { "enum": ["execution", "finalization", "completed", "timed-out", "aborted", "failed"] },
+            "started_at_ms": { "$ref": "#/$defs/nonNegativeInteger" },
+            "execution_deadline_ms": { "$ref": "#/$defs/nonNegativeInteger" },
+            "finalization_deadline_ms": { "type": ["integer", "null"], "minimum": 0 },
+            "finalization_grace_ms": { "$ref": "#/$defs/nonNegativeInteger" },
+            "last_activity_at_ms": { "$ref": "#/$defs/nonNegativeInteger" },
+            "checkpoint_available": { "type": "boolean" }
+          }
+        }
       }
     },
     "snapshotsData": {

packages/cli/src/commands/backend/run.ts

@@ -0,0 +1,21 @@
+import { Args, Command } from "@oclif/core";
+import { runCheckpointedBackend } from "@ultrafuzz/runtime";
+
+/** Internal campaign primitive; stdout/stderr are durable files, not CLI JSON. */
+export default class BackendRun extends Command {
+  static override summary = "Run a backend with a task-scoped completion checkpoint";
+  static override hidden = true;
+  static override strict = false;
+  static override args = { checkpoint: Args.string({ required: true }) };
+
+  async run(): Promise<void> {
+    const { args } = await this.parse(BackendRun);
+    const separator = this.argv.indexOf("--");
+    if (separator !== 1) throw new Error("usage: ultrafuzz backend run <checkpoint> -- <command> [args...]");
+    // Oclif's parsed argv can consume a backend's own second `--`. Preserve
+    // the command vector exactly; never reparse it as Ultrafuzz flags.
+    const command = this.argv.slice(separator + 1);
+    const code = await runCheckpointedBackend(args.checkpoint, command);
+    if (code !== 0) this.exit(code);
+  }
+}

packages/cli/src/commands/node.ts

@@ -1,9 +1,10 @@
 import { Args, Command, Flags } from "@oclif/core";
-import { getWorkflowNode, watchWorkflowNode, type WorkflowNodeValue } from "@ultrafuzz/runtime";
+import { getWorkflowNode, watchWorkflowNode, type RuntimeDiagnostic, type WorkflowNodeValue } from "@ultrafuzz/runtime";
 
 import {
   cliIo,
   commandFromRuntime,
+  diagnosticsText,
   emitCommandResult,
   emitWatchFailure,
   envelope,
@@ -59,8 +60,8 @@ export default class Node extends Command {
     const result = await watchWorkflowNode({
       ...query,
       intervalSeconds: flags.interval,
-      onSnapshot: (value) => {
-        emitWatchSnapshot(value, json);
+      onSnapshot: (value, diagnostics) => {
+        emitWatchSnapshot(value, json, diagnostics);
       }
     });
     if (!result.ok) {
@@ -69,13 +70,13 @@ export default class Node extends Command {
   }
 }
 
-function emitWatchSnapshot(value: WorkflowNodeValue, json: boolean): void {
+function emitWatchSnapshot(value: WorkflowNodeValue, json: boolean, diagnostics: RuntimeDiagnostic[]): void {
   const io = cliIo();
   if (!json) {
-    io.stdout.write(renderNode(value));
+    io.stdout.write(`${renderNode(value)}${diagnosticsText(diagnostics)}`);
     return;
   }
-  const result: CommandResult = { ok: true, command: "node", data: toCliWorkflowNodeData(value), diagnostics: [] };
+  const result: CommandResult = { ok: true, command: "node", data: toCliWorkflowNodeData(value), diagnostics };
   io.stdout.write(`${JSON.stringify(envelope("node", result))}\n`);
 }
 
@@ -92,6 +93,14 @@ function renderNode(value: WorkflowNodeValue): string {
     `Output: ${value.output.present ? `recorded (${value.output.source ?? "unknown source"})` : "not recorded"}`,
     `Updated: ${value.updated_at ?? "unknown"}`
   ];
+  const phase = value.backend_finalization;
+  if (phase !== undefined) {
+    lines.push(
+      `Backend phase: ${phase.phase}; checkpoint: ${phase.checkpoint_available ? "available" : "not available"}`,
+      `Execution deadline: ${new Date(phase.execution_deadline_ms).toISOString()}; finalization deadline: ${phase.finalization_deadline_ms === null ? "not entered" : new Date(phase.finalization_deadline_ms).toISOString()}`,
+      `Finalization grace: ${String(phase.finalization_grace_ms)}ms; last activity: ${new Date(phase.last_activity_at_ms).toISOString()}`
+    );
+  }
   for (const attempt of value.attempts) {
     lines.push(
       `- attempt ${attempt.attempt ?? "?"}: ${attempt.state ?? "unknown"}${attempt.cached ? " (cached)" : ""}${

packages/cli/test/backend-run.test.ts

@@ -0,0 +1,46 @@
+import assert from "node:assert/strict";
+import { execFile } from "node:child_process";
+import fs from "node:fs";
+import path from "node:path";
+import test from "node:test";
+import { fileURLToPath } from "node:url";
+import { promisify } from "node:util";
+import { withBackendFinalization, readBackendFinalizationStatus } from "@ultrafuzz/runtime";
+import { temporaryRoot } from "./temporary-root.js";
+
+test("backend run passes argv after -- to the child and publishes its completion checkpoint", async (t) => {
+  const root = temporaryRoot("backend-cli-", t);
+  const artifactDirectory = path.join(root, "artifacts");
+  const directory = path.join(root, "checkpoint");
+  fs.mkdirSync(artifactDirectory);
+  const entry = fileURLToPath(new URL("../../dist/index.js", import.meta.url));
+  await withBackendFinalization({
+    directory,
+    artifactDirectory,
+    binding: "cli-fixture",
+    timeoutMs: 5_000,
+    graceMs: 1_000,
+    args: { prompt: "test" },
+    generate: async () => {
+      const result = await promisify(execFile)(process.execPath, [
+        entry,
+        "backend",
+        "run",
+        directory,
+        "--",
+        process.execPath,
+        "-e",
+        "process.stdout.write(process.argv[1])",
+        "--",
+        "literal --argument"
+      ]);
+      assert.equal(result.stdout, "");
+      assert.equal(result.stderr, "");
+    }
+  });
+  const entries = fs.readdirSync(directory).filter((name) => fs.lstatSync(path.join(directory, name)).isDirectory());
+  const snapshot = entries[0];
+  assert.ok(snapshot !== undefined);
+  assert.equal(fs.readFileSync(path.join(directory, snapshot, "stdout.log"), "utf8"), "literal --argument");
+  assert.equal(readBackendFinalizationStatus(directory)?.checkpoint_available, true);
+});