fix(e2e): bound MCP discovery and verify backup transport

docs/manage-sandboxes/recover-rebuild-sandboxes.mdx

@@ -310,6 +310,10 @@ An unavailable or different version ends the rebuild with a nonzero status inste
 
 <AgentOnly variant="openclaw">
 
+Before backing up a running OpenClaw sandbox, rebuild verifies its maintenance window through OpenShell command transport.
+It waits within the maintenance deadline when command transport is not ready.
+If verification fails, rebuild stops before replacement and reports the failed maintenance-window check.
+
 Rebuild restores the complete credential-sanitized native `openclaw.json` file from the pre-rebuild snapshot.
 NemoClaw does not compare that file with a config hash, seal, last-good copy, recovery anchor, baseline, or selected-key registry.
 If the file cannot be restored safely, NemoClaw exits nonzero and does not report a successful rebuild.
@@ -521,6 +525,9 @@ NemoClaw fails closed when the selected gateway, replacement settings, durable s
 A same-name recreation started by `$$nemoclaw onboard` uses the same replacement journal. If that recreation is interrupted after the `Journaled replacement` message, rerun the original onboarding command with the same target settings. The active replacement can continue without adding `--resume`. Use `--resume` for interrupted onboarding steps that occur before a replacement journal exists.
 
 If any manifest-declared state directory or state file cannot be archived, NemoClaw reports each failed path and stops before deleting the original sandbox.
+State-directory discovery can fail because of an SSH command failure, unsafe entries, or an expired backup deadline.
+Use the reported reason to identify the failed condition.
+For an SSH access failure, restore access to the recorded OpenShell gateway and sandbox before retrying.
 This fail-closed behavior also applies to `rebuild --force`, even when the backup contains other usable entries.
 When an incomplete snapshot contains captured entries, NemoClaw retains it at the printed path but excludes it from `snapshot list` and snapshot restore selection.
 The retained snapshot may contain unsanitized credentials.

src/lib/actions/sandbox/process-recovery-openclaw-doctor.test.ts

@@ -402,6 +402,33 @@ describe("OpenClaw post-upgrade recovery doctor", () => {
     ]);
   });
 
+  it.each([true, false])(
+    "requires command transport after the privileged backup receipt (recovers: %s)",
+    async (recovers) => {
+      let currentMs = 0;
+      const execute = vi
+        .fn()
+        .mockResolvedValue({ status: recovers ? 0 : 255, stdout: "", stderr: "" })
+        .mockResolvedValueOnce({ status: 0, stdout: "", stderr: "" })
+        .mockResolvedValueOnce({ status: 255, stdout: "", stderr: "" });
+      const capture = vi.fn(() => ({ status: 0, output: "" }));
+      const result = await beginOpenClawBackupQuiesce("alpha", undefined, {
+        captureOpenshell: capture as never,
+        executeSandboxExecCommand: execute,
+        executePrivilegedSandboxCommand: vi.fn(() => ({ status: 0, stdout: "", stderr: "" })),
+        now: () => currentMs,
+        sleep: async () => {
+          currentMs += 30_000;
+        },
+      });
+
+      expect(execute.mock.calls.length).toBeGreaterThan(2);
+      expect(result.ok).toBe(recovers);
+      expect(capture).toHaveBeenCalledTimes(recovers ? 2 : 3);
+      expect(currentMs).toBe(recovers ? 30_000 : 6 * 60_000);
+    },
+  );
+
   it("promotes restored backup state through doctor before release", async () => {
     const execute = vi.fn(async (_sandboxName: string, _command: string) => ({
       status: 0,

src/lib/actions/sandbox/process-recovery.ts

@@ -818,7 +818,20 @@ export async function beginOpenClawPostRestoreDoctor(
           Math.max(1, Math.min(15_000, Math.floor(remainingMs))),
           runtimeSelection,
         );
-        return result?.status === 0;
+        if (result?.status !== 0) return false;
+        if (maintenanceKind !== "backup" || !deps.executePrivilegedSandboxCommand) return true;
+        // A Docker receipt can precede OpenShell transport readiness after start.
+        // Verify the same gateway-down window through the command transport before backup.
+        const transportRemainingMs = reconciliationDeadlineMs - deps.now();
+        if (!Number.isFinite(transportRemainingMs) || transportRemainingMs <= 0) return false;
+        const transport = await executeOpenClawDoctorNetworkCommand(
+          deps,
+          sandboxName,
+          buildOpenClawPostUpgradeDoctorWindowProbe(sandboxName, markerContent),
+          Math.max(1, Math.min(15_000, Math.floor(transportRemainingMs))),
+          runtimeSelection,
+        );
+        return transport?.status === 0;
       } catch {
         return false;
       }
@@ -858,7 +871,7 @@ export async function beginOpenClawPostRestoreDoctor(
       : stage === "restart"
         ? `OpenShell did not converge the recreated sandbox start into a verified ${maintenanceLabel} window`
         : maintenanceKind === "backup"
-          ? "startup did not prove backup quiescence with the gateway held down"
+          ? "startup did not prove backup quiescence and OpenShell command transport readiness"
           : "startup did not prove doctor completion with the gateway held down";
   return {
     ok: false,

src/lib/state/sandbox.ts

@@ -1191,10 +1191,19 @@ function describeStateDirDiscoveryFailure(
   result: ReturnType<typeof spawnSync>,
   invalidDirectories: readonly string[],
 ): { log: string; unreachable: boolean; error?: string } | null {
+  if (result.status === 65) {
+    return {
+      log: "SECURITY: State directory discovery rejected an unsafe entry",
+      unreachable: false,
+      error:
+        "State directory discovery rejected an unsafe entry (exit 65). No state archive was captured.",
+    };
+  }
   if (result.status !== 0) {
     return {
       log: `FAILED: SSH dir check exited ${String(result.status)} — cannot determine which dirs exist`,
       unreachable: isSshTransportFailure(result),
+      error: `SSH state-directory discovery failed (exit ${String(result.status)}${result.signal ? `, signal ${result.signal}` : ""}). No state archive was captured.`,
     };
   }
   if (invalidDirectories.length > 0) {
@@ -2208,6 +2217,8 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions =
       backedUpFiles,
       failedFiles: stateFiles.map((f) => f.path),
       unreachable: true,
+      error:
+        "Could not obtain SSH configuration for sandbox state backup. No state archive was captured.",
     };
   }
   _log(`SSH config obtained (${sshConfig.length} bytes)`);

test/e2e/docs/README.md

@@ -150,6 +150,12 @@ OpenClaw failure probes read only regular, single-link log files without followi
 They omit log content above 16 KiB or changed during the read, so truncation cannot split a credential before host redaction.
 Oversized files retain size and permission metadata for diagnosis.
 
+The OpenClaw `mcp-bridge` fixture sets `NEMOCLAW_MCP_TOOLS_LIST_TIMEOUT_MS=5000` during onboarding and rebuild.
+This gives tool discovery through its public tunnel a 5,000 ms budget without changing production defaults or tool-call timeouts.
+If the native tool call after credential rotation fails, the fixture captures MCP status, tool discovery, and credential-free request metadata before cleanup.
+The Telegram add/remove fixture also captures rebuild failure diagnostics after channel addition, before cleanup removes the sandbox.
+These diagnostics preserve the failing operation's result; they do not retry it.
+
 When the missing-custom-presets target fails before its expected policy rejection, it captures these bounded, redacted failure probes before cleanup.
 The probes also capture unexpected JavaScript failures; they do not change the onboarding result or the required policy rejection.
 Container probes use a resolved full container ID and never delete resources or retry onboarding.

test/e2e/live/channels-add-remove.test.ts

@@ -532,6 +532,12 @@ test(
       redactionValues: secretsToRedact,
       timeoutMs: REBUILD_TIMEOUT_MS,
     });
+    await captureSandboxFailureDiagnostics(host, rebuildAdd, {
+      sandboxName: SANDBOX_NAME,
+      artifactPrefix: "phase-3-rebuild-add-failure",
+      redactionValues: secretsToRedact,
+      captureGatewayLog: true,
+    });
     expect(resultText(rebuildAdd)).not.toContain("provider credential not found");
     assertExitZero(rebuildAdd, `nemoclaw ${SANDBOX_NAME} rebuild --yes after add`);
     expect(

test/e2e/live/mcp-bridge-onboard-env.ts

@@ -88,6 +88,9 @@ export function buildMcpBridgeExactMainEnv(options: {
     ...buildAvailabilityProbeEnv(baseEnv),
     ...qualificationEnv,
     ...envOverlay,
+    // Public-tunnel discovery can exceed OpenClaw's 1.5-second default.
+    // This existing sandbox override affects tools/list, not tool-call timeouts.
+    NEMOCLAW_MCP_TOOLS_LIST_TIMEOUT_MS: "5000",
   };
 }
 

test/e2e/live/mcp-bridge.test.ts

@@ -39,10 +39,10 @@ import {
 import {
   assertHermesMcpHttpResponse,
   buildHermesMcpChatProbeScript,
+  HERMES_MCP_FAILURE_CAPTURE_BYTES,
   buildHermesMcpRuntimeDiagnosticsScript,
   captureHermesMcpLifecycleFailure,
   readHermesGatewayIdentity,
-  HERMES_MCP_FAILURE_CAPTURE_BYTES,
 } from "./mcp-bridge-hermes-http.ts";
 import {
   assertHermesConfig,
@@ -67,14 +67,14 @@ import {
   readConcurrentMcpStatusAndConfirmHermesRegistration,
   MCP_BRIDGE_DENIED_TOOL_NAME,
   runDeniedMcpToolCall,
+  retryHermesGatewayDraining,
   runMcpProviderRewriteProbe,
   runOpenClawDeniedToolUpdateProof,
   runOpenClawPublicPinRefreshProof,
   restartBridgeWithoutHostSecret,
   rebuildWithoutMcpHostSecret,
   retryOpenClawBaselineScopeOnboardFailure,
   retryAfterConcurrentAddTransientFailure,
-  retryHermesGatewayDraining,
 } from "./mcp-bridge-reliability.ts";
 import {
   buildMcpDnsRebindingProbeScript,
@@ -526,6 +526,7 @@ async function assertRealAdapterToolCall(
     mcpUrl?: string;
   },
 ): Promise<void> {
+  const isHermes = options.agent === "hermes";
   const before = fakeMcp.requests.filter((request) => request.rpcMethod === "tools/call").length;
   const otherCallsBefore = options.otherEndpoint?.requests.filter(
     (request) => request.rpcMethod === "tools/call",
@@ -536,17 +537,8 @@ async function assertRealAdapterToolCall(
     messages: [{ role: "user", content: prompt }],
     max_tokens: 256,
   });
-  // ShellProbe redacts these values before it returns command output or writes
-  // artifacts. The HTTP assertion redacts them again before Vitest formats a
-  // bounded failure preview.
-  const hermesRedactionValues = [
-    HOST_SECRET,
-    ROTATED_HOST_SECRET,
-    COMPATIBLE_KEY,
-    TOOL_CHALLENGE,
-    prompt,
-    hermesPayload,
-  ];
+  const hermesRedactionValues: string[] = Object.values(MCP_BRIDGE_TEST_CREDENTIALS);
+  hermesRedactionValues.push(TOOL_CHALLENGE, prompt, hermesPayload);
   const command =
     options.agent === "openclaw"
       ? `nemoclaw-start openclaw agent --agent main --json --thinking off --session-id ${shellQuote(`mcp-e2e-native-${options.artifactName}`)} -m ${shellQuote(prompt)}`
@@ -556,35 +548,29 @@ async function assertRealAdapterToolCall(
             buildHermesMcpChatProbeScript(hermesPayload, options.resultToken),
           ].join("\n")
         : `nemoclaw-start dcode -n ${JSON.stringify(prompt)}`;
+  const script = trustedSandboxShellScript(["set -eu", command].join("\n"));
   const runToolCall = (artifactName: string) =>
-    sandbox.execShell(
-      options.sandboxName,
-      trustedSandboxShellScript(["set -eu", command].join("\n")),
-      {
-        artifactName,
-        env: buildAvailabilityProbeEnv(),
-        captureLimitBytes:
-          options.agent === "hermes" ? HERMES_MCP_FAILURE_CAPTURE_BYTES : undefined,
-        redactionValues: options.agent === "hermes" ? hermesRedactionValues : [],
-        timeoutMs: 5 * 60_000,
-      },
-    );
+    sandbox.execShell(options.sandboxName, script, {
+      artifactName,
+      env: buildAvailabilityProbeEnv(),
+      captureLimitBytes: isHermes ? HERMES_MCP_FAILURE_CAPTURE_BYTES : undefined,
+      redactionValues: isHermes ? hermesRedactionValues : [],
+      timeoutMs: 5 * 60_000,
+    });
   const initialResult = await runToolCall(options.artifactName);
-  const result =
-    options.agent === "hermes"
-      ? await retryHermesGatewayDraining({
-          initialResult,
-          retry: (attempt) =>
-            runToolCall(`${options.artifactName}-gateway-draining-retry-${attempt}`),
-        })
-      : initialResult;
-  const assertResponse =
-    options.agent === "hermes"
-      ? () => assertHermesMcpHttpResponse(result, hermesRedactionValues)
-      : () => {
-          expectExitZero(result, `${options.agent} real MCP tool call`);
-          expect(resultText(result)).toContain(options.resultToken);
-        };
+  const result = isHermes
+    ? await retryHermesGatewayDraining({
+        initialResult,
+        retry: (attempt) =>
+          runToolCall(`${options.artifactName}-gateway-draining-retry-${attempt}`),
+      })
+    : initialResult;
+  const assertResponse = isHermes
+    ? () => assertHermesMcpHttpResponse(result, hermesRedactionValues)
+    : () => {
+        expectExitZero(result, `${options.agent} real MCP tool call`);
+        expect(resultText(result)).toContain(options.resultToken);
+      };
   assertResponse();
   const calls = fakeMcp.requests.filter((request) => request.rpcMethod === "tools/call");
   const details = ["rpcId", "rpcToolName", "responseStatus", "responseHasResult"];
@@ -1036,12 +1022,26 @@ test(
       OPENCLAW_SANDBOX_NAME,
       mcpUrl,
     );
-    await assertRealAdapterToolCall(host, sandbox, fakeMcp, {
-      ...bridge,
-      resultToken: MCP_RESULT,
-      artifactName: "openclaw-real-mcp-tool-call-after-credential-rotation",
-      expectedSecret: ROTATED_HOST_SECRET,
-    });
+    await withMcpToolCallFailureEvidence(
+      () =>
+        assertRealAdapterToolCall(host, sandbox, fakeMcp, {
+          ...bridge,
+          resultToken: MCP_RESULT,
+          artifactName: "openclaw-real-mcp-tool-call-after-credential-rotation",
+          expectedSecret: ROTATED_HOST_SECRET,
+        }),
+      host,
+      {
+        artifacts,
+        artifactPrefix: "openclaw-credential-rotation",
+        sandboxName: OPENCLAW_SANDBOX_NAME,
+        serverName: SERVER_NAME,
+        credentialEnvName: "FAKE_MCP_SECRET",
+        requests: fakeMcp.requests,
+        expectedSecret: ROTATED_HOST_SECRET,
+        redactionValues: [COMPATIBLE_KEY, HOST_SECRET, ROTATED_HOST_SECRET],
+      },
+    );
     await assertSecretAbsentFromSandbox(
       sandbox,
       OPENCLAW_SANDBOX_NAME,

test/e2e/support/channels-add-remove-helpers.test.ts

@@ -1,7 +1,11 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import { describe, expect, it } from "vitest";
+import { describe, expect, it, vi } from "vitest";
+
+import { assertExitZero } from "../fixtures/clients/command.ts";
+import type { HostCliClient } from "../fixtures/clients/host.ts";
+import { captureSandboxFailureDiagnostics } from "../fixtures/sandbox-failure-diagnostics.ts";
 
 import { ArtifactSink } from "../fixtures/artifacts.ts";
 import {
@@ -76,3 +80,67 @@ describe("Telegram artifact credential detection", () => {
     );
   });
 });
+
+describe("Telegram rebuild failure diagnostics", () => {
+  it("preserves a failed backup when diagnostic transport is unavailable", async () => {
+    const host = {
+      openshellCommandPath: "/fixture/openshell",
+      command: vi
+        .fn<HostCliClient["command"]>()
+        .mockRejectedValue(new Error("diagnostic transport unavailable")),
+    };
+    const result = {
+      exitCode: 1,
+      timedOut: false,
+      stdout: "",
+      stderr: "backup failed before archive capture",
+    };
+    const token = "synthetic-telegram-diagnostic-secret";
+    await captureSandboxFailureDiagnostics(host, result, {
+      sandboxName: "alpha",
+      artifactPrefix: "phase-3-rebuild-add-failure",
+      redactionValues: [token],
+      captureGatewayLog: true,
+    });
+
+    expect(host.command).toHaveBeenCalledWith(
+      host.openshellCommandPath,
+      ["logs", "alpha", "-n", "200", "--source", "all", "--since", "2m"],
+      expect.objectContaining({
+        redactionValues: [token],
+        captureLimitBytes: 32_768,
+        timeoutMs: 30_000,
+      }),
+    );
+    expect(host.command).toHaveBeenCalledWith(
+      "cat",
+      expect.any(Array),
+      expect.objectContaining({
+        redactionValues: [token],
+        captureLimitBytes: 32_768,
+        timeoutMs: 5_000,
+      }),
+    );
+    expect(() => assertExitZero(result, "rebuild after Telegram add")).toThrow(
+      "backup failed before archive capture",
+    );
+  });
+
+  it("does not run failure diagnostics after a successful rebuild", async () => {
+    const host = {
+      openshellCommandPath: "/fixture/openshell",
+      command: vi.fn<HostCliClient["command"]>(),
+    };
+    await captureSandboxFailureDiagnostics(
+      host,
+      { exitCode: 0, timedOut: false },
+      {
+        sandboxName: "alpha",
+        artifactPrefix: "phase-3-rebuild-add-failure",
+        redactionValues: [],
+        captureGatewayLog: true,
+      },
+    );
+    expect(host.command).not.toHaveBeenCalled();
+  });
+});

test/e2e/support/mcp-bridge-onboard-env.test.ts

@@ -31,6 +31,17 @@ const ONBOARD_OPTIONS = {
 const SELECTED_REVISION = "c".repeat(40);
 const SELECTED_COHORT = "ghrun-123-4";
 const PLATFORM = "linux/amd64";
+
+it("uses the bounded discovery timeout for public-tunnel onboarding and rebuild", () => {
+  const baseEnv = { NEMOCLAW_MCP_TOOLS_LIST_TIMEOUT_MS: "999999" };
+  expect(
+    buildMcpBridgeOnboardEnv({ ...ONBOARD_OPTIONS, agent: "openclaw", baseEnv }),
+  ).toHaveProperty("NEMOCLAW_MCP_TOOLS_LIST_TIMEOUT_MS", "5000");
+  expect(buildMcpBridgeExactMainEnv({ baseEnv })).toHaveProperty(
+    "NEMOCLAW_MCP_TOOLS_LIST_TIMEOUT_MS",
+    "5000",
+  );
+});
 const selectedReferences = Object.fromEntries(
   SHIPPED_MANAGED_IMAGE_AGENTS.map((agent, agentIndex) => [
     agent,