fix(forward): handle spawn errors and early child exit

src/lib/actions/sandbox/connect-flow.test.ts

@@ -51,10 +51,10 @@ function runInferenceRouteThenDriftLiveIdentity(
 function awaitHermesRouteVerification(harness: ReturnType<typeof createConnectHarness>): void {
   harness.recoverHermesPortableOllamaInferenceSpy.mockImplementation((async (input: {
     verifyRoute: () => Promise<unknown>;
-    prepareProbeDependency?: () => { release: () => void };
+    prepareProbeDependency?: () => Promise<{ release: () => void }>;
   }) => {
     await input.verifyRoute();
-    input.prepareProbeDependency?.().release();
+    (await input.prepareProbeDependency?.())?.release();
     return "reused";
   }) as never);
 }

src/lib/actions/sandbox/connect-hermes-accepted-readiness.test.ts

@@ -109,10 +109,10 @@ function missingHermesHarness(
   });
   harness.recoverHermesPortableOllamaInferenceSpy.mockImplementation((async (input: {
     verifyRoute: () => Promise<unknown>;
-    prepareProbeDependency?: () => { release: () => void };
+    prepareProbeDependency?: () => Promise<{ release: () => void }>;
   }) => {
     await input.verifyRoute();
-    input.prepareProbeDependency?.().release();
+    (await input.prepareProbeDependency?.())?.release();
     return "reused";
   }) as never);
   return harness;

src/lib/actions/sandbox/connect.ts

@@ -401,7 +401,7 @@ async function runSandboxConnectProbe(
         probeTiming?.markFailureStage("forward");
         failHermesPortableForwardRecovery(sandboxName, "authority-drift");
       }
-      recoverHermesPortableForwardsForConnectProbeOrExit(
+      await recoverHermesPortableForwardsForConnectProbeOrExit(
         sandboxName,
         authority,
         probeTiming,
@@ -586,7 +586,7 @@ function describeHermesPortableForwardRecoveryFailure(
     case "forward-settlement-timed-out":
       return "The required recorded host forwards did not become healthy before the recovery deadline. Inspect the recorded forward state before retrying.";
     case "forward-mutation-failed":
-      return `NemoClaw could not confirm that OpenShell forward ${context.operation} completed for recorded host port ${String(context.port)}. Inspect the recorded forward state before retrying.`;
+      return `NemoClaw could not confirm that OpenShell forward ${context.operation} completed for recorded host port ${String(context.port)}.${context.startupFailure ? ` Startup failure: ${context.startupFailure}.` : ""} Inspect the recorded forward state before retrying.`;
   }
   switch (failure) {
     case "forward-occupied":
@@ -827,14 +827,14 @@ function verifyHermesPortableForwardsForConnectProbe(
   return verifyHermesPortableLaunchForwards(hermesPortableForwardInputForConnectProbe(input));
 }
 
-function recoverHermesPortableForwardsForConnectProbeOrExit(
+async function recoverHermesPortableForwardsForConnectProbeOrExit(
   sandboxName: string,
   authority: HermesPortableActiveLifecycleAuthority,
   probeTiming?: ProbeTimingRecorder,
   commandAuthority?: ReturnType<typeof qualifyHermesPortableOperatingCommandAuthority>,
-): void {
+): Promise<void> {
   try {
-    const prepared = prepareHermesPortableForwardsForConnectProbeMeasured(
+    const prepared = await prepareHermesPortableForwardsForConnectProbeMeasured(
       sandboxName,
       authority,
       probeTiming,
@@ -851,7 +851,7 @@ function recoverHermesPortableForwardsForConnectProbeOrExit(
   }
 }
 
-function prepareHermesPortableForwardsForConnectProbeMeasured(
+async function prepareHermesPortableForwardsForConnectProbeMeasured(
   sandboxName: string,
   authority: HermesPortableActiveLifecycleAuthority,
   probeTiming?: ProbeTimingRecorder,
@@ -866,7 +866,7 @@ function prepareHermesPortableForwardsForConnectProbeMeasured(
         readRegistry: registry.getSandbox,
         ...(commandAuthority ? { commandAuthority } : {}),
       });
-    return probeTiming ? probeTiming.measure("forward", prepare) : prepare();
+    return await (probeTiming ? probeTiming.measureAsync("forward", prepare) : prepare());
   } catch (error) {
     probeTiming?.setForwardAction("failed");
     probeTiming?.markFailureStage("forward");
@@ -1092,8 +1092,8 @@ async function verifyOrRecoverHermesPortableInferenceRouteForProbeOnlyOrExit(
         }
         return verified;
       },
-      prepareProbeDependency: () => {
-        preparedForwards = prepareHermesPortableForwardsForConnectProbeMeasured(
+      prepareProbeDependency: async () => {
+        preparedForwards = await prepareHermesPortableForwardsForConnectProbeMeasured(
           sandboxName,
           authority,
           options.probeTiming,
@@ -2331,7 +2331,7 @@ export async function prepareInteractiveSession(sandboxName: string): Promise<{
           undefined,
           portableAgentLifecycleAuthorityDeps(),
         );
-        recoverHermesPortableForwardsForConnectProbeOrExit(sandboxName, authority);
+        await recoverHermesPortableForwardsForConnectProbeOrExit(sandboxName, authority);
       }
       if (!hermesPortable && !(await settlePortablePairingOrExit(sandboxName))) {
         completeInteractiveSessionSetup(sandboxName, sb);
@@ -2615,7 +2615,7 @@ async function prepareConnectSandboxWithinLifecycleFence(
             probeTiming!.markFailureStage("authority");
             failHermesPortableReadinessAuthority(sandboxName);
           }
-          recoverHermesPortableForwardsForConnectProbeOrExit(
+          await recoverHermesPortableForwardsForConnectProbeOrExit(
             sandboxName,
             activeAuthority,
             probeTiming,

src/lib/actions/sandbox/forward-recovery-declared-ports.test.ts

@@ -189,7 +189,7 @@ describe("ensureDeclaredAgentForwardPortsHealthy", { timeout: 30_000 }, () => {
     });
     const { ensureSandboxPortForward } = await import("./forward-recovery");
 
-    expect(ensureSandboxPortForward("remote-box")).toBe(true);
+    expect(await ensureSandboxPortForward("remote-box")).toBe(true);
     expect(mocks.isForwardServiceListenerOwner).toHaveBeenCalledWith({
       executable: "/usr/local/bin/openshell",
       gatewayEndpoint: "https://127.0.0.1:8080",
@@ -210,7 +210,7 @@ describe("ensureDeclaredAgentForwardPortsHealthy", { timeout: 30_000 }, () => {
     mocks.isForwardServiceListenerOwner.mockReturnValue(false);
     const { ensureSandboxPortForward } = await import("./forward-recovery");
 
-    expect(ensureSandboxPortForward("foreign-listener")).toBe(false);
+    expect(await ensureSandboxPortForward("foreign-listener")).toBe(false);
     expect(mocks.isForwardServiceListenerOwner).toHaveBeenCalledOnce();
     expect(mocks.launchForwardService).not.toHaveBeenCalled();
   });
@@ -222,7 +222,7 @@ describe("ensureDeclaredAgentForwardPortsHealthy", { timeout: 30_000 }, () => {
       hermesApiPort: 8643,
     });
     const { ensureDeclaredAgentForwardPortsHealthy } = await import("./forward-recovery");
-    expect(ensureDeclaredAgentForwardPortsHealthy("beta", 18790)).toBe(true);
+    expect(await ensureDeclaredAgentForwardPortsHealthy("beta", 18790)).toBe(true);
     expect(mocks.runOpenshell).not.toHaveBeenCalled();
   });
 
@@ -234,7 +234,7 @@ describe("ensureDeclaredAgentForwardPortsHealthy", { timeout: 30_000 }, () => {
       hermesApiPort: 8643,
     });
     const { ensureDeclaredAgentForwardPortsHealthy } = await import("./forward-recovery");
-    expect(ensureDeclaredAgentForwardPortsHealthy("beta", 18790)).toBe(true);
+    expect(await ensureDeclaredAgentForwardPortsHealthy("beta", 18790)).toBe(true);
     expect(mocks.launchForwardService).toHaveBeenCalledWith(
       expect.objectContaining({ localPort: 8643, targetPort: 8643 }),
       expect.objectContaining({ verifyReady: expect.any(Function) }),
@@ -261,7 +261,9 @@ describe("ensureDeclaredAgentForwardPortsHealthy", { timeout: 30_000 }, () => {
     };
     const { ensureDeclaredAgentForwardPortsHealthy } = await import("./forward-recovery");
 
-    expect(ensureDeclaredAgentForwardPortsHealthy("beta", 18790, runtimeSelection)).toBe(true);
+    expect(await ensureDeclaredAgentForwardPortsHealthy("beta", 18790, runtimeSelection)).toBe(
+      true,
+    );
     expect(mocks.launchForwardService).toHaveBeenCalledWith(
       expect.objectContaining({
         gatewayEndpoint: "https://127.0.0.1:19080",
@@ -302,7 +304,7 @@ describe("ensureDeclaredAgentForwardPortsHealthy", { timeout: 30_000 }, () => {
       mocks.launchForwardService.mockImplementation((_target, options) => options.verifyReady?.());
       const { ensureDeclaredAgentForwardPortsHealthy } = await import("./forward-recovery");
 
-      expect(ensureDeclaredAgentForwardPortsHealthy("beta", 18790)).toBe(true);
+      expect(await ensureDeclaredAgentForwardPortsHealthy("beta", 18790)).toBe(true);
       expect(mocks.launchForwardService).toHaveBeenCalledWith(
         expect.objectContaining({
           gatewayEndpoint: endpoint,
@@ -319,7 +321,7 @@ describe("ensureDeclaredAgentForwardPortsHealthy", { timeout: 30_000 }, () => {
   it("keeps the default API port for a sandbox registered without one (#8543)", async () => {
     mocks.getSandbox.mockReturnValue({ agent: "hermes", dashboardPort: 18789 });
     const { ensureDeclaredAgentForwardPortsHealthy } = await import("./forward-recovery");
-    expect(ensureDeclaredAgentForwardPortsHealthy("beta", 18789)).toBe(true);
+    expect(await ensureDeclaredAgentForwardPortsHealthy("beta", 18789)).toBe(true);
     expect(mocks.runOpenshell).not.toHaveBeenCalled();
   });
 });
@@ -355,7 +357,7 @@ describe("a dashboard port held by a listener the sandbox does not own (#11149)"
     mocks.isForwardServiceListenerOwner.mockReturnValue(false);
     const { ensureSandboxPortForward } = await import("./forward-recovery");
 
-    expect(ensureSandboxPortForward("box", { isWsl: false })).toBe(false);
+    expect(await ensureSandboxPortForward("box", { isWsl: false })).toBe(false);
 
     expect(mocks.launchForwardService).not.toHaveBeenCalled();
     expect(mocks.runOpenshell).not.toHaveBeenCalled();
@@ -380,7 +382,7 @@ describe("a dashboard port held by a listener the sandbox does not own (#11149)"
       await import("./forward-recovery");
 
     expect(isSandboxForwardHealthy("box", { isWsl: false })).toBe(false);
-    expect(ensureSandboxPortForward("box", { afterSuccess, isWsl: false })).toBe(false);
+    expect(await ensureSandboxPortForward("box", { afterSuccess, isWsl: false })).toBe(false);
     expect(afterSuccess).not.toHaveBeenCalled();
     expect(mocks.captureOpenshell).not.toHaveBeenCalled();
     expect(mocks.runOpenshell).not.toHaveBeenCalled();
@@ -399,7 +401,7 @@ describe("a dashboard port held by a listener the sandbox does not own (#11149)"
     mocks.isLocalForwardReachable.mockReturnValue(false);
     const { ensureSandboxPortForward } = await import("./forward-recovery");
 
-    expect(ensureSandboxPortForward("box", { isWsl: false })).toBe(true);
+    expect(await ensureSandboxPortForward("box", { isWsl: false })).toBe(true);
 
     expect(mocks.launchForwardService).toHaveBeenCalledWith(
       expect.objectContaining({ gatewayName: "nemoclaw", workspace: "default" }),
@@ -426,7 +428,7 @@ describe("a dashboard port held by a listener the sandbox does not own (#11149)"
     });
     const { ensureSandboxPortForward } = await import("./forward-recovery");
 
-    expect(ensureSandboxPortForward("box", { isWsl: false })).toBe(false);
+    expect(await ensureSandboxPortForward("box", { isWsl: false })).toBe(false);
     expect(mocks.launchForwardService).toHaveBeenCalledOnce();
     expect(mocks.runOpenshell).not.toHaveBeenCalled();
   });
@@ -445,7 +447,7 @@ describe("a dashboard port held by a listener the sandbox does not own (#11149)"
     mocks.launchForwardService.mockImplementation((_target, options) => options.verifyReady?.());
     const { ensureSandboxPortForward } = await import("./forward-recovery");
 
-    expect(ensureSandboxPortForward("box", { isWsl: false })).toBe(false);
+    expect(await ensureSandboxPortForward("box", { isWsl: false })).toBe(false);
     expect(mocks.launchForwardService).toHaveBeenCalledOnce();
     expect(mocks.isForwardServiceListenerOwner).not.toHaveBeenCalled();
   });
@@ -465,7 +467,7 @@ describe("a dashboard port held by a listener the sandbox does not own (#11149)"
     mocks.launchForwardService.mockImplementation((_target, options) => options.verifyReady?.());
     const { ensureSandboxPortForward } = await import("./forward-recovery");
 
-    expect(ensureSandboxPortForward("box", { isWsl: false })).toBe(false);
+    expect(await ensureSandboxPortForward("box", { isWsl: false })).toBe(false);
     expect(mocks.launchForwardService).toHaveBeenCalledOnce();
     expect(mocks.isForwardServiceListenerOwner).toHaveBeenCalledOnce();
   });
@@ -494,7 +496,7 @@ describe("a dashboard port held by a listener the sandbox does not own (#11149)"
       await import("./forward-recovery");
 
     expect(describeSandboxForwardListener("box", { isWsl: false })).toBe("unverified");
-    expect(ensureSandboxPortForward("box", { isWsl: false })).toBe(false);
+    expect(await ensureSandboxPortForward("box", { isWsl: false })).toBe(false);
     expect(mocks.isForwardServiceListenerOwner).not.toHaveBeenCalled();
     expect(mocks.launchForwardService).not.toHaveBeenCalled();
   });
@@ -508,7 +510,7 @@ describe("a dashboard port held by a listener the sandbox does not own (#11149)"
       await import("./forward-recovery");
 
     expect(describeSandboxForwardListener("box", { isWsl: false })).toBe("unverified");
-    expect(ensureSandboxPortForward("box", { isWsl: false })).toBe(false);
+    expect(await ensureSandboxPortForward("box", { isWsl: false })).toBe(false);
     expect(mocks.launchForwardService).not.toHaveBeenCalled();
   });
 
@@ -520,7 +522,7 @@ describe("a dashboard port held by a listener the sandbox does not own (#11149)"
     });
     const { ensureSandboxPortForward } = await import("./forward-recovery");
 
-    expect(ensureSandboxPortForward("box", { isWsl: false })).toBe(false);
+    expect(await ensureSandboxPortForward("box", { isWsl: false })).toBe(false);
     expect(mocks.isForwardServiceListenerOwner).not.toHaveBeenCalled();
     expect(mocks.launchForwardService).not.toHaveBeenCalled();
   });

src/lib/actions/sandbox/forward-recovery.ts

@@ -388,10 +388,10 @@ export function teardownSandboxDashboardForward(
  * non-OpenClaw agents, then falls back to the active agent's declared port.
  * Returns true when the detached OpenShell service makes the port reachable.
  */
-export function ensureSandboxPortForward(
+export async function ensureSandboxPortForward(
   sandboxName: string,
   options: SandboxForwardRecoveryOptions = {},
-): boolean {
+): Promise<boolean> {
   const port = resolveSandboxDashboardPort(sandboxName);
   const remoteBindRequested = isRemoteDashboardBindRequested(process.env.NEMOCLAW_DASHBOARD_BIND);
   const allInterfaceBindRequired = remoteBindRequested || isWsl({ isWsl: options.isWsl });
@@ -515,7 +515,7 @@ function inspectSandboxPortForwardListener(
   }
 }
 
-export function ensureSandboxPortForwardForPort(
+export async function ensureSandboxPortForwardForPort(
   sandboxName: string,
   port: number,
   options: {
@@ -525,7 +525,7 @@ export function ensureSandboxPortForwardForPort(
     beforeStart?: () => boolean;
     runtimeSelection?: OpenShellRuntimeSelection;
   } = {},
-): boolean {
+): Promise<boolean> {
   const {
     afterSuccess = () => true,
     forwardTarget = String(port),
@@ -583,7 +583,7 @@ export function ensureSandboxPortForwardForPort(
       launchRuntime.workspace,
       authority.endpoint,
     );
-    launchForwardService(target, {
+    await launchForwardService(target, {
       sourceEnvironment: buildSelectedOpenShellSubprocessEnv(launchRuntime),
       verifyReady: () => {
         assertForwardGatewayAuthorityCurrent(gatewayName, authority);
@@ -604,10 +604,10 @@ export function ensureSandboxPortForwardForPort(
   }
 }
 
-export function ensureHermesDashboardPortForwardIfEnabled(
+export async function ensureHermesDashboardPortForwardIfEnabled(
   sandboxName: string,
   runtimeSelection?: OpenShellRuntimeSelection,
-): boolean | null {
+): Promise<boolean | null> {
   return ensureHermesDashboardPortForward(sandboxName, {
     isPortForwardHealthy: (name, port) =>
       isSandboxPortForwardHealthy(name, port, undefined, runtimeSelection),
@@ -625,10 +625,10 @@ function getSandboxMessagingHostForward(
   return getActiveMessagingHostForward(plan);
 }
 
-export function ensureMessagingHostForwardHealthy(
+export async function ensureMessagingHostForwardHealthy(
   sandboxName: string,
   runtimeSelection?: OpenShellRuntimeSelection,
-): boolean | null {
+): Promise<boolean | null> {
   const forward = getSandboxMessagingHostForward(sandboxName);
   if (!forward) return null;
   const health = isSandboxPortForwardHealthy(
@@ -641,11 +641,11 @@ export function ensureMessagingHostForwardHealthy(
   return ensureSandboxPortForwardForPort(sandboxName, forward.port, { runtimeSelection });
 }
 
-export function recoverMessagingHostForward(
+export async function recoverMessagingHostForward(
   sandboxName: string,
   { quiet, runtimeSelection }: { quiet: boolean; runtimeSelection?: OpenShellRuntimeSelection },
-): boolean | null {
-  const recovered = ensureMessagingHostForwardHealthy(sandboxName, runtimeSelection);
+): Promise<boolean | null> {
+  const recovered = await ensureMessagingHostForwardHealthy(sandboxName, runtimeSelection);
   if (!quiet && recovered === false) {
     console.error("  Messaging webhook port forward could not be re-established.");
   }
@@ -692,11 +692,11 @@ function resolveDeclaredAgentForwardPorts(
  * recorded API port, or recovery demands a port that belongs to a sibling
  * sandbox and reports a failure the sandbox cannot repair.
  */
-export function ensureDeclaredAgentForwardPortsHealthy(
+export async function ensureDeclaredAgentForwardPortsHealthy(
   sandboxName: string,
   primaryPort: number,
   runtimeSelection?: OpenShellRuntimeSelection,
-): boolean | null {
+): Promise<boolean | null> {
   const agent = agentRuntime.getSessionAgent(sandboxName);
   if (!agent) return null;
   const hermesDashboard = getHermesDashboardRecoveryConfig(sandboxName);
@@ -712,7 +712,7 @@ export function ensureDeclaredAgentForwardPortsHealthy(
   for (const port of ports) {
     const health = isSandboxPortForwardHealthy(sandboxName, port, undefined, runtimeSelection);
     if (health === true) continue;
-    if (!ensureSandboxPortForwardForPort(sandboxName, port, { runtimeSelection })) {
+    if (!(await ensureSandboxPortForwardForPort(sandboxName, port, { runtimeSelection }))) {
       allHealthy = false;
     }
   }
@@ -878,12 +878,12 @@ export function resolveSandboxLaunchForwardPorts(sandboxName: string): number[]
   return resolveSandboxLaunchForwardPortsFromAuthority(sandboxName, sandbox, agent, primaryPort);
 }
 
-export function recoverDeclaredAgentForwardPorts(
+export async function recoverDeclaredAgentForwardPorts(
   sandboxName: string,
   recoveryPort: number,
   { quiet, runtimeSelection }: { quiet: boolean; runtimeSelection?: OpenShellRuntimeSelection },
-): boolean | null {
-  const recovered = ensureDeclaredAgentForwardPortsHealthy(
+): Promise<boolean | null> {
+  const recovered = await ensureDeclaredAgentForwardPortsHealthy(
     sandboxName,
     recoveryPort,
     runtimeSelection,