From 45c138c104b58494add2a0e9aae4aefd324d65d4 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Tue, 11 Aug 2026 19:31:01 -0700 Subject: [PATCH 1/3] fix(recovery): expose verbose backup failures Signed-off-by: Senthil Ravichandran --- .../actions/sandbox/supervisor-relaunch.test.ts | 17 +++++++++++++++++ src/lib/actions/sandbox/supervisor-relaunch.ts | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/lib/actions/sandbox/supervisor-relaunch.test.ts b/src/lib/actions/sandbox/supervisor-relaunch.test.ts index 3a87a2be3d8..de8fff68946 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.test.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.test.ts @@ -486,6 +486,23 @@ describe("relaunchManagedSupervisorSession", () => { expect(deps.removeBackup).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery"); }); + it("reports a redacted verbose diagnostic for quiet recovery failures", () => { + vi.stubEnv("NEMOCLAW_REBUILD_VERBOSE", "1"); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + const deps = baseDeps({ + backupState: vi.fn(() => { + throw new Error("OPENAI_API_KEY=sk-recovery-secret backup finalization failed"); + }), + }); + + expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull(); + const output = errorSpy.mock.calls.flat().join("\n"); + expect(output).toContain("Trusted container recovery could not start"); + expect(output).toContain("OPENAI_API_KEY="); + expect(output).toContain("backup finalization failed"); + expect(output).not.toContain("sk-recovery-secret"); + }); + it("preserves the recreation diagnostic when state-backup cleanup throws", () => { vi.spyOn(console, "log").mockImplementation(() => undefined); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); diff --git a/src/lib/actions/sandbox/supervisor-relaunch.ts b/src/lib/actions/sandbox/supervisor-relaunch.ts index 587c5fc49ae..ada035db3e6 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.ts @@ -301,7 +301,7 @@ export function relaunchManagedSupervisorSession( // Preserve the recreation failure that stopped container recovery. } } - if (!quiet) { + if (!quiet || process.env.NEMOCLAW_REBUILD_VERBOSE === "1") { const detail = error instanceof Error ? error.message : String(error); console.error(` Trusted container recovery could not start: ${redactFull(redact(detail))}`); } From 10ab46d6ff2e0d312563c174b63aa71e1aca0fca Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Tue, 11 Aug 2026 20:05:09 -0700 Subject: [PATCH 2/3] fix(recovery): preserve repeated Docker recreation Signed-off-by: Senthil Ravichandran --- .../onboard/docker-gpu-patch-clone.test.ts | 46 +++++++++++++++++++ src/lib/onboard/docker-gpu-patch-clone.ts | 10 ++++ 2 files changed, 56 insertions(+) diff --git a/src/lib/onboard/docker-gpu-patch-clone.test.ts b/src/lib/onboard/docker-gpu-patch-clone.test.ts index 940f73bdab5..28d43714b6d 100644 --- a/src/lib/onboard/docker-gpu-patch-clone.test.ts +++ b/src/lib/onboard/docker-gpu-patch-clone.test.ts @@ -64,6 +64,28 @@ describe("Docker GPU clone envelope", () => { expect(shouldOmitOpenShellOciImageUser(inspect, NEMOCLAW_STARTUP_ARGV)).toBe(false); }); + it("preserves the exact workspace compatibility metadata across another recreation (#8662)", () => { + const inspect = openShellOciWorkspaceInspect(); + inspect.Config!.Env = inspect.Config!.Env!.filter( + (entry) => !entry.startsWith("OPENSHELL_OCI_IMAGE_USER="), + ); + + const args = buildDockerGpuCloneRunArgs(inspect, buildDockerGpuMode("startup-command"), { + openshellSandboxCommand: NEMOCLAW_STARTUP_ARGV, + }); + + expect(shouldOmitOpenShellOciImageUser(inspect, NEMOCLAW_STARTUP_ARGV)).toBe(false); + expect(args).toEqual( + expect.arrayContaining([ + "--env", + "OPENSHELL_SANDBOX_UID=", + "--env", + "OPENSHELL_SANDBOX_GID=", + ]), + ); + expect(args).not.toEqual(expect.arrayContaining(["--env", "OPENSHELL_OCI_IMAGE_USER=sandbox"])); + }); + it.each([ { name: "missing sandbox GID", @@ -95,6 +117,30 @@ describe("Docker GPU clone envelope", () => { name: "duplicate OCI user", mutate: (environment: string[]) => [...environment, "OPENSHELL_OCI_IMAGE_USER=sandbox"], }, + { + name: "already-applied metadata without a sandbox GID", + mutate: (environment: string[]) => + environment.filter( + (entry) => + !entry.startsWith("OPENSHELL_OCI_IMAGE_USER=") && entry !== "OPENSHELL_SANDBOX_GID=", + ), + }, + { + name: "already-applied metadata with a non-empty sandbox UID", + mutate: (environment: string[]) => + environment + .filter((entry) => !entry.startsWith("OPENSHELL_OCI_IMAGE_USER=")) + .map((entry) => + entry === "OPENSHELL_SANDBOX_UID=" ? "OPENSHELL_SANDBOX_UID=998" : entry, + ), + }, + { + name: "already-applied metadata with a duplicate sandbox UID", + mutate: (environment: string[]) => [ + ...environment.filter((entry) => !entry.startsWith("OPENSHELL_OCI_IMAGE_USER=")), + "OPENSHELL_SANDBOX_UID=", + ], + }, ])("rejects $name in OpenShell's protected identity metadata (#8662)", ({ mutate }) => { const inspect = openShellOciWorkspaceInspect(); inspect.Config!.Env = mutate(inspect.Config!.Env!); diff --git a/src/lib/onboard/docker-gpu-patch-clone.ts b/src/lib/onboard/docker-gpu-patch-clone.ts index 9433c06a88e..75e41f9f49e 100644 --- a/src/lib/onboard/docker-gpu-patch-clone.ts +++ b/src/lib/onboard/docker-gpu-patch-clone.ts @@ -375,6 +375,16 @@ function validateOpenShellOciIdentityMetadata(environment: readonly string[]): b // OpenShell through v0.0.85 did not publish OCI identity metadata. return false; } + if ( + ociUsers.length === 0 && + sandboxUids.length === 1 && + sandboxUids[0] === `${OPENSHELL_SANDBOX_UID_ENV}=` && + sandboxGids.length === 1 && + sandboxGids[0] === `${OPENSHELL_SANDBOX_GID_ENV}=` + ) { + // A prior reviewed recreation already removed only the OCI-user marker. + return false; + } const ociUserPrefix = `${OPENSHELL_OCI_IMAGE_USER_ENV}=`; if ( ociUsers.length !== 1 || From f8b8f20a793ce658d8998ca71a4e17db7f5f86f0 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Tue, 11 Aug 2026 20:53:18 -0700 Subject: [PATCH 3/3] fix(recovery): retry empty replacement probes Signed-off-by: Senthil Ravichandran --- .../actions/sandbox/process-recovery.test.ts | 50 +++++++++++++++++++ src/lib/actions/sandbox/process-recovery.ts | 20 +++++++- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/src/lib/actions/sandbox/process-recovery.test.ts b/src/lib/actions/sandbox/process-recovery.test.ts index e8b07d81201..28826ef3ecf 100644 --- a/src/lib/actions/sandbox/process-recovery.test.ts +++ b/src/lib/actions/sandbox/process-recovery.test.ts @@ -166,6 +166,56 @@ describe("recreated sandbox OpenShell readiness", () => { expect(sleeps).toEqual([3]); }); + it("keeps an empty read-only probe inconclusive after exact replacement re-registration", () => { + const captureOpenshellImpl = vi + .fn() + .mockReturnValueOnce({ + status: 1, + output: OPENSHELL_TRANSIENT_ERROR_PHASE_STDERR.trim(), + stdout: "", + stderr: OPENSHELL_TRANSIENT_ERROR_PHASE_STDERR, + }) + .mockReturnValueOnce({ status: 1, output: "", stdout: "", stderr: "" }) + .mockReturnValueOnce({ status: 0, output: "", stdout: "", stderr: "" }); + const beforeProbe = vi.fn(() => true); + const sleeps: number[] = []; + + expect( + waitForRecreatedSandboxOpenShellReady("recreated-box", { + beforeProbe, + captureOpenshellImpl, + intervalSeconds: 3, + sleepImpl: (seconds) => sleeps.push(seconds), + timeoutSeconds: 6, + }), + ).toBe(true); + expect(beforeProbe).toHaveBeenCalledTimes(3); + expect(captureOpenshellImpl).toHaveBeenCalledTimes(3); + expect(sleeps).toEqual([3, 3]); + }); + + it("keeps an empty first OpenShell failure terminal", () => { + const captureOpenshellImpl = vi.fn(() => ({ + status: 1, + output: "", + stdout: "", + stderr: "", + })); + const sleeps: number[] = []; + + expect( + waitForRecreatedSandboxOpenShellReady("recreated-box", { + beforeProbe: () => true, + captureOpenshellImpl, + intervalSeconds: 3, + sleepImpl: (seconds) => sleeps.push(seconds), + timeoutSeconds: 30, + }), + ).toBe(false); + expect(captureOpenshellImpl).toHaveBeenCalledOnce(); + expect(sleeps).toEqual([]); + }); + it("rides out a transient Error phase past the old 30s budget by default (#7227)", () => { // No timeoutSeconds option and no env override: the default recovery budget // must be large enough (120s, aligned with connect's readiness wait) to keep diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 12bd5be0195..96582bbadb0 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -818,6 +818,7 @@ function waitForRecreatedSandboxOpenShellReadyResult( ? Math.max(1, Math.floor(timeoutSeconds / intervalSeconds) + 1) : Math.max(1, Math.floor(timeoutSeconds) + 1); let lastOpenshellError: string | undefined; + let observedRetryableReRegistrationState = false; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { const preGuardRemainingMs = deadlineMs - now(); @@ -861,8 +862,25 @@ function waitForRecreatedSandboxOpenShellReadyResult( // mutation outcome to reconcile. Treat that exact timeout as inconclusive // and retry behind the pinned managed-health guard on the next iteration. // All other unexpected OpenShell failures remain definitive. + const retryableReRegistrationState = isRetryableOpenshellReRegistrationState( + result, + sandboxName, + ); + if (retryableReRegistrationState) observedRetryableReRegistrationState = true; + const emptyReadOnlyProbeAfterReRegistration = + observedRetryableReRegistrationState && + result.status === 1 && + !result.error && + String(result.stdout ?? "").trim() === "" && + String(result.stderr ?? "").trim() === ""; + // OpenShell can briefly return an empty exit-1 result after first exposing + // the exact replacement re-registration state. Keep that result + // inconclusive only inside the same bounded wait and behind the pinned + // managed-health guard. It is never accepted as Ready, and an empty first + // failure or any diagnostic-bearing unknown failure remains terminal. if ( - !isRetryableOpenshellReRegistrationState(result, sandboxName) && + !retryableReRegistrationState && + !emptyReadOnlyProbeAfterReRegistration && !isCommandTimeout(result) ) { return {