Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 50 additions & 0 deletions src/lib/actions/sandbox/process-recovery.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,56 @@ describe("recreated sandbox OpenShell readiness", () => {
expect(sleeps).toEqual([3]);
});

it("keeps an empty read-only probe inconclusive after exact replacement re-registration", () => {
const captureOpenshellImpl = vi
.fn()
.mockReturnValueOnce({
status: 1,
output: OPENSHELL_TRANSIENT_ERROR_PHASE_STDERR.trim(),
stdout: "",
stderr: OPENSHELL_TRANSIENT_ERROR_PHASE_STDERR,
})
.mockReturnValueOnce({ status: 1, output: "", stdout: "", stderr: "" })
.mockReturnValueOnce({ status: 0, output: "", stdout: "", stderr: "" });
const beforeProbe = vi.fn(() => true);
const sleeps: number[] = [];

expect(
waitForRecreatedSandboxOpenShellReady("recreated-box", {
beforeProbe,
captureOpenshellImpl,
intervalSeconds: 3,
sleepImpl: (seconds) => sleeps.push(seconds),
timeoutSeconds: 6,
}),
).toBe(true);
expect(beforeProbe).toHaveBeenCalledTimes(3);
expect(captureOpenshellImpl).toHaveBeenCalledTimes(3);
expect(sleeps).toEqual([3, 3]);
});

it("keeps an empty first OpenShell failure terminal", () => {
const captureOpenshellImpl = vi.fn(() => ({
status: 1,
output: "",
stdout: "",
stderr: "",
}));
const sleeps: number[] = [];

expect(
waitForRecreatedSandboxOpenShellReady("recreated-box", {
beforeProbe: () => true,
captureOpenshellImpl,
intervalSeconds: 3,
sleepImpl: (seconds) => sleeps.push(seconds),
timeoutSeconds: 30,
}),
).toBe(false);
expect(captureOpenshellImpl).toHaveBeenCalledOnce();
expect(sleeps).toEqual([]);
});

it("rides out a transient Error phase past the old 30s budget by default (#7227)", () => {
// No timeoutSeconds option and no env override: the default recovery budget
// must be large enough (120s, aligned with connect's readiness wait) to keep
Expand Down
20 changes: 19 additions & 1 deletion src/lib/actions/sandbox/process-recovery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -818,6 +818,7 @@ function waitForRecreatedSandboxOpenShellReadyResult(
? Math.max(1, Math.floor(timeoutSeconds / intervalSeconds) + 1)
: Math.max(1, Math.floor(timeoutSeconds) + 1);
let lastOpenshellError: string | undefined;
let observedRetryableReRegistrationState = false;

for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
const preGuardRemainingMs = deadlineMs - now();
Expand Down Expand Up @@ -861,8 +862,25 @@ function waitForRecreatedSandboxOpenShellReadyResult(
// mutation outcome to reconcile. Treat that exact timeout as inconclusive
// and retry behind the pinned managed-health guard on the next iteration.
// All other unexpected OpenShell failures remain definitive.
const retryableReRegistrationState = isRetryableOpenshellReRegistrationState(
result,
sandboxName,
);
if (retryableReRegistrationState) observedRetryableReRegistrationState = true;
const emptyReadOnlyProbeAfterReRegistration =
observedRetryableReRegistrationState &&
result.status === 1 &&
!result.error &&
String(result.stdout ?? "").trim() === "" &&
String(result.stderr ?? "").trim() === "";
// OpenShell can briefly return an empty exit-1 result after first exposing
// the exact replacement re-registration state. Keep that result
// inconclusive only inside the same bounded wait and behind the pinned
// managed-health guard. It is never accepted as Ready, and an empty first
// failure or any diagnostic-bearing unknown failure remains terminal.
if (
!isRetryableOpenshellReRegistrationState(result, sandboxName) &&
!retryableReRegistrationState &&
!emptyReadOnlyProbeAfterReRegistration &&
!isCommandTimeout(result)
) {
return {
Expand Down
17 changes: 17 additions & 0 deletions src/lib/actions/sandbox/supervisor-relaunch.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -486,6 +486,23 @@ describe("relaunchManagedSupervisorSession", () => {
expect(deps.removeBackup).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery");
});

it("reports a redacted verbose diagnostic for quiet recovery failures", () => {
vi.stubEnv("NEMOCLAW_REBUILD_VERBOSE", "1");
const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined);
const deps = baseDeps({
backupState: vi.fn(() => {
throw new Error("OPENAI_API_KEY=sk-recovery-secret backup finalization failed");
}),
});

expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull();
const output = errorSpy.mock.calls.flat().join("\n");
expect(output).toContain("Trusted container recovery could not start");
expect(output).toContain("OPENAI_API_KEY=<REDACTED>");
expect(output).toContain("backup finalization failed");
expect(output).not.toContain("sk-recovery-secret");
});

it("preserves the recreation diagnostic when state-backup cleanup throws", () => {
vi.spyOn(console, "log").mockImplementation(() => undefined);
const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined);
Expand Down
2 changes: 1 addition & 1 deletion src/lib/actions/sandbox/supervisor-relaunch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,7 @@ export function relaunchManagedSupervisorSession(
// Preserve the recreation failure that stopped container recovery.
}
}
if (!quiet) {
if (!quiet || process.env.NEMOCLAW_REBUILD_VERBOSE === "1") {
const detail = error instanceof Error ? error.message : String(error);
console.error(` Trusted container recovery could not start: ${redactFull(redact(detail))}`);
}
Expand Down
46 changes: 46 additions & 0 deletions src/lib/onboard/docker-gpu-patch-clone.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,28 @@ describe("Docker GPU clone envelope", () => {
expect(shouldOmitOpenShellOciImageUser(inspect, NEMOCLAW_STARTUP_ARGV)).toBe(false);
});

it("preserves the exact workspace compatibility metadata across another recreation (#8662)", () => {
const inspect = openShellOciWorkspaceInspect();
inspect.Config!.Env = inspect.Config!.Env!.filter(
(entry) => !entry.startsWith("OPENSHELL_OCI_IMAGE_USER="),
);

const args = buildDockerGpuCloneRunArgs(inspect, buildDockerGpuMode("startup-command"), {
openshellSandboxCommand: NEMOCLAW_STARTUP_ARGV,
});

expect(shouldOmitOpenShellOciImageUser(inspect, NEMOCLAW_STARTUP_ARGV)).toBe(false);
expect(args).toEqual(
expect.arrayContaining([
"--env",
"OPENSHELL_SANDBOX_UID=",
"--env",
"OPENSHELL_SANDBOX_GID=",
]),
);
expect(args).not.toEqual(expect.arrayContaining(["--env", "OPENSHELL_OCI_IMAGE_USER=sandbox"]));
});

it.each([
{
name: "missing sandbox GID",
Expand Down Expand Up @@ -95,6 +117,30 @@ describe("Docker GPU clone envelope", () => {
name: "duplicate OCI user",
mutate: (environment: string[]) => [...environment, "OPENSHELL_OCI_IMAGE_USER=sandbox"],
},
{
name: "already-applied metadata without a sandbox GID",
mutate: (environment: string[]) =>
environment.filter(
(entry) =>
!entry.startsWith("OPENSHELL_OCI_IMAGE_USER=") && entry !== "OPENSHELL_SANDBOX_GID=",
),
},
{
name: "already-applied metadata with a non-empty sandbox UID",
mutate: (environment: string[]) =>
environment
.filter((entry) => !entry.startsWith("OPENSHELL_OCI_IMAGE_USER="))
.map((entry) =>
entry === "OPENSHELL_SANDBOX_UID=" ? "OPENSHELL_SANDBOX_UID=998" : entry,
),
},
{
name: "already-applied metadata with a duplicate sandbox UID",
mutate: (environment: string[]) => [
...environment.filter((entry) => !entry.startsWith("OPENSHELL_OCI_IMAGE_USER=")),
"OPENSHELL_SANDBOX_UID=",
],
},
])("rejects $name in OpenShell's protected identity metadata (#8662)", ({ mutate }) => {
const inspect = openShellOciWorkspaceInspect();
inspect.Config!.Env = mutate(inspect.Config!.Env!);
Expand Down
10 changes: 10 additions & 0 deletions src/lib/onboard/docker-gpu-patch-clone.ts
Original file line number Diff line number Diff line change
Expand Up @@ -375,6 +375,16 @@ function validateOpenShellOciIdentityMetadata(environment: readonly string[]): b
// OpenShell through v0.0.85 did not publish OCI identity metadata.
return false;
}
if (
ociUsers.length === 0 &&
sandboxUids.length === 1 &&
sandboxUids[0] === `${OPENSHELL_SANDBOX_UID_ENV}=` &&
sandboxGids.length === 1 &&
sandboxGids[0] === `${OPENSHELL_SANDBOX_GID_ENV}=`
) {
// A prior reviewed recreation already removed only the OCI-user marker.
return false;
}
const ociUserPrefix = `${OPENSHELL_OCI_IMAGE_USER_ENV}=`;
if (
ociUsers.length !== 1 ||
Expand Down
Loading