From 9da84b9737b88df99131630c6b0012c115626b2c Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 17 Sep 2026 11:37:49 -0700 Subject: [PATCH 01/39] fix(onboard): use compatible receipt seed user Signed-off-by: Aaron Erickson --- .../onboard/managed-startup/docker-receipt-transfer.test.ts | 6 ++++-- src/lib/onboard/managed-startup/docker-receipt-transfer.ts | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/lib/onboard/managed-startup/docker-receipt-transfer.test.ts b/src/lib/onboard/managed-startup/docker-receipt-transfer.test.ts index ede6497cd79..7c48d801cef 100644 --- a/src/lib/onboard/managed-startup/docker-receipt-transfer.test.ts +++ b/src/lib/onboard/managed-startup/docker-receipt-transfer.test.ts @@ -51,19 +51,21 @@ describe("Docker daemon receipt transfer", () => { }); expect(calls.map((args) => args[0])).toEqual(["volume", "create", "cp", "rm"]); - expect(calls[1]).toEqual( + const seedArgs = calls[1] ?? []; + expect(seedArgs).toEqual( expect.arrayContaining([ "--network", "none", "--read-only", "--user", - "0:0", "--security-opt", "no-new-privileges", "--cap-drop", "ALL", ]), ); + expect(seedArgs.at(seedArgs.indexOf("--user") + 1)).toBe("0"); + expect(seedArgs).not.toContain("0:0"); expect(calls[2]).toEqual( expect.arrayContaining([ "-a", diff --git a/src/lib/onboard/managed-startup/docker-receipt-transfer.ts b/src/lib/onboard/managed-startup/docker-receipt-transfer.ts index addca58147d..cf4e2253de0 100644 --- a/src/lib/onboard/managed-startup/docker-receipt-transfer.ts +++ b/src/lib/onboard/managed-startup/docker-receipt-transfer.ts @@ -121,7 +121,7 @@ export function transferDockerReceiptToDaemon( "none", "--read-only", "--user", - "0:0", + "0", "--security-opt", "no-new-privileges", "--cap-drop", From 8003d334b8120445bcc2a30b7e79eaa0d8d6369d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 17 Sep 2026 12:46:44 -0700 Subject: [PATCH 02/39] fix(onboard): allow standalone gateway fallback on macOS Signed-off-by: Aaron Erickson --- ...er-driver-gateway-service-homebrew.test.ts | 18 ++++++---- .../onboard/docker-driver-gateway-service.ts | 33 +++---------------- 2 files changed, 15 insertions(+), 36 deletions(-) diff --git a/src/lib/onboard/docker-driver-gateway-service-homebrew.test.ts b/src/lib/onboard/docker-driver-gateway-service-homebrew.test.ts index d2395b0df6c..759658fdb5b 100644 --- a/src/lib/onboard/docker-driver-gateway-service-homebrew.test.ts +++ b/src/lib/onboard/docker-driver-gateway-service-homebrew.test.ts @@ -136,16 +136,17 @@ describe("OpenShell Homebrew service boundary", () => { }); it.each([ - [66, "Run curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash"], - [67, "could not grant temporary trust"], - [68, "could not remove temporary trust"], - [69, "Run curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash"], - ])("fails closed on Homebrew boundary status %i (#7707)", (status, expected) => { + [66], + [67], + [68], + [69], + ])("uses standalone fallback on Homebrew boundary status %i (#7707)", (status) => { const preparePortForServiceStart = vi.fn(); const prepareServiceEnv = vi.fn(); const validatePortOwnerForServiceStart = vi.fn(); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); - expect(() => + expect( startOpenShellGatewayUserService({ commandExists: () => true, homebrewFormulaOperation: () => spawnResult(status, "opaque Homebrew diagnostic"), @@ -154,7 +155,10 @@ describe("OpenShell Homebrew service boundary", () => { prepareServiceEnv, validatePortOwnerForServiceStart, }), - ).toThrow(expected); + ).toEqual({ attempted: false, started: false, reason: "service not installed" }); + expect(warn).toHaveBeenCalledWith( + " OpenShell Homebrew service inspection failed; continuing with standalone gateway fallback.", + ); expect(preparePortForServiceStart).not.toHaveBeenCalled(); expect(prepareServiceEnv).not.toHaveBeenCalled(); expect(validatePortOwnerForServiceStart).not.toHaveBeenCalled(); diff --git a/src/lib/onboard/docker-driver-gateway-service.ts b/src/lib/onboard/docker-driver-gateway-service.ts index ee720db4ce6..f7abcfb6622 100644 --- a/src/lib/onboard/docker-driver-gateway-service.ts +++ b/src/lib/onboard/docker-driver-gateway-service.ts @@ -463,10 +463,6 @@ function runSystemctlUser( } const OPENSHELL_HOMEBREW_FORMULA_ABSENT = 65; -const OPENSHELL_HOMEBREW_FORMULA_REPAIR = 66; -const OPENSHELL_HOMEBREW_TRUST_FAILED = 67; -const OPENSHELL_HOMEBREW_UNTRUST_FAILED = 68; -const OPENSHELL_HOMEBREW_OPERATION_FAILED = 69; function homebrewFormulaOperationScript(): string { return path.resolve(__dirname, "../../../scripts/install-openshell.sh"); @@ -524,27 +520,6 @@ const HOMEBREW_FORMULA_REPAIR_GUIDANCE = "OpenShell's Homebrew formula is installed but cannot satisfy NemoClaw's pinned checksum and temporary trust contract. " + "Run curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash, then rerun onboarding."; -function throwHomebrewFormulaOperationFailure(operation: string, result: CommandResult): never { - if (result.status === OPENSHELL_HOMEBREW_FORMULA_REPAIR) { - throw new OpenShellGatewayServiceTrustError(HOMEBREW_FORMULA_REPAIR_GUIDANCE); - } - if (result.status === OPENSHELL_HOMEBREW_TRUST_FAILED) { - throw new OpenShellGatewayServiceTrustError( - `Homebrew could not grant temporary trust for the checksum-verified OpenShell formula during ${operation}. ` + - "No service operation was performed.", - ); - } - if (result.status === OPENSHELL_HOMEBREW_UNTRUST_FAILED) { - throw new OpenShellGatewayServiceTrustError( - `Homebrew could not remove temporary trust for the OpenShell formula after ${operation}. ` + - "Stop and repair Homebrew trust before continuing.", - ); - } - throw new OpenShellGatewayServiceTrustError( - `OpenShell Homebrew ${operation} failed inside the checksum-verified temporary trust boundary.`, - ); -} - function runStopService( service: OpenShellGatewayUserServiceTarget, opts: Required> & @@ -625,10 +600,10 @@ function resolveOfficialHomebrewFormulaPaths( ); if (!info.ok) { if (info.status === OPENSHELL_HOMEBREW_FORMULA_ABSENT) return null; - if (info.status === OPENSHELL_HOMEBREW_OPERATION_FAILED) { - throw new OpenShellGatewayServiceTrustError(HOMEBREW_FORMULA_REPAIR_GUIDANCE); - } - throwHomebrewFormulaOperationFailure("formula identity inspection", info); + console.warn( + " OpenShell Homebrew service inspection failed; continuing with standalone gateway fallback.", + ); + return null; } try { const parsed = JSON.parse(info.stdout ?? "") as { From 63bdd967161a121abf64677ed05dd5332ee46097 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 17 Sep 2026 12:55:24 -0700 Subject: [PATCH 03/39] fix(macos): fall back after Homebrew service failures Signed-off-by: Aaron Erickson --- scripts/install-openshell.sh | 16 +++++++++++----- .../docker-driver-gateway-service.test.ts | 12 ++++++------ .../onboard/docker-driver-gateway-service.ts | 18 ++++++------------ test/helpers/openshell-release-fixtures.ts | 4 ++-- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/scripts/install-openshell.sh b/scripts/install-openshell.sh index 6688ff81da4..c16b492a0a8 100755 --- a/scripts/install-openshell.sh +++ b/scripts/install-openshell.sh @@ -823,8 +823,6 @@ install_macos_homebrew_formula() { run_trusted_openshell_homebrew_operation "$formula_operation_pin" -- \ brew "$install_action" --formula "$formula_ref" \ || homebrew_operation_status=$? - [ "$homebrew_operation_status" = "0" ] \ - || fail "OpenShell Homebrew ${install_action} failed inside the temporary formula trust boundary (status ${homebrew_operation_status})." brew_prefix="$(brew --prefix 2>/dev/null || true)" openshell_bin="${brew_prefix}/bin/openshell" @@ -832,11 +830,19 @@ install_macos_homebrew_formula() { openshell_bin="$(command -v openshell 2>/dev/null || true)" fi if [ -z "$openshell_bin" ] || [ ! -x "$openshell_bin" ]; then - fail "Homebrew completed but the openshell binary was not found on PATH." + fail "Homebrew did not install the openshell binary." fi + installed_version_output="$("$openshell_bin" --version 2>&1 || true)" + installed_version="$(printf '%s\n' "$installed_version_output" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)" + [ "$installed_version" = "$PIN_VERSION" ] \ + || fail "Homebrew did not install the pinned OpenShell ${PIN_VERSION} binary." require_openshell_messaging_features "$openshell_bin" - info "$("$openshell_bin" --version 2>&1 || echo openshell) installed" - info "OpenShell Homebrew service staged; onboarding will start it after gateway validation." + info "${installed_version_output:-openshell} installed" + if [ "$homebrew_operation_status" = "0" ]; then + info "OpenShell Homebrew service staged; onboarding will start it after gateway validation." + else + warn "OpenShell Homebrew ${install_action} returned status ${homebrew_operation_status} after installing the pinned binary; onboarding will try the service and fall back to standalone startup." + fi exit 0 } diff --git a/src/lib/onboard/docker-driver-gateway-service.test.ts b/src/lib/onboard/docker-driver-gateway-service.test.ts index a99b2d2db9e..f1466255aa4 100644 --- a/src/lib/onboard/docker-driver-gateway-service.test.ts +++ b/src/lib/onboard/docker-driver-gateway-service.test.ts @@ -544,7 +544,7 @@ describe("docker-driver-gateway-service", () => { }), verifySandboxBridgeGatewayReachableOrExit: vi.fn(), }), - ).rejects.toThrow("remains lifecycle authority"); + ).resolves.toBe(false); expect(register).toHaveBeenCalledOnce(); expect(sleepSeconds).not.toHaveBeenCalled(); expect(ready).not.toHaveBeenCalled(); @@ -596,7 +596,7 @@ describe("docker-driver-gateway-service", () => { }, ); - it("keeps Homebrew as lifecycle authority when managed startup fails (#7707)", async () => { + it("uses standalone fallback when Homebrew startup fails (#7707)", async () => { const stopService = vi.fn(); await expect( @@ -618,7 +618,7 @@ describe("docker-driver-gateway-service", () => { stopOpenShellGatewayUserService: stopService, verifySandboxBridgeGatewayReachableOrExit: vi.fn(), }), - ).rejects.toThrow("temporary trust failed"); + ).resolves.toBe(false); expect(stopService).not.toHaveBeenCalled(); }); @@ -662,7 +662,7 @@ describe("docker-driver-gateway-service", () => { ); }); - it("stops an unhealthy Homebrew service without changing lifecycle authority (#7707)", async () => { + it("uses standalone fallback when the Homebrew service stays unhealthy (#7707)", async () => { const clock = createVirtualClock(); const stopService = vi.fn(() => ({ attempted: true, @@ -693,8 +693,8 @@ describe("docker-driver-gateway-service", () => { stopOpenShellGatewayUserService: stopService, verifySandboxBridgeGatewayReachableOrExit: vi.fn(), }), - ).rejects.toThrow("Homebrew formula remains lifecycle authority"); - expect(stopService).toHaveBeenCalledOnce(); + ).resolves.toBe(false); + expect(stopService).not.toHaveBeenCalled(); }); it("blocks standalone fallback when managed service cleanup fails without permission (#8926)", async () => { diff --git a/src/lib/onboard/docker-driver-gateway-service.ts b/src/lib/onboard/docker-driver-gateway-service.ts index f7abcfb6622..9769483e617 100644 --- a/src/lib/onboard/docker-driver-gateway-service.ts +++ b/src/lib/onboard/docker-driver-gateway-service.ts @@ -1462,10 +1462,7 @@ export async function startPackageManagedDockerDriverGateway({ validatePortOwnerForServiceStart: validatePortOwnerForOpenShellGatewayUserServiceStart, }); } catch (error) { - if ( - error instanceof OpenShellGatewayServiceEnvironmentError || - error instanceof OpenShellGatewayServiceTrustError - ) { + if (error instanceof OpenShellGatewayServiceEnvironmentError) { throw error; } warn( @@ -1481,7 +1478,7 @@ export async function startPackageManagedDockerDriverGateway({ }; if (!serviceStart.started) { const detail = serviceStart.reason ? ` (${serviceStart.reason})` : ""; - if (serviceStart.standaloneFallbackBlocked || serviceStart.manager === "homebrew") { + if (serviceStart.standaloneFallbackBlocked && serviceStart.manager !== "homebrew") { const message = `OpenShell gateway managed service failed to start${detail}.`; printError(` ${message}`); if (exitOnFailure) process.exit(1); @@ -1491,7 +1488,9 @@ export async function startPackageManagedDockerDriverGateway({ ` OpenShell gateway managed service failed to start${detail}; using standalone fallback.`, ); reportLogs(); - if (serviceStart.attempted) stopBeforeStandaloneFallback(); + if (serviceStart.attempted && serviceStart.manager !== "homebrew") { + stopBeforeStandaloneFallback(); + } return false; } @@ -1534,12 +1533,7 @@ export async function startPackageManagedDockerDriverGateway({ ); reportLogs(); if (serviceStart.manager === "homebrew") { - stopBeforeStandaloneFallback(); - const authorityMessage = - "The installed OpenShell Homebrew formula remains lifecycle authority; " + - "run curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash before retrying onboarding."; - if (exitOnFailure) process.exit(1); - throw new OpenShellGatewayServiceTrustError(authorityMessage); + return false; } if (serviceStart.attempted) stopBeforeStandaloneFallback(); return false; diff --git a/test/helpers/openshell-release-fixtures.ts b/test/helpers/openshell-release-fixtures.ts index 8998d114ee8..a3c6022a974 100644 --- a/test/helpers/openshell-release-fixtures.ts +++ b/test/helpers/openshell-release-fixtures.ts @@ -32,8 +32,8 @@ export const OPENSHELL_MCP_FEATURE_MARKER = "allow_all_known_mcp_methods"; export const OPENSHELL_FEATURE_MARKERS = `${OPENSHELL_REWRITE_FEATURE_MARKERS} ${OPENSHELL_MCP_FEATURE_MARKER}`; export const BREW_OUTCOMES = [ ["0", "0", "reinstall", 0], - ["1", "1", "install", 1], - ["0", "1", "reinstall", 1], + ["1", "1", "install", 0], + ["0", "1", "reinstall", 0], ] as const; export function trustedFormulaBoundaryEvents(operation: string): string[] { From 6e3beacedc70f7fbe8006d57d35457a694451630 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 17 Sep 2026 13:06:06 -0700 Subject: [PATCH 04/39] fix(install): preserve local Docker context socket Signed-off-by: Aaron Erickson --- scripts/install.sh | 19 +++++++++++-- ...stall-preexisting-sandbox-recovery.test.ts | 28 +++++++++++++++++-- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index e9106c5aa25..8fdcdef5e11 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -1369,7 +1369,7 @@ installer_docker_host_has_supported_shape() { # Admit a usable socket and the default context before Docker or recovery effects. # Persisted JSON inspection can wait only for its missing Node.js prerequisite. validate_installer_docker_target_before_host_changes() { - local raw="${DOCKER_HOST-}" candidate active_context="" + local raw="${DOCKER_HOST-}" candidate active_context="" context_host="" installer_docker_host_has_supported_shape \ || error "DOCKER_HOST is not a supported absolute local Unix socket endpoint. Unset DOCKER_HOST or set it to an absolute local Unix socket URL, such as unix:///var/run/docker.sock. Then rerun the installer." candidate="${raw#"${raw%%[![:space:]]*}"}" @@ -1386,8 +1386,21 @@ validate_installer_docker_target_before_host_changes() { fi active_context="$(docker_active_context)" fi - [[ "$active_context" == default ]] \ - || error "The Docker context does not select the local default target. Unset DOCKER_CONTEXT or set it to default, and run 'docker context use default' if a non-default context is persisted. Then rerun the installer." + if [[ "$active_context" != default ]]; then + context_host="$(docker context inspect "$active_context" --format '{{.Endpoints.docker.Host}}' 2>/dev/null || true)" + [[ -n "$context_host" ]] \ + || error "The Docker context does not select a supported local Unix socket. Select a local Docker or Colima context, then rerun the installer." + DOCKER_HOST="$context_host" + installer_docker_host_has_supported_shape \ + || error "The Docker context does not select a supported local Unix socket. Select a local Docker or Colima context, then rerun the installer." + export DOCKER_HOST + unset DOCKER_CONTEXT + elif [[ -n "$candidate" ]]; then + unset DOCKER_CONTEXT + fi + # DOCKER_CONTEXT takes precedence over DOCKER_HOST in Docker clients. Once + # admission resolves a local socket, keep that socket authoritative through + # host preflight and automatic onboarding. } # Re-read persisted context after Node installation instead of trusting the temporary default. diff --git a/test/install/install-preexisting-sandbox-recovery.test.ts b/test/install/install-preexisting-sandbox-recovery.test.ts index d0625795551..ebb7efb7445 100644 --- a/test/install/install-preexisting-sandbox-recovery.test.ts +++ b/test/install/install-preexisting-sandbox-recovery.test.ts @@ -38,6 +38,7 @@ function runRecoveryBeforeOnboard( options: { detectedExpressPlatform?: string; dockerContext?: string; + dockerContextEndpoint?: string; dockerHost?: string; hostPreflightExitCode?: number; includeNodeOnPath?: boolean; @@ -185,6 +186,14 @@ exit 0 [[ "$*" == "info --format {{.Host.RemoteSocket.Path}}" ]] || return 99 printf '%s\n' "$PODMAN_SOCKET" } + docker() { + if [[ "\${1:-}" == "context" && "\${2:-}" == "inspect" ]]; then + [[ -n "$DOCKER_CONTEXT_ENDPOINT" ]] || return 1 + printf '%s\n' "$DOCKER_CONTEXT_ENDPOINT" + return 0 + fi + return 99 + } sleep() { printf 'sleep=%s\n' "$*" >> "${callLog}"; } step() { :; } install_nodejs() { @@ -221,6 +230,7 @@ exit 0 extraEnv: { DETECTED_EXPRESS_PLATFORM: options.detectedExpressPlatform ?? "", ...(options.dockerContext !== undefined ? { DOCKER_CONTEXT: options.dockerContext } : {}), + DOCKER_CONTEXT_ENDPOINT: options.dockerContextEndpoint ?? "", ...(dockerConfig === undefined ? {} : { DOCKER_CONFIG: dockerConfig }), ...(options.dockerHost !== undefined ? { DOCKER_HOST: options.dockerHost } : {}), NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE: "1", @@ -277,6 +287,19 @@ describe("install.sh pre-existing sandbox recovery ordering (#6114)", () => { expect(result.output).toContain("Existing sandboxes recovered; skipping generic onboarding"); }); + it("resolves a persisted local Colima context to its Unix socket", () => { + const result = runRecoveryBeforeOnboard(2, 0, { + dockerContextEndpoint: "unix:///Users/test/.colima/default/docker.sock", + persistedDockerContext: "colima", + recordRuntimeTarget: true, + }); + + expect(result.status, result.output).toBe(0); + expect(result.calls).toContain( + "cli-target=host:unix:///Users/test/.colima/default/docker.sock,context:unset argv=upgrade-sandboxes --auto", + ); + }); + it("treats a whitespace-only Docker host as unset", () => { const result = runRecoveryBeforeOnboard(2, 0, { dockerHost: " \t ", @@ -329,7 +352,7 @@ describe("install.sh pre-existing sandbox recovery ordering (#6114)", () => { expect(result.calls).not.toContain( 'restore=1 confirmed=["legacy-box"] argv=upgrade-sandboxes --auto', ); - expect(result.output).toContain("Docker context does not select the local default target"); + expect(result.output).toContain("Docker context does not select a supported local Unix socket"); }); it.each([ @@ -388,8 +411,7 @@ describe("install.sh pre-existing sandbox recovery ordering (#6114)", () => { expect(result.status).toBe(1); expect(result.calls).toEqual([]); - expect(result.output).toContain("Docker context does not select the local default target"); - expect(result.output).toContain("docker context use default"); + expect(result.output).toContain("Docker context does not select a supported local Unix socket"); }); it("rejects a non-local Podman socket before sandbox recovery", () => { From 817876dea92a3dca37b527266fc097a21b583076 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 17 Sep 2026 13:09:11 -0700 Subject: [PATCH 05/39] fix(macos): support Colima installer and dead forwards Signed-off-by: Aaron Erickson --- .../openshell/forward-cli-validation.test.ts | 9 +++++++++ src/lib/adapters/openshell/forward-cli.ts | 15 ++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/lib/adapters/openshell/forward-cli-validation.test.ts b/src/lib/adapters/openshell/forward-cli-validation.test.ts index bfcdf3a0202..2df96289840 100644 --- a/src/lib/adapters/openshell/forward-cli-validation.test.ts +++ b/src/lib/adapters/openshell/forward-cli-validation.test.ts @@ -94,6 +94,15 @@ describe("CLI OpenShell forward list parser", () => { expect(parseCliOpenShellForwardList("", noActiveForwards)).toEqual({ ok: true, rows: [] }); }); + it("accepts different dead forwards that reused one port", () => { + expect( + parseCliOpenShellForwardList( + `${listHeader}\nnemochow 127.0.0.1 18789 6236 dead\nnemoclaw-inbox 127.0.0.1 18789 81718 dead`, + "", + ), + ).toMatchObject({ ok: true }); + }); + it.each([ ["empty output", ""], ["missing header", "demo 127.0.0.1 18789 4312 running"], diff --git a/src/lib/adapters/openshell/forward-cli.ts b/src/lib/adapters/openshell/forward-cli.ts index 4eab99d8ef7..21e46e137d7 100644 --- a/src/lib/adapters/openshell/forward-cli.ts +++ b/src/lib/adapters/openshell/forward-cli.ts @@ -208,7 +208,8 @@ export function parseCliOpenShellForwardList( } if (lines.length === 0) return { ok: false, error: SCHEMA_ERROR }; const rows: CliOpenShellLegacyForwardRow[] = []; - const occupiedPorts = new Set(); + const runningPorts = new Set(); + const rowIdentities = new Set(); for (const line of lines) { const columns = line.split(/\s+/u); if (columns.length !== 5) return { ok: false, error: SCHEMA_ERROR }; @@ -224,12 +225,16 @@ export function parseCliOpenShellForwardList( port > 65_535 || !/^[1-9]\d*$/u.test(pidText ?? "") || !Number.isSafeInteger(pid) || - (status !== "running" && status !== "dead") || - occupiedPorts.has(port) + (status !== "running" && status !== "dead") ) { return { ok: false, error: SCHEMA_ERROR }; } - occupiedPorts.add(port); + const rowIdentity = `${sandboxName}\u0000${bind}\u0000${String(port)}\u0000${String(pid)}\u0000${status}`; + if (rowIdentities.has(rowIdentity) || (status === "running" && runningPorts.has(port))) { + return { ok: false, error: SCHEMA_ERROR }; + } + rowIdentities.add(rowIdentity); + if (status === "running") runningPorts.add(port); rows.push({ sandboxName, bind, port, pid, status }); } return { ok: true, rows }; @@ -1011,7 +1016,7 @@ function classifyRegistry( forward: OpenShellForwardIdentity, rows: readonly CliOpenShellLegacyForwardRow[], ): CliOpenShellForwardRegistryMatch { - const samePort = rows.filter((row) => row.port === forward.port); + const samePort = rows.filter((row) => row.status === "running" && row.port === forward.port); const sameTarget = samePort.filter( (row) => row.sandboxName === forward.sandboxName && row.bind === forward.localHost, ); From 21340e5ceeb48dda27cf18a4d273093159c8ee44 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 17 Sep 2026 14:00:34 -0700 Subject: [PATCH 06/39] fix(onboard): delegate forwards to OpenShell Signed-off-by: Aaron Erickson --- .../adapters/openshell/forward-cli-args.ts | 17 ++ .../openshell/forward-cli-validation.test.ts | 9 - src/lib/adapters/openshell/forward-cli.ts | 15 +- src/lib/onboard.ts | 1 + src/lib/onboard/dashboard-port.test.ts | 22 ++ src/lib/onboard/dashboard-port.ts | 8 +- src/lib/onboard/dashboard.ts | 132 +++------- src/lib/onboard/hermes-api-port.test.ts | 15 ++ src/lib/onboard/hermes-api-port.ts | 14 +- .../onboard/sandbox-create/orchestration.ts | 5 - ...ard-finalization-dashboard-forward.test.ts | 226 ++++++------------ 11 files changed, 179 insertions(+), 285 deletions(-) diff --git a/src/lib/adapters/openshell/forward-cli-args.ts b/src/lib/adapters/openshell/forward-cli-args.ts index c3ac53143ce..bdc518e520e 100644 --- a/src/lib/adapters/openshell/forward-cli-args.ts +++ b/src/lib/adapters/openshell/forward-cli-args.ts @@ -23,3 +23,20 @@ export function buildCliOpenShellForwardServiceArgs(forward: OpenShellForwardIde `${forward.localHost}:${String(forward.port)}`, ]; } + +/** Build the OpenShell-owned background forward command without a shell. */ +export function buildCliOpenShellForwardStartArgs(forward: OpenShellForwardIdentity): string[] { + return [ + "forward", + "start", + "-d", + `${forward.localHost}:${String(forward.port)}`, + forward.sandboxName, + "--gateway", + forward.gatewayName, + "--gateway-endpoint", + forward.gatewayEndpoint, + "--workspace", + forward.workspace, + ]; +} diff --git a/src/lib/adapters/openshell/forward-cli-validation.test.ts b/src/lib/adapters/openshell/forward-cli-validation.test.ts index 2df96289840..bfcdf3a0202 100644 --- a/src/lib/adapters/openshell/forward-cli-validation.test.ts +++ b/src/lib/adapters/openshell/forward-cli-validation.test.ts @@ -94,15 +94,6 @@ describe("CLI OpenShell forward list parser", () => { expect(parseCliOpenShellForwardList("", noActiveForwards)).toEqual({ ok: true, rows: [] }); }); - it("accepts different dead forwards that reused one port", () => { - expect( - parseCliOpenShellForwardList( - `${listHeader}\nnemochow 127.0.0.1 18789 6236 dead\nnemoclaw-inbox 127.0.0.1 18789 81718 dead`, - "", - ), - ).toMatchObject({ ok: true }); - }); - it.each([ ["empty output", ""], ["missing header", "demo 127.0.0.1 18789 4312 running"], diff --git a/src/lib/adapters/openshell/forward-cli.ts b/src/lib/adapters/openshell/forward-cli.ts index 21e46e137d7..4eab99d8ef7 100644 --- a/src/lib/adapters/openshell/forward-cli.ts +++ b/src/lib/adapters/openshell/forward-cli.ts @@ -208,8 +208,7 @@ export function parseCliOpenShellForwardList( } if (lines.length === 0) return { ok: false, error: SCHEMA_ERROR }; const rows: CliOpenShellLegacyForwardRow[] = []; - const runningPorts = new Set(); - const rowIdentities = new Set(); + const occupiedPorts = new Set(); for (const line of lines) { const columns = line.split(/\s+/u); if (columns.length !== 5) return { ok: false, error: SCHEMA_ERROR }; @@ -225,16 +224,12 @@ export function parseCliOpenShellForwardList( port > 65_535 || !/^[1-9]\d*$/u.test(pidText ?? "") || !Number.isSafeInteger(pid) || - (status !== "running" && status !== "dead") + (status !== "running" && status !== "dead") || + occupiedPorts.has(port) ) { return { ok: false, error: SCHEMA_ERROR }; } - const rowIdentity = `${sandboxName}\u0000${bind}\u0000${String(port)}\u0000${String(pid)}\u0000${status}`; - if (rowIdentities.has(rowIdentity) || (status === "running" && runningPorts.has(port))) { - return { ok: false, error: SCHEMA_ERROR }; - } - rowIdentities.add(rowIdentity); - if (status === "running") runningPorts.add(port); + occupiedPorts.add(port); rows.push({ sandboxName, bind, port, pid, status }); } return { ok: true, rows }; @@ -1016,7 +1011,7 @@ function classifyRegistry( forward: OpenShellForwardIdentity, rows: readonly CliOpenShellLegacyForwardRow[], ): CliOpenShellForwardRegistryMatch { - const samePort = rows.filter((row) => row.status === "running" && row.port === forward.port); + const samePort = rows.filter((row) => row.port === forward.port); const sameTarget = samePort.filter( (row) => row.sandboxName === forward.sandboxName && row.bind === forward.localHost, ); diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 74d6f596412..381113666d8 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2477,6 +2477,7 @@ const { stopAllDashboardForwards, } = onboardDashboard.createOnboardDashboardHelpers({ runCaptureOpenshell, + runOpenshell, runCapture, cliName, agentProductName, diff --git a/src/lib/onboard/dashboard-port.test.ts b/src/lib/onboard/dashboard-port.test.ts index 6e7c1ea85bc..9c995dfc761 100644 --- a/src/lib/onboard/dashboard-port.test.ts +++ b/src/lib/onboard/dashboard-port.test.ts @@ -493,6 +493,28 @@ describe("dashboard port reservation", () => { assert.deepEqual(released, [18790]); }); + it("keeps the requested port without inspecting or reserving forwards", async () => { + const reservePort = vi.fn(); + const result = await reserveCreateSandboxDashboardPort( + { + sandboxName: "cursor", + controlUiPort: 18789, + chatUiUrlEnv: null, + persistedPort: null, + agentForwardPort: null, + }, + reservePort, + ); + + expect(result).toEqual({ + preferredPort: 18789, + effectivePort: 18789, + chatUiUrl: "http://127.0.0.1:18789", + reservation: null, + }); + expect(reservePort).not.toHaveBeenCalled(); + }); + it("defers a persisted port reservation only for the exact owned forward", async () => { const reservePort = vi.fn(); const observeForwardPorts = observePorts("cursor", new Map([[18789, "owned"]])); diff --git a/src/lib/onboard/dashboard-port.ts b/src/lib/onboard/dashboard-port.ts index 59641c7e0ab..f74c35fcf55 100644 --- a/src/lib/onboard/dashboard-port.ts +++ b/src/lib/onboard/dashboard-port.ts @@ -463,7 +463,7 @@ export type ReserveCreateSandboxDashboardPortInput = Omit< ObservedCreateSandboxDashboardPortInput, "forwardObservations" > & { - observeForwardPorts: OpenShellForwardPortObserver; + observeForwardPorts?: OpenShellForwardPortObserver; }; export interface CreateSandboxDashboardPortResult { @@ -671,6 +671,12 @@ export async function reserveCreateSandboxDashboardPort( reservePort: (port: number) => Promise = reserveDashboardPort, ): Promise { const preferredPort = preferredCreateSandboxDashboardPort(input); + if (!input.observeForwardPorts) { + return { + ...createSandboxDashboardPortResult(input, preferredPort, preferredPort), + reservation: null, + }; + } const observed = await findAvailableDashboardPortFromObserver( input.sandboxName, preferredPort, diff --git a/src/lib/onboard/dashboard.ts b/src/lib/onboard/dashboard.ts index 2bf20ed6d69..a41e30d3fed 100644 --- a/src/lib/onboard/dashboard.ts +++ b/src/lib/onboard/dashboard.ts @@ -13,6 +13,7 @@ import { openShellForwardIdentity, type OpenShellForwardRuntimeAuthority, } from "../adapters/openshell/forward-runtime"; +import { buildCliOpenShellForwardStartArgs } from "../adapters/openshell/forward-cli-args"; import { createCliOpenShellSandboxTransferExecutor } from "../adapters/openshell/sandbox-transfer-cli"; import type { OpenShellSandboxTransferExecutor } from "../adapters/openshell/sandbox-transfer"; import type { AgentDefinition } from "../agent/defs"; @@ -34,10 +35,7 @@ import { } from "./dashboard-forward-control"; import { createOpenShellForwardPortObserver, - findAvailableDashboardPortFromObserver, getPersistedDashboardPort, - getRegistryOccupiedDashboardPorts, - getRegistryOccupiedHermesApiPorts, type ListSandboxesFn, type OpenShellForwardPortObserver, } from "./dashboard-port"; @@ -58,6 +56,10 @@ type DashboardForwardRuntimeAuthority = OpenShellForwardRuntimeAuthority; export interface OnboardDashboardDeps { runCaptureOpenshell(args: string[], opts?: Record): string | null; + runOpenshell?( + args: string[], + opts?: Record, + ): { status: number | null; error?: Error }; runCapture?: typeof defaultRunCapture; cliName(): string; agentProductName(): string; @@ -388,20 +390,6 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa process.exit(1); } - function forwardResultMessage( - result: - | Awaited> - | Awaited>, - ): string { - if ("error" in result) return result.error.message; - if ("observation" in result) { - return result.observation.state === "foreign" - ? "The host port is owned by a foreign listener." - : "NemoClaw could not prove the forward state."; - } - return "NemoClaw could not reconcile the forward state."; - } - async function reconcileForward( sandboxName: string, authority: DashboardForwardRuntimeAuthority, @@ -409,34 +397,36 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa label: string, revalidateSandboxIdentity?: (operation: string) => void, ): Promise { - const adapter = forwardAdapterForAuthority(authority); - const assertCurrent = () => - assertForwardGatewayCurrent( - authority, - revalidateSandboxIdentity, - `accept ${label} forward ${String(forward.port)} for sandbox '${sandboxName}'`, - ); - let result = await adapter.startForward({ forward, assertCurrent }); - if (result.state === "refused" && result.observation.state === "stale") { - const retirement = await adapter.retireLegacyForward({ - forward, - assertCurrent, - authorize: async () => { - await assertForwardGatewayCurrent( - authority, - revalidateSandboxIdentity, - `retire legacy ${label} forward ${String(forward.port)} for sandbox '${sandboxName}'`, - ); - }, + await assertForwardGatewayCurrent( + authority, + revalidateSandboxIdentity, + `start ${label} forward ${String(forward.port)} for sandbox '${sandboxName}'`, + ); + const startArgs = buildCliOpenShellForwardStartArgs(forward); + if (deps.runOpenshell) { + const result = deps.runOpenshell(startArgs, { + ignoreError: true, + suppressOutput: true, + stdio: ["ignore", "ignore", "ignore"], + timeout: 15_000, + killProcessTreeOnTimeout: true, }); - if (retirement.state !== "retired" && retirement.state !== "not_needed") { - throw new Error(forwardResultMessage(retirement)); + if (result.error || result.status !== 0) { + throw new Error( + `OpenShell forward start failed${result.status === null ? "" : ` with exit ${String(result.status)}`}.`, + ); } - result = await adapter.startForward({ forward, assertCurrent }); - } - if (result.state !== "started" && result.state !== "reused") { - throw new Error(forwardResultMessage(result)); + } else { + deps.runCaptureOpenshell(startArgs, { + timeout: 15_000, + killProcessTreeOnTimeout: true, + }); } + await assertForwardGatewayCurrent( + authority, + revalidateSandboxIdentity, + `accept ${label} forward ${String(forward.port)} for sandbox '${sandboxName}'`, + ); } async function ensureDashboardForward( @@ -445,8 +435,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa options: DashboardForwardOptions = {}, ): Promise { chatUiUrl ||= `http://127.0.0.1:${CONTROL_UI_PORT}`; - const { rollbackSandboxOnFailure, allowPortReallocation, reuseExistingForward } = - normalizeDashboardForwardOptions(options); + const { rollbackSandboxOnFailure } = normalizeDashboardForwardOptions(options); const { revalidateSandboxIdentity } = options; const preferredPort = Number(getDashboardForwardPort(chatUiUrl)); const forwardGateway = resolveForwardServiceGateway(sandboxName, options); @@ -454,60 +443,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa throw new Error(`ForwardTcp authority is unavailable for '${sandboxName}'`); } const authority = getForwardRuntimeAuthority(sandboxName, forwardGateway); - const forwardAdapter = forwardAdapterForAuthority(authority); - const observeForwardPorts = createOpenShellForwardPortObserver({ - adapter: forwardAdapter, - forwardForPort: (port) => forwardIdentity(sandboxName, authority, port, "dashboard"), - assertCurrent: () => - assertForwardGatewayCurrent( - authority, - revalidateSandboxIdentity, - `inspect dashboard forwards for sandbox '${sandboxName}'`, - ), - }); - const persistedPort = getPersistedDashboardPort(sandboxName, listSandboxes); - const registryOccupiedPorts = new Map([ - ...getRegistryOccupiedDashboardPorts(sandboxName, listSandboxes), - ...getRegistryOccupiedHermesApiPorts(sandboxName, listSandboxes), - ]); - const fixedPort = persistedPort === preferredPort || reuseExistingForward; - if (fixedPort && registryOccupiedPorts.has(String(preferredPort))) { - throw new Error( - `Port ${String(preferredPort)} is not available for '${sandboxName}'; another sandbox registered it.`, - ); - } - let actualPort: number; - try { - actualPort = ( - await findAvailableDashboardPortFromObserver( - sandboxName, - preferredPort, - observeForwardPorts, - registryOccupiedPorts, - ) - ).port; - } catch (err) { - if (!rollbackSandboxOnFailure) throw err; - rollbackSandboxAndExit(sandboxName, err, options.gatewayName); - } - - if (actualPort !== preferredPort) { - if (!allowPortReallocation || fixedPort) { - throw new Error( - `Port ${preferredPort} is not available for '${sandboxName}' and cannot be reallocated.`, - ); - } - if (rollbackSandboxOnFailure) { - const err = new Error( - `Dashboard port ${preferredPort} became host-bound during sandbox build; ` + - `cannot reallocate to ${actualPort} after the sandbox has been created with ` + - `CHAT_UI_URL=${preferredPort}. Free the port and re-run \`${deps.cliName()} onboard\`, ` + - `or pass \`--control-ui-port \` to pick a different dashboard port.`, - ); - rollbackSandboxAndExit(sandboxName, err, options.gatewayName); - } - console.warn(` ! Port ${preferredPort} is taken. Using port ${actualPort} instead.`); - } + const actualPort = preferredPort; const actualGateway = resolveForwardServiceGateway(sandboxName, options); let fwdOk = false; diff --git a/src/lib/onboard/hermes-api-port.test.ts b/src/lib/onboard/hermes-api-port.test.ts index 1b998480fb8..9f1da750b13 100644 --- a/src/lib/onboard/hermes-api-port.test.ts +++ b/src/lib/onboard/hermes-api-port.test.ts @@ -153,6 +153,21 @@ describe("findAvailableHermesApiPortFromObservations", () => { }); describe("reserveCreateSandboxHermesApiPort", () => { + it("keeps the requested API port without inspecting or reserving forwards", async () => { + const env: NodeJS.ProcessEnv = { [HERMES_API_PORT_ENV]: "8644" }; + const reservePort = vi.fn(); + + await expect( + reserveCreateSandboxHermesApiPort({ + sandboxName: "alpha", + env, + getSandbox: () => undefined, + reservePort, + }), + ).resolves.toEqual({ effectivePort: 8644, reservation: null }); + expect(reservePort).not.toHaveBeenCalled(); + }); + it("keeps concurrent selections distinct before either host forward exists", async () => { const firstRelease = vi.fn(async () => undefined); const secondRelease = vi.fn(async () => undefined); diff --git a/src/lib/onboard/hermes-api-port.ts b/src/lib/onboard/hermes-api-port.ts index 9016646134f..b3a37451748 100644 --- a/src/lib/onboard/hermes-api-port.ts +++ b/src/lib/onboard/hermes-api-port.ts @@ -52,7 +52,7 @@ export interface HermesApiPortReservationInput { sandboxName: string; env: NodeJS.ProcessEnv; getSandbox(name: string): HermesApiPortSandboxLookup | null | undefined; - observeForwardPorts: OpenShellForwardPortObserver; + observeForwardPorts?: OpenShellForwardPortObserver; reservePort?(port: number): Promise; warn(message: string): void; } @@ -221,7 +221,7 @@ export async function reserveCreateSandboxHermesApiPort(options: { env?: NodeJS.ProcessEnv; getSandbox?: (name: string) => HermesApiPortSandboxLookup | null | undefined; allowRegisteredOverride?: boolean; - observeForwardPorts: OpenShellForwardPortObserver; + observeForwardPorts?: OpenShellForwardPortObserver; registryOccupiedPorts?: ReadonlyMap; reservePort?: (port: number) => Promise; warn?: (message: string) => void; @@ -242,6 +242,11 @@ export async function reserveCreateSandboxHermesApiPort(options: { const registryOccupiedPorts = new Map( options.registryOccupiedPorts ?? getRegistryOccupiedHermesApiPorts(options.sandboxName), ); + if (!options.observeForwardPorts) { + const effectivePort = pinnedPort ?? HERMES_OPENAI_API_PORT; + env[HERMES_API_PORT_ENV] = String(effectivePort); + return { effectivePort, reservation: null }; + } const reserveSelectedPort = async ( effectivePort: number, observations: readonly OpenShellForwardObservation[], @@ -327,7 +332,10 @@ export function createHermesApiPortReservationScope(): HermesApiPortReservationS }); this.effectivePort = selection.effectivePort; this.current = selection.reservation; - deferredOwnedForwardPort = selection.reservation === null ? selection.effectivePort : null; + deferredOwnedForwardPort = + input.observeForwardPorts && selection.reservation === null + ? selection.effectivePort + : null; }, async rebindAfterOwnedForwardDelete(input) { if ( diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index 8d2f3268ef9..dab6a4cb19e 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -1524,7 +1524,6 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche ensureAgentFixedForward, ensureDashboardForward, filterEnabledChannelsByAgent, - forwardObserver, formatSandboxAgentName, formatSandboxBuildEstimateNote, getDashboardForwardPort, @@ -1670,7 +1669,6 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche agent, ); const isManagedDcodeAgent = usesManagedDcodeIdentity(agent?.name, fromDockerfile); - const observeLoopbackForwardPorts = forwardObserver(sandboxName, "loopback"); let effectivePort = 0, chatUiUrl = "", hermesApiPortReservationInput = { @@ -1678,7 +1676,6 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche sandboxName, env: process.env, getSandbox: registry.getSandbox, - observeForwardPorts: observeLoopbackForwardPorts, warn: (message: string) => console.warn(message), }; if (manageDashboard) { @@ -1689,12 +1686,10 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche persistedPort: registry.getSandbox(sandboxName)?.dashboardPort ?? null, agentForwardPort: dashboardRuntime.getAgentPrimaryForwardPort(agent, DASHBOARD_PORT), defaultPort: DASHBOARD_PORT, - observeForwardPorts: forwardObserver(sandboxName), warn: (message: string) => console.warn(message), }); ({ effectivePort, chatUiUrl } = dashboardSelection); dashboardPortReservationScope.current = dashboardSelection.reservation; - dashboardPortReservationScope.deferOwnedForwardPort(effectivePort); } const hermesDashboardForwarding = onboardHermesDashboard.createHermesDashboardOnboardForwarding( { diff --git a/test/onboarding/onboard-finalization-dashboard-forward.test.ts b/test/onboarding/onboard-finalization-dashboard-forward.test.ts index 596f122f2b9..548fd67c646 100644 --- a/test/onboarding/onboard-finalization-dashboard-forward.test.ts +++ b/test/onboarding/onboard-finalization-dashboard-forward.test.ts @@ -9,7 +9,6 @@ import type { OpenShellForwardAdapter, OpenShellForwardIdentity, OpenShellForwardObservation, - OpenShellForwardStartResult, RetireLegacyOpenShellForwardRequest, StartOpenShellForwardRequest, } from "../../src/lib/adapters/openshell/forward"; @@ -22,17 +21,6 @@ type ForwardObservationWithIdentity = Extract< >; type ForwardState = ForwardObservationWithIdentity["state"]; -function deferred(): { - promise: Promise; - resolve: (value: T) => void; -} { - let resolve!: (value: T) => void; - const promise = new Promise((resolvePromise) => { - resolve = resolvePromise; - }); - return { promise, resolve }; -} - function observation( forward: OpenShellForwardIdentity, state: ForwardState, @@ -125,8 +113,21 @@ function harness(options: { retireLegacyForward, verifyForwardRelease: vi.fn(async () => ({ state: "released" as const })), }; + const runCaptureOpenshell = vi.fn(() => ""); + const runOpenshell = vi.fn((args: string[]) => { + if ( + args[0] === "forward" && + args[1] === "start" && + options.startFailurePort !== undefined && + args[3]?.endsWith(`:${String(options.startFailurePort)}`) + ) { + return { status: 1 }; + } + return { status: 0 }; + }); const helpers = createOnboardDashboardHelpers({ - runCaptureOpenshell: vi.fn(() => ""), + runCaptureOpenshell, + runOpenshell, cliName: () => "nemoclaw", agentProductName: () => "NemoClaw", getProviderLabel: (provider) => provider, @@ -142,7 +143,15 @@ function harness(options: { resolveForwardGatewayName: (sandbox) => sandbox?.gatewayName ?? "nemoclaw", forwardAdapterForAuthority: vi.fn(() => adapter), }); - return { helpers, observeForwards, retireLegacyForward, startForward, states }; + return { + helpers, + observeForwards, + retireLegacyForward, + runCaptureOpenshell, + runOpenshell, + startForward, + states, + }; } afterEach(() => { @@ -210,79 +219,47 @@ describe("finalization dashboard ForwardTcp reconciliation", () => { }, ); - it("launches the persisted dashboard port and publishes its URL", async () => { + it("starts the persisted dashboard forward with one OpenShell command", async () => { vi.stubEnv("CHAT_UI_URL", undefined); const test = harness({ listSandboxes: () => ({ sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], }), + initialStates: new Map([[18_790, "indeterminate"]]), }); await expect(test.helpers.ensureFinalizationDashboardForward("reonboard-test")).resolves.toBe( 18_790, ); - expect(test.startForward).toHaveBeenCalledOnce(); - expect(test.startForward.mock.calls[0]?.[0].forward).toMatchObject({ - sandboxName: "reonboard-test", - port: 18_790, - }); - expect(process.env.CHAT_UI_URL).toBe("http://127.0.0.1:18790"); - }); - - it.each(["foreign", "indeterminate"] as const)( - "leaves a %s persisted listener untouched with zero mutation attempts", - async (state) => { - vi.stubEnv("CHAT_UI_URL", undefined); - const test = harness({ - listSandboxes: () => ({ - sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], - }), - initialStates: new Map([[18_790, state]]), - }); - - await expect( - test.helpers.ensureFinalizationDashboardForward("reonboard-test"), - ).rejects.toThrow(/cannot be reallocated|could not prove/u); - expect(test.startForward).not.toHaveBeenCalled(); - expect(test.retireLegacyForward).not.toHaveBeenCalled(); - }, - ); - - it("retires an exact stale forward before one replacement start", async () => { - vi.stubEnv("CHAT_UI_URL", undefined); - const test = harness({ - listSandboxes: () => ({ - sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], - }), - initialStates: new Map([[18_790, "stale"]]), - }); - - await expect(test.helpers.ensureFinalizationDashboardForward("reonboard-test")).resolves.toBe( - 18_790, - ); - expect(test.retireLegacyForward).toHaveBeenCalledOnce(); - expect(test.startForward).toHaveBeenCalledTimes(2); - }); - - it("reuses an exactly owned dashboard forward", async () => { - vi.stubEnv("CHAT_UI_URL", undefined); - const test = harness({ - listSandboxes: () => ({ - sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], - }), - initialStates: new Map([[18_790, "owned"]]), - }); - - await expect(test.helpers.ensureFinalizationDashboardForward("reonboard-test")).resolves.toBe( - 18_790, + expect(test.runOpenshell).toHaveBeenCalledWith( + [ + "forward", + "start", + "-d", + "127.0.0.1:18790", + "reonboard-test", + "--gateway", + "nemoclaw", + "--gateway-endpoint", + "https://127.0.0.1:8080", + "--workspace", + "default", + ], + { + ignoreError: true, + suppressOutput: true, + stdio: ["ignore", "ignore", "ignore"], + timeout: 15_000, + killProcessTreeOnTimeout: true, + }, ); - expect(test.startForward).toHaveBeenCalledOnce(); - await expect(test.startForward.mock.results[0]?.value).resolves.toMatchObject({ - state: "reused", - }); + expect(test.observeForwards).not.toHaveBeenCalled(); + expect(test.retireLegacyForward).not.toHaveBeenCalled(); + expect(test.startForward).not.toHaveBeenCalled(); + expect(process.env.CHAT_UI_URL).toBe("http://127.0.0.1:18790"); }); - it("does not reuse a port registered by another sandbox", async () => { + it("lets OpenShell decide whether a registered port can be forwarded", async () => { vi.stubEnv("CHAT_UI_URL", undefined); const test = harness({ listSandboxes: () => ({ @@ -293,102 +270,32 @@ describe("finalization dashboard ForwardTcp reconciliation", () => { }), }); - await expect(test.helpers.ensureFinalizationDashboardForward("reonboard-test")).rejects.toThrow( - /another sandbox registered it/u, + await expect(test.helpers.ensureFinalizationDashboardForward("reonboard-test")).resolves.toBe( + 18_790, ); - expect(test.startForward).not.toHaveBeenCalled(); + expect(test.runOpenshell).toHaveBeenCalledOnce(); + expect(test.observeForwards).not.toHaveBeenCalled(); }); - it.each(["openclaw", "hermes"])( - "reuses registered forwards for %s without creating another listener", - async (name) => { - vi.stubEnv("CHAT_UI_URL", undefined); - const ports = name === "openclaw" ? [18_790] : [18_790, 8_643]; - const test = harness({ - listSandboxes: () => ({ - sandboxes: [ - { name: "reonboard-test", dashboardPort: 18_790, hermesApiPort: 8_643 }, - { name: "sibling", dashboardPort: 18_789, hermesApiPort: 8_642 }, - ], - }), - initialStates: new Map(ports.map((port) => [port, "owned" as const])), - }); - - await expect( - test.helpers.ensureFinalizationAgentDashboardForward("reonboard-test", loadAgent(name)), - ).resolves.toBe(18_790); - expect(test.startForward.mock.calls.map(([request]) => request.forward.port)).toEqual(ports); - }, - ); - - it("awaits every forward start and propagates a rejected start", async () => { + it("issues one OpenShell start command for each Hermes forward", async () => { vi.stubEnv("CHAT_UI_URL", undefined); const test = harness({ listSandboxes: () => ({ sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790, hermesApiPort: 8_643 }], }), }); - const first = deferred(); - const second = deferred(); - test.startForward - .mockImplementationOnce(() => first.promise) - .mockImplementationOnce(() => second.promise); - - let settled = false; - const finalization = test.helpers - .ensureFinalizationAgentDashboardForward("reonboard-test", loadAgent("hermes")) - .then((value) => { - settled = true; - return value; - }); - await vi.waitFor(() => expect(test.startForward).toHaveBeenCalledTimes(1)); - expect(settled).toBe(false); - const firstForward = test.startForward.mock.calls[0]?.[0].forward; - expect(firstForward).toBeDefined(); - first.resolve({ state: "reused", forward: firstForward! }); - await vi.waitFor(() => expect(test.startForward).toHaveBeenCalledTimes(2)); - expect(settled).toBe(false); - const secondForward = test.startForward.mock.calls[1]?.[0].forward; - expect(secondForward).toBeDefined(); - second.resolve({ state: "reused", forward: secondForward! }); - await expect(finalization).resolves.toBe(18_790); - const failure = harness({ - listSandboxes: () => ({ - sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790, hermesApiPort: 8_643 }], - }), - }); - failure.startForward.mockRejectedValueOnce(new Error("forward startup rejected")); await expect( - failure.helpers.ensureFinalizationAgentDashboardForward( - "reonboard-test", - loadAgent("hermes"), - ), - ).rejects.toThrow(/forward startup rejected/u); + test.helpers.ensureFinalizationAgentDashboardForward("reonboard-test", loadAgent("hermes")), + ).resolves.toBe(18_790); + expect(test.runOpenshell.mock.calls.map(([args]) => args[3])).toEqual([ + "127.0.0.1:18790", + "127.0.0.1:8643", + ]); + expect(test.observeForwards).not.toHaveBeenCalled(); }); - it.each([18_790, 8_643])( - "establishes missing Hermes forward %s on its recorded port", - async (missingPort) => { - vi.stubEnv("CHAT_UI_URL", undefined); - const otherPort = missingPort === 18_790 ? 8_643 : 18_790; - const test = harness({ - listSandboxes: () => ({ - sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790, hermesApiPort: 8_643 }], - }), - initialStates: new Map([[otherPort, "owned"]]), - }); - - await expect( - test.helpers.ensureFinalizationAgentDashboardForward("reonboard-test", loadAgent("hermes")), - ).resolves.toBe(18_790); - expect( - test.startForward.mock.calls.filter(([request]) => request.forward.port === missingPort), - ).toHaveLength(1); - }, - ); - - it("fails the agent forward when the adapter reports a command failure", async () => { + it("reports an OpenShell command failure without inspecting forward state", async () => { vi.spyOn(console, "warn").mockImplementation(() => undefined); const test = harness({ listSandboxes: () => ({ sandboxes: [{ name: "reonboard-test" }] }), @@ -398,7 +305,8 @@ describe("finalization dashboard ForwardTcp reconciliation", () => { await expect( test.helpers.ensureAgentFixedForward("reonboard-test", 8_642, "Hermes API"), ).resolves.toBe(false); - expect(test.startForward).toHaveBeenCalledOnce(); + expect(test.runOpenshell).toHaveBeenCalledOnce(); + expect(test.observeForwards).not.toHaveBeenCalled(); }); it("honors an explicit dashboard URL", async () => { @@ -408,6 +316,6 @@ describe("finalization dashboard ForwardTcp reconciliation", () => { await expect(test.helpers.ensureFinalizationDashboardForward("reonboard-test")).resolves.toBe( 19_001, ); - expect(test.startForward.mock.calls[0]?.[0].forward.port).toBe(19_001); + expect(test.runOpenshell.mock.calls[0]?.[0]?.[3]).toBe("127.0.0.1:19001"); }); }); From 4b0ab3ab345a34689c49ec9e4c3051e42d3b5de0 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 17 Sep 2026 15:40:45 -0700 Subject: [PATCH 07/39] fix(onboard): keep receipt compatibility change scoped Signed-off-by: Aaron Erickson --- scripts/install-openshell.sh | 16 +- scripts/install.sh | 19 +- .../adapters/openshell/forward-cli-args.ts | 17 -- src/lib/onboard.ts | 1 - src/lib/onboard/dashboard-port.test.ts | 22 -- src/lib/onboard/dashboard-port.ts | 8 +- src/lib/onboard/dashboard.ts | 132 +++++++--- ...er-driver-gateway-service-homebrew.test.ts | 18 +- .../docker-driver-gateway-service.test.ts | 12 +- .../onboard/docker-driver-gateway-service.ts | 51 +++- src/lib/onboard/hermes-api-port.test.ts | 15 -- src/lib/onboard/hermes-api-port.ts | 14 +- .../onboard/sandbox-create/orchestration.ts | 5 + test/helpers/openshell-release-fixtures.ts | 4 +- ...stall-preexisting-sandbox-recovery.test.ts | 28 +-- ...ard-finalization-dashboard-forward.test.ts | 226 ++++++++++++------ 16 files changed, 333 insertions(+), 255 deletions(-) diff --git a/scripts/install-openshell.sh b/scripts/install-openshell.sh index c16b492a0a8..6688ff81da4 100755 --- a/scripts/install-openshell.sh +++ b/scripts/install-openshell.sh @@ -823,6 +823,8 @@ install_macos_homebrew_formula() { run_trusted_openshell_homebrew_operation "$formula_operation_pin" -- \ brew "$install_action" --formula "$formula_ref" \ || homebrew_operation_status=$? + [ "$homebrew_operation_status" = "0" ] \ + || fail "OpenShell Homebrew ${install_action} failed inside the temporary formula trust boundary (status ${homebrew_operation_status})." brew_prefix="$(brew --prefix 2>/dev/null || true)" openshell_bin="${brew_prefix}/bin/openshell" @@ -830,19 +832,11 @@ install_macos_homebrew_formula() { openshell_bin="$(command -v openshell 2>/dev/null || true)" fi if [ -z "$openshell_bin" ] || [ ! -x "$openshell_bin" ]; then - fail "Homebrew did not install the openshell binary." + fail "Homebrew completed but the openshell binary was not found on PATH." fi - installed_version_output="$("$openshell_bin" --version 2>&1 || true)" - installed_version="$(printf '%s\n' "$installed_version_output" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)" - [ "$installed_version" = "$PIN_VERSION" ] \ - || fail "Homebrew did not install the pinned OpenShell ${PIN_VERSION} binary." require_openshell_messaging_features "$openshell_bin" - info "${installed_version_output:-openshell} installed" - if [ "$homebrew_operation_status" = "0" ]; then - info "OpenShell Homebrew service staged; onboarding will start it after gateway validation." - else - warn "OpenShell Homebrew ${install_action} returned status ${homebrew_operation_status} after installing the pinned binary; onboarding will try the service and fall back to standalone startup." - fi + info "$("$openshell_bin" --version 2>&1 || echo openshell) installed" + info "OpenShell Homebrew service staged; onboarding will start it after gateway validation." exit 0 } diff --git a/scripts/install.sh b/scripts/install.sh index 8fdcdef5e11..e9106c5aa25 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -1369,7 +1369,7 @@ installer_docker_host_has_supported_shape() { # Admit a usable socket and the default context before Docker or recovery effects. # Persisted JSON inspection can wait only for its missing Node.js prerequisite. validate_installer_docker_target_before_host_changes() { - local raw="${DOCKER_HOST-}" candidate active_context="" context_host="" + local raw="${DOCKER_HOST-}" candidate active_context="" installer_docker_host_has_supported_shape \ || error "DOCKER_HOST is not a supported absolute local Unix socket endpoint. Unset DOCKER_HOST or set it to an absolute local Unix socket URL, such as unix:///var/run/docker.sock. Then rerun the installer." candidate="${raw#"${raw%%[![:space:]]*}"}" @@ -1386,21 +1386,8 @@ validate_installer_docker_target_before_host_changes() { fi active_context="$(docker_active_context)" fi - if [[ "$active_context" != default ]]; then - context_host="$(docker context inspect "$active_context" --format '{{.Endpoints.docker.Host}}' 2>/dev/null || true)" - [[ -n "$context_host" ]] \ - || error "The Docker context does not select a supported local Unix socket. Select a local Docker or Colima context, then rerun the installer." - DOCKER_HOST="$context_host" - installer_docker_host_has_supported_shape \ - || error "The Docker context does not select a supported local Unix socket. Select a local Docker or Colima context, then rerun the installer." - export DOCKER_HOST - unset DOCKER_CONTEXT - elif [[ -n "$candidate" ]]; then - unset DOCKER_CONTEXT - fi - # DOCKER_CONTEXT takes precedence over DOCKER_HOST in Docker clients. Once - # admission resolves a local socket, keep that socket authoritative through - # host preflight and automatic onboarding. + [[ "$active_context" == default ]] \ + || error "The Docker context does not select the local default target. Unset DOCKER_CONTEXT or set it to default, and run 'docker context use default' if a non-default context is persisted. Then rerun the installer." } # Re-read persisted context after Node installation instead of trusting the temporary default. diff --git a/src/lib/adapters/openshell/forward-cli-args.ts b/src/lib/adapters/openshell/forward-cli-args.ts index bdc518e520e..c3ac53143ce 100644 --- a/src/lib/adapters/openshell/forward-cli-args.ts +++ b/src/lib/adapters/openshell/forward-cli-args.ts @@ -23,20 +23,3 @@ export function buildCliOpenShellForwardServiceArgs(forward: OpenShellForwardIde `${forward.localHost}:${String(forward.port)}`, ]; } - -/** Build the OpenShell-owned background forward command without a shell. */ -export function buildCliOpenShellForwardStartArgs(forward: OpenShellForwardIdentity): string[] { - return [ - "forward", - "start", - "-d", - `${forward.localHost}:${String(forward.port)}`, - forward.sandboxName, - "--gateway", - forward.gatewayName, - "--gateway-endpoint", - forward.gatewayEndpoint, - "--workspace", - forward.workspace, - ]; -} diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 381113666d8..74d6f596412 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2477,7 +2477,6 @@ const { stopAllDashboardForwards, } = onboardDashboard.createOnboardDashboardHelpers({ runCaptureOpenshell, - runOpenshell, runCapture, cliName, agentProductName, diff --git a/src/lib/onboard/dashboard-port.test.ts b/src/lib/onboard/dashboard-port.test.ts index 9c995dfc761..6e7c1ea85bc 100644 --- a/src/lib/onboard/dashboard-port.test.ts +++ b/src/lib/onboard/dashboard-port.test.ts @@ -493,28 +493,6 @@ describe("dashboard port reservation", () => { assert.deepEqual(released, [18790]); }); - it("keeps the requested port without inspecting or reserving forwards", async () => { - const reservePort = vi.fn(); - const result = await reserveCreateSandboxDashboardPort( - { - sandboxName: "cursor", - controlUiPort: 18789, - chatUiUrlEnv: null, - persistedPort: null, - agentForwardPort: null, - }, - reservePort, - ); - - expect(result).toEqual({ - preferredPort: 18789, - effectivePort: 18789, - chatUiUrl: "http://127.0.0.1:18789", - reservation: null, - }); - expect(reservePort).not.toHaveBeenCalled(); - }); - it("defers a persisted port reservation only for the exact owned forward", async () => { const reservePort = vi.fn(); const observeForwardPorts = observePorts("cursor", new Map([[18789, "owned"]])); diff --git a/src/lib/onboard/dashboard-port.ts b/src/lib/onboard/dashboard-port.ts index f74c35fcf55..59641c7e0ab 100644 --- a/src/lib/onboard/dashboard-port.ts +++ b/src/lib/onboard/dashboard-port.ts @@ -463,7 +463,7 @@ export type ReserveCreateSandboxDashboardPortInput = Omit< ObservedCreateSandboxDashboardPortInput, "forwardObservations" > & { - observeForwardPorts?: OpenShellForwardPortObserver; + observeForwardPorts: OpenShellForwardPortObserver; }; export interface CreateSandboxDashboardPortResult { @@ -671,12 +671,6 @@ export async function reserveCreateSandboxDashboardPort( reservePort: (port: number) => Promise = reserveDashboardPort, ): Promise { const preferredPort = preferredCreateSandboxDashboardPort(input); - if (!input.observeForwardPorts) { - return { - ...createSandboxDashboardPortResult(input, preferredPort, preferredPort), - reservation: null, - }; - } const observed = await findAvailableDashboardPortFromObserver( input.sandboxName, preferredPort, diff --git a/src/lib/onboard/dashboard.ts b/src/lib/onboard/dashboard.ts index a41e30d3fed..2bf20ed6d69 100644 --- a/src/lib/onboard/dashboard.ts +++ b/src/lib/onboard/dashboard.ts @@ -13,7 +13,6 @@ import { openShellForwardIdentity, type OpenShellForwardRuntimeAuthority, } from "../adapters/openshell/forward-runtime"; -import { buildCliOpenShellForwardStartArgs } from "../adapters/openshell/forward-cli-args"; import { createCliOpenShellSandboxTransferExecutor } from "../adapters/openshell/sandbox-transfer-cli"; import type { OpenShellSandboxTransferExecutor } from "../adapters/openshell/sandbox-transfer"; import type { AgentDefinition } from "../agent/defs"; @@ -35,7 +34,10 @@ import { } from "./dashboard-forward-control"; import { createOpenShellForwardPortObserver, + findAvailableDashboardPortFromObserver, getPersistedDashboardPort, + getRegistryOccupiedDashboardPorts, + getRegistryOccupiedHermesApiPorts, type ListSandboxesFn, type OpenShellForwardPortObserver, } from "./dashboard-port"; @@ -56,10 +58,6 @@ type DashboardForwardRuntimeAuthority = OpenShellForwardRuntimeAuthority; export interface OnboardDashboardDeps { runCaptureOpenshell(args: string[], opts?: Record): string | null; - runOpenshell?( - args: string[], - opts?: Record, - ): { status: number | null; error?: Error }; runCapture?: typeof defaultRunCapture; cliName(): string; agentProductName(): string; @@ -390,6 +388,20 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa process.exit(1); } + function forwardResultMessage( + result: + | Awaited> + | Awaited>, + ): string { + if ("error" in result) return result.error.message; + if ("observation" in result) { + return result.observation.state === "foreign" + ? "The host port is owned by a foreign listener." + : "NemoClaw could not prove the forward state."; + } + return "NemoClaw could not reconcile the forward state."; + } + async function reconcileForward( sandboxName: string, authority: DashboardForwardRuntimeAuthority, @@ -397,36 +409,34 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa label: string, revalidateSandboxIdentity?: (operation: string) => void, ): Promise { - await assertForwardGatewayCurrent( - authority, - revalidateSandboxIdentity, - `start ${label} forward ${String(forward.port)} for sandbox '${sandboxName}'`, - ); - const startArgs = buildCliOpenShellForwardStartArgs(forward); - if (deps.runOpenshell) { - const result = deps.runOpenshell(startArgs, { - ignoreError: true, - suppressOutput: true, - stdio: ["ignore", "ignore", "ignore"], - timeout: 15_000, - killProcessTreeOnTimeout: true, + const adapter = forwardAdapterForAuthority(authority); + const assertCurrent = () => + assertForwardGatewayCurrent( + authority, + revalidateSandboxIdentity, + `accept ${label} forward ${String(forward.port)} for sandbox '${sandboxName}'`, + ); + let result = await adapter.startForward({ forward, assertCurrent }); + if (result.state === "refused" && result.observation.state === "stale") { + const retirement = await adapter.retireLegacyForward({ + forward, + assertCurrent, + authorize: async () => { + await assertForwardGatewayCurrent( + authority, + revalidateSandboxIdentity, + `retire legacy ${label} forward ${String(forward.port)} for sandbox '${sandboxName}'`, + ); + }, }); - if (result.error || result.status !== 0) { - throw new Error( - `OpenShell forward start failed${result.status === null ? "" : ` with exit ${String(result.status)}`}.`, - ); + if (retirement.state !== "retired" && retirement.state !== "not_needed") { + throw new Error(forwardResultMessage(retirement)); } - } else { - deps.runCaptureOpenshell(startArgs, { - timeout: 15_000, - killProcessTreeOnTimeout: true, - }); + result = await adapter.startForward({ forward, assertCurrent }); + } + if (result.state !== "started" && result.state !== "reused") { + throw new Error(forwardResultMessage(result)); } - await assertForwardGatewayCurrent( - authority, - revalidateSandboxIdentity, - `accept ${label} forward ${String(forward.port)} for sandbox '${sandboxName}'`, - ); } async function ensureDashboardForward( @@ -435,7 +445,8 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa options: DashboardForwardOptions = {}, ): Promise { chatUiUrl ||= `http://127.0.0.1:${CONTROL_UI_PORT}`; - const { rollbackSandboxOnFailure } = normalizeDashboardForwardOptions(options); + const { rollbackSandboxOnFailure, allowPortReallocation, reuseExistingForward } = + normalizeDashboardForwardOptions(options); const { revalidateSandboxIdentity } = options; const preferredPort = Number(getDashboardForwardPort(chatUiUrl)); const forwardGateway = resolveForwardServiceGateway(sandboxName, options); @@ -443,7 +454,60 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa throw new Error(`ForwardTcp authority is unavailable for '${sandboxName}'`); } const authority = getForwardRuntimeAuthority(sandboxName, forwardGateway); - const actualPort = preferredPort; + const forwardAdapter = forwardAdapterForAuthority(authority); + const observeForwardPorts = createOpenShellForwardPortObserver({ + adapter: forwardAdapter, + forwardForPort: (port) => forwardIdentity(sandboxName, authority, port, "dashboard"), + assertCurrent: () => + assertForwardGatewayCurrent( + authority, + revalidateSandboxIdentity, + `inspect dashboard forwards for sandbox '${sandboxName}'`, + ), + }); + const persistedPort = getPersistedDashboardPort(sandboxName, listSandboxes); + const registryOccupiedPorts = new Map([ + ...getRegistryOccupiedDashboardPorts(sandboxName, listSandboxes), + ...getRegistryOccupiedHermesApiPorts(sandboxName, listSandboxes), + ]); + const fixedPort = persistedPort === preferredPort || reuseExistingForward; + if (fixedPort && registryOccupiedPorts.has(String(preferredPort))) { + throw new Error( + `Port ${String(preferredPort)} is not available for '${sandboxName}'; another sandbox registered it.`, + ); + } + let actualPort: number; + try { + actualPort = ( + await findAvailableDashboardPortFromObserver( + sandboxName, + preferredPort, + observeForwardPorts, + registryOccupiedPorts, + ) + ).port; + } catch (err) { + if (!rollbackSandboxOnFailure) throw err; + rollbackSandboxAndExit(sandboxName, err, options.gatewayName); + } + + if (actualPort !== preferredPort) { + if (!allowPortReallocation || fixedPort) { + throw new Error( + `Port ${preferredPort} is not available for '${sandboxName}' and cannot be reallocated.`, + ); + } + if (rollbackSandboxOnFailure) { + const err = new Error( + `Dashboard port ${preferredPort} became host-bound during sandbox build; ` + + `cannot reallocate to ${actualPort} after the sandbox has been created with ` + + `CHAT_UI_URL=${preferredPort}. Free the port and re-run \`${deps.cliName()} onboard\`, ` + + `or pass \`--control-ui-port \` to pick a different dashboard port.`, + ); + rollbackSandboxAndExit(sandboxName, err, options.gatewayName); + } + console.warn(` ! Port ${preferredPort} is taken. Using port ${actualPort} instead.`); + } const actualGateway = resolveForwardServiceGateway(sandboxName, options); let fwdOk = false; diff --git a/src/lib/onboard/docker-driver-gateway-service-homebrew.test.ts b/src/lib/onboard/docker-driver-gateway-service-homebrew.test.ts index 759658fdb5b..d2395b0df6c 100644 --- a/src/lib/onboard/docker-driver-gateway-service-homebrew.test.ts +++ b/src/lib/onboard/docker-driver-gateway-service-homebrew.test.ts @@ -136,17 +136,16 @@ describe("OpenShell Homebrew service boundary", () => { }); it.each([ - [66], - [67], - [68], - [69], - ])("uses standalone fallback on Homebrew boundary status %i (#7707)", (status) => { + [66, "Run curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash"], + [67, "could not grant temporary trust"], + [68, "could not remove temporary trust"], + [69, "Run curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash"], + ])("fails closed on Homebrew boundary status %i (#7707)", (status, expected) => { const preparePortForServiceStart = vi.fn(); const prepareServiceEnv = vi.fn(); const validatePortOwnerForServiceStart = vi.fn(); - const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); - expect( + expect(() => startOpenShellGatewayUserService({ commandExists: () => true, homebrewFormulaOperation: () => spawnResult(status, "opaque Homebrew diagnostic"), @@ -155,10 +154,7 @@ describe("OpenShell Homebrew service boundary", () => { prepareServiceEnv, validatePortOwnerForServiceStart, }), - ).toEqual({ attempted: false, started: false, reason: "service not installed" }); - expect(warn).toHaveBeenCalledWith( - " OpenShell Homebrew service inspection failed; continuing with standalone gateway fallback.", - ); + ).toThrow(expected); expect(preparePortForServiceStart).not.toHaveBeenCalled(); expect(prepareServiceEnv).not.toHaveBeenCalled(); expect(validatePortOwnerForServiceStart).not.toHaveBeenCalled(); diff --git a/src/lib/onboard/docker-driver-gateway-service.test.ts b/src/lib/onboard/docker-driver-gateway-service.test.ts index f1466255aa4..a99b2d2db9e 100644 --- a/src/lib/onboard/docker-driver-gateway-service.test.ts +++ b/src/lib/onboard/docker-driver-gateway-service.test.ts @@ -544,7 +544,7 @@ describe("docker-driver-gateway-service", () => { }), verifySandboxBridgeGatewayReachableOrExit: vi.fn(), }), - ).resolves.toBe(false); + ).rejects.toThrow("remains lifecycle authority"); expect(register).toHaveBeenCalledOnce(); expect(sleepSeconds).not.toHaveBeenCalled(); expect(ready).not.toHaveBeenCalled(); @@ -596,7 +596,7 @@ describe("docker-driver-gateway-service", () => { }, ); - it("uses standalone fallback when Homebrew startup fails (#7707)", async () => { + it("keeps Homebrew as lifecycle authority when managed startup fails (#7707)", async () => { const stopService = vi.fn(); await expect( @@ -618,7 +618,7 @@ describe("docker-driver-gateway-service", () => { stopOpenShellGatewayUserService: stopService, verifySandboxBridgeGatewayReachableOrExit: vi.fn(), }), - ).resolves.toBe(false); + ).rejects.toThrow("temporary trust failed"); expect(stopService).not.toHaveBeenCalled(); }); @@ -662,7 +662,7 @@ describe("docker-driver-gateway-service", () => { ); }); - it("uses standalone fallback when the Homebrew service stays unhealthy (#7707)", async () => { + it("stops an unhealthy Homebrew service without changing lifecycle authority (#7707)", async () => { const clock = createVirtualClock(); const stopService = vi.fn(() => ({ attempted: true, @@ -693,8 +693,8 @@ describe("docker-driver-gateway-service", () => { stopOpenShellGatewayUserService: stopService, verifySandboxBridgeGatewayReachableOrExit: vi.fn(), }), - ).resolves.toBe(false); - expect(stopService).not.toHaveBeenCalled(); + ).rejects.toThrow("Homebrew formula remains lifecycle authority"); + expect(stopService).toHaveBeenCalledOnce(); }); it("blocks standalone fallback when managed service cleanup fails without permission (#8926)", async () => { diff --git a/src/lib/onboard/docker-driver-gateway-service.ts b/src/lib/onboard/docker-driver-gateway-service.ts index 9769483e617..ee720db4ce6 100644 --- a/src/lib/onboard/docker-driver-gateway-service.ts +++ b/src/lib/onboard/docker-driver-gateway-service.ts @@ -463,6 +463,10 @@ function runSystemctlUser( } const OPENSHELL_HOMEBREW_FORMULA_ABSENT = 65; +const OPENSHELL_HOMEBREW_FORMULA_REPAIR = 66; +const OPENSHELL_HOMEBREW_TRUST_FAILED = 67; +const OPENSHELL_HOMEBREW_UNTRUST_FAILED = 68; +const OPENSHELL_HOMEBREW_OPERATION_FAILED = 69; function homebrewFormulaOperationScript(): string { return path.resolve(__dirname, "../../../scripts/install-openshell.sh"); @@ -520,6 +524,27 @@ const HOMEBREW_FORMULA_REPAIR_GUIDANCE = "OpenShell's Homebrew formula is installed but cannot satisfy NemoClaw's pinned checksum and temporary trust contract. " + "Run curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash, then rerun onboarding."; +function throwHomebrewFormulaOperationFailure(operation: string, result: CommandResult): never { + if (result.status === OPENSHELL_HOMEBREW_FORMULA_REPAIR) { + throw new OpenShellGatewayServiceTrustError(HOMEBREW_FORMULA_REPAIR_GUIDANCE); + } + if (result.status === OPENSHELL_HOMEBREW_TRUST_FAILED) { + throw new OpenShellGatewayServiceTrustError( + `Homebrew could not grant temporary trust for the checksum-verified OpenShell formula during ${operation}. ` + + "No service operation was performed.", + ); + } + if (result.status === OPENSHELL_HOMEBREW_UNTRUST_FAILED) { + throw new OpenShellGatewayServiceTrustError( + `Homebrew could not remove temporary trust for the OpenShell formula after ${operation}. ` + + "Stop and repair Homebrew trust before continuing.", + ); + } + throw new OpenShellGatewayServiceTrustError( + `OpenShell Homebrew ${operation} failed inside the checksum-verified temporary trust boundary.`, + ); +} + function runStopService( service: OpenShellGatewayUserServiceTarget, opts: Required> & @@ -600,10 +625,10 @@ function resolveOfficialHomebrewFormulaPaths( ); if (!info.ok) { if (info.status === OPENSHELL_HOMEBREW_FORMULA_ABSENT) return null; - console.warn( - " OpenShell Homebrew service inspection failed; continuing with standalone gateway fallback.", - ); - return null; + if (info.status === OPENSHELL_HOMEBREW_OPERATION_FAILED) { + throw new OpenShellGatewayServiceTrustError(HOMEBREW_FORMULA_REPAIR_GUIDANCE); + } + throwHomebrewFormulaOperationFailure("formula identity inspection", info); } try { const parsed = JSON.parse(info.stdout ?? "") as { @@ -1462,7 +1487,10 @@ export async function startPackageManagedDockerDriverGateway({ validatePortOwnerForServiceStart: validatePortOwnerForOpenShellGatewayUserServiceStart, }); } catch (error) { - if (error instanceof OpenShellGatewayServiceEnvironmentError) { + if ( + error instanceof OpenShellGatewayServiceEnvironmentError || + error instanceof OpenShellGatewayServiceTrustError + ) { throw error; } warn( @@ -1478,7 +1506,7 @@ export async function startPackageManagedDockerDriverGateway({ }; if (!serviceStart.started) { const detail = serviceStart.reason ? ` (${serviceStart.reason})` : ""; - if (serviceStart.standaloneFallbackBlocked && serviceStart.manager !== "homebrew") { + if (serviceStart.standaloneFallbackBlocked || serviceStart.manager === "homebrew") { const message = `OpenShell gateway managed service failed to start${detail}.`; printError(` ${message}`); if (exitOnFailure) process.exit(1); @@ -1488,9 +1516,7 @@ export async function startPackageManagedDockerDriverGateway({ ` OpenShell gateway managed service failed to start${detail}; using standalone fallback.`, ); reportLogs(); - if (serviceStart.attempted && serviceStart.manager !== "homebrew") { - stopBeforeStandaloneFallback(); - } + if (serviceStart.attempted) stopBeforeStandaloneFallback(); return false; } @@ -1533,7 +1559,12 @@ export async function startPackageManagedDockerDriverGateway({ ); reportLogs(); if (serviceStart.manager === "homebrew") { - return false; + stopBeforeStandaloneFallback(); + const authorityMessage = + "The installed OpenShell Homebrew formula remains lifecycle authority; " + + "run curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash before retrying onboarding."; + if (exitOnFailure) process.exit(1); + throw new OpenShellGatewayServiceTrustError(authorityMessage); } if (serviceStart.attempted) stopBeforeStandaloneFallback(); return false; diff --git a/src/lib/onboard/hermes-api-port.test.ts b/src/lib/onboard/hermes-api-port.test.ts index 9f1da750b13..1b998480fb8 100644 --- a/src/lib/onboard/hermes-api-port.test.ts +++ b/src/lib/onboard/hermes-api-port.test.ts @@ -153,21 +153,6 @@ describe("findAvailableHermesApiPortFromObservations", () => { }); describe("reserveCreateSandboxHermesApiPort", () => { - it("keeps the requested API port without inspecting or reserving forwards", async () => { - const env: NodeJS.ProcessEnv = { [HERMES_API_PORT_ENV]: "8644" }; - const reservePort = vi.fn(); - - await expect( - reserveCreateSandboxHermesApiPort({ - sandboxName: "alpha", - env, - getSandbox: () => undefined, - reservePort, - }), - ).resolves.toEqual({ effectivePort: 8644, reservation: null }); - expect(reservePort).not.toHaveBeenCalled(); - }); - it("keeps concurrent selections distinct before either host forward exists", async () => { const firstRelease = vi.fn(async () => undefined); const secondRelease = vi.fn(async () => undefined); diff --git a/src/lib/onboard/hermes-api-port.ts b/src/lib/onboard/hermes-api-port.ts index b3a37451748..9016646134f 100644 --- a/src/lib/onboard/hermes-api-port.ts +++ b/src/lib/onboard/hermes-api-port.ts @@ -52,7 +52,7 @@ export interface HermesApiPortReservationInput { sandboxName: string; env: NodeJS.ProcessEnv; getSandbox(name: string): HermesApiPortSandboxLookup | null | undefined; - observeForwardPorts?: OpenShellForwardPortObserver; + observeForwardPorts: OpenShellForwardPortObserver; reservePort?(port: number): Promise; warn(message: string): void; } @@ -221,7 +221,7 @@ export async function reserveCreateSandboxHermesApiPort(options: { env?: NodeJS.ProcessEnv; getSandbox?: (name: string) => HermesApiPortSandboxLookup | null | undefined; allowRegisteredOverride?: boolean; - observeForwardPorts?: OpenShellForwardPortObserver; + observeForwardPorts: OpenShellForwardPortObserver; registryOccupiedPorts?: ReadonlyMap; reservePort?: (port: number) => Promise; warn?: (message: string) => void; @@ -242,11 +242,6 @@ export async function reserveCreateSandboxHermesApiPort(options: { const registryOccupiedPorts = new Map( options.registryOccupiedPorts ?? getRegistryOccupiedHermesApiPorts(options.sandboxName), ); - if (!options.observeForwardPorts) { - const effectivePort = pinnedPort ?? HERMES_OPENAI_API_PORT; - env[HERMES_API_PORT_ENV] = String(effectivePort); - return { effectivePort, reservation: null }; - } const reserveSelectedPort = async ( effectivePort: number, observations: readonly OpenShellForwardObservation[], @@ -332,10 +327,7 @@ export function createHermesApiPortReservationScope(): HermesApiPortReservationS }); this.effectivePort = selection.effectivePort; this.current = selection.reservation; - deferredOwnedForwardPort = - input.observeForwardPorts && selection.reservation === null - ? selection.effectivePort - : null; + deferredOwnedForwardPort = selection.reservation === null ? selection.effectivePort : null; }, async rebindAfterOwnedForwardDelete(input) { if ( diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index dab6a4cb19e..8d2f3268ef9 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -1524,6 +1524,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche ensureAgentFixedForward, ensureDashboardForward, filterEnabledChannelsByAgent, + forwardObserver, formatSandboxAgentName, formatSandboxBuildEstimateNote, getDashboardForwardPort, @@ -1669,6 +1670,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche agent, ); const isManagedDcodeAgent = usesManagedDcodeIdentity(agent?.name, fromDockerfile); + const observeLoopbackForwardPorts = forwardObserver(sandboxName, "loopback"); let effectivePort = 0, chatUiUrl = "", hermesApiPortReservationInput = { @@ -1676,6 +1678,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche sandboxName, env: process.env, getSandbox: registry.getSandbox, + observeForwardPorts: observeLoopbackForwardPorts, warn: (message: string) => console.warn(message), }; if (manageDashboard) { @@ -1686,10 +1689,12 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche persistedPort: registry.getSandbox(sandboxName)?.dashboardPort ?? null, agentForwardPort: dashboardRuntime.getAgentPrimaryForwardPort(agent, DASHBOARD_PORT), defaultPort: DASHBOARD_PORT, + observeForwardPorts: forwardObserver(sandboxName), warn: (message: string) => console.warn(message), }); ({ effectivePort, chatUiUrl } = dashboardSelection); dashboardPortReservationScope.current = dashboardSelection.reservation; + dashboardPortReservationScope.deferOwnedForwardPort(effectivePort); } const hermesDashboardForwarding = onboardHermesDashboard.createHermesDashboardOnboardForwarding( { diff --git a/test/helpers/openshell-release-fixtures.ts b/test/helpers/openshell-release-fixtures.ts index a3c6022a974..8998d114ee8 100644 --- a/test/helpers/openshell-release-fixtures.ts +++ b/test/helpers/openshell-release-fixtures.ts @@ -32,8 +32,8 @@ export const OPENSHELL_MCP_FEATURE_MARKER = "allow_all_known_mcp_methods"; export const OPENSHELL_FEATURE_MARKERS = `${OPENSHELL_REWRITE_FEATURE_MARKERS} ${OPENSHELL_MCP_FEATURE_MARKER}`; export const BREW_OUTCOMES = [ ["0", "0", "reinstall", 0], - ["1", "1", "install", 0], - ["0", "1", "reinstall", 0], + ["1", "1", "install", 1], + ["0", "1", "reinstall", 1], ] as const; export function trustedFormulaBoundaryEvents(operation: string): string[] { diff --git a/test/install/install-preexisting-sandbox-recovery.test.ts b/test/install/install-preexisting-sandbox-recovery.test.ts index ebb7efb7445..d0625795551 100644 --- a/test/install/install-preexisting-sandbox-recovery.test.ts +++ b/test/install/install-preexisting-sandbox-recovery.test.ts @@ -38,7 +38,6 @@ function runRecoveryBeforeOnboard( options: { detectedExpressPlatform?: string; dockerContext?: string; - dockerContextEndpoint?: string; dockerHost?: string; hostPreflightExitCode?: number; includeNodeOnPath?: boolean; @@ -186,14 +185,6 @@ exit 0 [[ "$*" == "info --format {{.Host.RemoteSocket.Path}}" ]] || return 99 printf '%s\n' "$PODMAN_SOCKET" } - docker() { - if [[ "\${1:-}" == "context" && "\${2:-}" == "inspect" ]]; then - [[ -n "$DOCKER_CONTEXT_ENDPOINT" ]] || return 1 - printf '%s\n' "$DOCKER_CONTEXT_ENDPOINT" - return 0 - fi - return 99 - } sleep() { printf 'sleep=%s\n' "$*" >> "${callLog}"; } step() { :; } install_nodejs() { @@ -230,7 +221,6 @@ exit 0 extraEnv: { DETECTED_EXPRESS_PLATFORM: options.detectedExpressPlatform ?? "", ...(options.dockerContext !== undefined ? { DOCKER_CONTEXT: options.dockerContext } : {}), - DOCKER_CONTEXT_ENDPOINT: options.dockerContextEndpoint ?? "", ...(dockerConfig === undefined ? {} : { DOCKER_CONFIG: dockerConfig }), ...(options.dockerHost !== undefined ? { DOCKER_HOST: options.dockerHost } : {}), NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE: "1", @@ -287,19 +277,6 @@ describe("install.sh pre-existing sandbox recovery ordering (#6114)", () => { expect(result.output).toContain("Existing sandboxes recovered; skipping generic onboarding"); }); - it("resolves a persisted local Colima context to its Unix socket", () => { - const result = runRecoveryBeforeOnboard(2, 0, { - dockerContextEndpoint: "unix:///Users/test/.colima/default/docker.sock", - persistedDockerContext: "colima", - recordRuntimeTarget: true, - }); - - expect(result.status, result.output).toBe(0); - expect(result.calls).toContain( - "cli-target=host:unix:///Users/test/.colima/default/docker.sock,context:unset argv=upgrade-sandboxes --auto", - ); - }); - it("treats a whitespace-only Docker host as unset", () => { const result = runRecoveryBeforeOnboard(2, 0, { dockerHost: " \t ", @@ -352,7 +329,7 @@ describe("install.sh pre-existing sandbox recovery ordering (#6114)", () => { expect(result.calls).not.toContain( 'restore=1 confirmed=["legacy-box"] argv=upgrade-sandboxes --auto', ); - expect(result.output).toContain("Docker context does not select a supported local Unix socket"); + expect(result.output).toContain("Docker context does not select the local default target"); }); it.each([ @@ -411,7 +388,8 @@ describe("install.sh pre-existing sandbox recovery ordering (#6114)", () => { expect(result.status).toBe(1); expect(result.calls).toEqual([]); - expect(result.output).toContain("Docker context does not select a supported local Unix socket"); + expect(result.output).toContain("Docker context does not select the local default target"); + expect(result.output).toContain("docker context use default"); }); it("rejects a non-local Podman socket before sandbox recovery", () => { diff --git a/test/onboarding/onboard-finalization-dashboard-forward.test.ts b/test/onboarding/onboard-finalization-dashboard-forward.test.ts index 548fd67c646..596f122f2b9 100644 --- a/test/onboarding/onboard-finalization-dashboard-forward.test.ts +++ b/test/onboarding/onboard-finalization-dashboard-forward.test.ts @@ -9,6 +9,7 @@ import type { OpenShellForwardAdapter, OpenShellForwardIdentity, OpenShellForwardObservation, + OpenShellForwardStartResult, RetireLegacyOpenShellForwardRequest, StartOpenShellForwardRequest, } from "../../src/lib/adapters/openshell/forward"; @@ -21,6 +22,17 @@ type ForwardObservationWithIdentity = Extract< >; type ForwardState = ForwardObservationWithIdentity["state"]; +function deferred(): { + promise: Promise; + resolve: (value: T) => void; +} { + let resolve!: (value: T) => void; + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise; + }); + return { promise, resolve }; +} + function observation( forward: OpenShellForwardIdentity, state: ForwardState, @@ -113,21 +125,8 @@ function harness(options: { retireLegacyForward, verifyForwardRelease: vi.fn(async () => ({ state: "released" as const })), }; - const runCaptureOpenshell = vi.fn(() => ""); - const runOpenshell = vi.fn((args: string[]) => { - if ( - args[0] === "forward" && - args[1] === "start" && - options.startFailurePort !== undefined && - args[3]?.endsWith(`:${String(options.startFailurePort)}`) - ) { - return { status: 1 }; - } - return { status: 0 }; - }); const helpers = createOnboardDashboardHelpers({ - runCaptureOpenshell, - runOpenshell, + runCaptureOpenshell: vi.fn(() => ""), cliName: () => "nemoclaw", agentProductName: () => "NemoClaw", getProviderLabel: (provider) => provider, @@ -143,15 +142,7 @@ function harness(options: { resolveForwardGatewayName: (sandbox) => sandbox?.gatewayName ?? "nemoclaw", forwardAdapterForAuthority: vi.fn(() => adapter), }); - return { - helpers, - observeForwards, - retireLegacyForward, - runCaptureOpenshell, - runOpenshell, - startForward, - states, - }; + return { helpers, observeForwards, retireLegacyForward, startForward, states }; } afterEach(() => { @@ -219,47 +210,79 @@ describe("finalization dashboard ForwardTcp reconciliation", () => { }, ); - it("starts the persisted dashboard forward with one OpenShell command", async () => { + it("launches the persisted dashboard port and publishes its URL", async () => { vi.stubEnv("CHAT_UI_URL", undefined); const test = harness({ listSandboxes: () => ({ sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], }), - initialStates: new Map([[18_790, "indeterminate"]]), }); await expect(test.helpers.ensureFinalizationDashboardForward("reonboard-test")).resolves.toBe( 18_790, ); - expect(test.runOpenshell).toHaveBeenCalledWith( - [ - "forward", - "start", - "-d", - "127.0.0.1:18790", - "reonboard-test", - "--gateway", - "nemoclaw", - "--gateway-endpoint", - "https://127.0.0.1:8080", - "--workspace", - "default", - ], - { - ignoreError: true, - suppressOutput: true, - stdio: ["ignore", "ignore", "ignore"], - timeout: 15_000, - killProcessTreeOnTimeout: true, - }, - ); - expect(test.observeForwards).not.toHaveBeenCalled(); - expect(test.retireLegacyForward).not.toHaveBeenCalled(); - expect(test.startForward).not.toHaveBeenCalled(); + expect(test.startForward).toHaveBeenCalledOnce(); + expect(test.startForward.mock.calls[0]?.[0].forward).toMatchObject({ + sandboxName: "reonboard-test", + port: 18_790, + }); expect(process.env.CHAT_UI_URL).toBe("http://127.0.0.1:18790"); }); - it("lets OpenShell decide whether a registered port can be forwarded", async () => { + it.each(["foreign", "indeterminate"] as const)( + "leaves a %s persisted listener untouched with zero mutation attempts", + async (state) => { + vi.stubEnv("CHAT_UI_URL", undefined); + const test = harness({ + listSandboxes: () => ({ + sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], + }), + initialStates: new Map([[18_790, state]]), + }); + + await expect( + test.helpers.ensureFinalizationDashboardForward("reonboard-test"), + ).rejects.toThrow(/cannot be reallocated|could not prove/u); + expect(test.startForward).not.toHaveBeenCalled(); + expect(test.retireLegacyForward).not.toHaveBeenCalled(); + }, + ); + + it("retires an exact stale forward before one replacement start", async () => { + vi.stubEnv("CHAT_UI_URL", undefined); + const test = harness({ + listSandboxes: () => ({ + sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], + }), + initialStates: new Map([[18_790, "stale"]]), + }); + + await expect(test.helpers.ensureFinalizationDashboardForward("reonboard-test")).resolves.toBe( + 18_790, + ); + expect(test.retireLegacyForward).toHaveBeenCalledOnce(); + expect(test.startForward).toHaveBeenCalledTimes(2); + }); + + it("reuses an exactly owned dashboard forward", async () => { + vi.stubEnv("CHAT_UI_URL", undefined); + const test = harness({ + listSandboxes: () => ({ + sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], + }), + initialStates: new Map([[18_790, "owned"]]), + }); + + await expect(test.helpers.ensureFinalizationDashboardForward("reonboard-test")).resolves.toBe( + 18_790, + ); + expect(test.startForward).toHaveBeenCalledOnce(); + await expect(test.startForward.mock.results[0]?.value).resolves.toMatchObject({ + state: "reused", + }); + }); + + it("does not reuse a port registered by another sandbox", async () => { vi.stubEnv("CHAT_UI_URL", undefined); const test = harness({ listSandboxes: () => ({ @@ -270,32 +293,102 @@ describe("finalization dashboard ForwardTcp reconciliation", () => { }), }); - await expect(test.helpers.ensureFinalizationDashboardForward("reonboard-test")).resolves.toBe( - 18_790, + await expect(test.helpers.ensureFinalizationDashboardForward("reonboard-test")).rejects.toThrow( + /another sandbox registered it/u, ); - expect(test.runOpenshell).toHaveBeenCalledOnce(); - expect(test.observeForwards).not.toHaveBeenCalled(); + expect(test.startForward).not.toHaveBeenCalled(); }); - it("issues one OpenShell start command for each Hermes forward", async () => { + it.each(["openclaw", "hermes"])( + "reuses registered forwards for %s without creating another listener", + async (name) => { + vi.stubEnv("CHAT_UI_URL", undefined); + const ports = name === "openclaw" ? [18_790] : [18_790, 8_643]; + const test = harness({ + listSandboxes: () => ({ + sandboxes: [ + { name: "reonboard-test", dashboardPort: 18_790, hermesApiPort: 8_643 }, + { name: "sibling", dashboardPort: 18_789, hermesApiPort: 8_642 }, + ], + }), + initialStates: new Map(ports.map((port) => [port, "owned" as const])), + }); + + await expect( + test.helpers.ensureFinalizationAgentDashboardForward("reonboard-test", loadAgent(name)), + ).resolves.toBe(18_790); + expect(test.startForward.mock.calls.map(([request]) => request.forward.port)).toEqual(ports); + }, + ); + + it("awaits every forward start and propagates a rejected start", async () => { vi.stubEnv("CHAT_UI_URL", undefined); const test = harness({ listSandboxes: () => ({ sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790, hermesApiPort: 8_643 }], }), }); + const first = deferred(); + const second = deferred(); + test.startForward + .mockImplementationOnce(() => first.promise) + .mockImplementationOnce(() => second.promise); + + let settled = false; + const finalization = test.helpers + .ensureFinalizationAgentDashboardForward("reonboard-test", loadAgent("hermes")) + .then((value) => { + settled = true; + return value; + }); + await vi.waitFor(() => expect(test.startForward).toHaveBeenCalledTimes(1)); + expect(settled).toBe(false); + const firstForward = test.startForward.mock.calls[0]?.[0].forward; + expect(firstForward).toBeDefined(); + first.resolve({ state: "reused", forward: firstForward! }); + await vi.waitFor(() => expect(test.startForward).toHaveBeenCalledTimes(2)); + expect(settled).toBe(false); + const secondForward = test.startForward.mock.calls[1]?.[0].forward; + expect(secondForward).toBeDefined(); + second.resolve({ state: "reused", forward: secondForward! }); + await expect(finalization).resolves.toBe(18_790); + const failure = harness({ + listSandboxes: () => ({ + sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790, hermesApiPort: 8_643 }], + }), + }); + failure.startForward.mockRejectedValueOnce(new Error("forward startup rejected")); await expect( - test.helpers.ensureFinalizationAgentDashboardForward("reonboard-test", loadAgent("hermes")), - ).resolves.toBe(18_790); - expect(test.runOpenshell.mock.calls.map(([args]) => args[3])).toEqual([ - "127.0.0.1:18790", - "127.0.0.1:8643", - ]); - expect(test.observeForwards).not.toHaveBeenCalled(); + failure.helpers.ensureFinalizationAgentDashboardForward( + "reonboard-test", + loadAgent("hermes"), + ), + ).rejects.toThrow(/forward startup rejected/u); }); - it("reports an OpenShell command failure without inspecting forward state", async () => { + it.each([18_790, 8_643])( + "establishes missing Hermes forward %s on its recorded port", + async (missingPort) => { + vi.stubEnv("CHAT_UI_URL", undefined); + const otherPort = missingPort === 18_790 ? 8_643 : 18_790; + const test = harness({ + listSandboxes: () => ({ + sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790, hermesApiPort: 8_643 }], + }), + initialStates: new Map([[otherPort, "owned"]]), + }); + + await expect( + test.helpers.ensureFinalizationAgentDashboardForward("reonboard-test", loadAgent("hermes")), + ).resolves.toBe(18_790); + expect( + test.startForward.mock.calls.filter(([request]) => request.forward.port === missingPort), + ).toHaveLength(1); + }, + ); + + it("fails the agent forward when the adapter reports a command failure", async () => { vi.spyOn(console, "warn").mockImplementation(() => undefined); const test = harness({ listSandboxes: () => ({ sandboxes: [{ name: "reonboard-test" }] }), @@ -305,8 +398,7 @@ describe("finalization dashboard ForwardTcp reconciliation", () => { await expect( test.helpers.ensureAgentFixedForward("reonboard-test", 8_642, "Hermes API"), ).resolves.toBe(false); - expect(test.runOpenshell).toHaveBeenCalledOnce(); - expect(test.observeForwards).not.toHaveBeenCalled(); + expect(test.startForward).toHaveBeenCalledOnce(); }); it("honors an explicit dashboard URL", async () => { @@ -316,6 +408,6 @@ describe("finalization dashboard ForwardTcp reconciliation", () => { await expect(test.helpers.ensureFinalizationDashboardForward("reonboard-test")).resolves.toBe( 19_001, ); - expect(test.runOpenshell.mock.calls[0]?.[0]?.[3]).toBe("127.0.0.1:19001"); + expect(test.startForward.mock.calls[0]?.[0].forward.port).toBe(19_001); }); }); From 73d4e2bcb131452e3f260c851a0d4c58aa3b6f7d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 17 Sep 2026 16:32:42 -0700 Subject: [PATCH 08/39] fix(images): refresh managed startup runtime hash Signed-off-by: Aaron Erickson --- test/mcp/mcp-tool-discovery-image-contract.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/mcp/mcp-tool-discovery-image-contract.test.ts b/test/mcp/mcp-tool-discovery-image-contract.test.ts index 2f8aa1188b7..50b47c0a25c 100644 --- a/test/mcp/mcp-tool-discovery-image-contract.test.ts +++ b/test/mcp/mcp-tool-discovery-image-contract.test.ts @@ -15,7 +15,7 @@ const repoRoot = path.join(import.meta.dirname, "../.."); const runtimeRoot = "/usr/local/lib/nemoclaw/mcp-tool-discovery-runtime"; const managedStartupRuntimeBundle = "managed-startup-image-runtime.bundle"; const reviewedRuntimeHashOverrides: Readonly> = { - [managedStartupRuntimeBundle]: "5bbb0af04fa24bb66e2fe3efdb6af4d1081e1350ffffcdcaba1be4150f198109", + [managedStartupRuntimeBundle]: "f00a03aa854a37e94ac3d592ad75470e89530a20956c5e59473fedf6affdb775", }; const dockerfiles = [ "Dockerfile", From b460e242b052d1828aba0f0bfd174f2a73de34b7 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 17 Sep 2026 17:14:27 -0700 Subject: [PATCH 09/39] test(images): consolidate reviewed runtime digest Signed-off-by: Aaron Erickson --- test/mcp/mcp-tool-discovery-image-contract.test.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/test/mcp/mcp-tool-discovery-image-contract.test.ts b/test/mcp/mcp-tool-discovery-image-contract.test.ts index 50b47c0a25c..1e7b4c7a4ff 100644 --- a/test/mcp/mcp-tool-discovery-image-contract.test.ts +++ b/test/mcp/mcp-tool-discovery-image-contract.test.ts @@ -13,10 +13,6 @@ import { expectManagedToolDiscoveryRuntimeImageContract } from "../support/manag const repoRoot = path.join(import.meta.dirname, "../.."); const runtimeRoot = "/usr/local/lib/nemoclaw/mcp-tool-discovery-runtime"; -const managedStartupRuntimeBundle = "managed-startup-image-runtime.bundle"; -const reviewedRuntimeHashOverrides: Readonly> = { - [managedStartupRuntimeBundle]: "f00a03aa854a37e94ac3d592ad75470e89530a20956c5e59473fedf6affdb775", -}; const dockerfiles = [ "Dockerfile", "agents/hermes/Dockerfile", @@ -215,7 +211,7 @@ describe("MCP tool discovery image contract", () => { // source-shape-contract: security -- Exact reviewed runtime digests reject substituted executable and license artifacts before managed image construction. it.each([ { - expectedHash: "0c07b731d2f32a9419605bae4f84329c8d7440528eed2ac6dbcd5835724961e9", + expectedHash: "f00a03aa854a37e94ac3d592ad75470e89530a20956c5e59473fedf6affdb775", relativePath: "managed-startup-image-runtime.bundle", }, { @@ -239,9 +235,7 @@ describe("MCP tool discovery image contract", () => { .createHash("sha256") .update(fs.readFileSync(path.join(bundleRoot, relativePath))) .digest("hex"); - expect(actualHash, relativePath).toBe( - reviewedRuntimeHashOverrides[relativePath] ?? expectedHash, - ); + expect(actualHash, relativePath).toBe(expectedHash); }); it("executes the reviewed MCP discovery runtime artifact", () => { From 83e75d353679ff9cd57d5b7a57aeba2366d0e5e8 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 17 Sep 2026 17:28:27 -0700 Subject: [PATCH 10/39] test(e2e): cover Docker 27 receipt transfer Signed-off-by: Aaron Erickson --- .../docker-engine-27-receipt-transfer-e2e.ts | 300 ++++++++++++++++++ .../managed-image-multiarch-startup.test.ts | 28 +- 2 files changed, 325 insertions(+), 3 deletions(-) create mode 100644 scripts/checks/docker-engine-27-receipt-transfer-e2e.ts diff --git a/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts b/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts new file mode 100644 index 00000000000..e54c78261f9 --- /dev/null +++ b/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts @@ -0,0 +1,300 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { + dockerDaemonReceiptMount, + transferDockerReceiptToDaemon, +} from "../../src/lib/onboard/managed-startup/docker-receipt-transfer.ts"; + +const DIND_IMAGE = + "docker.io/library/docker:27.5.1-dind@sha256:aa3df78ecf320f5fafdce71c659f1629e96e9de0968305fe1de670e0ca9176ce"; +const RECEIPT_IMAGE = + "docker.io/library/alpine:3.20@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc"; +const RECEIPT_VOLUME_DIRECTORY = "/run/nemoclaw/managed-startup-receipt-transfer"; + +type CommandResult = { + readonly error?: Error; + readonly status: number | null; + readonly stderr: string; + readonly stdout: string; +}; + +function runDocker(args: readonly string[]): CommandResult { + const result = spawnSync("docker", [...args], { + encoding: "utf8", + killSignal: "SIGKILL", + maxBuffer: 8 * 1024 * 1024, + timeout: 240_000, + }); + return { + ...(result.error ? { error: result.error } : {}), + status: result.status, + stderr: result.stderr ?? "", + stdout: result.stdout ?? "", + }; +} + +function commandDetail(result: CommandResult): string { + return `${result.stderr} ${result.stdout} ${result.error?.message ?? ""}`.trim().slice(-1_600); +} + +function requireSuccess(result: CommandResult, operation: string): string { + if (result.status !== 0) { + throw new Error(`${operation} failed: ${commandDetail(result)}`); + } + return result.stdout.trim(); +} + +function requireCondition(condition: unknown, detail: string): asserts condition { + if (!condition) throw new Error(detail); +} + +async function waitForDocker27( + innerDocker: (args: readonly string[]) => CommandResult, +): Promise { + for (let attempt = 0; attempt < 90; attempt += 1) { + if (innerDocker(["info"]).status === 0) return; + await new Promise((resolve) => setTimeout(resolve, 1_000)); + } + throw new Error("Docker Engine 27 daemon did not become ready"); +} + +function seedName(args: readonly string[]): string { + const nameIndex = args.indexOf("--name"); + requireCondition(nameIndex >= 0 && args[nameIndex + 1], "receipt seed command omitted --name"); + return args[nameIndex + 1]; +} + +function assertSeedIsolation( + innerDocker: (args: readonly string[]) => CommandResult, + name: string, +): void { + const inspected = JSON.parse( + requireSuccess(innerDocker(["inspect", name]), "inspect receipt seed"), + ); + requireCondition( + Array.isArray(inspected) && inspected.length === 1, + "receipt seed inspect changed shape", + ); + const seed = inspected[0] as { + Config?: { User?: unknown }; + HostConfig?: { + CapDrop?: unknown; + Mounts?: unknown; + NetworkMode?: unknown; + ReadonlyRootfs?: unknown; + SecurityOpt?: unknown; + }; + }; + requireCondition(seed.Config?.User === "0", "receipt seed did not use numeric root"); + requireCondition(seed.HostConfig?.NetworkMode === "none", "receipt seed retained networking"); + requireCondition(seed.HostConfig?.ReadonlyRootfs === true, "receipt seed root was writable"); + requireCondition( + Array.isArray(seed.HostConfig?.SecurityOpt) && + seed.HostConfig.SecurityOpt.includes("no-new-privileges"), + "receipt seed omitted no-new-privileges", + ); + requireCondition( + Array.isArray(seed.HostConfig?.CapDrop) && seed.HostConfig.CapDrop.includes("ALL"), + "receipt seed retained capabilities", + ); + requireCondition( + Array.isArray(seed.HostConfig?.Mounts) && + seed.HostConfig.Mounts.some( + (mount) => + typeof mount === "object" && + mount !== null && + Reflect.get(mount, "Type") === "volume" && + Reflect.get(mount, "Target") === RECEIPT_VOLUME_DIRECTORY, + ), + "receipt seed omitted the daemon volume", + ); +} + +function requireAbsent(result: CommandResult, resource: string): void { + requireCondition(result.status !== 0, `${resource} remained after receipt-transfer cleanup`); +} + +async function verifyDockerEngine27ReceiptTransfer(): Promise { + const suffix = randomUUID().replaceAll("-", ""); + const daemonName = `nemoclaw-receipt-engine27-${suffix}`; + const legacySeed = `nemoclaw-receipt-legacy-seed-${suffix}`; + const legacyVolume = `nemoclaw-receipt-legacy-volume-${suffix}`; + const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-receipt-engine27-")); + const fixtureReceipt = path.join(fixtureRoot, "receipt"); + const daemonReceipt = `/nemoclaw-receipt-${suffix}`; + let successfulVolume: string | null = null; + + fs.mkdirSync(fixtureReceipt, { mode: 0o700 }); + fs.writeFileSync(path.join(fixtureReceipt, "receipt.json"), "verified\n", { mode: 0o400 }); + + const innerDocker = (args: readonly string[]): CommandResult => + runDocker(["exec", daemonName, "docker", ...args]); + + try { + requireSuccess( + runDocker([ + "run", + "--privileged", + "--detach", + "--name", + daemonName, + "--env", + "DOCKER_TLS_CERTDIR=", + DIND_IMAGE, + "--host=unix:///var/run/docker.sock", + ]), + "start isolated Docker Engine 27 daemon", + ); + await waitForDocker27(innerDocker); + const engineVersion = requireSuccess( + innerDocker(["version", "--format", "{{.Server.Version}}"]), + "read isolated Docker version", + ); + requireCondition( + engineVersion === "27.5.1", + `unexpected Docker Engine version ${engineVersion}`, + ); + + requireSuccess(innerDocker(["pull", RECEIPT_IMAGE]), "pull digest-pinned receipt image"); + requireSuccess( + runDocker(["cp", fixtureReceipt, `${daemonName}:${daemonReceipt}`]), + "stage protected receipt in Docker Engine 27 client", + ); + + requireSuccess(innerDocker(["volume", "create", legacyVolume]), "create legacy probe volume"); + requireSuccess( + innerDocker([ + "create", + "--name", + legacySeed, + "--pull", + "never", + "--network", + "none", + "--read-only", + "--user", + "0:0", + "--security-opt", + "no-new-privileges", + "--cap-drop", + "ALL", + "--mount", + `type=volume,src=${legacyVolume},dst=${RECEIPT_VOLUME_DIRECTORY}`, + RECEIPT_IMAGE, + ]), + "create legacy receipt seed", + ); + const legacyCopy = innerDocker([ + "cp", + "-a", + daemonReceipt, + `${legacySeed}:${RECEIPT_VOLUME_DIRECTORY}/receipt`, + ]); + requireCondition(legacyCopy.status !== 0, "Docker Engine 27 unexpectedly accepted --user 0:0"); + requireCondition( + /unable to find entry "0:0" in passwd(?: database)?/u.test(commandDetail(legacyCopy)), + `Docker Engine 27 returned an unexpected legacy failure: ${commandDetail(legacyCopy)}`, + ); + requireSuccess(innerDocker(["rm", "-f", legacySeed]), "remove legacy receipt seed"); + requireSuccess(innerDocker(["volume", "rm", legacyVolume]), "remove legacy receipt volume"); + + let successfulSeed = ""; + const dockerRun = (args: readonly string[]): CommandResult => { + const result = innerDocker(args); + if (args[0] === "create" && result.status === 0) { + successfulSeed = seedName(args); + assertSeedIsolation(innerDocker, successfulSeed); + } + return result; + }; + const receipt = transferDockerReceiptToDaemon({ + image: RECEIPT_IMAGE, + receiptPath: daemonReceipt, + destinations: ["/run/nemoclaw/receipt"], + dockerOptions: {}, + dockerRun, + }); + successfulVolume = receipt.volumeName; + requireCondition(successfulSeed.length > 0, "receipt transfer did not create a seed"); + requireAbsent(innerDocker(["container", "inspect", successfulSeed]), "successful receipt seed"); + requireSuccess( + innerDocker([ + "run", + "--rm", + "--network", + "none", + "--read-only", + "--user", + "0", + "--security-opt", + "no-new-privileges", + "--cap-drop", + "ALL", + "--mount", + dockerDaemonReceiptMount(receipt, "/run/nemoclaw/receipt"), + RECEIPT_IMAGE, + "sh", + "-ceu", + [ + 'test "$(cat /run/nemoclaw/receipt/receipt.json)" = verified', + 'test "$(stat -c %u:%g:%a /run/nemoclaw/receipt)" = 0:0:700', + 'test "$(stat -c %u:%g:%a /run/nemoclaw/receipt/receipt.json)" = 0:0:400', + "! touch /run/nemoclaw/receipt/write-denied", + ].join(" && "), + ]), + "verify Docker Engine 27 receipt volume", + ); + requireSuccess(innerDocker(["volume", "rm", receipt.volumeName]), "remove receipt volume"); + successfulVolume = null; + + let failedSeed = ""; + let failedVolume = ""; + const failingDockerRun = (args: readonly string[]): CommandResult => { + if (args[0] === "volume" && args[1] === "create") failedVolume = String(args[2] ?? ""); + if (args[0] === "create") failedSeed = seedName(args); + return innerDocker(args); + }; + const missingReceipt = `/nemoclaw-missing-receipt-${suffix}`; + let transferFailure: unknown; + try { + transferDockerReceiptToDaemon({ + image: RECEIPT_IMAGE, + receiptPath: missingReceipt, + destinations: ["/run/nemoclaw/receipt"], + dockerOptions: {}, + dockerRun: failingDockerRun, + }); + } catch (error) { + transferFailure = error; + } + requireCondition(transferFailure instanceof Error, "missing receipt transfer did not fail"); + requireCondition( + transferFailure.message.includes(`host receipt ${missingReceipt}`), + "receipt failure omitted the retained host receipt", + ); + requireCondition( + failedSeed.length > 0 && failedVolume.length > 0, + "failed transfer omitted resources", + ); + requireAbsent(innerDocker(["container", "inspect", failedSeed]), "failed receipt seed"); + requireAbsent(innerDocker(["volume", "inspect", failedVolume]), "failed receipt volume"); + } finally { + innerDocker(["rm", "-f", legacySeed]); + innerDocker(["volume", "rm", legacyVolume]); + if (successfulVolume) innerDocker(["volume", "rm", successfulVolume]); + runDocker(["rm", "-f", daemonName]); + fs.rmSync(fixtureRoot, { force: true, recursive: true }); + } +} + +void verifyDockerEngine27ReceiptTransfer().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); diff --git a/test/e2e/live/managed-image-multiarch-startup.test.ts b/test/e2e/live/managed-image-multiarch-startup.test.ts index 3d52f079e8d..c8da987382a 100644 --- a/test/e2e/live/managed-image-multiarch-startup.test.ts +++ b/test/e2e/live/managed-image-multiarch-startup.test.ts @@ -15,6 +15,7 @@ import { protectedManagedImageDispatchEnvironment, readRegularArtifact, } from "./managed-image-multiarch-startup-helpers.ts"; +import { trustedShellCommand } from "../fixtures/shell-probe.ts"; test( "binds protected all-agent direct startup to the exact multiarch dispatch (#7744)", @@ -24,10 +25,11 @@ test( "validate protected activation and dispatch identity", "validate exact all-agent managed-image contracts", "validate direct-start evidence binding", + "verify Docker Engine 27 receipt transfer", ], }, }, - ({ progress }) => { + async ({ progress, shellProbe }) => { progress.phase("validate protected activation and dispatch identity"); const dispatch = protectedManagedImageDispatchEnvironment(); @@ -56,9 +58,29 @@ test( }, ); + progress.phase("verify Docker Engine 27 receipt transfer"); + const docker27Result = await shellProbe.run( + trustedShellCommand({ + command: process.execPath, + args: ["--import", "tsx", "scripts/checks/docker-engine-27-receipt-transfer-e2e.ts"], + reason: "verify the Docker Engine 27 receipt archive-copy boundary", + }), + { + artifactName: "docker-engine-27-receipt-transfer", + cwd: dispatch.workspace, + env: { + ...(process.env.DOCKER_CONFIG ? { DOCKER_CONFIG: process.env.DOCKER_CONFIG } : {}), + ...(process.env.HOME ? { HOME: process.env.HOME } : {}), + }, + timeoutMs: 240_000, + }, + ); + expect( - evidence.contractSha256 === - `sha256:${createHash("sha256").update(contractBytes).digest("hex")}` && + docker27Result.exitCode === 0 && + !docker27Result.timedOut && + evidence.contractSha256 === + `sha256:${createHash("sha256").update(contractBytes).digest("hex")}` && JSON.stringify(evidence.contracts) === JSON.stringify(contracts), ).toBe(true); }, From 9e7a93f6a958ae0cdd72bf37437092acdf8c1e90 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 17 Sep 2026 17:38:54 -0700 Subject: [PATCH 11/39] test(e2e): add fast Docker 27 seed contract Signed-off-by: Aaron Erickson --- .../docker-engine-27-receipt-transfer-e2e.ts | 18 +++++++---- ...ed-image-multiarch-startup-helpers.test.ts | 30 +++++++++++++++++++ 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts b/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts index e54c78261f9..8a5814b429a 100644 --- a/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts +++ b/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts @@ -6,6 +6,7 @@ import { randomUUID } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { fileURLToPath } from "node:url"; import { dockerDaemonReceiptMount, @@ -82,7 +83,12 @@ function assertSeedIsolation( Array.isArray(inspected) && inspected.length === 1, "receipt seed inspect changed shape", ); - const seed = inspected[0] as { + validateDockerEngine27SeedIsolation(inspected[0]); +} + +export function validateDockerEngine27SeedIsolation(value: unknown): void { + requireCondition(typeof value === "object" && value !== null, "receipt seed inspect is invalid"); + const seed = value as { Config?: { User?: unknown }; HostConfig?: { CapDrop?: unknown; @@ -294,7 +300,9 @@ async function verifyDockerEngine27ReceiptTransfer(): Promise { } } -void verifyDockerEngine27ReceiptTransfer().catch((error: unknown) => { - console.error(error instanceof Error ? error.message : String(error)); - process.exitCode = 1; -}); +if (path.resolve(process.argv[1] ?? "") === fileURLToPath(import.meta.url)) { + void verifyDockerEngine27ReceiptTransfer().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts b/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts index 76cf813d7e9..5d73bc8903a 100644 --- a/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts +++ b/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts @@ -7,6 +7,7 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { validateDockerEngine27SeedIsolation } from "../../../scripts/checks/docker-engine-27-receipt-transfer-e2e.ts"; import { protectedManagedImageDispatchEnvironment, readRegularArtifact, @@ -38,6 +39,35 @@ afterEach(() => { }); describe("protected managed-image startup helpers", () => { + it("requires numeric root and every receipt seed isolation control", () => { + const secureSeed = { + Config: { User: "0" }, + HostConfig: { + CapDrop: ["ALL"], + Mounts: [ + { + Type: "volume", + Target: "/run/nemoclaw/managed-startup-receipt-transfer", + }, + ], + NetworkMode: "none", + ReadonlyRootfs: true, + SecurityOpt: ["no-new-privileges"], + }, + }; + + expect(() => validateDockerEngine27SeedIsolation(secureSeed)).not.toThrow(); + expect(() => + validateDockerEngine27SeedIsolation({ ...secureSeed, Config: { User: "0:0" } }), + ).toThrow("receipt seed did not use numeric root"); + expect(() => + validateDockerEngine27SeedIsolation({ + ...secureSeed, + HostConfig: { ...secureSeed.HostConfig, CapDrop: [] }, + }), + ).toThrow("receipt seed retained capabilities"); + }); + it("parses exact protected dispatch identity", () => { expect(protectedManagedImageDispatchEnvironment()).toMatchObject({ baseSha: sha, From 83823571c0eb7bbf477bd051c2ed887de44d3bd9 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 17 Sep 2026 18:27:41 -0700 Subject: [PATCH 12/39] test(e2e): harden Docker 27 probe cleanup Signed-off-by: Aaron Erickson --- .../docker-engine-27-receipt-transfer-e2e.ts | 104 ++++++++++++++++-- ...managed-image-multiarch-startup-helpers.ts | 20 ++++ .../managed-image-multiarch-startup.test.ts | 65 ++++++++--- test/e2e/mock-parity.json | 1 + ...ed-image-multiarch-startup-helpers.test.ts | 31 +++++- 5 files changed, 192 insertions(+), 29 deletions(-) diff --git a/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts b/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts index 8a5814b429a..cb6f0705670 100644 --- a/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts +++ b/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts @@ -18,6 +18,17 @@ const DIND_IMAGE = const RECEIPT_IMAGE = "docker.io/library/alpine:3.20@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc"; const RECEIPT_VOLUME_DIRECTORY = "/run/nemoclaw/managed-startup-receipt-transfer"; +const DAEMON_OWNER_LABEL = "io.nvidia.nemoclaw.e2e.docker27-receipt"; +export const DOCKER_ENGINE_27_OPERATION_TIMEOUT_MS = 120_000; +export const DOCKER_ENGINE_27_CLEANUP_TIMEOUT_MS = 30_000; +export const DOCKER_ENGINE_27_READINESS_ATTEMPTS = 45; +export const DOCKER_ENGINE_27_READINESS_COMMAND_TIMEOUT_MS = 5_000; +export const DOCKER_ENGINE_27_READINESS_INTERVAL_MS = 1_000; +export const DOCKER_ENGINE_27_MINIMUM_PROBE_TIMEOUT_MS = + DOCKER_ENGINE_27_READINESS_ATTEMPTS * + (DOCKER_ENGINE_27_READINESS_COMMAND_TIMEOUT_MS + DOCKER_ENGINE_27_READINESS_INTERVAL_MS) + + DOCKER_ENGINE_27_OPERATION_TIMEOUT_MS + + DOCKER_ENGINE_27_CLEANUP_TIMEOUT_MS; type CommandResult = { readonly error?: Error; @@ -31,7 +42,37 @@ function runDocker(args: readonly string[]): CommandResult { encoding: "utf8", killSignal: "SIGKILL", maxBuffer: 8 * 1024 * 1024, - timeout: 240_000, + timeout: DOCKER_ENGINE_27_OPERATION_TIMEOUT_MS, + }); + return { + ...(result.error ? { error: result.error } : {}), + status: result.status, + stderr: result.stderr ?? "", + stdout: result.stdout ?? "", + }; +} + +function runDockerReadiness(args: readonly string[]): CommandResult { + const result = spawnSync("docker", [...args], { + encoding: "utf8", + killSignal: "SIGKILL", + maxBuffer: 1024 * 1024, + timeout: DOCKER_ENGINE_27_READINESS_COMMAND_TIMEOUT_MS, + }); + return { + ...(result.error ? { error: result.error } : {}), + status: result.status, + stderr: result.stderr ?? "", + stdout: result.stdout ?? "", + }; +} + +function runDockerCleanup(args: readonly string[]): CommandResult { + const result = spawnSync("docker", [...args], { + encoding: "utf8", + killSignal: "SIGKILL", + maxBuffer: 1024 * 1024, + timeout: DOCKER_ENGINE_27_CLEANUP_TIMEOUT_MS, }); return { ...(result.error ? { error: result.error } : {}), @@ -56,12 +97,12 @@ function requireCondition(condition: unknown, detail: string): asserts condition if (!condition) throw new Error(detail); } -async function waitForDocker27( - innerDocker: (args: readonly string[]) => CommandResult, -): Promise { - for (let attempt = 0; attempt < 90; attempt += 1) { - if (innerDocker(["info"]).status === 0) return; - await new Promise((resolve) => setTimeout(resolve, 1_000)); +async function waitForDocker27(daemonName: string): Promise { + for (let attempt = 0; attempt < DOCKER_ENGINE_27_READINESS_ATTEMPTS; attempt += 1) { + if (runDockerReadiness(["exec", daemonName, "docker", "info"]).status === 0) return; + await new Promise((resolve) => + setTimeout(resolve, DOCKER_ENGINE_27_READINESS_INTERVAL_MS), + ); } throw new Error("Docker Engine 27 daemon did not become ready"); } @@ -127,9 +168,31 @@ function requireAbsent(result: CommandResult, resource: string): void { requireCondition(result.status !== 0, `${resource} remained after receipt-transfer cleanup`); } -async function verifyDockerEngine27ReceiptTransfer(): Promise { +export function cleanupDockerEngine27ReceiptDaemon(daemonName: string): void { + const inspected = runDockerCleanup([ + "container", + "inspect", + "--format", + `{{ index .Config.Labels "${DAEMON_OWNER_LABEL}" }}`, + daemonName, + ]); + if (inspected.status !== 0) { + requireCondition( + /No such (?:container|object)/iu.test(commandDetail(inspected)), + `could not inspect Docker Engine 27 daemon during cleanup: ${commandDetail(inspected)}`, + ); + return; + } + requireCondition( + inspected.stdout.trim() === daemonName, + "refusing to remove a Docker Engine 27 daemon with mismatched ownership", + ); + requireSuccess(runDockerCleanup(["rm", "-f", daemonName]), "remove Docker Engine 27 daemon"); + requireAbsent(runDockerCleanup(["container", "inspect", daemonName]), "Docker Engine 27 daemon"); +} + +async function verifyDockerEngine27ReceiptTransfer(daemonName: string): Promise { const suffix = randomUUID().replaceAll("-", ""); - const daemonName = `nemoclaw-receipt-engine27-${suffix}`; const legacySeed = `nemoclaw-receipt-legacy-seed-${suffix}`; const legacyVolume = `nemoclaw-receipt-legacy-volume-${suffix}`; const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-receipt-engine27-")); @@ -151,6 +214,8 @@ async function verifyDockerEngine27ReceiptTransfer(): Promise { "--detach", "--name", daemonName, + "--label", + `${DAEMON_OWNER_LABEL}=${daemonName}`, "--env", "DOCKER_TLS_CERTDIR=", DIND_IMAGE, @@ -158,7 +223,7 @@ async function verifyDockerEngine27ReceiptTransfer(): Promise { ]), "start isolated Docker Engine 27 daemon", ); - await waitForDocker27(innerDocker); + await waitForDocker27(daemonName); const engineVersion = requireSuccess( innerDocker(["version", "--format", "{{.Server.Version}}"]), "read isolated Docker version", @@ -295,13 +360,28 @@ async function verifyDockerEngine27ReceiptTransfer(): Promise { innerDocker(["rm", "-f", legacySeed]); innerDocker(["volume", "rm", legacyVolume]); if (successfulVolume) innerDocker(["volume", "rm", successfulVolume]); - runDocker(["rm", "-f", daemonName]); + cleanupDockerEngine27ReceiptDaemon(daemonName); fs.rmSync(fixtureRoot, { force: true, recursive: true }); } } +function parseDaemonName(args: readonly string[]): string { + const index = args.indexOf("--daemon-name"); + const daemonName = index < 0 ? "" : String(args[index + 1] ?? ""); + requireCondition( + /^nemoclaw-receipt-engine27-[a-z0-9-]{1,80}$/u.test(daemonName), + "Docker Engine 27 probe requires a safe owned daemon name", + ); + return daemonName; +} + if (path.resolve(process.argv[1] ?? "") === fileURLToPath(import.meta.url)) { - void verifyDockerEngine27ReceiptTransfer().catch((error: unknown) => { + const args = process.argv.slice(2); + const daemonName = parseDaemonName(args); + const operation = args.includes("--cleanup-only") + ? Promise.resolve().then(() => cleanupDockerEngine27ReceiptDaemon(daemonName)) + : verifyDockerEngine27ReceiptTransfer(daemonName); + void operation.catch((error: unknown) => { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; }); diff --git a/test/e2e/live/managed-image-multiarch-startup-helpers.ts b/test/e2e/live/managed-image-multiarch-startup-helpers.ts index 27b011abe80..4fb69cbbf92 100644 --- a/test/e2e/live/managed-image-multiarch-startup-helpers.ts +++ b/test/e2e/live/managed-image-multiarch-startup-helpers.ts @@ -25,6 +25,26 @@ export interface ProtectedManagedImageDispatchEnvironment { workspace: string; } +export const DOCKER_ENGINE_27_PROBE_TIMEOUT_MS = 10 * 60_000; + +export function dockerEngine27ReceiptDaemonName(runId: number, runAttempt: number): string { + return `nemoclaw-receipt-engine27-${String(runId)}-${String(runAttempt)}`; +} + +export function shouldRunDockerEngine27ReceiptProbe( + platform: ProtectedManagedImagePlatform, +): boolean { + return platform === "linux/amd64"; +} + +export function registerDockerEngine27ReceiptCleanup( + cleanup: { trackDisposable(name: string, dispose: () => void): void }, + daemonName: string, + dispose: () => void, +): void { + cleanup.trackDisposable(`remove owned Docker Engine 27 daemon ${daemonName}`, dispose); +} + function requiredEnvironment(name: string): string { const value = process.env[name]; if (!value) throw new Error(`${name} is required`); diff --git a/test/e2e/live/managed-image-multiarch-startup.test.ts b/test/e2e/live/managed-image-multiarch-startup.test.ts index c8da987382a..2f93c01e7c5 100644 --- a/test/e2e/live/managed-image-multiarch-startup.test.ts +++ b/test/e2e/live/managed-image-multiarch-startup.test.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; import path from "node:path"; @@ -12,8 +13,12 @@ import { } from "../../../scripts/checks/protected-managed-image-contract.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { + DOCKER_ENGINE_27_PROBE_TIMEOUT_MS, + dockerEngine27ReceiptDaemonName, protectedManagedImageDispatchEnvironment, readRegularArtifact, + registerDockerEngine27ReceiptCleanup, + shouldRunDockerEngine27ReceiptProbe, } from "./managed-image-multiarch-startup-helpers.ts"; import { trustedShellCommand } from "../fixtures/shell-probe.ts"; @@ -29,7 +34,7 @@ test( ], }, }, - async ({ progress, shellProbe }) => { + async ({ cleanup, progress, shellProbe }) => { progress.phase("validate protected activation and dispatch identity"); const dispatch = protectedManagedImageDispatchEnvironment(); @@ -59,22 +64,50 @@ test( ); progress.phase("verify Docker Engine 27 receipt transfer"); - const docker27Result = await shellProbe.run( - trustedShellCommand({ - command: process.execPath, - args: ["--import", "tsx", "scripts/checks/docker-engine-27-receipt-transfer-e2e.ts"], - reason: "verify the Docker Engine 27 receipt archive-copy boundary", - }), - { - artifactName: "docker-engine-27-receipt-transfer", - cwd: dispatch.workspace, - env: { - ...(process.env.DOCKER_CONFIG ? { DOCKER_CONFIG: process.env.DOCKER_CONFIG } : {}), - ...(process.env.HOME ? { HOME: process.env.HOME } : {}), + const docker27DaemonName = dockerEngine27ReceiptDaemonName(dispatch.runId, dispatch.runAttempt); + registerDockerEngine27ReceiptCleanup(cleanup, docker27DaemonName, () => { + execFileSync( + process.execPath, + [ + "--import", + "tsx", + "scripts/checks/docker-engine-27-receipt-transfer-e2e.ts", + "--cleanup-only", + "--daemon-name", + docker27DaemonName, + ], + { + cwd: dispatch.workspace, + killSignal: "SIGKILL", + stdio: "ignore", + timeout: 45_000, }, - timeoutMs: 240_000, - }, - ); + ); + }); + const docker27Result = shouldRunDockerEngine27ReceiptProbe(dispatch.platform) + ? await shellProbe.run( + trustedShellCommand({ + command: process.execPath, + args: [ + "--import", + "tsx", + "scripts/checks/docker-engine-27-receipt-transfer-e2e.ts", + "--daemon-name", + docker27DaemonName, + ], + reason: "verify the Docker Engine 27 receipt archive-copy boundary", + }), + { + artifactName: "docker-engine-27-receipt-transfer", + cwd: dispatch.workspace, + env: { + ...(process.env.DOCKER_CONFIG ? { DOCKER_CONFIG: process.env.DOCKER_CONFIG } : {}), + ...(process.env.HOME ? { HOME: process.env.HOME } : {}), + }, + timeoutMs: DOCKER_ENGINE_27_PROBE_TIMEOUT_MS, + }, + ) + : { exitCode: 0, timedOut: false }; expect( docker27Result.exitCode === 0 && diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 0c57b000fc8..1fe7c4dde46 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -37,6 +37,7 @@ }, { "live": "test/e2e/live/managed-image-multiarch-startup.test.ts", + "liveSources": ["test/e2e/live/managed-image-multiarch-startup-helpers.ts"], "fast": [ "test/e2e/support/base-image-publication.test.ts", "test/e2e/support/managed-image-multiarch-startup-helpers.test.ts", diff --git a/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts b/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts index 5d73bc8903a..7e723dce417 100644 --- a/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts +++ b/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts @@ -7,10 +7,17 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { validateDockerEngine27SeedIsolation } from "../../../scripts/checks/docker-engine-27-receipt-transfer-e2e.ts"; import { + DOCKER_ENGINE_27_MINIMUM_PROBE_TIMEOUT_MS, + validateDockerEngine27SeedIsolation, +} from "../../../scripts/checks/docker-engine-27-receipt-transfer-e2e.ts"; +import { + DOCKER_ENGINE_27_PROBE_TIMEOUT_MS, + dockerEngine27ReceiptDaemonName, protectedManagedImageDispatchEnvironment, readRegularArtifact, + registerDockerEngine27ReceiptCleanup, + shouldRunDockerEngine27ReceiptProbe, } from "../live/managed-image-multiarch-startup-helpers.ts"; const sha = "a".repeat(40); @@ -39,6 +46,28 @@ afterEach(() => { }); describe("protected managed-image startup helpers", () => { + it("runs one bounded Docker 27 probe and registers exact cleanup first", () => { + const daemonName = dockerEngine27ReceiptDaemonName(123, 4); + const dispose = vi.fn(); + const trackDisposable = vi.fn(); + + registerDockerEngine27ReceiptCleanup({ trackDisposable }, daemonName, dispose); + const registeredCleanup = trackDisposable.mock.calls[0]?.[1]; + registeredCleanup?.(); + + expect(daemonName).toBe("nemoclaw-receipt-engine27-123-4"); + expect(shouldRunDockerEngine27ReceiptProbe("linux/amd64")).toBe(true); + expect(shouldRunDockerEngine27ReceiptProbe("linux/arm64")).toBe(false); + expect(DOCKER_ENGINE_27_PROBE_TIMEOUT_MS).toBeGreaterThanOrEqual( + DOCKER_ENGINE_27_MINIMUM_PROBE_TIMEOUT_MS, + ); + expect(trackDisposable).toHaveBeenCalledWith( + `remove owned Docker Engine 27 daemon ${daemonName}`, + dispose, + ); + expect(dispose).toHaveBeenCalledOnce(); + }); + it("requires numeric root and every receipt seed isolation control", () => { const secureSeed = { Config: { User: "0" }, From 166969ca048f663cfd1d2c00a4916e08ede8ea68 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 17 Sep 2026 19:25:03 -0700 Subject: [PATCH 13/39] test(e2e): bound Docker 27 probe lifecycle Signed-off-by: Aaron Erickson --- .../docker-engine-27-receipt-transfer-e2e.ts | 44 ++++++++++++-- ...managed-image-multiarch-startup-helpers.ts | 20 ------- .../managed-image-multiarch-startup.test.ts | 57 +++++++++---------- ...ed-image-multiarch-startup-helpers.test.ts | 28 ++------- 4 files changed, 70 insertions(+), 79 deletions(-) diff --git a/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts b/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts index cb6f0705670..7c7fe43ee41 100644 --- a/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts +++ b/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts @@ -19,16 +19,26 @@ const RECEIPT_IMAGE = "docker.io/library/alpine:3.20@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc"; const RECEIPT_VOLUME_DIRECTORY = "/run/nemoclaw/managed-startup-receipt-transfer"; const DAEMON_OWNER_LABEL = "io.nvidia.nemoclaw.e2e.docker27-receipt"; -export const DOCKER_ENGINE_27_OPERATION_TIMEOUT_MS = 120_000; -export const DOCKER_ENGINE_27_CLEANUP_TIMEOUT_MS = 30_000; +export const DOCKER_ENGINE_27_OPERATION_TIMEOUT_MS = 30_000; +export const DOCKER_ENGINE_27_PULL_TIMEOUT_MS = 120_000; +export const DOCKER_ENGINE_27_CLEANUP_TIMEOUT_MS = 15_000; export const DOCKER_ENGINE_27_READINESS_ATTEMPTS = 45; export const DOCKER_ENGINE_27_READINESS_COMMAND_TIMEOUT_MS = 5_000; export const DOCKER_ENGINE_27_READINESS_INTERVAL_MS = 1_000; +export const DOCKER_ENGINE_27_MAX_OPERATION_COUNT = 28; +export const DOCKER_ENGINE_27_PULL_COUNT = 2; +export const DOCKER_ENGINE_27_CLEANUP_OPERATION_COUNT = 3; +export const DOCKER_ENGINE_27_PROCESS_ALLOWANCE_MS = 60_000; export const DOCKER_ENGINE_27_MINIMUM_PROBE_TIMEOUT_MS = DOCKER_ENGINE_27_READINESS_ATTEMPTS * (DOCKER_ENGINE_27_READINESS_COMMAND_TIMEOUT_MS + DOCKER_ENGINE_27_READINESS_INTERVAL_MS) + - DOCKER_ENGINE_27_OPERATION_TIMEOUT_MS + - DOCKER_ENGINE_27_CLEANUP_TIMEOUT_MS; + DOCKER_ENGINE_27_MAX_OPERATION_COUNT * DOCKER_ENGINE_27_OPERATION_TIMEOUT_MS + + DOCKER_ENGINE_27_PULL_COUNT * DOCKER_ENGINE_27_PULL_TIMEOUT_MS + + DOCKER_ENGINE_27_CLEANUP_OPERATION_COUNT * DOCKER_ENGINE_27_CLEANUP_TIMEOUT_MS + + DOCKER_ENGINE_27_PROCESS_ALLOWANCE_MS; +export const DOCKER_ENGINE_27_MINIMUM_CLEANUP_PROCESS_TIMEOUT_MS = + DOCKER_ENGINE_27_CLEANUP_OPERATION_COUNT * DOCKER_ENGINE_27_CLEANUP_TIMEOUT_MS + + DOCKER_ENGINE_27_PROCESS_ALLOWANCE_MS / 2; type CommandResult = { readonly error?: Error; @@ -52,6 +62,21 @@ function runDocker(args: readonly string[]): CommandResult { }; } +function runDockerPull(args: readonly string[]): CommandResult { + const result = spawnSync("docker", [...args], { + encoding: "utf8", + killSignal: "SIGKILL", + maxBuffer: 8 * 1024 * 1024, + timeout: DOCKER_ENGINE_27_PULL_TIMEOUT_MS, + }); + return { + ...(result.error ? { error: result.error } : {}), + status: result.status, + stderr: result.stderr ?? "", + stdout: result.stdout ?? "", + }; +} + function runDockerReadiness(args: readonly string[]): CommandResult { const result = spawnSync("docker", [...args], { encoding: "utf8", @@ -207,9 +232,15 @@ async function verifyDockerEngine27ReceiptTransfer(daemonName: string): Promise< runDocker(["exec", daemonName, "docker", ...args]); try { + requireSuccess( + runDockerPull(["pull", DIND_IMAGE]), + "pull digest-pinned Docker Engine 27 image", + ); requireSuccess( runDocker([ "run", + "--pull", + "never", "--privileged", "--detach", "--name", @@ -233,7 +264,10 @@ async function verifyDockerEngine27ReceiptTransfer(daemonName: string): Promise< `unexpected Docker Engine version ${engineVersion}`, ); - requireSuccess(innerDocker(["pull", RECEIPT_IMAGE]), "pull digest-pinned receipt image"); + requireSuccess( + runDockerPull(["exec", daemonName, "docker", "pull", RECEIPT_IMAGE]), + "pull digest-pinned receipt image", + ); requireSuccess( runDocker(["cp", fixtureReceipt, `${daemonName}:${daemonReceipt}`]), "stage protected receipt in Docker Engine 27 client", diff --git a/test/e2e/live/managed-image-multiarch-startup-helpers.ts b/test/e2e/live/managed-image-multiarch-startup-helpers.ts index 4fb69cbbf92..27b011abe80 100644 --- a/test/e2e/live/managed-image-multiarch-startup-helpers.ts +++ b/test/e2e/live/managed-image-multiarch-startup-helpers.ts @@ -25,26 +25,6 @@ export interface ProtectedManagedImageDispatchEnvironment { workspace: string; } -export const DOCKER_ENGINE_27_PROBE_TIMEOUT_MS = 10 * 60_000; - -export function dockerEngine27ReceiptDaemonName(runId: number, runAttempt: number): string { - return `nemoclaw-receipt-engine27-${String(runId)}-${String(runAttempt)}`; -} - -export function shouldRunDockerEngine27ReceiptProbe( - platform: ProtectedManagedImagePlatform, -): boolean { - return platform === "linux/amd64"; -} - -export function registerDockerEngine27ReceiptCleanup( - cleanup: { trackDisposable(name: string, dispose: () => void): void }, - daemonName: string, - dispose: () => void, -): void { - cleanup.trackDisposable(`remove owned Docker Engine 27 daemon ${daemonName}`, dispose); -} - function requiredEnvironment(name: string): string { const value = process.env[name]; if (!value) throw new Error(`${name} is required`); diff --git a/test/e2e/live/managed-image-multiarch-startup.test.ts b/test/e2e/live/managed-image-multiarch-startup.test.ts index 2f93c01e7c5..83bde0e002c 100644 --- a/test/e2e/live/managed-image-multiarch-startup.test.ts +++ b/test/e2e/live/managed-image-multiarch-startup.test.ts @@ -13,12 +13,8 @@ import { } from "../../../scripts/checks/protected-managed-image-contract.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { - DOCKER_ENGINE_27_PROBE_TIMEOUT_MS, - dockerEngine27ReceiptDaemonName, protectedManagedImageDispatchEnvironment, readRegularArtifact, - registerDockerEngine27ReceiptCleanup, - shouldRunDockerEngine27ReceiptProbe, } from "./managed-image-multiarch-startup-helpers.ts"; import { trustedShellCommand } from "../fixtures/shell-probe.ts"; @@ -64,8 +60,8 @@ test( ); progress.phase("verify Docker Engine 27 receipt transfer"); - const docker27DaemonName = dockerEngine27ReceiptDaemonName(dispatch.runId, dispatch.runAttempt); - registerDockerEngine27ReceiptCleanup(cleanup, docker27DaemonName, () => { + const docker27DaemonName = `nemoclaw-receipt-engine27-${String(dispatch.runId)}-${String(dispatch.runAttempt)}`; + cleanup.trackDisposable(`remove owned Docker Engine 27 daemon ${docker27DaemonName}`, () => { execFileSync( process.execPath, [ @@ -80,34 +76,35 @@ test( cwd: dispatch.workspace, killSignal: "SIGKILL", stdio: "ignore", - timeout: 45_000, + timeout: 90_000, }, ); }); - const docker27Result = shouldRunDockerEngine27ReceiptProbe(dispatch.platform) - ? await shellProbe.run( - trustedShellCommand({ - command: process.execPath, - args: [ - "--import", - "tsx", - "scripts/checks/docker-engine-27-receipt-transfer-e2e.ts", - "--daemon-name", - docker27DaemonName, - ], - reason: "verify the Docker Engine 27 receipt archive-copy boundary", - }), - { - artifactName: "docker-engine-27-receipt-transfer", - cwd: dispatch.workspace, - env: { - ...(process.env.DOCKER_CONFIG ? { DOCKER_CONFIG: process.env.DOCKER_CONFIG } : {}), - ...(process.env.HOME ? { HOME: process.env.HOME } : {}), + const docker27Result = + dispatch.platform === "linux/amd64" + ? await shellProbe.run( + trustedShellCommand({ + command: process.execPath, + args: [ + "--import", + "tsx", + "scripts/checks/docker-engine-27-receipt-transfer-e2e.ts", + "--daemon-name", + docker27DaemonName, + ], + reason: "verify the Docker Engine 27 receipt archive-copy boundary", + }), + { + artifactName: "docker-engine-27-receipt-transfer", + cwd: dispatch.workspace, + env: { + ...(process.env.DOCKER_CONFIG ? { DOCKER_CONFIG: process.env.DOCKER_CONFIG } : {}), + ...(process.env.HOME ? { HOME: process.env.HOME } : {}), + }, + timeoutMs: 30 * 60_000, }, - timeoutMs: DOCKER_ENGINE_27_PROBE_TIMEOUT_MS, - }, - ) - : { exitCode: 0, timedOut: false }; + ) + : { exitCode: 0, timedOut: false }; expect( docker27Result.exitCode === 0 && diff --git a/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts b/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts index 7e723dce417..3ce85d7007b 100644 --- a/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts +++ b/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts @@ -8,16 +8,13 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { + DOCKER_ENGINE_27_MINIMUM_CLEANUP_PROCESS_TIMEOUT_MS, DOCKER_ENGINE_27_MINIMUM_PROBE_TIMEOUT_MS, validateDockerEngine27SeedIsolation, } from "../../../scripts/checks/docker-engine-27-receipt-transfer-e2e.ts"; import { - DOCKER_ENGINE_27_PROBE_TIMEOUT_MS, - dockerEngine27ReceiptDaemonName, protectedManagedImageDispatchEnvironment, readRegularArtifact, - registerDockerEngine27ReceiptCleanup, - shouldRunDockerEngine27ReceiptProbe, } from "../live/managed-image-multiarch-startup-helpers.ts"; const sha = "a".repeat(40); @@ -46,26 +43,9 @@ afterEach(() => { }); describe("protected managed-image startup helpers", () => { - it("runs one bounded Docker 27 probe and registers exact cleanup first", () => { - const daemonName = dockerEngine27ReceiptDaemonName(123, 4); - const dispose = vi.fn(); - const trackDisposable = vi.fn(); - - registerDockerEngine27ReceiptCleanup({ trackDisposable }, daemonName, dispose); - const registeredCleanup = trackDisposable.mock.calls[0]?.[1]; - registeredCleanup?.(); - - expect(daemonName).toBe("nemoclaw-receipt-engine27-123-4"); - expect(shouldRunDockerEngine27ReceiptProbe("linux/amd64")).toBe(true); - expect(shouldRunDockerEngine27ReceiptProbe("linux/arm64")).toBe(false); - expect(DOCKER_ENGINE_27_PROBE_TIMEOUT_MS).toBeGreaterThanOrEqual( - DOCKER_ENGINE_27_MINIMUM_PROBE_TIMEOUT_MS, - ); - expect(trackDisposable).toHaveBeenCalledWith( - `remove owned Docker Engine 27 daemon ${daemonName}`, - dispose, - ); - expect(dispose).toHaveBeenCalledOnce(); + it("bounds the complete Docker 27 probe and external cleanup", () => { + expect(30 * 60_000).toBeGreaterThanOrEqual(DOCKER_ENGINE_27_MINIMUM_PROBE_TIMEOUT_MS); + expect(90_000).toBeGreaterThanOrEqual(DOCKER_ENGINE_27_MINIMUM_CLEANUP_PROCESS_TIMEOUT_MS); }); it("requires numeric root and every receipt seed isolation control", () => { From acbf8e4ecfeff76d92700b8691a924048068cc6d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 17 Sep 2026 20:11:02 -0700 Subject: [PATCH 14/39] test(e2e): consolidate Docker 27 probe execution Signed-off-by: Aaron Erickson --- .../docker-engine-27-receipt-transfer-e2e.ts | 91 +++++++------------ .../managed-image-multiarch-startup.test.ts | 9 +- 2 files changed, 38 insertions(+), 62 deletions(-) diff --git a/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts b/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts index 7c7fe43ee41..1a1ff0835ea 100644 --- a/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts +++ b/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts @@ -47,57 +47,24 @@ type CommandResult = { readonly stdout: string; }; -function runDocker(args: readonly string[]): CommandResult { - const result = spawnSync("docker", [...args], { - encoding: "utf8", - killSignal: "SIGKILL", - maxBuffer: 8 * 1024 * 1024, - timeout: DOCKER_ENGINE_27_OPERATION_TIMEOUT_MS, - }); - return { - ...(result.error ? { error: result.error } : {}), - status: result.status, - stderr: result.stderr ?? "", - stdout: result.stdout ?? "", - }; -} +type DockerCommandProfile = "cleanup" | "operation" | "pull" | "readiness"; -function runDockerPull(args: readonly string[]): CommandResult { +function runDocker( + args: readonly string[], + profile: DockerCommandProfile = "operation", +): CommandResult { const result = spawnSync("docker", [...args], { encoding: "utf8", killSignal: "SIGKILL", - maxBuffer: 8 * 1024 * 1024, - timeout: DOCKER_ENGINE_27_PULL_TIMEOUT_MS, - }); - return { - ...(result.error ? { error: result.error } : {}), - status: result.status, - stderr: result.stderr ?? "", - stdout: result.stdout ?? "", - }; -} - -function runDockerReadiness(args: readonly string[]): CommandResult { - const result = spawnSync("docker", [...args], { - encoding: "utf8", - killSignal: "SIGKILL", - maxBuffer: 1024 * 1024, - timeout: DOCKER_ENGINE_27_READINESS_COMMAND_TIMEOUT_MS, - }); - return { - ...(result.error ? { error: result.error } : {}), - status: result.status, - stderr: result.stderr ?? "", - stdout: result.stdout ?? "", - }; -} - -function runDockerCleanup(args: readonly string[]): CommandResult { - const result = spawnSync("docker", [...args], { - encoding: "utf8", - killSignal: "SIGKILL", - maxBuffer: 1024 * 1024, - timeout: DOCKER_ENGINE_27_CLEANUP_TIMEOUT_MS, + maxBuffer: profile === "operation" || profile === "pull" ? 8 * 1024 * 1024 : 1024 * 1024, + timeout: + profile === "pull" + ? DOCKER_ENGINE_27_PULL_TIMEOUT_MS + : profile === "readiness" + ? DOCKER_ENGINE_27_READINESS_COMMAND_TIMEOUT_MS + : profile === "cleanup" + ? DOCKER_ENGINE_27_CLEANUP_TIMEOUT_MS + : DOCKER_ENGINE_27_OPERATION_TIMEOUT_MS, }); return { ...(result.error ? { error: result.error } : {}), @@ -124,7 +91,7 @@ function requireCondition(condition: unknown, detail: string): asserts condition async function waitForDocker27(daemonName: string): Promise { for (let attempt = 0; attempt < DOCKER_ENGINE_27_READINESS_ATTEMPTS; attempt += 1) { - if (runDockerReadiness(["exec", daemonName, "docker", "info"]).status === 0) return; + if (runDocker(["exec", daemonName, "docker", "info"], "readiness").status === 0) return; await new Promise((resolve) => setTimeout(resolve, DOCKER_ENGINE_27_READINESS_INTERVAL_MS), ); @@ -194,13 +161,16 @@ function requireAbsent(result: CommandResult, resource: string): void { } export function cleanupDockerEngine27ReceiptDaemon(daemonName: string): void { - const inspected = runDockerCleanup([ - "container", - "inspect", - "--format", - `{{ index .Config.Labels "${DAEMON_OWNER_LABEL}" }}`, - daemonName, - ]); + const inspected = runDocker( + [ + "container", + "inspect", + "--format", + `{{ index .Config.Labels "${DAEMON_OWNER_LABEL}" }}`, + daemonName, + ], + "cleanup", + ); if (inspected.status !== 0) { requireCondition( /No such (?:container|object)/iu.test(commandDetail(inspected)), @@ -212,8 +182,11 @@ export function cleanupDockerEngine27ReceiptDaemon(daemonName: string): void { inspected.stdout.trim() === daemonName, "refusing to remove a Docker Engine 27 daemon with mismatched ownership", ); - requireSuccess(runDockerCleanup(["rm", "-f", daemonName]), "remove Docker Engine 27 daemon"); - requireAbsent(runDockerCleanup(["container", "inspect", daemonName]), "Docker Engine 27 daemon"); + requireSuccess(runDocker(["rm", "-f", daemonName], "cleanup"), "remove Docker Engine 27 daemon"); + requireAbsent( + runDocker(["container", "inspect", daemonName], "cleanup"), + "Docker Engine 27 daemon", + ); } async function verifyDockerEngine27ReceiptTransfer(daemonName: string): Promise { @@ -233,7 +206,7 @@ async function verifyDockerEngine27ReceiptTransfer(daemonName: string): Promise< try { requireSuccess( - runDockerPull(["pull", DIND_IMAGE]), + runDocker(["pull", DIND_IMAGE], "pull"), "pull digest-pinned Docker Engine 27 image", ); requireSuccess( @@ -265,7 +238,7 @@ async function verifyDockerEngine27ReceiptTransfer(daemonName: string): Promise< ); requireSuccess( - runDockerPull(["exec", daemonName, "docker", "pull", RECEIPT_IMAGE]), + runDocker(["exec", daemonName, "docker", "pull", RECEIPT_IMAGE], "pull"), "pull digest-pinned receipt image", ); requireSuccess( diff --git a/test/e2e/live/managed-image-multiarch-startup.test.ts b/test/e2e/live/managed-image-multiarch-startup.test.ts index 83bde0e002c..f2228fc3340 100644 --- a/test/e2e/live/managed-image-multiarch-startup.test.ts +++ b/test/e2e/live/managed-image-multiarch-startup.test.ts @@ -104,11 +104,14 @@ test( timeoutMs: 30 * 60_000, }, ) - : { exitCode: 0, timedOut: false }; + : null; expect( - docker27Result.exitCode === 0 && - !docker27Result.timedOut && + (docker27Result === null + ? dispatch.platform === "linux/arm64" + : docker27Result.exitCode === 0 && + !docker27Result.timedOut && + dispatch.platform === "linux/amd64") && evidence.contractSha256 === `sha256:${createHash("sha256").update(contractBytes).digest("hex")}` && JSON.stringify(evidence.contracts) === JSON.stringify(contracts), From 42334efdd942a0bf3f3e5a7d542ab883e1b9b0a6 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 17 Sep 2026 20:56:49 -0700 Subject: [PATCH 15/39] test(e2e): isolate Docker 27 probe per platform Signed-off-by: Aaron Erickson --- .../live/managed-image-multiarch-startup-helpers.ts | 8 ++++++++ test/e2e/live/managed-image-multiarch-startup.test.ts | 7 ++++++- .../managed-image-multiarch-startup-helpers.test.ts | 10 ++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/test/e2e/live/managed-image-multiarch-startup-helpers.ts b/test/e2e/live/managed-image-multiarch-startup-helpers.ts index 27b011abe80..c7ecf4cff51 100644 --- a/test/e2e/live/managed-image-multiarch-startup-helpers.ts +++ b/test/e2e/live/managed-image-multiarch-startup-helpers.ts @@ -25,6 +25,14 @@ export interface ProtectedManagedImageDispatchEnvironment { workspace: string; } +export function dockerEngine27ReceiptDaemonName( + runId: number, + runAttempt: number, + platform: ProtectedManagedImagePlatform, +): string { + return `nemoclaw-receipt-engine27-${String(runId)}-${String(runAttempt)}-${platform.replace("/", "-")}`; +} + function requiredEnvironment(name: string): string { const value = process.env[name]; if (!value) throw new Error(`${name} is required`); diff --git a/test/e2e/live/managed-image-multiarch-startup.test.ts b/test/e2e/live/managed-image-multiarch-startup.test.ts index f2228fc3340..b8c44c080f9 100644 --- a/test/e2e/live/managed-image-multiarch-startup.test.ts +++ b/test/e2e/live/managed-image-multiarch-startup.test.ts @@ -13,6 +13,7 @@ import { } from "../../../scripts/checks/protected-managed-image-contract.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { + dockerEngine27ReceiptDaemonName, protectedManagedImageDispatchEnvironment, readRegularArtifact, } from "./managed-image-multiarch-startup-helpers.ts"; @@ -60,7 +61,11 @@ test( ); progress.phase("verify Docker Engine 27 receipt transfer"); - const docker27DaemonName = `nemoclaw-receipt-engine27-${String(dispatch.runId)}-${String(dispatch.runAttempt)}`; + const docker27DaemonName = dockerEngine27ReceiptDaemonName( + dispatch.runId, + dispatch.runAttempt, + dispatch.platform, + ); cleanup.trackDisposable(`remove owned Docker Engine 27 daemon ${docker27DaemonName}`, () => { execFileSync( process.execPath, diff --git a/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts b/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts index 3ce85d7007b..821105bc0fd 100644 --- a/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts +++ b/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts @@ -13,6 +13,7 @@ import { validateDockerEngine27SeedIsolation, } from "../../../scripts/checks/docker-engine-27-receipt-transfer-e2e.ts"; import { + dockerEngine27ReceiptDaemonName, protectedManagedImageDispatchEnvironment, readRegularArtifact, } from "../live/managed-image-multiarch-startup-helpers.ts"; @@ -43,6 +44,15 @@ afterEach(() => { }); describe("protected managed-image startup helpers", () => { + it("isolates Docker 27 daemon ownership across matrix platforms", () => { + const amd64 = dockerEngine27ReceiptDaemonName(123, 4, "linux/amd64"); + const arm64 = dockerEngine27ReceiptDaemonName(123, 4, "linux/arm64"); + + expect(amd64).toBe("nemoclaw-receipt-engine27-123-4-linux-amd64"); + expect(arm64).toBe("nemoclaw-receipt-engine27-123-4-linux-arm64"); + expect(amd64).not.toBe(arm64); + }); + it("bounds the complete Docker 27 probe and external cleanup", () => { expect(30 * 60_000).toBeGreaterThanOrEqual(DOCKER_ENGINE_27_MINIMUM_PROBE_TIMEOUT_MS); expect(90_000).toBeGreaterThanOrEqual(DOCKER_ENGINE_27_MINIMUM_CLEANUP_PROCESS_TIMEOUT_MS); From ebe900ed851f069e964a18b7b1dc2e7cddcda80d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 17 Sep 2026 21:48:01 -0700 Subject: [PATCH 16/39] test(e2e): preserve Docker 27 probe evidence Signed-off-by: Aaron Erickson --- .../docker-engine-27-receipt-transfer-e2e.ts | 54 ++++++++++++++++--- .../managed-image-multiarch-startup.test.ts | 54 +++++++++---------- ...ed-image-multiarch-startup-helpers.test.ts | 16 ++++++ 3 files changed, 88 insertions(+), 36 deletions(-) diff --git a/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts b/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts index 1a1ff0835ea..37af891743d 100644 --- a/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts +++ b/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts @@ -189,6 +189,41 @@ export function cleanupDockerEngine27ReceiptDaemon(daemonName: string): void { ); } +function errorDetail(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +export function finalizeDockerEngine27ReceiptProbe( + primaryError: unknown | null, + cleanupDaemon: () => void, + cleanupFixture: () => void, +): void { + const cleanupErrors: unknown[] = []; + try { + cleanupDaemon(); + } catch (error) { + cleanupErrors.push(error); + } + try { + cleanupFixture(); + } catch (error) { + cleanupErrors.push(error); + } + if (primaryError !== null) { + if (cleanupErrors.length === 0) throw primaryError; + throw new AggregateError( + [primaryError, ...cleanupErrors], + `Docker Engine 27 receipt probe failed: ${errorDetail(primaryError)}; cleanup also failed: ${cleanupErrors.map(errorDetail).join("; ")}`, + ); + } + if (cleanupErrors.length > 0) { + throw new AggregateError( + cleanupErrors, + `Docker Engine 27 receipt probe cleanup failed: ${cleanupErrors.map(errorDetail).join("; ")}`, + ); + } +} + async function verifyDockerEngine27ReceiptTransfer(daemonName: string): Promise { const suffix = randomUUID().replaceAll("-", ""); const legacySeed = `nemoclaw-receipt-legacy-seed-${suffix}`; @@ -196,6 +231,7 @@ async function verifyDockerEngine27ReceiptTransfer(daemonName: string): Promise< const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-receipt-engine27-")); const fixtureReceipt = path.join(fixtureRoot, "receipt"); const daemonReceipt = `/nemoclaw-receipt-${suffix}`; + let primaryError: unknown | null = null; let successfulVolume: string | null = null; fs.mkdirSync(fixtureReceipt, { mode: 0o700 }); @@ -363,13 +399,19 @@ async function verifyDockerEngine27ReceiptTransfer(daemonName: string): Promise< ); requireAbsent(innerDocker(["container", "inspect", failedSeed]), "failed receipt seed"); requireAbsent(innerDocker(["volume", "inspect", failedVolume]), "failed receipt volume"); - } finally { - innerDocker(["rm", "-f", legacySeed]); - innerDocker(["volume", "rm", legacyVolume]); - if (successfulVolume) innerDocker(["volume", "rm", successfulVolume]); - cleanupDockerEngine27ReceiptDaemon(daemonName); - fs.rmSync(fixtureRoot, { force: true, recursive: true }); + } catch (error) { + primaryError = error; } + finalizeDockerEngine27ReceiptProbe( + primaryError, + () => { + innerDocker(["rm", "-f", legacySeed]); + innerDocker(["volume", "rm", legacyVolume]); + if (successfulVolume) innerDocker(["volume", "rm", successfulVolume]); + cleanupDockerEngine27ReceiptDaemon(daemonName); + }, + () => fs.rmSync(fixtureRoot, { force: true, recursive: true }), + ); } function parseDaemonName(args: readonly string[]): string { diff --git a/test/e2e/live/managed-image-multiarch-startup.test.ts b/test/e2e/live/managed-image-multiarch-startup.test.ts index b8c44c080f9..8a629a65cb9 100644 --- a/test/e2e/live/managed-image-multiarch-startup.test.ts +++ b/test/e2e/live/managed-image-multiarch-startup.test.ts @@ -85,38 +85,32 @@ test( }, ); }); - const docker27Result = - dispatch.platform === "linux/amd64" - ? await shellProbe.run( - trustedShellCommand({ - command: process.execPath, - args: [ - "--import", - "tsx", - "scripts/checks/docker-engine-27-receipt-transfer-e2e.ts", - "--daemon-name", - docker27DaemonName, - ], - reason: "verify the Docker Engine 27 receipt archive-copy boundary", - }), - { - artifactName: "docker-engine-27-receipt-transfer", - cwd: dispatch.workspace, - env: { - ...(process.env.DOCKER_CONFIG ? { DOCKER_CONFIG: process.env.DOCKER_CONFIG } : {}), - ...(process.env.HOME ? { HOME: process.env.HOME } : {}), - }, - timeoutMs: 30 * 60_000, - }, - ) - : null; + const docker27Result = await shellProbe.run( + trustedShellCommand({ + command: process.execPath, + args: [ + "--import", + "tsx", + "scripts/checks/docker-engine-27-receipt-transfer-e2e.ts", + "--daemon-name", + docker27DaemonName, + ], + reason: "verify the Docker Engine 27 receipt archive-copy boundary", + }), + { + artifactName: "docker-engine-27-receipt-transfer", + cwd: dispatch.workspace, + env: { + ...(process.env.DOCKER_CONFIG ? { DOCKER_CONFIG: process.env.DOCKER_CONFIG } : {}), + ...(process.env.HOME ? { HOME: process.env.HOME } : {}), + }, + timeoutMs: 30 * 60_000, + }, + ); expect( - (docker27Result === null - ? dispatch.platform === "linux/arm64" - : docker27Result.exitCode === 0 && - !docker27Result.timedOut && - dispatch.platform === "linux/amd64") && + docker27Result.exitCode === 0 && + !docker27Result.timedOut && evidence.contractSha256 === `sha256:${createHash("sha256").update(contractBytes).digest("hex")}` && JSON.stringify(evidence.contracts) === JSON.stringify(contracts), diff --git a/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts b/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts index 821105bc0fd..e335a4940f5 100644 --- a/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts +++ b/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts @@ -10,6 +10,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { DOCKER_ENGINE_27_MINIMUM_CLEANUP_PROCESS_TIMEOUT_MS, DOCKER_ENGINE_27_MINIMUM_PROBE_TIMEOUT_MS, + finalizeDockerEngine27ReceiptProbe, validateDockerEngine27SeedIsolation, } from "../../../scripts/checks/docker-engine-27-receipt-transfer-e2e.ts"; import { @@ -44,6 +45,21 @@ afterEach(() => { }); describe("protected managed-image startup helpers", () => { + it("preserves the probe failure and attempts fixture cleanup after daemon cleanup fails", () => { + const cleanupFixture = vi.fn(); + + expect(() => + finalizeDockerEngine27ReceiptProbe( + new Error("receipt transfer failed"), + () => { + throw new Error("daemon cleanup failed"); + }, + cleanupFixture, + ), + ).toThrow(/receipt transfer failed.*daemon cleanup failed/u); + expect(cleanupFixture).toHaveBeenCalledOnce(); + }); + it("isolates Docker 27 daemon ownership across matrix platforms", () => { const amd64 = dockerEngine27ReceiptDaemonName(123, 4, "linux/amd64"); const arm64 = dockerEngine27ReceiptDaemonName(123, 4, "linux/arm64"); From 56a6b1c2232ef3a5d6be27c79dc42422ea77301f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aaron=20Erickson=20=F0=9F=A6=9E?= Date: Thu, 17 Sep 2026 22:19:12 -0700 Subject: [PATCH 17/39] feat(installer): add force-fresh macOS reset (#12041) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Outcome The macOS installer now accepts `--force-fresh-install` when an operator wants to discard every NemoClaw and OpenShell sandbox, gateway, credential, configuration, and recovery record before reinstalling. The reset stages the selected candidate outside the active state root, keeps model caches, and starts onboarding with fresh state. ## Reason An interrupted install can leave a new NemoClaw CLI with old OpenShell services, stale registry entries, recovery artifacts, or conflicting shims. The ordinary installer must preserve uncertain state, so repeated repair attempts can stop at a different guard each time. An explicit destructive reset gives operators a bounded alternative to reconciling every partial-install shape. ## Changes - Add `--force-fresh-install` and `NEMOCLAW_FORCE_FRESH_INSTALL=1` to the public bootstrap and versioned installer. - Stack the reset on PR #12015 and retain its Docker 27 receipt compatibility. Resolve a persisted local Colima context to its Unix socket before reset or reinstall effects. - Bypass implicit maintained-version downgrade inspection only when the destructive reset is explicit. The selected candidate is already cloned before this decision. - Build the selected candidate in the bootstrap checkout before cleanup. Use that candidate to run the existing all-gateway uninstaller with `--destroy-user-data`, so a broken installed CLI is not part of the reset authority. - On macOS, stop and remove the Homebrew OpenShell formula and remove standalone user-local OpenShell binaries before the pinned reinstall. - If Homebrew reports its known status-69 post-install failure after placing OpenShell, rerun the existing pinned version, formula, driver, and capability verifier. Continue only when that independent verifier passes. - Skip pre-upgrade backup and recovery for this destructive path. Preserve model caches and unrelated `nemoclaw-acp` files. - Reject the option outside macOS and reject a cleanup candidate staged inside the state root. - Add isolated installer tests for accepted input, operation order, cleanup command construction, Homebrew removal, failure propagation, platform scope, source isolation, and unrelated-shim preservation. ## Verification - Exact head `c25a8816f26e598963115b9272ced2c5604ac17a`: `npx vitest run test/installer-integration/install-force-fresh.test.ts` — 15 tests passed. - Exact head `c25a8816f26e598963115b9272ced2c5604ac17a`: `bash scripts/check-installer-hash.sh` — all pinned OpenShell release assets and protected installer templates passed. - Focused force-reset and Colima recovery tests — 48 tests passed. - `npm exec vitest run -- --project integration test/mcp/mcp-tool-discovery-image-contract.test.ts` — 19 tests passed, including the contract that failed on the prior `main`-based candidate. - `npm exec vitest run -- --project integration test/install/install-macos-arch-guard.test.ts test/install/install-hugging-face-guidance.test.ts` — 6 tests passed. - `npm exec vitest run -- --project integration test/automation/pull-requests/growth-guardrails.test.ts` — 7 tests passed. - `npm run test:projects:check` — project membership passed. - `npm run test:titles:check` — test-title validation passed. - `bash -n install.sh && bash -n scripts/install.sh` — passed. - `shellcheck install.sh scripts/install.sh` — passed. - `npm run build:cli` — passed. - Commit hooks — formatting, lint, repository checks, ShellCheck, secret scan, source-shape, and growth checks passed. - Live destructive macOS/Colima install from exact PR head `c25a8816f26e598963115b9272ced2c5604ac17a` — exited 0 after 388 seconds. The run reproduced Homebrew status 69, passed the pinned-runtime retry, detected Colima, disabled sandbox GPU, created `hermes`, and verified the OpenShell gateway, dashboard forward, and inference route. - Post-install inspection — installed source/build identity matches `c25a8816f26e598963115b9272ced2c5604ac17a`; OpenShell and the Homebrew formula are `0.0.116`; gateway `nemoclaw-18084` and sandbox `hermes` are healthy; stale recovery paths are absent; the existing 2.9 GB Ollama model cache remains. - Exact-head CI — all required checks passed, including 12/12 CLI shards, installer integration, build/typecheck, static checks, CodeQL, OpenClaw managed-image build/contract/direct-startup/publication, and real all-agent managed runtime activation. - Diff inspection — no secrets, API keys, or credentials are present. ## Review notes This draft changes the sensitive installer and bootstrap paths at exact head `c25a8816f26e598963115b9272ced2c5604ac17a` and is stacked on PR #12015. The author completed a local self-review against `NVIDIA/NemoClaw`; no independent pre-publication review exists. The destructive reset removed prior NemoClaw/OpenShell sandboxes, gateways, credentials, configuration, and recovery state from the test Mac; that deleted state is not recoverable without an external backup. Model caches and an unrelated `nemoclaw-acp` file were preserved. Exact-head CI is green; independent review remains required. --- Signed-off-by: Aaron Erickson --------- Signed-off-by: Aaron Erickson --- install.sh | 17 +- scripts/install.sh | 283 +++++++++++++- ...stall-preexisting-sandbox-recovery.test.ts | 28 +- .../install-force-fresh.test.ts | 359 ++++++++++++++++++ 4 files changed, 678 insertions(+), 9 deletions(-) create mode 100644 test/installer-integration/install-force-fresh.test.ts diff --git a/install.sh b/install.sh index 4b469a1a435..44b7d830da1 100755 --- a/install.sh +++ b/install.sh @@ -262,7 +262,9 @@ exec_installer_from_ref() { clone_nemoclaw_ref "$ref" "$source_root" selected_commit="$(git -C "$source_root" rev-parse HEAD)" - guard_implicit_maintained_downgrade "$source_root" "$ref" + if ! bootstrap_force_fresh_install_requested "$@"; then + guard_implicit_maintained_downgrade "$source_root" "$ref" + fi payload_script="${source_root}/scripts/install.sh" legacy_script="${source_root}/install.sh" @@ -284,6 +286,17 @@ exec_installer_from_ref() { NEMOCLAW_INSTALL_TAG="$ref" bash "$legacy_script" "$@" } +bootstrap_force_fresh_install_requested() { + local arg + case "${NEMOCLAW_FORCE_FRESH_INSTALL:-}" in + 1 | true | TRUE | yes | YES | y | Y) return 0 ;; + esac + for arg in "$@"; do + [[ "$arg" == "--force-fresh-install" ]] && return 0 + done + return 1 +} + require_supported_platform() { # macOS ships only an Apple Silicon (aarch64) OpenShell gateway build, so an # Intel Mac (x86_64 Darwin) install always fails once that binary is fetched. @@ -313,6 +326,7 @@ bootstrap_usage() { printf " Use only with NEMOCLAW_AGENT=hermes, no registered sandboxes, no local model profile,\n" printf " and the build, cloud, or routed NVIDIA hosted provider\n" printf " --fresh Discard any failed/interrupted onboarding session and start over\n" + printf " --force-fresh-install Destroy all NemoClaw and OpenShell state, then reinstall\n" printf " --force-station-install Validate an unrecognized Station GB300 release profile without onboarding\n" printf " --version, -v Print installer version and exit\n" printf " --help, -h Show this help message and exit\n\n" @@ -326,6 +340,7 @@ bootstrap_usage() { printf " Use only with NEMOCLAW_AGENT=hermes, no registered sandboxes, no local model profile,\n" printf " and the build, cloud, or routed NVIDIA hosted provider\n" printf " NEMOCLAW_FRESH=1 Same as --fresh\n" + printf " NEMOCLAW_FORCE_FRESH_INSTALL=1 Same as --force-fresh-install\n" printf " NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 Same as --yes-i-accept-third-party-software\n" printf " NEMOCLAW_NO_EXPRESS=1 Skip express install prompt on supported platforms\n" printf " NEMOCLAW_SANDBOX_NAME Sandbox name to create/use\n" diff --git a/scripts/install.sh b/scripts/install.sh index e9106c5aa25..604bfb46abf 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -1056,6 +1056,7 @@ usage() { printf " Use only with NEMOCLAW_AGENT=hermes, no registered sandboxes, no local model profile,\n" printf " and the build, cloud, or routed NVIDIA hosted provider\n" printf " --fresh Discard any failed/interrupted onboarding session and start over\n" + printf " --force-fresh-install Destroy all NemoClaw and OpenShell state, then reinstall\n" printf " --station-deepseek Use DeepSeek V4 Flash for DGX Station express install (interactive terminal required)\n" printf " --force-station-install Validate an unrecognized Station GB300 release profile without onboarding\n" printf " --version, -v Print installer version and exit\n" @@ -1069,6 +1070,7 @@ usage() { printf " and the build, cloud, or routed NVIDIA hosted provider\n" printf " NEMOCLAW_NON_INTERACTIVE_SUDO_MODE=prompt Allow sudo prompts during non-interactive onboarding\n" printf " NEMOCLAW_FRESH=1 Same as --fresh\n" + printf " NEMOCLAW_FORCE_FRESH_INSTALL=1 Same as --force-fresh-install\n" printf " NEMOCLAW_NO_EXPRESS=1 Skip the Express prompt on detected platforms\n" printf " NEMOCLAW_SANDBOX_NAME Sandbox name to create/use\n" printf " HF_TOKEN Optional Hugging Face read token for managed-vLLM downloads\n" @@ -1369,7 +1371,7 @@ installer_docker_host_has_supported_shape() { # Admit a usable socket and the default context before Docker or recovery effects. # Persisted JSON inspection can wait only for its missing Node.js prerequisite. validate_installer_docker_target_before_host_changes() { - local raw="${DOCKER_HOST-}" candidate active_context="" + local raw="${DOCKER_HOST-}" candidate active_context="" context_host="" installer_docker_host_has_supported_shape \ || error "DOCKER_HOST is not a supported absolute local Unix socket endpoint. Unset DOCKER_HOST or set it to an absolute local Unix socket URL, such as unix:///var/run/docker.sock. Then rerun the installer." candidate="${raw#"${raw%%[![:space:]]*}"}" @@ -1386,8 +1388,21 @@ validate_installer_docker_target_before_host_changes() { fi active_context="$(docker_active_context)" fi - [[ "$active_context" == default ]] \ - || error "The Docker context does not select the local default target. Unset DOCKER_CONTEXT or set it to default, and run 'docker context use default' if a non-default context is persisted. Then rerun the installer." + if [[ "$active_context" != default ]]; then + context_host="$(docker context inspect "$active_context" --format '{{.Endpoints.docker.Host}}' 2>/dev/null || true)" + [[ -n "$context_host" ]] \ + || error "The Docker context does not select a supported local Unix socket. Select a local Docker or Colima context, then rerun the installer." + DOCKER_HOST="$context_host" + installer_docker_host_has_supported_shape \ + || error "The Docker context does not select a supported local Unix socket. Select a local Docker or Colima context, then rerun the installer." + export DOCKER_HOST + unset DOCKER_CONTEXT + elif [[ -n "$candidate" ]]; then + unset DOCKER_CONTEXT + fi + # DOCKER_CONTEXT takes precedence over DOCKER_HOST in Docker clients. Once + # admission resolves a local socket, keep that socket authoritative through + # host preflight and automatic onboarding. } # Re-read persisted context after Node installation instead of trusting the temporary default. @@ -2287,7 +2302,14 @@ maybe_install_openshell_during_install() { fi if ! _NEMOCLAW_OPENSHELL_INSTALL_METHOD="$macos_install_method" \ spin "Installing OpenShell CLI" bash "${NEMOCLAW_SOURCE_ROOT}/scripts/install-openshell.sh"; then - return 1 + if [[ "$platform" == "Darwin" && "$macos_install_method" == "homebrew" ]] \ + && truthy_env "${FORCE_FRESH_INSTALL:-}" \ + && _NEMOCLAW_OPENSHELL_INSTALL_METHOD="$macos_install_method" \ + spin "Verifying the installed OpenShell CLI" bash "${NEMOCLAW_SOURCE_ROOT}/scripts/install-openshell.sh"; then + warn "Homebrew reported an install failure after placing OpenShell; the pinned OpenShell verifier passed, so force-fresh installation will continue." + else + return 1 + fi fi if [[ "$platform" == "Darwin" ]]; then observed_install_method="$(observed_macos_openshell_install_method)" || return 1 @@ -2434,6 +2456,11 @@ preflight_nemoclaw_acp_shim() { fi is_installer_managed_cli_shim "$shim_path" "nemoclaw-acp" "$cli_path" && return 0 is_npm_managed_nemoclaw_acp_link "$shim_path" "$cli_path" && return 0 + if truthy_env "${FORCE_FRESH_INSTALL:-}" && [[ "$cli_path" != "$shim_path" ]]; then + _NEMOCLAW_FORCE_FRESH_SKIP_ACP_SHIM=true + warn "Leaving unrelated $shim_path unchanged; the force-fresh install will not publish a nemoclaw-acp shim there." + return 0 + fi error "Installation stopped because $shim_path already exists and is not a NemoClaw-managed shim. NemoClaw left it unchanged. Move or remove that path, then rerun the installer." } @@ -2524,6 +2551,11 @@ ensure_cli_shim() { npm_bin="$(resolve_npm_bin)" || true shim_path="${NEMOCLAW_SHIM_DIR}/${cli_bin}" + if [[ "$cli_bin" == "nemoclaw-acp" ]] \ + && [[ "${_NEMOCLAW_FORCE_FRESH_SKIP_ACP_SHIM:-false}" == true ]]; then + return 0 + fi + if [[ -z "$npm_bin" || ! -x "$npm_bin/$cli_bin" ]]; then return 1 fi @@ -3463,6 +3495,234 @@ run_preupgrade_backup() { NEMOCLAW_REQUIRE_ALL_SANDBOX_BACKUPS=1 "$current_cli_runner" backup-all 2>&1 } +force_fresh_install_has_existing_state() { + if [[ -e "$(nemoclaw_state_root)" ]] \ + || [[ -e "${HOME}/.config/nemoclaw" ]] \ + || [[ -e "${HOME}/.config/openshell" ]] \ + || [[ -e "${HOME}/.local/state/nemoclaw" ]] \ + || [[ -e "${HOME}/.local/bin/nemoclaw" ]] \ + || [[ -e "${HOME}/.local/bin/openshell" ]] \ + || command_exists nemoclaw \ + || command_exists openshell; then + return 0 + fi + command_exists brew \ + && brew list --formula nvidia/openshell/openshell >/dev/null 2>&1 +} + +force_fresh_install_source_root() { + local source_root state_root state_parent + source_root="$(cd "${NEMOCLAW_SOURCE_ROOT}" && pwd -P)" \ + || error "Could not resolve the staged source for the force-fresh install." + state_root="$(nemoclaw_state_root)" \ + || error "Could not resolve the NemoClaw state root for the force-fresh install." + if [[ -d "$state_root" ]]; then + state_root="$(cd "$state_root" && pwd -P)" \ + || error "Could not resolve the existing NemoClaw state root." + else + state_parent="$(dirname "$state_root")" + state_root="$(cd "$state_parent" && pwd -P)/$(basename "$state_root")" \ + || error "Could not resolve the NemoClaw state parent directory." + fi + case "$source_root" in + "$state_root" | "$state_root"/*) + error "The force-fresh installer must run from the versioned bootstrap checkout outside ${state_root}. Use the public curl installer with --force-fresh-install." + ;; + esac + [[ -d "${source_root}/.git" && ! -L "${source_root}/.git" ]] \ + || error "The force-fresh installer requires a staged NemoClaw Git checkout." + printf '%s' "$source_root" +} + +prepare_force_fresh_uninstaller() { + local source_root="$1" + if [[ -z "${NEMOCLAW_AGENT:-}" || "${NEMOCLAW_AGENT}" == "openclaw" ]]; then + spin "Preparing OpenClaw package for force-fresh cleanup" bash -c "$(declare -f info warn resolve_openclaw_version pre_extract_openclaw); pre_extract_openclaw \"\$1\"" _ "$source_root" \ + || warn "Pre-extraction failed — npm install may fail if the OpenClaw tarball is broken" + fi + spin "Preparing the force-fresh uninstaller" bash -c \ + "cd \"$source_root\" && node scripts/lib/openshell-sdk-install.mts prepare && npm install --ignore-scripts --prefer-offline --include=optional --@nvidia:registry=https://npm.pkg.github.com && node scripts/lib/openshell-sdk-install.mts check && npm run --if-present build:cli" + [[ -s "${source_root}/dist/lib/actions/uninstall/run-plan.js" ]] \ + || error "The staged force-fresh uninstaller did not build." +} + +run_force_fresh_uninstaller() { + local source_root="$1" node_bin + node_bin="$(command -v node 2>/dev/null || true)" + [[ -n "$node_bin" && -x "$node_bin" ]] \ + || error "Node.js is required for the force-fresh uninstaller." + NEMOCLAW_UNINSTALL_DESTROY_USER_DATA=1 "$node_bin" \ + "${source_root}/bin/nemoclaw.js" internal uninstall run-plan \ + --yes --destroy-user-data --all-gateway-ports +} + +remove_force_fresh_state_path() { + local target="$1" + case "$target" in + "${HOME}/.nemoclaw" | "${HOME}/.config/nemoclaw" | "${HOME}/.config/openshell" | "${HOME}/.local/state/nemoclaw") ;; + *) error "Refusing force-fresh cleanup outside the supported user state roots: ${target}" ;; + esac + if [[ -L "$target" ]]; then + rm -f -- "$target" + else + rm -rf -- "$target" + fi +} + +remove_force_fresh_docker_resources() { + docker info >/dev/null 2>&1 \ + || error "Docker is not reachable. Start Colima, then rerun the force-fresh installer." + + local container_inventory labeled_container_ids volume_inventory network_inventory image_inventory + container_inventory="$(docker ps -a --format '{{.ID}} {{.Image}} {{.Names}}')" \ + || error "Could not inspect Docker containers during force-fresh cleanup." + labeled_container_ids="$(docker ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract)" \ + || error "Could not inspect labeled NemoClaw containers during force-fresh cleanup." + volume_inventory="$(docker volume ls --format '{{.Name}}')" \ + || error "Could not inspect Docker volumes during force-fresh cleanup." + network_inventory="$(docker network ls --format '{{.ID}} {{.Name}}')" \ + || error "Could not inspect Docker networks during force-fresh cleanup." + image_inventory="$(docker images --format '{{.ID}} {{.Repository}}')" \ + || error "Could not inspect Docker images during force-fresh cleanup." + + local id image name removed_container_ids=" " + while read -r id image name; do + [[ "$id" =~ ^[0-9a-f]{12,64}$ ]] || continue + case "$removed_container_ids" in + *" $id "*) continue ;; + esac + case "$name" in + openshell-* | nemoclaw-*) + docker rm -f "$id" >/dev/null + removed_container_ids="${removed_container_ids}${id} " + ;; + *) + case "$image" in + nemoclaw-* | openshell/* | ghcr.io/nvidia/nemoclaw | ghcr.io/nvidia/nemoclaw/* | ghcr.io/nvidia/nemoclaw-*) + docker rm -f "$id" >/dev/null + removed_container_ids="${removed_container_ids}${id} " + ;; + esac + ;; + esac + done <<<"$container_inventory" + while IFS= read -r id; do + [[ "$id" =~ ^[0-9a-f]{12,64}$ ]] || continue + case "$removed_container_ids" in + *" $id "*) continue ;; + esac + docker rm -f "$id" >/dev/null + removed_container_ids="${removed_container_ids}${id} " + done <<<"$labeled_container_ids" + + local resource + while IFS= read -r resource; do + case "$resource" in + openshell-* | nemoclaw-*) docker volume rm -f -- "$resource" >/dev/null ;; + esac + done <<<"$volume_inventory" + while read -r id name; do + [[ "$id" =~ ^[0-9a-f]{12,64}$ ]] || continue + case "$name" in + openshell-* | nemoclaw-*) docker network rm "$id" >/dev/null ;; + esac + done <<<"$network_inventory" + local removed_image_ids=" " + while read -r id image; do + [[ "$id" =~ ^[0-9a-f]{12,64}$ ]] || continue + case "$removed_image_ids" in + *" $id "*) continue ;; + esac + case "$image" in + nemoclaw-* | openshell/* | ghcr.io/nvidia/nemoclaw | ghcr.io/nvidia/nemoclaw/* | ghcr.io/nvidia/nemoclaw-*) + docker rmi -f "$id" >/dev/null + removed_image_ids="${removed_image_ids}${id} " + ;; + esac + done <<<"$image_inventory" +} + +remove_force_fresh_openshell_resources() { + if command_exists openshell; then + local gateway_name + # shellcheck disable=SC2016 # JavaScript template literal is evaluated by Node.js. + while IFS= read -r gateway_name; do + [[ "$gateway_name" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$ ]] || continue + openshell sandbox delete --all -g "$gateway_name" >/dev/null 2>&1 || true + done < <(openshell gateway list -o json 2>/dev/null | node -e ' +let input = ""; +process.stdin.setEncoding("utf8"); +process.stdin.on("data", (chunk) => { input += chunk; }); +process.stdin.on("end", () => { + try { + const rows = JSON.parse(input); + if (!Array.isArray(rows)) return; + for (const row of rows) { + if (row && typeof row.name === "string") process.stdout.write(`${row.name}\n`); + } + } catch {} +}); +') + fi + pkill -TERM -x openshell-gateway >/dev/null 2>&1 || true + sleep 1 + pkill -KILL -x openshell-gateway >/dev/null 2>&1 || true +} + +remove_force_fresh_install_state() { + remove_force_fresh_openshell_resources + remove_force_fresh_docker_resources + npm uninstall -g nemoclaw >/dev/null 2>&1 || true + remove_force_fresh_state_path "${HOME}/.nemoclaw" + remove_force_fresh_state_path "${HOME}/.config/nemoclaw" + remove_force_fresh_state_path "${HOME}/.config/openshell" + remove_force_fresh_state_path "${HOME}/.local/state/nemoclaw" +} + +remove_macos_openshell_for_force_fresh_install() { + local formula="nvidia/openshell/openshell" openshell_path="" + openshell_path="$(command -v openshell 2>/dev/null || true)" + if command_exists brew && brew list --formula "$formula" >/dev/null 2>&1; then + brew services stop "$formula" >/dev/null 2>&1 || true + brew uninstall --force "$formula" \ + || error "Homebrew could not remove OpenShell during the force-fresh install. Rerun the same command after Homebrew is healthy." + elif [[ "$openshell_path" == /opt/homebrew/* || "$openshell_path" == /usr/local/* ]]; then + error "The force-fresh installer found a system OpenShell binary that Homebrew did not identify as ${formula}. Remove that installation explicitly, then rerun." + fi + local binary + for binary in openshell openshell-gateway openshell-sandbox openshell-driver-vm; do + rm -f -- "${HOME}/.local/bin/${binary}" \ + || error "Could not remove ${HOME}/.local/bin/${binary} during the force-fresh install." + done + hash -r 2>/dev/null || true +} + +run_force_fresh_install_reset() { + local source_root + warn "Force-fresh install selected. NemoClaw will destroy all NemoClaw/OpenShell sandboxes, gateways, credentials, configuration, and recovery state on this host. Model caches are kept." + if force_fresh_install_has_existing_state; then + source_root="$(force_fresh_install_source_root)" + prepare_force_fresh_uninstaller "$source_root" + if ! run_force_fresh_uninstaller "$source_root"; then + warn "Managed uninstall could not reconcile all partial state; continuing with the explicit force-fresh cleanup." + fi + remove_force_fresh_install_state + remove_macos_openshell_for_force_fresh_install + else + info "No existing NemoClaw or OpenShell installation was found; continuing with a clean install." + fi + FRESH=1 + export NEMOCLAW_FRESH=1 + export NEMOCLAW_REINSTALL_CLI=1 + _PREEXISTING_SANDBOX_COUNT=0 +} + +validate_force_fresh_install_platform() { + truthy_env "${FORCE_FRESH_INSTALL:-}" || return 0 + [[ "$(uname -s)" == "Darwin" ]] \ + || error "--force-fresh-install currently supports macOS only." +} + # Return nonzero when OpenShell is absent or its version command fails. installed_openshell_version() { command_exists openshell || return 1 @@ -7032,6 +7292,9 @@ install_nemoclaw_before_onboarding() { bash "${SCRIPT_DIR}/setup-jetson.sh" prepare_installer_node_runtime + if truthy_env "${FORCE_FRESH_INSTALL:-}"; then + run_force_fresh_install_reset + fi ensure_station_express_pair step 2 "${_CLI_DISPLAY} CLI" @@ -7040,7 +7303,9 @@ install_nemoclaw_before_onboarding() { # install.sh stays focused on dependency setup. fix_npm_permissions preflight_nemoclaw_acp_shim - preinstall_backup_and_retire_legacy_gateway + if ! truthy_env "${FORCE_FRESH_INSTALL:-}"; then + preinstall_backup_and_retire_legacy_gateway + fi install_nemoclaw verify_nemoclaw require_reportable_openshell_version @@ -7066,6 +7331,7 @@ main() { ACCEPT_THIRD_PARTY_SOFTWARE="" DEFER_ONBOARDING="" FRESH="" + FORCE_FRESH_INSTALL="" STATION_DEEPSEEK="" FORCE_STATION_INSTALL="" LOCAL_MODEL_RUNTIME="" @@ -7085,6 +7351,7 @@ main() { --yes-i-accept-third-party-software) ACCEPT_THIRD_PARTY_SOFTWARE=1 ;; --defer-onboarding) DEFER_ONBOARDING=1 ;; --fresh) FRESH=1 ;; + --force-fresh-install) FORCE_FRESH_INSTALL=1 ;; --station-deepseek) STATION_DEEPSEEK=1 ;; --force-station-install) FORCE_STATION_INSTALL=1 ;; --local-model-runtime=*) @@ -7129,6 +7396,12 @@ main() { ACCEPT_THIRD_PARTY_SOFTWARE="${ACCEPT_THIRD_PARTY_SOFTWARE:-${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-}}" DEFER_ONBOARDING="${DEFER_ONBOARDING:-${NEMOCLAW_DEFER_ONBOARDING:-}}" FRESH="${FRESH:-${NEMOCLAW_FRESH:-}}" + FORCE_FRESH_INSTALL="${FORCE_FRESH_INSTALL:-${NEMOCLAW_FORCE_FRESH_INSTALL:-}}" + if truthy_env "${FORCE_FRESH_INSTALL:-}"; then + validate_force_fresh_install_platform + FRESH=1 + export FORCE_FRESH_INSTALL NEMOCLAW_FORCE_FRESH_INSTALL=1 + fi if [ -n "${LOCAL_MODEL_RUNTIME:-}" ]; then export NEMOCLAW_ENABLE_LOCAL_MODEL_PROFILE=1 export NEMOCLAW_LOCAL_MODEL_RUNTIME="$LOCAL_MODEL_RUNTIME" diff --git a/test/install/install-preexisting-sandbox-recovery.test.ts b/test/install/install-preexisting-sandbox-recovery.test.ts index d0625795551..ebb7efb7445 100644 --- a/test/install/install-preexisting-sandbox-recovery.test.ts +++ b/test/install/install-preexisting-sandbox-recovery.test.ts @@ -38,6 +38,7 @@ function runRecoveryBeforeOnboard( options: { detectedExpressPlatform?: string; dockerContext?: string; + dockerContextEndpoint?: string; dockerHost?: string; hostPreflightExitCode?: number; includeNodeOnPath?: boolean; @@ -185,6 +186,14 @@ exit 0 [[ "$*" == "info --format {{.Host.RemoteSocket.Path}}" ]] || return 99 printf '%s\n' "$PODMAN_SOCKET" } + docker() { + if [[ "\${1:-}" == "context" && "\${2:-}" == "inspect" ]]; then + [[ -n "$DOCKER_CONTEXT_ENDPOINT" ]] || return 1 + printf '%s\n' "$DOCKER_CONTEXT_ENDPOINT" + return 0 + fi + return 99 + } sleep() { printf 'sleep=%s\n' "$*" >> "${callLog}"; } step() { :; } install_nodejs() { @@ -221,6 +230,7 @@ exit 0 extraEnv: { DETECTED_EXPRESS_PLATFORM: options.detectedExpressPlatform ?? "", ...(options.dockerContext !== undefined ? { DOCKER_CONTEXT: options.dockerContext } : {}), + DOCKER_CONTEXT_ENDPOINT: options.dockerContextEndpoint ?? "", ...(dockerConfig === undefined ? {} : { DOCKER_CONFIG: dockerConfig }), ...(options.dockerHost !== undefined ? { DOCKER_HOST: options.dockerHost } : {}), NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE: "1", @@ -277,6 +287,19 @@ describe("install.sh pre-existing sandbox recovery ordering (#6114)", () => { expect(result.output).toContain("Existing sandboxes recovered; skipping generic onboarding"); }); + it("resolves a persisted local Colima context to its Unix socket", () => { + const result = runRecoveryBeforeOnboard(2, 0, { + dockerContextEndpoint: "unix:///Users/test/.colima/default/docker.sock", + persistedDockerContext: "colima", + recordRuntimeTarget: true, + }); + + expect(result.status, result.output).toBe(0); + expect(result.calls).toContain( + "cli-target=host:unix:///Users/test/.colima/default/docker.sock,context:unset argv=upgrade-sandboxes --auto", + ); + }); + it("treats a whitespace-only Docker host as unset", () => { const result = runRecoveryBeforeOnboard(2, 0, { dockerHost: " \t ", @@ -329,7 +352,7 @@ describe("install.sh pre-existing sandbox recovery ordering (#6114)", () => { expect(result.calls).not.toContain( 'restore=1 confirmed=["legacy-box"] argv=upgrade-sandboxes --auto', ); - expect(result.output).toContain("Docker context does not select the local default target"); + expect(result.output).toContain("Docker context does not select a supported local Unix socket"); }); it.each([ @@ -388,8 +411,7 @@ describe("install.sh pre-existing sandbox recovery ordering (#6114)", () => { expect(result.status).toBe(1); expect(result.calls).toEqual([]); - expect(result.output).toContain("Docker context does not select the local default target"); - expect(result.output).toContain("docker context use default"); + expect(result.output).toContain("Docker context does not select a supported local Unix socket"); }); it("rejects a non-local Podman socket before sandbox recovery", () => { diff --git a/test/installer-integration/install-force-fresh.test.ts b/test/installer-integration/install-force-fresh.test.ts new file mode 100644 index 00000000000..aea95f087f9 --- /dev/null +++ b/test/installer-integration/install-force-fresh.test.ts @@ -0,0 +1,359 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { expect, it, onTestFinished } from "vitest"; + +import { createInstallerCheckout } from "../helpers/installer-run-fixture"; +import { + INSTALLER_PAYLOAD, + TEST_SYSTEM_PATH, + writeExecutable, +} from "../helpers/installer-sourced-env"; + +const INSTALLER = path.join(import.meta.dirname, "../..", "install.sh"); + +function installerCheckout(prefix: string) { + const checkout = createInstallerCheckout(prefix); + onTestFinished(() => checkout.remove()); + return checkout; +} + +function callBootstrapFunction(command: string, env: Record = {}) { + return spawnSync("bash", ["-c", `source "${INSTALLER}" 2>/dev/null; ${command}`], { + cwd: path.join(import.meta.dirname, "../.."), + encoding: "utf-8", + env: { + HOME: os.tmpdir(), + PATH: TEST_SYSTEM_PATH, + ...env, + }, + }); +} + +function callPayloadFunction(command: string, env: Record = {}) { + return spawnSync("bash", ["-c", `source "${INSTALLER_PAYLOAD}" 2>/dev/null; ${command}`], { + cwd: path.join(import.meta.dirname, "../.."), + encoding: "utf-8", + env: { + HOME: os.tmpdir(), + PATH: TEST_SYSTEM_PATH, + ...env, + }, + }); +} + +it("documents the destructive force-fresh install option", () => { + const result = spawnSync("bash", [INSTALLER, "--help"], { + cwd: path.join(import.meta.dirname, "../.."), + encoding: "utf-8", + }); + + expect(result.status).toBe(0); + const output = `${result.stdout}${result.stderr}`; + expect(output).toContain("--force-fresh-install"); + expect(output).toContain("NEMOCLAW_FORCE_FRESH_INSTALL=1"); +}); + +it("recognizes force-fresh intent from the flag or environment", () => { + const result = callBootstrapFunction(` + bootstrap_force_fresh_install_requested --force-fresh-install && printf 'flag\\n' + NEMOCLAW_FORCE_FRESH_INSTALL=1 + bootstrap_force_fresh_install_requested && printf 'environment\\n' + unset NEMOCLAW_FORCE_FRESH_INSTALL + if bootstrap_force_fresh_install_requested; then exit 9; fi + printf 'absent\\n' + `); + + expect(result.status).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual(["flag", "environment", "absent"]); +}); + +it("detects a Homebrew-only OpenShell installation", () => { + const { root: tmp, binDir: fakeBin } = installerCheckout("nemoclaw-force-fresh-detect-"); + writeExecutable( + path.join(fakeBin, "brew"), + `#!/usr/bin/env bash +if [ "$*" = "list --formula nvidia/openshell/openshell" ]; then exit 0; fi +exit 1 +`, + ); + + const result = callPayloadFunction("force_fresh_install_has_existing_state && printf 'present'", { + HOME: tmp, + PATH: `${fakeBin}:${TEST_SYSTEM_PATH}`, + }); + + expect(result.status).toBe(0); + expect(result.stdout).toBe("present"); +}); + +it("runs destructive cleanup through the staged candidate CLI", () => { + const { root: tmp, binDir: fakeBin } = installerCheckout("nemoclaw-force-fresh-cli-"); + const logPath = path.join(tmp, "node.log"); + const sourceRoot = path.join(tmp, "candidate"); + fs.mkdirSync(path.join(sourceRoot, "bin"), { recursive: true }); + fs.writeFileSync(path.join(sourceRoot, "bin", "nemoclaw.js"), "// staged candidate\n"); + writeExecutable( + path.join(fakeBin, "node"), + `#!/usr/bin/env bash +printf 'destroy=%s args=%s\n' "\${NEMOCLAW_UNINSTALL_DESTROY_USER_DATA:-}" "$*" > "$FORCE_FRESH_LOG" +`, + ); + + const result = callPayloadFunction(`run_force_fresh_uninstaller ${JSON.stringify(sourceRoot)}`, { + FORCE_FRESH_LOG: logPath, + HOME: tmp, + PATH: `${fakeBin}:${TEST_SYSTEM_PATH}`, + }); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(fs.readFileSync(logPath, "utf-8")).toBe( + `destroy=1 args=${path.join(sourceRoot, "bin", "nemoclaw.js")} internal uninstall run-plan --yes --destroy-user-data --all-gateway-ports\n`, + ); +}); + +it("stops when the staged candidate uninstaller fails", () => { + const { root: tmp, binDir: fakeBin } = installerCheckout("nemoclaw-force-fresh-failure-"); + const sourceRoot = path.join(tmp, "candidate"); + fs.mkdirSync(path.join(sourceRoot, "bin"), { recursive: true }); + fs.writeFileSync(path.join(sourceRoot, "bin", "nemoclaw.js"), "// staged candidate\n"); + writeExecutable(path.join(fakeBin, "node"), "#!/usr/bin/env bash\nexit 9\n"); + + const result = callPayloadFunction(`run_force_fresh_uninstaller ${JSON.stringify(sourceRoot)}`, { + HOME: tmp, + PATH: `${fakeBin}:${TEST_SYSTEM_PATH}`, + }); + + expect(result.status).not.toBe(0); +}); + +it("removes only NemoClaw and OpenShell Docker resources", () => { + const { root: tmp, binDir: fakeBin } = installerCheckout("nemoclaw-force-fresh-docker-"); + const logPath = path.join(tmp, "docker.log"); + writeExecutable( + path.join(fakeBin, "docker"), + `#!/usr/bin/env bash +case "$*" in + info) exit 0 ;; + "ps -a --format {{.ID}} {{.Image}} {{.Names}}") + printf '%s\n' 'aaaaaaaaaaaa unrelated openshell-default--demo' '999999999999 82913f8e3281 random-name' 'bbbbbbbbbbbb unrelated keep-me' + ;; + "ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract") printf '%s\n' '999999999999' ;; + "volume ls --format {{.Name}}") printf '%s\n' 'openshell-cluster-nemoclaw' 'keep-volume' ;; + "network ls --format {{.ID}} {{.Name}}") printf '%s\n' 'cccccccccccc nemoclaw-net' 'dddddddddddd keep-net' ;; + "images --format {{.ID}} {{.Repository}}") printf '%s\n' 'eeeeeeeeeeee ghcr.io/nvidia/nemoclaw/hermes-sandbox' 'ffffffffffff unrelated' ;; + *) printf '%s\n' "$*" >> "$FORCE_FRESH_LOG" ;; +esac +`, + ); + + const result = callPayloadFunction("remove_force_fresh_docker_resources", { + FORCE_FRESH_LOG: logPath, + HOME: tmp, + PATH: `${fakeBin}:${TEST_SYSTEM_PATH}`, + }); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(fs.readFileSync(logPath, "utf-8").trim().split("\n")).toEqual([ + "rm -f aaaaaaaaaaaa", + "rm -f 999999999999", + "volume rm -f -- openshell-cluster-nemoclaw", + "network rm cccccccccccc", + "rmi -f eeeeeeeeeeee", + ]); +}); + +it("removes Homebrew and standalone OpenShell installations", () => { + const { root: tmp, binDir: fakeBin } = installerCheckout("nemoclaw-force-fresh-brew-"); + const localBin = path.join(tmp, ".local", "bin"); + const logPath = path.join(tmp, "brew.log"); + fs.mkdirSync(localBin, { recursive: true }); + writeExecutable(path.join(localBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n"); + writeExecutable(path.join(localBin, "openshell-gateway"), "#!/usr/bin/env bash\nexit 0\n"); + writeExecutable(path.join(localBin, "openshell-sandbox"), "#!/usr/bin/env bash\nexit 0\n"); + writeExecutable(path.join(localBin, "openshell-driver-vm"), "#!/usr/bin/env bash\nexit 0\n"); + writeExecutable( + path.join(fakeBin, "brew"), + `#!/usr/bin/env bash +printf '%s\n' "$*" >> "$FORCE_FRESH_LOG" +exit 0 +`, + ); + + const result = callPayloadFunction("remove_macos_openshell_for_force_fresh_install", { + FORCE_FRESH_LOG: logPath, + HOME: tmp, + PATH: `${localBin}:${fakeBin}:${TEST_SYSTEM_PATH}`, + }); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(fs.readFileSync(logPath, "utf-8").trim().split("\n")).toEqual([ + "list --formula nvidia/openshell/openshell", + "services stop nvidia/openshell/openshell", + "uninstall --force nvidia/openshell/openshell", + ]); + expect(fs.existsSync(path.join(localBin, "openshell"))).toBe(false); + expect(fs.existsSync(path.join(localBin, "openshell-gateway"))).toBe(false); + expect(fs.existsSync(path.join(localBin, "openshell-sandbox"))).toBe(false); + expect(fs.existsSync(path.join(localBin, "openshell-driver-vm"))).toBe(false); +}, 30_000); + +it("stops when Homebrew cannot remove OpenShell", () => { + const { root: tmp, binDir: fakeBin } = installerCheckout("nemoclaw-force-fresh-brew-fail-"); + writeExecutable( + path.join(fakeBin, "brew"), + `#!/usr/bin/env bash +if [ "$1" = "list" ]; then exit 0; fi +if [ "$1" = "services" ]; then exit 0; fi +exit 7 +`, + ); + + const result = callPayloadFunction("remove_macos_openshell_for_force_fresh_install", { + HOME: tmp, + PATH: `${fakeBin}:${TEST_SYSTEM_PATH}`, + }); + + expect(result.status).not.toBe(0); + expect(`${result.stdout}${result.stderr}`).toContain("Homebrew could not remove OpenShell"); +}); + +it("runs cleanup before selecting fresh onboarding", () => { + const result = callPayloadFunction(` + warn() { :; } + info() { :; } + force_fresh_install_has_existing_state() { return 0; } + force_fresh_install_source_root() { printf '/tmp/staged-candidate'; } + prepare_force_fresh_uninstaller() { printf 'prepare:%s\\n' "$1"; } + run_force_fresh_uninstaller() { printf 'uninstall:%s\\n' "$1"; } + remove_force_fresh_install_state() { printf 'force-state-reset\\n'; } + remove_macos_openshell_for_force_fresh_install() { printf 'openshell-reset\\n'; } + run_force_fresh_install_reset + printf 'fresh=%s env=%s reinstall=%s count=%s\\n' "$FRESH" "$NEMOCLAW_FRESH" "$NEMOCLAW_REINSTALL_CLI" "$_PREEXISTING_SANDBOX_COUNT" + `); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual([ + "prepare:/tmp/staged-candidate", + "uninstall:/tmp/staged-candidate", + "force-state-reset", + "openshell-reset", + "fresh=1 env=1 reinstall=1 count=0", + ]); +}); + +it("continues force cleanup when managed uninstall rejects partial state", () => { + const result = callPayloadFunction(` + warn() { :; } + info() { :; } + force_fresh_install_has_existing_state() { return 0; } + force_fresh_install_source_root() { printf '/tmp/staged-candidate'; } + prepare_force_fresh_uninstaller() { :; } + run_force_fresh_uninstaller() { return 9; } + remove_force_fresh_install_state() { printf 'force-state-reset\\n'; } + remove_macos_openshell_for_force_fresh_install() { printf 'openshell-reset\\n'; } + run_force_fresh_install_reset + `); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual(["force-state-reset", "openshell-reset"]); +}); + +it("continues force-fresh installation when Homebrew fails after the pinned runtime lands", () => { + const result = callPayloadFunction(` + warn() { printf 'warn:%s\n' "$*"; } + uname() { printf 'Darwin'; } + observed_macos_openshell_install_method() { printf 'homebrew\n'; } + spin_count=0 + spin() { + spin_count=$((spin_count + 1)) + printf 'spin:%s\n' "$1" + [ "$spin_count" -eq 2 ] + } + prefer_homebrew_openshell() { printf 'prefer:%s\n' "$1"; } + install_nemoclaw_openshell_gateway_user_service() { printf 'service\n'; } + FORCE_FRESH_INSTALL=1 + maybe_install_openshell_during_install force + `); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual([ + "spin:Installing OpenShell CLI", + "spin:Verifying the installed OpenShell CLI", + "warn:Homebrew reported an install failure after placing OpenShell; the pinned OpenShell verifier passed, so force-fresh installation will continue.", + "prefer:verified-install", + "service", + ]); +}); + +it("rejects force cleanup outside supported state roots", () => { + const result = callPayloadFunction('remove_force_fresh_state_path "$HOME/Documents"', { + HOME: os.tmpdir(), + }); + + expect(result.status).not.toBe(0); + expect(`${result.stdout}${result.stderr}`).toContain( + "Refusing force-fresh cleanup outside the supported user state roots", + ); +}); + +it("rejects force-fresh installation outside macOS", () => { + const result = callPayloadFunction(` + error() { printf '%s' "$*" >&2; return 1; } + uname() { printf 'Linux'; } + FORCE_FRESH_INSTALL=1 + validate_force_fresh_install_platform + `); + + expect(result.status).not.toBe(0); + expect(`${result.stdout}${result.stderr}`).toContain( + "--force-fresh-install currently supports macOS only", + ); +}); + +it("rejects a staged source inside the state root", () => { + const { root: tmp } = installerCheckout("nemoclaw-force-fresh-source-"); + const sourceRoot = path.join(tmp, ".nemoclaw", "source"); + fs.mkdirSync(path.join(sourceRoot, ".git"), { recursive: true }); + + const result = callPayloadFunction( + `NEMOCLAW_SOURCE_ROOT=${JSON.stringify(sourceRoot)}; force_fresh_install_source_root`, + { HOME: tmp }, + ); + + expect(result.status).not.toBe(0); + expect(`${result.stdout}${result.stderr}`).toContain( + "must run from the versioned bootstrap checkout", + ); +}, 30_000); + +it("keeps an unrelated ACP shim", () => { + const { root: tmp } = installerCheckout("nemoclaw-force-fresh-acp-"); + const shimDir = path.join(tmp, ".local", "bin"); + const npmBin = path.join(tmp, "npm", "bin"); + fs.mkdirSync(shimDir, { recursive: true }); + fs.mkdirSync(npmBin, { recursive: true }); + fs.writeFileSync(path.join(shimDir, "nemoclaw-acp"), "foreign executable\n"); + + const result = callPayloadFunction( + ` + warn() { :; } + resolve_npm_bin() { printf '${npmBin}'; } + preflight_nemoclaw_acp_shim + printf '%s' "$_NEMOCLAW_FORCE_FRESH_SKIP_ACP_SHIM" + `, + { + FORCE_FRESH_INSTALL: "1", + HOME: tmp, + NEMOCLAW_SHIM_DIR: shimDir, + }, + ); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout).toBe("true"); + expect(fs.readFileSync(path.join(shimDir, "nemoclaw-acp"), "utf-8")).toBe("foreign executable\n"); +}, 30_000); From 7e397243a333a869ac3f906e26d1452e50a8e39f Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 18 Sep 2026 04:40:51 -0700 Subject: [PATCH 18/39] fix(installer): fail closed during force-fresh reset Signed-off-by: Aaron Erickson --- .github/workflows/e2e.yaml | 11 ++ docs/reference/commands.mdx | 13 ++ scripts/install.sh | 143 ++--------------- ...d-image-protected-runtime-workflow.test.ts | 10 ++ ...stall-preexisting-sandbox-recovery.test.ts | 14 ++ .../install-force-fresh.test.ts | 145 +++++++++++------- ...aged-image-multiarch-workflow-boundary.mts | 16 ++ 7 files changed, 168 insertions(+), 184 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 6653278563f..100591fe62b 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -4602,6 +4602,17 @@ jobs: run: >- npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/managed-image-multiarch-startup.test.ts + - name: Remove owned Docker Engine 27 receipt daemon + if: always() + shell: bash + run: | + set -euo pipefail + platform="${NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM//\//-}" + daemon_name="nemoclaw-receipt-engine27-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${platform}" + node --import tsx scripts/checks/docker-engine-27-receipt-transfer-e2e.ts \ + --cleanup-only \ + --daemon-name "$daemon_name" + - name: Validate OpenClaw managed-image security boundary shell: bash run: >- diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index de4b63e0aa8..1034479e026 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -108,6 +108,19 @@ The global `$$nemoclaw doctor` command remains available without a sandbox name. The hosted installer accepts options after `bash -s --`. These options control installation and the onboarding run that follows it. +### `--force-fresh-install` + +On macOS, remove every NemoClaw- and OpenShell-managed sandbox, gateway, credential, configuration file, and recovery record before reinstalling and starting fresh onboarding. This destructive reset keeps downloaded model caches, but the removed state cannot be recovered without an external backup. Unlike `--fresh`, it removes the existing managed installation instead of only requesting fresh onboarding state. + +Use the hosted installer so the selected candidate is staged outside the state it removes: + +```bash +curl -fsSL https://www.nvidia.com/nemoclaw.sh | \ + bash -s -- --force-fresh-install +``` + +The equivalent environment variable is `NEMOCLAW_FORCE_FRESH_INSTALL=1`. The installer rejects this option on non-macOS hosts and stops without deleting remaining state if the authoritative whole-host uninstall cannot complete. + ### `--local-model-runtime=vllm` Enable the fixed vLLM local model profile. The flag accepts only `vllm`. It makes the remaining onboarding non-interactive and disables Express profile selection. diff --git a/scripts/install.sh b/scripts/install.sh index 604bfb46abf..a2b5d3161e1 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -1411,7 +1411,6 @@ complete_deferred_installer_docker_context_validation() { _INSTALLER_DOCKER_CONTEXT_VALIDATION_DEFERRED="" unset DOCKER_CONTEXT validate_installer_docker_target_before_host_changes - export DOCKER_CONTEXT=default } MIN_NODE_VERSION="22.19.0" @@ -3506,8 +3505,17 @@ force_fresh_install_has_existing_state() { || command_exists openshell; then return 0 fi - command_exists brew \ - && brew list --formula nvidia/openshell/openshell >/dev/null 2>&1 + if command_exists brew \ + && brew list --formula nvidia/openshell/openshell >/dev/null 2>&1; then + return 0 + fi + if command_exists docker \ + && docker info >/dev/null 2>&1 \ + && docker ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract 2>/dev/null \ + | grep -q .; then + return 0 + fi + return 1 } force_fresh_install_source_root() { @@ -3556,129 +3564,6 @@ run_force_fresh_uninstaller() { --yes --destroy-user-data --all-gateway-ports } -remove_force_fresh_state_path() { - local target="$1" - case "$target" in - "${HOME}/.nemoclaw" | "${HOME}/.config/nemoclaw" | "${HOME}/.config/openshell" | "${HOME}/.local/state/nemoclaw") ;; - *) error "Refusing force-fresh cleanup outside the supported user state roots: ${target}" ;; - esac - if [[ -L "$target" ]]; then - rm -f -- "$target" - else - rm -rf -- "$target" - fi -} - -remove_force_fresh_docker_resources() { - docker info >/dev/null 2>&1 \ - || error "Docker is not reachable. Start Colima, then rerun the force-fresh installer." - - local container_inventory labeled_container_ids volume_inventory network_inventory image_inventory - container_inventory="$(docker ps -a --format '{{.ID}} {{.Image}} {{.Names}}')" \ - || error "Could not inspect Docker containers during force-fresh cleanup." - labeled_container_ids="$(docker ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract)" \ - || error "Could not inspect labeled NemoClaw containers during force-fresh cleanup." - volume_inventory="$(docker volume ls --format '{{.Name}}')" \ - || error "Could not inspect Docker volumes during force-fresh cleanup." - network_inventory="$(docker network ls --format '{{.ID}} {{.Name}}')" \ - || error "Could not inspect Docker networks during force-fresh cleanup." - image_inventory="$(docker images --format '{{.ID}} {{.Repository}}')" \ - || error "Could not inspect Docker images during force-fresh cleanup." - - local id image name removed_container_ids=" " - while read -r id image name; do - [[ "$id" =~ ^[0-9a-f]{12,64}$ ]] || continue - case "$removed_container_ids" in - *" $id "*) continue ;; - esac - case "$name" in - openshell-* | nemoclaw-*) - docker rm -f "$id" >/dev/null - removed_container_ids="${removed_container_ids}${id} " - ;; - *) - case "$image" in - nemoclaw-* | openshell/* | ghcr.io/nvidia/nemoclaw | ghcr.io/nvidia/nemoclaw/* | ghcr.io/nvidia/nemoclaw-*) - docker rm -f "$id" >/dev/null - removed_container_ids="${removed_container_ids}${id} " - ;; - esac - ;; - esac - done <<<"$container_inventory" - while IFS= read -r id; do - [[ "$id" =~ ^[0-9a-f]{12,64}$ ]] || continue - case "$removed_container_ids" in - *" $id "*) continue ;; - esac - docker rm -f "$id" >/dev/null - removed_container_ids="${removed_container_ids}${id} " - done <<<"$labeled_container_ids" - - local resource - while IFS= read -r resource; do - case "$resource" in - openshell-* | nemoclaw-*) docker volume rm -f -- "$resource" >/dev/null ;; - esac - done <<<"$volume_inventory" - while read -r id name; do - [[ "$id" =~ ^[0-9a-f]{12,64}$ ]] || continue - case "$name" in - openshell-* | nemoclaw-*) docker network rm "$id" >/dev/null ;; - esac - done <<<"$network_inventory" - local removed_image_ids=" " - while read -r id image; do - [[ "$id" =~ ^[0-9a-f]{12,64}$ ]] || continue - case "$removed_image_ids" in - *" $id "*) continue ;; - esac - case "$image" in - nemoclaw-* | openshell/* | ghcr.io/nvidia/nemoclaw | ghcr.io/nvidia/nemoclaw/* | ghcr.io/nvidia/nemoclaw-*) - docker rmi -f "$id" >/dev/null - removed_image_ids="${removed_image_ids}${id} " - ;; - esac - done <<<"$image_inventory" -} - -remove_force_fresh_openshell_resources() { - if command_exists openshell; then - local gateway_name - # shellcheck disable=SC2016 # JavaScript template literal is evaluated by Node.js. - while IFS= read -r gateway_name; do - [[ "$gateway_name" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$ ]] || continue - openshell sandbox delete --all -g "$gateway_name" >/dev/null 2>&1 || true - done < <(openshell gateway list -o json 2>/dev/null | node -e ' -let input = ""; -process.stdin.setEncoding("utf8"); -process.stdin.on("data", (chunk) => { input += chunk; }); -process.stdin.on("end", () => { - try { - const rows = JSON.parse(input); - if (!Array.isArray(rows)) return; - for (const row of rows) { - if (row && typeof row.name === "string") process.stdout.write(`${row.name}\n`); - } - } catch {} -}); -') - fi - pkill -TERM -x openshell-gateway >/dev/null 2>&1 || true - sleep 1 - pkill -KILL -x openshell-gateway >/dev/null 2>&1 || true -} - -remove_force_fresh_install_state() { - remove_force_fresh_openshell_resources - remove_force_fresh_docker_resources - npm uninstall -g nemoclaw >/dev/null 2>&1 || true - remove_force_fresh_state_path "${HOME}/.nemoclaw" - remove_force_fresh_state_path "${HOME}/.config/nemoclaw" - remove_force_fresh_state_path "${HOME}/.config/openshell" - remove_force_fresh_state_path "${HOME}/.local/state/nemoclaw" -} - remove_macos_openshell_for_force_fresh_install() { local formula="nvidia/openshell/openshell" openshell_path="" openshell_path="$(command -v openshell 2>/dev/null || true)" @@ -3703,10 +3588,8 @@ run_force_fresh_install_reset() { if force_fresh_install_has_existing_state; then source_root="$(force_fresh_install_source_root)" prepare_force_fresh_uninstaller "$source_root" - if ! run_force_fresh_uninstaller "$source_root"; then - warn "Managed uninstall could not reconcile all partial state; continuing with the explicit force-fresh cleanup." - fi - remove_force_fresh_install_state + run_force_fresh_uninstaller "$source_root" \ + || error "Force-fresh cleanup stopped because the authoritative whole-host uninstall did not complete. The installer preserved remaining state for a safe retry." remove_macos_openshell_for_force_fresh_install else info "No existing NemoClaw or OpenShell installation was found; continuing with a clean install." diff --git a/test/e2e/support/managed-image-protected-runtime-workflow.test.ts b/test/e2e/support/managed-image-protected-runtime-workflow.test.ts index eb85e057b6b..6d87ff97ce1 100644 --- a/test/e2e/support/managed-image-protected-runtime-workflow.test.ts +++ b/test/e2e/support/managed-image-protected-runtime-workflow.test.ts @@ -98,6 +98,16 @@ describe("protected managed-image runtime workflow", () => { expect(validateManagedImageProtectedRuntimeWorkflow(value)).toEqual([]); }); + it("requires cancellation cleanup for the derived Docker Engine 27 receipt daemon", () => { + const value = workflow(); + const cleanup = namedMultiarchStep(value, "Remove owned Docker Engine 27 receipt daemon"); + cleanup.if = "${{ !cancelled() }}"; + + expect(validateManagedImageMultiarchWorkflow(value)).toContain( + "managed-image-multiarch-startup Docker Engine 27 receipt daemon cleanup must always run", + ); + }); + // source-shape-contract: security -- Both protected jobs must execute the shared Hermes resolver from trusted workflow code it.each([ [ diff --git a/test/install/install-preexisting-sandbox-recovery.test.ts b/test/install/install-preexisting-sandbox-recovery.test.ts index ebb7efb7445..ab49cddd18a 100644 --- a/test/install/install-preexisting-sandbox-recovery.test.ts +++ b/test/install/install-preexisting-sandbox-recovery.test.ts @@ -335,6 +335,20 @@ describe("install.sh pre-existing sandbox recovery ordering (#6114)", () => { expect(result.output).not.toContain("Docker context does not select the local default target"); }); + it("keeps a deferred persisted Colima socket authoritative during recovery", () => { + const result = runRecoveryBeforeOnboard(2, 0, { + dockerContextEndpoint: "unix:///Users/test/.colima/default/docker.sock", + includeNodeOnPath: false, + persistedDockerContext: "colima", + recordRuntimeTarget: true, + }); + + expect(result.status, result.output).toBe(0); + expect(result.calls).toContain( + "cli-target=host:unix:///Users/test/.colima/default/docker.sock,context:unset argv=upgrade-sandboxes --auto", + ); + }); + it("rejects a deferred remote Docker context before recovery", () => { const result = runRecoveryBeforeOnboard(2, 0, { includeNodeOnPath: false, diff --git a/test/installer-integration/install-force-fresh.test.ts b/test/installer-integration/install-force-fresh.test.ts index aea95f087f9..0e819ec4d70 100644 --- a/test/installer-integration/install-force-fresh.test.ts +++ b/test/installer-integration/install-force-fresh.test.ts @@ -7,7 +7,7 @@ import os from "node:os"; import path from "node:path"; import { expect, it, onTestFinished } from "vitest"; -import { createInstallerCheckout } from "../helpers/installer-run-fixture"; +import { createInstallerCheckout, runInstallerSourcedBody } from "../helpers/installer-run-fixture"; import { INSTALLER_PAYLOAD, TEST_SYSTEM_PATH, @@ -91,6 +91,49 @@ exit 1 expect(result.stdout).toBe("present"); }); +it("detects Docker-only state only through an authoritative managed-image label", () => { + const { root: tmp, binDir: fakeBin } = installerCheckout("nemoclaw-force-fresh-detect-docker-"); + writeExecutable( + path.join(fakeBin, "docker"), + `#!/usr/bin/env bash +case "$*" in + info) exit 0 ;; + "ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract") printf '0123456789ab\n' ;; + *) exit 1 ;; +esac +`, + ); + + const result = callPayloadFunction("force_fresh_install_has_existing_state && printf 'present'", { + HOME: tmp, + PATH: `${fakeBin}:${TEST_SYSTEM_PATH}`, + }); + + expect(result.status).toBe(0); + expect(result.stdout).toBe("present"); +}); + +it("does not treat an unlabeled prefix-matching Docker container as owned state", () => { + const { root: tmp, binDir: fakeBin } = installerCheckout("nemoclaw-force-fresh-ignore-docker-"); + writeExecutable( + path.join(fakeBin, "docker"), + `#!/usr/bin/env bash +case "$*" in + info) exit 0 ;; + "ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract") exit 0 ;; + *) exit 1 ;; +esac +`, + ); + + const result = callPayloadFunction("force_fresh_install_has_existing_state", { + HOME: tmp, + PATH: `${fakeBin}:${TEST_SYSTEM_PATH}`, + }); + + expect(result.status).not.toBe(0); +}); + it("runs destructive cleanup through the staged candidate CLI", () => { const { root: tmp, binDir: fakeBin } = installerCheckout("nemoclaw-force-fresh-cli-"); const logPath = path.join(tmp, "node.log"); @@ -131,42 +174,6 @@ it("stops when the staged candidate uninstaller fails", () => { expect(result.status).not.toBe(0); }); -it("removes only NemoClaw and OpenShell Docker resources", () => { - const { root: tmp, binDir: fakeBin } = installerCheckout("nemoclaw-force-fresh-docker-"); - const logPath = path.join(tmp, "docker.log"); - writeExecutable( - path.join(fakeBin, "docker"), - `#!/usr/bin/env bash -case "$*" in - info) exit 0 ;; - "ps -a --format {{.ID}} {{.Image}} {{.Names}}") - printf '%s\n' 'aaaaaaaaaaaa unrelated openshell-default--demo' '999999999999 82913f8e3281 random-name' 'bbbbbbbbbbbb unrelated keep-me' - ;; - "ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract") printf '%s\n' '999999999999' ;; - "volume ls --format {{.Name}}") printf '%s\n' 'openshell-cluster-nemoclaw' 'keep-volume' ;; - "network ls --format {{.ID}} {{.Name}}") printf '%s\n' 'cccccccccccc nemoclaw-net' 'dddddddddddd keep-net' ;; - "images --format {{.ID}} {{.Repository}}") printf '%s\n' 'eeeeeeeeeeee ghcr.io/nvidia/nemoclaw/hermes-sandbox' 'ffffffffffff unrelated' ;; - *) printf '%s\n' "$*" >> "$FORCE_FRESH_LOG" ;; -esac -`, - ); - - const result = callPayloadFunction("remove_force_fresh_docker_resources", { - FORCE_FRESH_LOG: logPath, - HOME: tmp, - PATH: `${fakeBin}:${TEST_SYSTEM_PATH}`, - }); - - expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); - expect(fs.readFileSync(logPath, "utf-8").trim().split("\n")).toEqual([ - "rm -f aaaaaaaaaaaa", - "rm -f 999999999999", - "volume rm -f -- openshell-cluster-nemoclaw", - "network rm cccccccccccc", - "rmi -f eeeeeeeeeeee", - ]); -}); - it("removes Homebrew and standalone OpenShell installations", () => { const { root: tmp, binDir: fakeBin } = installerCheckout("nemoclaw-force-fresh-brew-"); const localBin = path.join(tmp, ".local", "bin"); @@ -230,7 +237,6 @@ it("runs cleanup before selecting fresh onboarding", () => { force_fresh_install_source_root() { printf '/tmp/staged-candidate'; } prepare_force_fresh_uninstaller() { printf 'prepare:%s\\n' "$1"; } run_force_fresh_uninstaller() { printf 'uninstall:%s\\n' "$1"; } - remove_force_fresh_install_state() { printf 'force-state-reset\\n'; } remove_macos_openshell_for_force_fresh_install() { printf 'openshell-reset\\n'; } run_force_fresh_install_reset printf 'fresh=%s env=%s reinstall=%s count=%s\\n' "$FRESH" "$NEMOCLAW_FRESH" "$NEMOCLAW_REINSTALL_CLI" "$_PREEXISTING_SANDBOX_COUNT" @@ -240,13 +246,12 @@ it("runs cleanup before selecting fresh onboarding", () => { expect(result.stdout.trim().split("\n")).toEqual([ "prepare:/tmp/staged-candidate", "uninstall:/tmp/staged-candidate", - "force-state-reset", "openshell-reset", "fresh=1 env=1 reinstall=1 count=0", ]); }); -it("continues force cleanup when managed uninstall rejects partial state", () => { +it("stops before package removal when managed uninstall rejects partial state", () => { const result = callPayloadFunction(` warn() { :; } info() { :; } @@ -254,13 +259,56 @@ it("continues force cleanup when managed uninstall rejects partial state", () => force_fresh_install_source_root() { printf '/tmp/staged-candidate'; } prepare_force_fresh_uninstaller() { :; } run_force_fresh_uninstaller() { return 9; } - remove_force_fresh_install_state() { printf 'force-state-reset\\n'; } remove_macos_openshell_for_force_fresh_install() { printf 'openshell-reset\\n'; } run_force_fresh_install_reset `); - expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); - expect(result.stdout.trim().split("\n")).toEqual(["force-state-reset", "openshell-reset"]); + expect(result.status).not.toBe(0); + expect(result.stdout).not.toContain("openshell-reset"); + expect(`${result.stdout}${result.stderr}`).toContain( + "authoritative whole-host uninstall did not complete", + ); +}); + +it("routes the public force-fresh flag through reset before CLI installation", () => { + const run = runInstallerSourcedBody(` + set -e + order="" + record() { order="\${order}$1 "; } + apply_persisted_automatic_gateway_port() { :; } + validate_deferred_hermes_onboarding_request() { :; } + load_station_vllm_conflict_helpers() { :; } + consume_station_local_vllm_resume() { return 1; } + resolve_nemoclaw_gateway_port() { printf '8080'; } + preflight_explicit_express_flags() { :; } + validate_installer_docker_target_before_host_changes() { :; } + print_banner() { :; } + preflight_usage_notice_prompt() { :; } + prepare_installer_host() { :; } + bash() { :; } + prepare_installer_node_runtime() { :; } + run_force_fresh_install_reset() { record reset; } + ensure_station_express_pair() { :; } + step() { :; } + fix_npm_permissions() { :; } + preflight_nemoclaw_acp_shim() { :; } + preinstall_backup_and_retire_legacy_gateway() { record legacy-backup; } + install_nemoclaw() { record install; } + verify_nemoclaw() { _CLI_PATH=""; } + require_reportable_openshell_version() { :; } + command_exists() { return 1; } + warn() { :; } + finalize_install() { :; } + clear_station_resume_after_completed_onboarding() { :; } + uname() { printf 'Darwin'; } + main --force-fresh-install --non-interactive --yes-i-accept-third-party-software + printf '%s' "$order" + `); + onTestFinished(run.remove); + + expect(run.result.status, run.output).toBe(0); + expect(run.result.stdout.trim()).toBe("reset install"); + expect(run.result.stdout).not.toContain("legacy-backup"); }); it("continues force-fresh installation when Homebrew fails after the pinned runtime lands", () => { @@ -290,17 +338,6 @@ it("continues force-fresh installation when Homebrew fails after the pinned runt ]); }); -it("rejects force cleanup outside supported state roots", () => { - const result = callPayloadFunction('remove_force_fresh_state_path "$HOME/Documents"', { - HOME: os.tmpdir(), - }); - - expect(result.status).not.toBe(0); - expect(`${result.stdout}${result.stderr}`).toContain( - "Refusing force-fresh cleanup outside the supported user state roots", - ); -}); - it("rejects force-fresh installation outside macOS", () => { const result = callPayloadFunction(` error() { printf '%s' "$*" >&2; return 1; } diff --git a/tools/e2e/managed-image-multiarch-workflow-boundary.mts b/tools/e2e/managed-image-multiarch-workflow-boundary.mts index eb5d3e6d241..b1ffd41706e 100644 --- a/tools/e2e/managed-image-multiarch-workflow-boundary.mts +++ b/tools/e2e/managed-image-multiarch-workflow-boundary.mts @@ -389,6 +389,21 @@ export function validateManagedImageMultiarchWorkflow(workflow: WorkflowRecord): "tools/e2e/live-vitest-invocation.mts run", `--test-path ${DIRECT_TEST_PATH}`, ]); + const receiptDaemonCleanup = requireStep( + errors, + steps, + "Remove owned Docker Engine 27 receipt daemon", + ); + if (receiptDaemonCleanup?.if !== "always()") { + errors.push(`${JOB_ID} Docker Engine 27 receipt daemon cleanup must always run`); + } + requireFragments(errors, receiptDaemonCleanup, [ + 'platform="${NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM//\\//-}"', + 'daemon_name="nemoclaw-receipt-engine27-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${platform}"', + "scripts/checks/docker-engine-27-receipt-transfer-e2e.ts", + "--cleanup-only", + '--daemon-name "$daemon_name"', + ]); const cacheUpload = requireStep( errors, steps, @@ -449,6 +464,7 @@ export function validateManagedImageMultiarchWorkflow(workflow: WorkflowRecord): "Run every exact managed-image contract directly", "Remove isolated protected managed-image registry", "Validate protected managed-image evidence", + "Remove owned Docker Engine 27 receipt daemon", "Validate OpenClaw managed-image security boundary", "Validate managed-image glibc probe lifecycle", "Remove protected managed-image cohort resources", From b9e09bb14676bda3908a91364feefc3c2fd304fc Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 18 Sep 2026 05:21:33 -0700 Subject: [PATCH 19/39] test(e2e): prove Docker receipt cleanup isolation Signed-off-by: Aaron Erickson --- .../docker-engine-27-receipt-transfer-e2e.ts | 29 ++++++++++++---- scripts/install.sh | 2 +- ...ed-image-multiarch-startup-helpers.test.ts | 34 +++++++++++++++++++ .../install-force-fresh.test.ts | 16 +++++++++ 4 files changed, 73 insertions(+), 8 deletions(-) diff --git a/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts b/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts index 37af891743d..05ca2a31558 100644 --- a/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts +++ b/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts @@ -40,7 +40,7 @@ export const DOCKER_ENGINE_27_MINIMUM_CLEANUP_PROCESS_TIMEOUT_MS = DOCKER_ENGINE_27_CLEANUP_OPERATION_COUNT * DOCKER_ENGINE_27_CLEANUP_TIMEOUT_MS + DOCKER_ENGINE_27_PROCESS_ALLOWANCE_MS / 2; -type CommandResult = { +export type CommandResult = { readonly error?: Error; readonly status: number | null; readonly stderr: string; @@ -127,12 +127,14 @@ export function validateDockerEngine27SeedIsolation(value: unknown): void { CapDrop?: unknown; Mounts?: unknown; NetworkMode?: unknown; + Privileged?: unknown; ReadonlyRootfs?: unknown; SecurityOpt?: unknown; }; }; requireCondition(seed.Config?.User === "0", "receipt seed did not use numeric root"); requireCondition(seed.HostConfig?.NetworkMode === "none", "receipt seed retained networking"); + requireCondition(seed.HostConfig?.Privileged === false, "receipt seed was privileged"); requireCondition(seed.HostConfig?.ReadonlyRootfs === true, "receipt seed root was writable"); requireCondition( Array.isArray(seed.HostConfig?.SecurityOpt) && @@ -156,8 +158,12 @@ export function validateDockerEngine27SeedIsolation(value: unknown): void { ); } -function requireAbsent(result: CommandResult, resource: string): void { - requireCondition(result.status !== 0, `${resource} remained after receipt-transfer cleanup`); +export function requireDockerResourceAbsent(result: CommandResult, resource: string): void { + const detail = commandDetail(result); + requireCondition( + result.status !== 0 && /\bno such (?:container|object|volume)\b/iu.test(detail), + `${resource} absence was not proven after receipt-transfer cleanup: ${detail}`, + ); } export function cleanupDockerEngine27ReceiptDaemon(daemonName: string): void { @@ -183,7 +189,7 @@ export function cleanupDockerEngine27ReceiptDaemon(daemonName: string): void { "refusing to remove a Docker Engine 27 daemon with mismatched ownership", ); requireSuccess(runDocker(["rm", "-f", daemonName], "cleanup"), "remove Docker Engine 27 daemon"); - requireAbsent( + requireDockerResourceAbsent( runDocker(["container", "inspect", daemonName], "cleanup"), "Docker Engine 27 daemon", ); @@ -337,7 +343,10 @@ async function verifyDockerEngine27ReceiptTransfer(daemonName: string): Promise< }); successfulVolume = receipt.volumeName; requireCondition(successfulSeed.length > 0, "receipt transfer did not create a seed"); - requireAbsent(innerDocker(["container", "inspect", successfulSeed]), "successful receipt seed"); + requireDockerResourceAbsent( + innerDocker(["container", "inspect", successfulSeed]), + "successful receipt seed", + ); requireSuccess( innerDocker([ "run", @@ -397,8 +406,14 @@ async function verifyDockerEngine27ReceiptTransfer(daemonName: string): Promise< failedSeed.length > 0 && failedVolume.length > 0, "failed transfer omitted resources", ); - requireAbsent(innerDocker(["container", "inspect", failedSeed]), "failed receipt seed"); - requireAbsent(innerDocker(["volume", "inspect", failedVolume]), "failed receipt volume"); + requireDockerResourceAbsent( + innerDocker(["container", "inspect", failedSeed]), + "failed receipt seed", + ); + requireDockerResourceAbsent( + innerDocker(["volume", "inspect", failedVolume]), + "failed receipt volume", + ); } catch (error) { primaryError = error; } diff --git a/scripts/install.sh b/scripts/install.sh index a2b5d3161e1..c868f65ce3f 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -3590,10 +3590,10 @@ run_force_fresh_install_reset() { prepare_force_fresh_uninstaller "$source_root" run_force_fresh_uninstaller "$source_root" \ || error "Force-fresh cleanup stopped because the authoritative whole-host uninstall did not complete. The installer preserved remaining state for a safe retry." - remove_macos_openshell_for_force_fresh_install else info "No existing NemoClaw or OpenShell installation was found; continuing with a clean install." fi + remove_macos_openshell_for_force_fresh_install FRESH=1 export NEMOCLAW_FRESH=1 export NEMOCLAW_REINSTALL_CLI=1 diff --git a/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts b/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts index e335a4940f5..f2bf29716ef 100644 --- a/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts +++ b/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts @@ -11,6 +11,7 @@ import { DOCKER_ENGINE_27_MINIMUM_CLEANUP_PROCESS_TIMEOUT_MS, DOCKER_ENGINE_27_MINIMUM_PROBE_TIMEOUT_MS, finalizeDockerEngine27ReceiptProbe, + requireDockerResourceAbsent, validateDockerEngine27SeedIsolation, } from "../../../scripts/checks/docker-engine-27-receipt-transfer-e2e.ts"; import { @@ -86,6 +87,7 @@ describe("protected managed-image startup helpers", () => { }, ], NetworkMode: "none", + Privileged: false, ReadonlyRootfs: true, SecurityOpt: ["no-new-privileges"], }, @@ -95,6 +97,12 @@ describe("protected managed-image startup helpers", () => { expect(() => validateDockerEngine27SeedIsolation({ ...secureSeed, Config: { User: "0:0" } }), ).toThrow("receipt seed did not use numeric root"); + expect(() => + validateDockerEngine27SeedIsolation({ + ...secureSeed, + HostConfig: { ...secureSeed.HostConfig, Privileged: true }, + }), + ).toThrow("receipt seed was privileged"); expect(() => validateDockerEngine27SeedIsolation({ ...secureSeed, @@ -103,6 +111,32 @@ describe("protected managed-image startup helpers", () => { ).toThrow("receipt seed retained capabilities"); }); + it("proves cleanup only from Docker's explicit absence response", () => { + expect(() => + requireDockerResourceAbsent( + { status: 1, stderr: "Error: No such container: receipt-seed", stdout: "" }, + "receipt seed", + ), + ).not.toThrow(); + expect(() => + requireDockerResourceAbsent( + { + error: new Error("spawnSync docker ETIMEDOUT"), + status: null, + stderr: "", + stdout: "", + }, + "receipt seed", + ), + ).toThrow("receipt seed absence was not proven"); + expect(() => + requireDockerResourceAbsent( + { status: 1, stderr: "permission denied", stdout: "" }, + "receipt seed", + ), + ).toThrow("receipt seed absence was not proven"); + }); + it("parses exact protected dispatch identity", () => { expect(protectedManagedImageDispatchEnvironment()).toMatchObject({ baseSha: sha, diff --git a/test/installer-integration/install-force-fresh.test.ts b/test/installer-integration/install-force-fresh.test.ts index 0e819ec4d70..ccd9ee12974 100644 --- a/test/installer-integration/install-force-fresh.test.ts +++ b/test/installer-integration/install-force-fresh.test.ts @@ -251,6 +251,22 @@ it("runs cleanup before selecting fresh onboarding", () => { ]); }); +it("removes standalone OpenShell helpers even when no other state is detected", () => { + const result = callPayloadFunction(` + warn() { :; } + info() { :; } + force_fresh_install_has_existing_state() { return 1; } + prepare_force_fresh_uninstaller() { printf 'unexpected-prepare\n'; } + run_force_fresh_uninstaller() { printf 'unexpected-uninstall\n'; } + remove_macos_openshell_for_force_fresh_install() { printf 'openshell-reset\n'; } + run_force_fresh_install_reset + `); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout.trim()).toBe("openshell-reset"); + expect(result.stdout).not.toContain("unexpected-"); +}); + it("stops before package removal when managed uninstall rejects partial state", () => { const result = callPayloadFunction(` warn() { :; } From dedcb9a7885b3b4ef6743b1e73581979b431bbc9 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 18 Sep 2026 06:09:43 -0700 Subject: [PATCH 20/39] fix(installer): bound force-fresh preparation Signed-off-by: Aaron Erickson --- install.sh | 4 +- scripts/install.sh | 88 ++++++++++++++++++- .../install-downgrade-guard.test.ts | 12 +++ .../install-force-fresh.test.ts | 28 ++++++ 4 files changed, 125 insertions(+), 7 deletions(-) diff --git a/install.sh b/install.sh index 44b7d830da1..ac8afd7d27f 100755 --- a/install.sh +++ b/install.sh @@ -262,9 +262,7 @@ exec_installer_from_ref() { clone_nemoclaw_ref "$ref" "$source_root" selected_commit="$(git -C "$source_root" rev-parse HEAD)" - if ! bootstrap_force_fresh_install_requested "$@"; then - guard_implicit_maintained_downgrade "$source_root" "$ref" - fi + guard_implicit_maintained_downgrade "$source_root" "$ref" payload_script="${source_root}/scripts/install.sh" legacy_script="${source_root}/install.sh" diff --git a/scripts/install.sh b/scripts/install.sh index c868f65ce3f..af12f059b90 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -1354,6 +1354,80 @@ spin() { return $status } +FORCE_FRESH_PREPARE_TIMEOUT_SECONDS=900 + +installer_command_group_is_alive() { + local command_pid="$1" + kill -0 -- "-$command_pid" 2>/dev/null || kill -0 "$command_pid" 2>/dev/null +} + +terminate_installer_command_group() { + local command_pid="$1" grace_ticks + installer_command_group_is_alive "$command_pid" || { + wait "$command_pid" 2>/dev/null || true + return + } + kill -TERM -- "-$command_pid" 2>/dev/null || kill -TERM "$command_pid" 2>/dev/null || true + for ((grace_ticks = 0; grace_ticks < 10; grace_ticks++)); do + installer_command_group_is_alive "$command_pid" || break + sleep 0.1 + done + if installer_command_group_is_alive "$command_pid"; then + kill -KILL -- "-$command_pid" 2>/dev/null || kill -KILL "$command_pid" 2>/dev/null || true + fi + wait "$command_pid" 2>/dev/null || true +} + +run_bounded_installer_command() ( + local label="$1" timeout_seconds="$2" log command_pid="" status=0 ticks=0 pending_signal=0 + shift 2 + [[ "$timeout_seconds" =~ ^[1-9][0-9]*$ ]] \ + || error "Installer command timeout must be a positive integer." + log="$(mktemp)" + # Invoked indirectly by the signal and exit traps below. + # shellcheck disable=SC2329 + cleanup_bounded_installer_command() { + local cleanup_status="$1" + trap 'cleanup_status=130' INT + trap 'cleanup_status=143' TERM + trap - EXIT + [[ -z "$command_pid" ]] || terminate_installer_command_group "$command_pid" + rm -f "$log" + exit "$cleanup_status" + } + trap 'pending_signal=130' INT + trap 'pending_signal=143' TERM + trap 'cleanup_bounded_installer_command "$?"' EXIT + info "$label" + set -m + "$@" >"$log" 2>&1 & + command_pid=$! + set +m + trap 'cleanup_bounded_installer_command 130' INT + trap 'cleanup_bounded_installer_command 143' TERM + ((pending_signal == 0)) || cleanup_bounded_installer_command "$pending_signal" + while installer_command_group_is_alive "$command_pid"; do + if ((ticks >= timeout_seconds * 10)); then + printf '[ERROR] Timed out during %s after %s seconds. Existing state was preserved; force-fresh cleanup did not start. Retry when package installation is healthy.\n' "$label" "$timeout_seconds" >&2 + return 124 + fi + sleep 0.1 + ticks=$((ticks + 1)) + done + if wait "$command_pid"; then + status=0 + else + status=$? + fi + command_pid="" + if ((status != 0)); then + cat "$log" >&2 + fi + rm -f "$log" + trap - EXIT + return "$status" +) + command_exists() { command -v "$1" &>/dev/null; } # Apply the gateway's Unix-socket constraints before Node or CLI modules are available. @@ -3543,13 +3617,19 @@ force_fresh_install_source_root() { } prepare_force_fresh_uninstaller() { - local source_root="$1" + local source_root="$1" prepare_status=0 if [[ -z "${NEMOCLAW_AGENT:-}" || "${NEMOCLAW_AGENT}" == "openclaw" ]]; then - spin "Preparing OpenClaw package for force-fresh cleanup" bash -c "$(declare -f info warn resolve_openclaw_version pre_extract_openclaw); pre_extract_openclaw \"\$1\"" _ "$source_root" \ + run_bounded_installer_command "Preparing OpenClaw package for force-fresh cleanup" "$FORCE_FRESH_PREPARE_TIMEOUT_SECONDS" bash -c "$(declare -f info warn resolve_openclaw_version pre_extract_openclaw); pre_extract_openclaw \"\$1\"" _ "$source_root" \ || warn "Pre-extraction failed — npm install may fail if the OpenClaw tarball is broken" fi - spin "Preparing the force-fresh uninstaller" bash -c \ - "cd \"$source_root\" && node scripts/lib/openshell-sdk-install.mts prepare && npm install --ignore-scripts --prefer-offline --include=optional --@nvidia:registry=https://npm.pkg.github.com && node scripts/lib/openshell-sdk-install.mts check && npm run --if-present build:cli" + run_bounded_installer_command "Preparing the force-fresh uninstaller" "$FORCE_FRESH_PREPARE_TIMEOUT_SECONDS" bash -c \ + "cd \"$source_root\" && node scripts/lib/openshell-sdk-install.mts prepare && npm install --ignore-scripts --prefer-offline --include=optional --@nvidia:registry=https://npm.pkg.github.com && node scripts/lib/openshell-sdk-install.mts check && npm run --if-present build:cli" \ + || prepare_status=$? + if ((prepare_status == 124)); then + error "Force-fresh uninstaller preparation timed out before cleanup. Existing state was preserved; rerun when package installation is healthy." + elif ((prepare_status != 0)); then + error "The staged force-fresh uninstaller could not be prepared. Existing state was preserved; cleanup did not start." + fi [[ -s "${source_root}/dist/lib/actions/uninstall/run-plan.js" ]] \ || error "The staged force-fresh uninstaller did not build." } diff --git a/test/installer-integration/install-downgrade-guard.test.ts b/test/installer-integration/install-downgrade-guard.test.ts index 805aa85a1fb..e7045b7947a 100644 --- a/test/installer-integration/install-downgrade-guard.test.ts +++ b/test/installer-integration/install-downgrade-guard.test.ts @@ -228,6 +228,18 @@ describe("public installer downgrade guard", () => { expect(fs.existsSync(payloadMarker)).toBe(false); }); + it("keeps the installed CLI when force-fresh would implicitly downgrade lkg", () => { + const { result, payloadMarker } = runInstall("0.0.118", "0.0.109", { + NEMOCLAW_FORCE_FRESH_INSTALL: "1", + }); + + expect(result.status).toBe(1); + expect(`${result.stdout}${result.stderr}`).toContain( + "Refusing to replace installed NemoClaw v0.0.118 with maintained lkg v0.0.109.", + ); + expect(fs.existsSync(payloadMarker)).toBe(false); + }); + it.each([ ["Hermes", "hermes"], ["Deep Agents", "langchain-deepagents-code"], diff --git a/test/installer-integration/install-force-fresh.test.ts b/test/installer-integration/install-force-fresh.test.ts index ccd9ee12974..d8c4e01ddeb 100644 --- a/test/installer-integration/install-force-fresh.test.ts +++ b/test/installer-integration/install-force-fresh.test.ts @@ -286,6 +286,34 @@ it("stops before package removal when managed uninstall rejects partial state", ); }); +it("bounds staged uninstaller preparation before destructive cleanup", () => { + const { root: tmp } = installerCheckout("nemoclaw-force-fresh-timeout-"); + const cleanupMarker = path.join(tmp, "cleanup-started"); + const startedAt = performance.now(); + const result = callPayloadFunction( + ` + NEMOCLAW_AGENT=hermes + FORCE_FRESH_PREPARE_TIMEOUT_SECONDS=1 + force_fresh_install_has_existing_state() { return 0; } + force_fresh_install_source_root() { printf '/tmp/staged-candidate'; } + bash() { trap '' TERM; while :; do sleep 1; done; } + run_force_fresh_uninstaller() { touch "$CLEANUP_MARKER"; } + remove_macos_openshell_for_force_fresh_install() { touch "$CLEANUP_MARKER"; } + run_force_fresh_install_reset + `, + { CLEANUP_MARKER: cleanupMarker, HOME: tmp }, + ); + const elapsedMs = performance.now() - startedAt; + + expect(result.status).not.toBe(0); + expect(elapsedMs).toBeGreaterThanOrEqual(900); + expect(elapsedMs).toBeLessThan(10_000); + expect(`${result.stdout}${result.stderr}`).toContain( + "Force-fresh uninstaller preparation timed out before cleanup", + ); + expect(fs.existsSync(cleanupMarker)).toBe(false); +}, 15_000); + it("routes the public force-fresh flag through reset before CLI installation", () => { const run = runInstallerSourcedBody(` set -e From 1483fa0cddcc302918ec6710d68428496c28631d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 18 Sep 2026 07:17:24 -0700 Subject: [PATCH 21/39] fix(e2e): centralize Docker 27 daemon identity Signed-off-by: Aaron Erickson --- .github/workflows/e2e.yaml | 6 +-- install.sh | 11 ---- .../docker-engine-27-receipt-transfer-e2e.ts | 51 ++++++++++++++++--- ...managed-image-multiarch-startup-helpers.ts | 8 --- .../managed-image-multiarch-startup.test.ts | 16 ++++-- ...ed-image-multiarch-startup-helpers.test.ts | 21 +++++++- .../install-force-fresh.test.ts | 26 ---------- ...aged-image-multiarch-workflow-boundary.mts | 6 +-- 8 files changed, 80 insertions(+), 65 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 100591fe62b..949ac06b0fd 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -4607,11 +4607,11 @@ jobs: shell: bash run: | set -euo pipefail - platform="${NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM//\//-}" - daemon_name="nemoclaw-receipt-engine27-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${platform}" node --import tsx scripts/checks/docker-engine-27-receipt-transfer-e2e.ts \ --cleanup-only \ - --daemon-name "$daemon_name" + --run-id "$GITHUB_RUN_ID" \ + --run-attempt "$GITHUB_RUN_ATTEMPT" \ + --platform "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM" - name: Validate OpenClaw managed-image security boundary shell: bash diff --git a/install.sh b/install.sh index ac8afd7d27f..b43a979df54 100755 --- a/install.sh +++ b/install.sh @@ -284,17 +284,6 @@ exec_installer_from_ref() { NEMOCLAW_INSTALL_TAG="$ref" bash "$legacy_script" "$@" } -bootstrap_force_fresh_install_requested() { - local arg - case "${NEMOCLAW_FORCE_FRESH_INSTALL:-}" in - 1 | true | TRUE | yes | YES | y | Y) return 0 ;; - esac - for arg in "$@"; do - [[ "$arg" == "--force-fresh-install" ]] && return 0 - done - return 1 -} - require_supported_platform() { # macOS ships only an Apple Silicon (aarch64) OpenShell gateway build, so an # Intel Mac (x86_64 Darwin) install always fails once that binary is fetched. diff --git a/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts b/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts index 05ca2a31558..f650a089fa7 100644 --- a/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts +++ b/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts @@ -40,6 +40,33 @@ export const DOCKER_ENGINE_27_MINIMUM_CLEANUP_PROCESS_TIMEOUT_MS = DOCKER_ENGINE_27_CLEANUP_OPERATION_COUNT * DOCKER_ENGINE_27_CLEANUP_TIMEOUT_MS + DOCKER_ENGINE_27_PROCESS_ALLOWANCE_MS / 2; +export type DockerEngine27Platform = "linux/amd64" | "linux/arm64"; + +export function dockerEngine27ReceiptDaemonName( + runId: number | string, + runAttempt: number | string, + platform: DockerEngine27Platform, +): string { + const run = String(runId); + const attempt = String(runAttempt); + requireCondition(/^[1-9][0-9]{0,15}$/u.test(run), "Docker Engine 27 run ID is invalid"); + requireCondition(/^[1-9][0-9]{0,5}$/u.test(attempt), "Docker Engine 27 run attempt is invalid"); + requireCondition( + platform === "linux/amd64" || platform === "linux/arm64", + "Docker Engine 27 platform is invalid", + ); + return `nemoclaw-receipt-engine27-${run}-${attempt}-${platform.replace("/", "-")}`; +} + +export function dockerEngine27ReceiptIdentityArguments( + runId: number | string, + runAttempt: number | string, + platform: DockerEngine27Platform, +): string[] { + dockerEngine27ReceiptDaemonName(runId, runAttempt, platform); + return ["--run-id", String(runId), "--run-attempt", String(runAttempt), "--platform", platform]; +} + export type CommandResult = { readonly error?: Error; readonly status: number | null; @@ -429,19 +456,29 @@ async function verifyDockerEngine27ReceiptTransfer(daemonName: string): Promise< ); } -function parseDaemonName(args: readonly string[]): string { - const index = args.indexOf("--daemon-name"); - const daemonName = index < 0 ? "" : String(args[index + 1] ?? ""); +function requiredArgument(args: readonly string[], name: string): string { + const index = args.indexOf(name); + const value = index < 0 ? "" : String(args[index + 1] ?? ""); + requireCondition(value.length > 0, `Docker Engine 27 probe requires ${name}`); + return value; +} + +function daemonNameFromArguments(args: readonly string[]): string { + const platform = requiredArgument(args, "--platform"); requireCondition( - /^nemoclaw-receipt-engine27-[a-z0-9-]{1,80}$/u.test(daemonName), - "Docker Engine 27 probe requires a safe owned daemon name", + platform === "linux/amd64" || platform === "linux/arm64", + "Docker Engine 27 platform is invalid", + ); + return dockerEngine27ReceiptDaemonName( + requiredArgument(args, "--run-id"), + requiredArgument(args, "--run-attempt"), + platform, ); - return daemonName; } if (path.resolve(process.argv[1] ?? "") === fileURLToPath(import.meta.url)) { const args = process.argv.slice(2); - const daemonName = parseDaemonName(args); + const daemonName = daemonNameFromArguments(args); const operation = args.includes("--cleanup-only") ? Promise.resolve().then(() => cleanupDockerEngine27ReceiptDaemon(daemonName)) : verifyDockerEngine27ReceiptTransfer(daemonName); diff --git a/test/e2e/live/managed-image-multiarch-startup-helpers.ts b/test/e2e/live/managed-image-multiarch-startup-helpers.ts index c7ecf4cff51..27b011abe80 100644 --- a/test/e2e/live/managed-image-multiarch-startup-helpers.ts +++ b/test/e2e/live/managed-image-multiarch-startup-helpers.ts @@ -25,14 +25,6 @@ export interface ProtectedManagedImageDispatchEnvironment { workspace: string; } -export function dockerEngine27ReceiptDaemonName( - runId: number, - runAttempt: number, - platform: ProtectedManagedImagePlatform, -): string { - return `nemoclaw-receipt-engine27-${String(runId)}-${String(runAttempt)}-${platform.replace("/", "-")}`; -} - function requiredEnvironment(name: string): string { const value = process.env[name]; if (!value) throw new Error(`${name} is required`); diff --git a/test/e2e/live/managed-image-multiarch-startup.test.ts b/test/e2e/live/managed-image-multiarch-startup.test.ts index 8a629a65cb9..b6db4e77544 100644 --- a/test/e2e/live/managed-image-multiarch-startup.test.ts +++ b/test/e2e/live/managed-image-multiarch-startup.test.ts @@ -11,9 +11,12 @@ import { parseProtectedManagedImageContracts, parseProtectedManagedImageEvidence, } from "../../../scripts/checks/protected-managed-image-contract.ts"; -import { expect, test } from "../fixtures/e2e-test.ts"; import { dockerEngine27ReceiptDaemonName, + dockerEngine27ReceiptIdentityArguments, +} from "../../../scripts/checks/docker-engine-27-receipt-transfer-e2e.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { protectedManagedImageDispatchEnvironment, readRegularArtifact, } from "./managed-image-multiarch-startup-helpers.ts"; @@ -66,6 +69,11 @@ test( dispatch.runAttempt, dispatch.platform, ); + const docker27IdentityArgs = dockerEngine27ReceiptIdentityArguments( + dispatch.runId, + dispatch.runAttempt, + dispatch.platform, + ); cleanup.trackDisposable(`remove owned Docker Engine 27 daemon ${docker27DaemonName}`, () => { execFileSync( process.execPath, @@ -74,8 +82,7 @@ test( "tsx", "scripts/checks/docker-engine-27-receipt-transfer-e2e.ts", "--cleanup-only", - "--daemon-name", - docker27DaemonName, + ...docker27IdentityArgs, ], { cwd: dispatch.workspace, @@ -92,8 +99,7 @@ test( "--import", "tsx", "scripts/checks/docker-engine-27-receipt-transfer-e2e.ts", - "--daemon-name", - docker27DaemonName, + ...docker27IdentityArgs, ], reason: "verify the Docker Engine 27 receipt archive-copy boundary", }), diff --git a/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts b/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts index f2bf29716ef..9b44d27e858 100644 --- a/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts +++ b/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts @@ -10,12 +10,13 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { DOCKER_ENGINE_27_MINIMUM_CLEANUP_PROCESS_TIMEOUT_MS, DOCKER_ENGINE_27_MINIMUM_PROBE_TIMEOUT_MS, + dockerEngine27ReceiptDaemonName, + dockerEngine27ReceiptIdentityArguments, finalizeDockerEngine27ReceiptProbe, requireDockerResourceAbsent, validateDockerEngine27SeedIsolation, } from "../../../scripts/checks/docker-engine-27-receipt-transfer-e2e.ts"; import { - dockerEngine27ReceiptDaemonName, protectedManagedImageDispatchEnvironment, readRegularArtifact, } from "../live/managed-image-multiarch-startup-helpers.ts"; @@ -61,13 +62,29 @@ describe("protected managed-image startup helpers", () => { expect(cleanupFixture).toHaveBeenCalledOnce(); }); - it("isolates Docker 27 daemon ownership across matrix platforms", () => { + it("derives one Docker 27 identity for probe and cleanup on each matrix platform", () => { const amd64 = dockerEngine27ReceiptDaemonName(123, 4, "linux/amd64"); const arm64 = dockerEngine27ReceiptDaemonName(123, 4, "linux/arm64"); + const sharedIdentity = dockerEngine27ReceiptIdentityArguments(123, 4, "linux/amd64"); + const probeArgs = ["docker-engine-27-receipt-transfer-e2e.ts", ...sharedIdentity]; + const cleanupArgs = [ + "docker-engine-27-receipt-transfer-e2e.ts", + "--cleanup-only", + ...sharedIdentity, + ]; expect(amd64).toBe("nemoclaw-receipt-engine27-123-4-linux-amd64"); expect(arm64).toBe("nemoclaw-receipt-engine27-123-4-linux-arm64"); expect(amd64).not.toBe(arm64); + expect(cleanupArgs.filter((arg) => arg !== "--cleanup-only")).toEqual(probeArgs); + expect(sharedIdentity).toEqual([ + "--run-id", + "123", + "--run-attempt", + "4", + "--platform", + "linux/amd64", + ]); }); it("bounds the complete Docker 27 probe and external cleanup", () => { diff --git a/test/installer-integration/install-force-fresh.test.ts b/test/installer-integration/install-force-fresh.test.ts index d8c4e01ddeb..3a4e64f1530 100644 --- a/test/installer-integration/install-force-fresh.test.ts +++ b/test/installer-integration/install-force-fresh.test.ts @@ -22,18 +22,6 @@ function installerCheckout(prefix: string) { return checkout; } -function callBootstrapFunction(command: string, env: Record = {}) { - return spawnSync("bash", ["-c", `source "${INSTALLER}" 2>/dev/null; ${command}`], { - cwd: path.join(import.meta.dirname, "../.."), - encoding: "utf-8", - env: { - HOME: os.tmpdir(), - PATH: TEST_SYSTEM_PATH, - ...env, - }, - }); -} - function callPayloadFunction(command: string, env: Record = {}) { return spawnSync("bash", ["-c", `source "${INSTALLER_PAYLOAD}" 2>/dev/null; ${command}`], { cwd: path.join(import.meta.dirname, "../.."), @@ -58,20 +46,6 @@ it("documents the destructive force-fresh install option", () => { expect(output).toContain("NEMOCLAW_FORCE_FRESH_INSTALL=1"); }); -it("recognizes force-fresh intent from the flag or environment", () => { - const result = callBootstrapFunction(` - bootstrap_force_fresh_install_requested --force-fresh-install && printf 'flag\\n' - NEMOCLAW_FORCE_FRESH_INSTALL=1 - bootstrap_force_fresh_install_requested && printf 'environment\\n' - unset NEMOCLAW_FORCE_FRESH_INSTALL - if bootstrap_force_fresh_install_requested; then exit 9; fi - printf 'absent\\n' - `); - - expect(result.status).toBe(0); - expect(result.stdout.trim().split("\n")).toEqual(["flag", "environment", "absent"]); -}); - it("detects a Homebrew-only OpenShell installation", () => { const { root: tmp, binDir: fakeBin } = installerCheckout("nemoclaw-force-fresh-detect-"); writeExecutable( diff --git a/tools/e2e/managed-image-multiarch-workflow-boundary.mts b/tools/e2e/managed-image-multiarch-workflow-boundary.mts index b1ffd41706e..0cf6c7d6447 100644 --- a/tools/e2e/managed-image-multiarch-workflow-boundary.mts +++ b/tools/e2e/managed-image-multiarch-workflow-boundary.mts @@ -398,11 +398,11 @@ export function validateManagedImageMultiarchWorkflow(workflow: WorkflowRecord): errors.push(`${JOB_ID} Docker Engine 27 receipt daemon cleanup must always run`); } requireFragments(errors, receiptDaemonCleanup, [ - 'platform="${NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM//\\//-}"', - 'daemon_name="nemoclaw-receipt-engine27-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${platform}"', "scripts/checks/docker-engine-27-receipt-transfer-e2e.ts", "--cleanup-only", - '--daemon-name "$daemon_name"', + '--run-id "$GITHUB_RUN_ID"', + '--run-attempt "$GITHUB_RUN_ATTEMPT"', + '--platform "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM"', ]); const cacheUpload = requireStep( errors, From 2b0fb35d0299b3e8039b45ba2673f06c5129ebbf Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 18 Sep 2026 08:06:31 -0700 Subject: [PATCH 22/39] fix(uninstall): own force-fresh residual cleanup Signed-off-by: Aaron Erickson --- scripts/install.sh | 10 +-- .../internal/uninstall/run-plan.test.ts | 6 +- src/commands/internal/uninstall/run-plan.ts | 2 + .../uninstall/all-gateway-ports.test.ts | 8 +- .../actions/uninstall/all-gateway-ports.ts | 1 + .../uninstall/run-plan-docker-scope.test.ts | 60 +++++++++++++ .../run-plan-homebrew-openshell.test.ts | 66 +++++++++----- src/lib/actions/uninstall/run-plan.ts | 89 ++++++++++++++++++- .../docker-receipt-transfer.ts | 4 +- .../install-force-fresh.test.ts | 44 ++++----- 10 files changed, 233 insertions(+), 57 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index af12f059b90..5bff331b2fd 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -3575,6 +3575,9 @@ force_fresh_install_has_existing_state() { || [[ -e "${HOME}/.local/state/nemoclaw" ]] \ || [[ -e "${HOME}/.local/bin/nemoclaw" ]] \ || [[ -e "${HOME}/.local/bin/openshell" ]] \ + || [[ -e "${HOME}/.local/bin/openshell-gateway" ]] \ + || [[ -e "${HOME}/.local/bin/openshell-sandbox" ]] \ + || [[ -e "${HOME}/.local/bin/openshell-driver-vm" ]] \ || command_exists nemoclaw \ || command_exists openshell; then return 0 @@ -3641,7 +3644,7 @@ run_force_fresh_uninstaller() { || error "Node.js is required for the force-fresh uninstaller." NEMOCLAW_UNINSTALL_DESTROY_USER_DATA=1 "$node_bin" \ "${source_root}/bin/nemoclaw.js" internal uninstall run-plan \ - --yes --destroy-user-data --all-gateway-ports + --yes --destroy-user-data --force-fresh-reset --all-gateway-ports } remove_macos_openshell_for_force_fresh_install() { @@ -3654,11 +3657,6 @@ remove_macos_openshell_for_force_fresh_install() { elif [[ "$openshell_path" == /opt/homebrew/* || "$openshell_path" == /usr/local/* ]]; then error "The force-fresh installer found a system OpenShell binary that Homebrew did not identify as ${formula}. Remove that installation explicitly, then rerun." fi - local binary - for binary in openshell openshell-gateway openshell-sandbox openshell-driver-vm; do - rm -f -- "${HOME}/.local/bin/${binary}" \ - || error "Could not remove ${HOME}/.local/bin/${binary} during the force-fresh install." - done hash -r 2>/dev/null || true } diff --git a/src/commands/internal/uninstall/run-plan.test.ts b/src/commands/internal/uninstall/run-plan.test.ts index 1eec8cf174e..89492bcd8fc 100644 --- a/src/commands/internal/uninstall/run-plan.test.ts +++ b/src/commands/internal/uninstall/run-plan.test.ts @@ -37,7 +37,9 @@ describe("internal uninstall command", () => { process.exitCode = undefined; const instance = new InternalUninstallRunPlanCommand([], {} as Config); Object.defineProperty(instance, "parse", { - value: vi.fn().mockResolvedValue({ flags: { yes: true, "keep-openshell": true } }), + value: vi.fn().mockResolvedValue({ + flags: { yes: true, "force-fresh-reset": true, "keep-openshell": true }, + }), }); const command = instance.run().then(() => { finished = true; @@ -45,7 +47,7 @@ describe("internal uninstall command", () => { try { await Promise.race([started, command]); expect(mocks.runUninstallPlanProduction).toHaveBeenCalledExactlyOnceWith( - expect.objectContaining({ assumeYes: true, keepOpenShell: true }), + expect.objectContaining({ assumeYes: true, forceFreshReset: true, keepOpenShell: true }), expect.objectContaining({ backupAllBeforeUninstall: expect.any(Function), withSandboxMutationLock: expect.any(Function), diff --git a/src/commands/internal/uninstall/run-plan.ts b/src/commands/internal/uninstall/run-plan.ts index e74af1470f5..3a91633e8fe 100644 --- a/src/commands/internal/uninstall/run-plan.ts +++ b/src/commands/internal/uninstall/run-plan.ts @@ -30,6 +30,7 @@ export default class InternalUninstallRunPlanCommand extends NemoClawCommand { "Uninstall every gateway port on this host, not only the port NEMOCLAW_GATEWAY_PORT selects", }), "all-gateway-ports-child": Flags.boolean({ hidden: true }), + "force-fresh-reset": Flags.boolean({ hidden: true }), "keep-openshell": Flags.boolean({ description: "Leave the openshell binary installed" }), "delete-models": Flags.boolean({ description: @@ -51,6 +52,7 @@ export default class InternalUninstallRunPlanCommand extends NemoClawCommand { assumeYes: flags.yes ?? false, deleteModels: flags["delete-models"] ?? false, destroyUserData: flags["destroy-user-data"] ?? false, + forceFreshReset: flags["force-fresh-reset"] ?? false, gatewayName: flags.gateway, keepOpenShell: flags["keep-openshell"] ?? false, }; diff --git a/src/lib/actions/uninstall/all-gateway-ports.test.ts b/src/lib/actions/uninstall/all-gateway-ports.test.ts index c6cb52f7921..229aaf7a869 100644 --- a/src/lib/actions/uninstall/all-gateway-ports.test.ts +++ b/src/lib/actions/uninstall/all-gateway-ports.test.ts @@ -352,7 +352,12 @@ describe("uninstall across every gateway port (#7791)", () => { ], [ "every passthrough flag", - { deleteModels: true, destroyUserData: true, keepOpenShell: true }, + { + deleteModels: true, + destroyUserData: true, + forceFreshReset: true, + keepOpenShell: true, + }, [ "internal", "uninstall", @@ -361,6 +366,7 @@ describe("uninstall across every gateway port (#7791)", () => { "--all-gateway-ports-child", "--delete-models", "--destroy-user-data", + "--force-fresh-reset", "--keep-openshell", ], ], diff --git a/src/lib/actions/uninstall/all-gateway-ports.ts b/src/lib/actions/uninstall/all-gateway-ports.ts index 62e4bd96ea2..3fe3cf3e28d 100644 --- a/src/lib/actions/uninstall/all-gateway-ports.ts +++ b/src/lib/actions/uninstall/all-gateway-ports.ts @@ -68,6 +68,7 @@ export function uninstallChildArgs(options: UninstallRunOptions): string[] { const args = ["internal", "uninstall", "run-plan", "--yes", "--all-gateway-ports-child"]; if (options.deleteModels) args.push("--delete-models"); if (options.destroyUserData) args.push("--destroy-user-data"); + if (options.forceFreshReset) args.push("--force-fresh-reset"); if (options.keepOpenShell) args.push("--keep-openshell"); return args; } diff --git a/src/lib/actions/uninstall/run-plan-docker-scope.test.ts b/src/lib/actions/uninstall/run-plan-docker-scope.test.ts index c8aa8d3260d..b1b8ea642d0 100644 --- a/src/lib/actions/uninstall/run-plan-docker-scope.test.ts +++ b/src/lib/actions/uninstall/run-plan-docker-scope.test.ts @@ -151,4 +151,64 @@ describe("uninstall Docker resource scope", () => { expect(calls).toContainEqual(["rmi", "-f", "i-openshell"]); }); + + it.each([ + ["removes", 0, 0], + ["fails closed on", 1, 1], + ] as const)( + "%s retained managed-startup receipt volumes during force-fresh cleanup", + async (_scenario, removalStatus, expectedExitCode) => { + const owned = `nemoclaw-managed-startup-receipt-volume-${"a".repeat(32)}`; + const unrelated = "nemoclaw-managed-startup-receipt-volume-user-data"; + const calls: string[][] = []; + let ownedPresent = true; + const runDocker = vi.fn((args: string[]) => { + calls.push(args); + const command = args.join(" "); + if (command === "info") return ok(); + if (command === "ps -a --format {{.ID}} {{.Image}} {{.Names}}") return ok(); + if (command === "images --format {{.ID}} {{.Repository}}:{{.Tag}}") return ok(); + if (command === "volume ls --format {{.Name}}") { + return ok(`${ownedPresent ? `${owned}\n` : ""}${unrelated}\n`); + } + if (command === `volume rm -f ${owned}`) { + if (removalStatus === 0) ownedPresent = false; + return { status: removalStatus, stdout: removalStatus === 0 ? owned : "", stderr: "" }; + } + if (command.startsWith("volume inspect ")) { + return { status: 1, stdout: "", stderr: "Error: no such volume" }; + } + return ok(); + }); + + const result = await runUninstallPlan( + { + assumeYes: true, + deleteModels: false, + destroyUserData: true, + forceFreshReset: true, + keepOpenShell: true, + }, + { + commandExists: () => true, + env: { HOME: "/tmp/nemoclaw-force-fresh-receipts" } as NodeJS.ProcessEnv, + existsSync: () => false, + hasPortableRuntimeCleanup: () => false, + isTty: false, + kill: () => true, + log: () => undefined, + rmSync: vi.fn(), + run: (command, args) => + command === "openshell" && args.join(" ") === "gateway list -o json" + ? ok(JSON.stringify([{ name: "nemoclaw" }])) + : ok(), + runDocker, + }, + ); + + expect(result.exitCode).toBe(expectedExitCode); + expect(calls).toContainEqual(["volume", "rm", "-f", owned]); + expect(calls).not.toContainEqual(["volume", "rm", "-f", unrelated]); + }, + ); }); diff --git a/src/lib/actions/uninstall/run-plan-homebrew-openshell.test.ts b/src/lib/actions/uninstall/run-plan-homebrew-openshell.test.ts index 40aa566beea..89ad21a6b41 100644 --- a/src/lib/actions/uninstall/run-plan-homebrew-openshell.test.ts +++ b/src/lib/actions/uninstall/run-plan-homebrew-openshell.test.ts @@ -21,9 +21,9 @@ function ok(stdout = ""): RunResult { return { status: 0, stdout, stderr: "" }; } -async function runUninstallPlan(deps: UninstallRunDeps) { +async function runUninstallPlan(deps: UninstallRunDeps, forceFreshReset = false) { return await runUninstallPlanBase( - { assumeYes: true, deleteModels: false, keepOpenShell: false }, + { assumeYes: true, deleteModels: false, forceFreshReset, keepOpenShell: false }, { resolveGatewayTeardownAuthority: ({ gatewayName, gatewayPort }) => ({ gatewayName, @@ -43,6 +43,7 @@ async function runUninstallPlan(deps: UninstallRunDeps) { async function uninstallOpenShell(options: { brewAvailable: boolean; brewStatus: number | null; + forceFreshReset?: boolean; platform?: NodeJS.Platform; }) { const home = "/tmp/nemoclaw-uninstall-test"; @@ -56,28 +57,31 @@ async function uninstallOpenShell(options: { removed.push(target); return ok(); }; - const result = await runUninstallPlan({ - commandExists: (command) => - command === "openshell" || (command === "brew" && options.brewAvailable), - env: { HOME: home } as NodeJS.ProcessEnv, - existsSync: (target) => existing.has(String(target)), - hasPortableRuntimeCleanup: () => false, - isTty: true, - log: (line) => logs.push(line), - platform: options.platform ?? "darwin", - rmSync: vi.fn((target) => remove(String(target))), - run: vi.fn((command, args) => { - calls.push([command, ...args]); - return command === "sudo" && args[0] === "rm" && args[1] === "-f" - ? remove(args[2]) - : command === "openshell" && args[0] === "gateway" && args[1] === "list" - ? ok(JSON.stringify([{ name: "nemoclaw" }])) - : command === "brew" && args[0] === "list" - ? { status: options.brewStatus, stdout: "", stderr: "" } - : ok(); - }), - runDocker: () => ok(), - }); + const result = await runUninstallPlan( + { + commandExists: (command) => + command === "openshell" || (command === "brew" && options.brewAvailable), + env: { HOME: home } as NodeJS.ProcessEnv, + existsSync: (target) => existing.has(String(target)), + hasPortableRuntimeCleanup: () => false, + isTty: true, + log: (line) => logs.push(line), + platform: options.platform ?? "darwin", + rmSync: vi.fn((target) => remove(String(target))), + run: vi.fn((command, args) => { + calls.push([command, ...args]); + return command === "sudo" && args[0] === "rm" && args[1] === "-f" + ? remove(args[2]) + : command === "openshell" && args[0] === "gateway" && args[1] === "list" + ? ok(JSON.stringify([{ name: "nemoclaw" }])) + : command === "brew" && args[0] === "list" + ? { status: options.brewStatus, stdout: "", stderr: "" } + : ok(); + }), + runDocker: () => ok(), + }, + options.forceFreshReset, + ); return { calls, executablePaths, logs, remaining: [...existing], removed, result }; } @@ -98,6 +102,20 @@ it("retains a Homebrew-managed OpenShell and reports its removal command (#8882) ); }); +it("lets force-fresh remove only managed user-local OpenShell binaries", async () => { + const { executablePaths, remaining, removed, result } = await uninstallOpenShell({ + brewAvailable: true, + brewStatus: 0, + forceFreshReset: true, + }); + const userLocal = executablePaths.filter((target) => target.includes("/.local/bin/")); + const system = executablePaths.filter((target) => target.startsWith("/usr/local/bin/")); + + expect(result.exitCode).toBe(0); + expect(new Set(removed)).toEqual(new Set(userLocal)); + expect(new Set(remaining)).toEqual(new Set(system)); +}); + it.each([ { label: "Homebrew is unavailable", diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index 961976ba567..d12210e960e 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -48,6 +48,7 @@ import { MCP_LIFECYCLE_LOCK_DIRNAME, } from "../../inference/serving/managed-runtime-receipts"; import { buildDockerGatewayDebEnvFile } from "../../onboard/docker-driver-gateway-env"; +import { MANAGED_STARTUP_RECEIPT_VOLUME_PREFIX } from "../../onboard/managed-startup/docker-receipt-transfer"; import { getTrustedActiveOpenShellGatewayUserServiceIdentity, getNemoclawOpenShellGatewayUserServicePath, @@ -157,6 +158,7 @@ export interface UninstallRunOptions { assumeYes: boolean; deleteModels: boolean; destroyUserData?: boolean; + forceFreshReset?: boolean; gatewayName?: string; keepOpenShell: boolean; } @@ -854,6 +856,24 @@ function reportRetainedMacOsOpenShell(runtime: UninstallRuntime): void { ); } +function removeForceFreshUserLocalOpenShell( + paths: UninstallPaths, + runtime: UninstallRuntime, +): boolean { + const userBin = path.resolve( + runtime.env.XDG_BIN_HOME || path.join(runtime.env.HOME || os.homedir(), ".local", "bin"), + ); + try { + for (const target of paths.openshellInstallPaths) { + if (path.dirname(path.resolve(target)) === userBin) removePath(target, runtime); + } + return true; + } catch (error) { + runtime.error(`Could not remove managed user-local OpenShell binaries: ${formatError(error)}`); + return false; + } +} + async function deletePortableOpenShellSandbox( runtime: UninstallRuntime, sandboxName: string, @@ -2362,6 +2382,54 @@ function removeDockerVolume(name: string, runtime: UninstallRuntime): void { else runtime.warn(`Failed to remove Docker volume ${name}`); } +const MANAGED_STARTUP_RECEIPT_VOLUME_PATTERN = new RegExp( + `^${MANAGED_STARTUP_RECEIPT_VOLUME_PREFIX}-[0-9a-f]{32}$`, + "u", +); + +function removeForceFreshReceiptVolumes(runtime: UninstallRuntime): boolean { + const inventory = runtime.runDocker(["volume", "ls", "--format", "{{.Name}}"], { + env: runtime.env, + }); + if (inventory.status !== 0) { + runtime.error( + "Could not inventory managed-startup receipt volumes during force-fresh cleanup.", + ); + return false; + } + const volumes = splitNonEmptyLines(inventory.stdout).filter((name) => + MANAGED_STARTUP_RECEIPT_VOLUME_PATTERN.test(name), + ); + for (const volume of volumes) { + const removed = runtime.runDocker(["volume", "rm", "-f", volume], { + env: runtime.env, + stdio: "ignore", + }); + if (removed.status !== 0) { + runtime.error(`Managed-startup receipt volume '${volume}' could not be removed.`); + return false; + } + runtime.log(`Removed managed-startup receipt volume ${volume}`); + } + const remaining = runtime.runDocker(["volume", "ls", "--format", "{{.Name}}"], { + env: runtime.env, + }); + if (remaining.status !== 0) { + runtime.error("Could not verify managed-startup receipt volume cleanup."); + return false; + } + const retained = splitNonEmptyLines(remaining.stdout).find((name) => + MANAGED_STARTUP_RECEIPT_VOLUME_PATTERN.test(name), + ); + if (retained) { + runtime.error( + `Managed-startup receipt volume '${retained}' remains after force-fresh cleanup.`, + ); + return false; + } + return true; +} + function parseOllamaModelInventory(output: string): string[] { const rows = output .split(/\r?\n/u) @@ -4117,6 +4185,22 @@ async function executePreparedPlan( step.actions.forEach((action) => { if (action.kind === "delete-docker-volume") removeDockerVolume(action.name, runtime); }); + if (options.forceFreshReset) { + if (scopedToSelectedGateway) { + runtime.error( + "Force-fresh cleanup preserved receipt volumes because another gateway environment remains.", + ); + return { ok: false, scopedToSelectedGateway }; + } + if (!removeForceFreshReceiptVolumes(runtime)) { + return { ok: false, scopedToSelectedGateway }; + } + } + } else if (options.forceFreshReset && runtime.commandExists("docker")) { + runtime.error( + "Docker is installed but unavailable; force-fresh cleanup cannot prove that receipt volumes are absent.", + ); + return { ok: false, scopedToSelectedGateway }; } } else if (step.name === "Model stores") { if ( @@ -4136,9 +4220,12 @@ async function executePreparedPlan( for (const pattern of paths.runtimeTempGlobs) removeGlob(pattern, runtime); if (preserveSharedOpenShell) { runtime.log(binaryKeepMessage); - } else if (GATEWAY_PORT !== DEFAULT_GATEWAY_PORT) { + } else if (GATEWAY_PORT !== DEFAULT_GATEWAY_PORT && !options.forceFreshReset) { runtime.log("Keeping OpenShell binaries used by the default gateway service."); } else if (runtime.platform === "darwin") { + if (options.forceFreshReset && !removeForceFreshUserLocalOpenShell(paths, runtime)) { + return { ok: false, scopedToSelectedGateway }; + } reportRetainedMacOsOpenShell(runtime); } else { paths.openshellInstallPaths.forEach((target) => diff --git a/src/lib/onboard/managed-startup/docker-receipt-transfer.ts b/src/lib/onboard/managed-startup/docker-receipt-transfer.ts index cf4e2253de0..a3e00f283da 100644 --- a/src/lib/onboard/managed-startup/docker-receipt-transfer.ts +++ b/src/lib/onboard/managed-startup/docker-receipt-transfer.ts @@ -10,7 +10,7 @@ import { cleanupTempDir } from "../temp-files"; const RECEIPT_VOLUME_DIRECTORY = "/run/nemoclaw/managed-startup-receipt-transfer"; const RECEIPT_SEED_PREFIX = "nemoclaw-managed-startup-receipt-seed"; -const RECEIPT_VOLUME_PREFIX = "nemoclaw-managed-startup-receipt-volume"; +export const MANAGED_STARTUP_RECEIPT_VOLUME_PREFIX = "nemoclaw-managed-startup-receipt-volume"; export type DockerDaemonReceipt = { readonly hostPath: string; @@ -100,7 +100,7 @@ export function transferDockerReceiptToDaemon( options: DockerReceiptTransferOptions, ): DockerDaemonReceipt { const dockerRun = options.dockerRun; - const volumeName = receiptName(RECEIPT_VOLUME_PREFIX); + const volumeName = receiptName(MANAGED_STARTUP_RECEIPT_VOLUME_PREFIX); const seedName = receiptName(RECEIPT_SEED_PREFIX); let volumeCreated = false; let seedCreated = false; diff --git a/test/installer-integration/install-force-fresh.test.ts b/test/installer-integration/install-force-fresh.test.ts index 3a4e64f1530..d896d164ff5 100644 --- a/test/installer-integration/install-force-fresh.test.ts +++ b/test/installer-integration/install-force-fresh.test.ts @@ -129,7 +129,7 @@ printf 'destroy=%s args=%s\n' "\${NEMOCLAW_UNINSTALL_DESTROY_USER_DATA:-}" "$*" expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); expect(fs.readFileSync(logPath, "utf-8")).toBe( - `destroy=1 args=${path.join(sourceRoot, "bin", "nemoclaw.js")} internal uninstall run-plan --yes --destroy-user-data --all-gateway-ports\n`, + `destroy=1 args=${path.join(sourceRoot, "bin", "nemoclaw.js")} internal uninstall run-plan --yes --destroy-user-data --force-fresh-reset --all-gateway-ports\n`, ); }); @@ -148,7 +148,7 @@ it("stops when the staged candidate uninstaller fails", () => { expect(result.status).not.toBe(0); }); -it("removes Homebrew and standalone OpenShell installations", () => { +it("removes only Homebrew after the canonical uninstaller handles user-local binaries", () => { const { root: tmp, binDir: fakeBin } = installerCheckout("nemoclaw-force-fresh-brew-"); const localBin = path.join(tmp, ".local", "bin"); const logPath = path.join(tmp, "brew.log"); @@ -177,10 +177,10 @@ exit 0 "services stop nvidia/openshell/openshell", "uninstall --force nvidia/openshell/openshell", ]); - expect(fs.existsSync(path.join(localBin, "openshell"))).toBe(false); - expect(fs.existsSync(path.join(localBin, "openshell-gateway"))).toBe(false); - expect(fs.existsSync(path.join(localBin, "openshell-sandbox"))).toBe(false); - expect(fs.existsSync(path.join(localBin, "openshell-driver-vm"))).toBe(false); + expect(fs.existsSync(path.join(localBin, "openshell"))).toBe(true); + expect(fs.existsSync(path.join(localBin, "openshell-gateway"))).toBe(true); + expect(fs.existsSync(path.join(localBin, "openshell-sandbox"))).toBe(true); + expect(fs.existsSync(path.join(localBin, "openshell-driver-vm"))).toBe(true); }, 30_000); it("stops when Homebrew cannot remove OpenShell", () => { @@ -225,21 +225,23 @@ it("runs cleanup before selecting fresh onboarding", () => { ]); }); -it("removes standalone OpenShell helpers even when no other state is detected", () => { - const result = callPayloadFunction(` - warn() { :; } - info() { :; } - force_fresh_install_has_existing_state() { return 1; } - prepare_force_fresh_uninstaller() { printf 'unexpected-prepare\n'; } - run_force_fresh_uninstaller() { printf 'unexpected-uninstall\n'; } - remove_macos_openshell_for_force_fresh_install() { printf 'openshell-reset\n'; } - run_force_fresh_install_reset - `); - - expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); - expect(result.stdout.trim()).toBe("openshell-reset"); - expect(result.stdout).not.toContain("unexpected-"); -}); +it.each(["openshell-gateway", "openshell-sandbox", "openshell-driver-vm"])( + "routes a standalone %s helper through canonical cleanup", + (binary) => { + const { root: tmp } = installerCheckout("nemoclaw-force-fresh-helper-detect-"); + const localBin = path.join(tmp, ".local", "bin"); + fs.mkdirSync(localBin, { recursive: true }); + writeExecutable(path.join(localBin, binary), "#!/usr/bin/env bash\nexit 0\n"); + + const result = callPayloadFunction( + "force_fresh_install_has_existing_state && printf 'present'", + { HOME: tmp }, + ); + + expect(result.status).toBe(0); + expect(result.stdout).toBe("present"); + }, +); it("stops before package removal when managed uninstall rejects partial state", () => { const result = callPayloadFunction(` From c8cbe1a001a0829caa7e160f40307d819ac13f57 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 18 Sep 2026 08:14:20 -0700 Subject: [PATCH 23/39] refactor(uninstall): isolate force-fresh cleanup Signed-off-by: Aaron Erickson --- .../uninstall/run-plan-docker-scope.test.ts | 31 ++--- src/lib/actions/uninstall/run-plan.ts | 109 +++++++----------- src/lib/actions/uninstall/runtime-commands.ts | 65 +++++++++++ 3 files changed, 121 insertions(+), 84 deletions(-) diff --git a/src/lib/actions/uninstall/run-plan-docker-scope.test.ts b/src/lib/actions/uninstall/run-plan-docker-scope.test.ts index b1b8ea642d0..cf1ecc11305 100644 --- a/src/lib/actions/uninstall/run-plan-docker-scope.test.ts +++ b/src/lib/actions/uninstall/run-plan-docker-scope.test.ts @@ -162,23 +162,26 @@ describe("uninstall Docker resource scope", () => { const unrelated = "nemoclaw-managed-startup-receipt-volume-user-data"; const calls: string[][] = []; let ownedPresent = true; + const routes: Record RunResult> = { + info: () => ok(), + "ps -a --format {{.ID}} {{.Image}} {{.Names}}": () => ok(), + "images --format {{.ID}} {{.Repository}}:{{.Tag}}": () => ok(), + "volume ls --format {{.Name}}": () => + ok(`${ownedPresent ? `${owned}\n` : ""}${unrelated}\n`), + [`volume rm -f ${owned}`]: () => { + ownedPresent = removalStatus === 0 ? false : ownedPresent; + return { status: removalStatus, stdout: removalStatus === 0 ? owned : "", stderr: "" }; + }, + "volume inspect openshell-cluster-nemoclaw": () => ({ + status: 1, + stdout: "", + stderr: "Error: no such volume", + }), + }; const runDocker = vi.fn((args: string[]) => { calls.push(args); const command = args.join(" "); - if (command === "info") return ok(); - if (command === "ps -a --format {{.ID}} {{.Image}} {{.Names}}") return ok(); - if (command === "images --format {{.ID}} {{.Repository}}:{{.Tag}}") return ok(); - if (command === "volume ls --format {{.Name}}") { - return ok(`${ownedPresent ? `${owned}\n` : ""}${unrelated}\n`); - } - if (command === `volume rm -f ${owned}`) { - if (removalStatus === 0) ownedPresent = false; - return { status: removalStatus, stdout: removalStatus === 0 ? owned : "", stderr: "" }; - } - if (command.startsWith("volume inspect ")) { - return { status: 1, stdout: "", stderr: "Error: no such volume" }; - } - return ok(); + return (routes[command] ?? (() => ok()))(); }); const result = await runUninstallPlan( diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index d12210e960e..52d33ee2e6a 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -48,7 +48,6 @@ import { MCP_LIFECYCLE_LOCK_DIRNAME, } from "../../inference/serving/managed-runtime-receipts"; import { buildDockerGatewayDebEnvFile } from "../../onboard/docker-driver-gateway-env"; -import { MANAGED_STARTUP_RECEIPT_VOLUME_PREFIX } from "../../onboard/managed-startup/docker-receipt-transfer"; import { getTrustedActiveOpenShellGatewayUserServiceIdentity, getNemoclawOpenShellGatewayUserServicePath, @@ -130,6 +129,7 @@ import { isModelRouterPid, isOllamaAuthProxyPid, pidExists, + removeForceFreshReceiptVolumes, } from "./runtime-commands"; import { buildUninstallPlan, @@ -2382,52 +2382,44 @@ function removeDockerVolume(name: string, runtime: UninstallRuntime): void { else runtime.warn(`Failed to remove Docker volume ${name}`); } -const MANAGED_STARTUP_RECEIPT_VOLUME_PATTERN = new RegExp( - `^${MANAGED_STARTUP_RECEIPT_VOLUME_PREFIX}-[0-9a-f]{32}$`, - "u", -); - -function removeForceFreshReceiptVolumes(runtime: UninstallRuntime): boolean { - const inventory = runtime.runDocker(["volume", "ls", "--format", "{{.Name}}"], { - env: runtime.env, - }); - if (inventory.status !== 0) { +function executeDockerResourceStep( + options: UninstallRunOptions, + runtime: UninstallRuntime, + scopedToSelectedGateway: boolean, + externallySupervised: boolean, + volumeNames: readonly string[], +): boolean { + if (externallySupervised) { + runtime.log( + "Kept Docker containers, images, and volumes used by the externally supervised gateway.", + ); + return true; + } + if (!dockerIsAvailable(runtime)) { + if (!options.forceFreshReset || !runtime.commandExists("docker")) return true; runtime.error( - "Could not inventory managed-startup receipt volumes during force-fresh cleanup.", + "Docker is installed but unavailable; force-fresh cleanup cannot prove that receipt volumes are absent.", ); return false; } - const volumes = splitNonEmptyLines(inventory.stdout).filter((name) => - MANAGED_STARTUP_RECEIPT_VOLUME_PATTERN.test(name), + removeDockerContainers( + runtime, + scopedToSelectedGateway ? options.gatewayName || resolveGatewayName(GATEWAY_PORT) : undefined, ); - for (const volume of volumes) { - const removed = runtime.runDocker(["volume", "rm", "-f", volume], { - env: runtime.env, - stdio: "ignore", - }); - if (removed.status !== 0) { - runtime.error(`Managed-startup receipt volume '${volume}' could not be removed.`); - return false; - } - runtime.log(`Removed managed-startup receipt volume ${volume}`); - } - const remaining = runtime.runDocker(["volume", "ls", "--format", "{{.Name}}"], { - env: runtime.env, - }); - if (remaining.status !== 0) { - runtime.error("Could not verify managed-startup receipt volume cleanup."); - return false; + if (scopedToSelectedGateway) { + runtime.log("Sibling gateways remain; kept shared Docker images."); + } else { + removeDockerImages(runtime); } - const retained = splitNonEmptyLines(remaining.stdout).find((name) => - MANAGED_STARTUP_RECEIPT_VOLUME_PATTERN.test(name), - ); - if (retained) { + for (const volumeName of volumeNames) removeDockerVolume(volumeName, runtime); + if (!options.forceFreshReset) return true; + if (scopedToSelectedGateway) { runtime.error( - `Managed-startup receipt volume '${retained}' remains after force-fresh cleanup.`, + "Force-fresh cleanup preserved receipt volumes because another gateway environment remains.", ); return false; } - return true; + return removeForceFreshReceiptVolumes(runtime); } function parseOllamaModelInventory(output: string): string[] { @@ -4166,40 +4158,17 @@ async function executePreparedPlan( otherGatewayPorts, ); } else if (step.name === "Docker resources") { - if (externallySupervised) { - runtime.log( - "Kept Docker containers, images, and volumes used by the externally supervised gateway.", - ); - } else if (dockerIsAvailable(runtime)) { - removeDockerContainers( + if ( + !executeDockerResourceStep( + options, runtime, - scopedToSelectedGateway - ? options.gatewayName || resolveGatewayName(GATEWAY_PORT) - : undefined, - ); - if (scopedToSelectedGateway) { - runtime.log("Sibling gateways remain; kept shared Docker images."); - } else { - removeDockerImages(runtime); - } - step.actions.forEach((action) => { - if (action.kind === "delete-docker-volume") removeDockerVolume(action.name, runtime); - }); - if (options.forceFreshReset) { - if (scopedToSelectedGateway) { - runtime.error( - "Force-fresh cleanup preserved receipt volumes because another gateway environment remains.", - ); - return { ok: false, scopedToSelectedGateway }; - } - if (!removeForceFreshReceiptVolumes(runtime)) { - return { ok: false, scopedToSelectedGateway }; - } - } - } else if (options.forceFreshReset && runtime.commandExists("docker")) { - runtime.error( - "Docker is installed but unavailable; force-fresh cleanup cannot prove that receipt volumes are absent.", - ); + scopedToSelectedGateway, + externallySupervised, + step.actions.flatMap((action) => + action.kind === "delete-docker-volume" ? [action.name] : [], + ), + ) + ) { return { ok: false, scopedToSelectedGateway }; } } else if (step.name === "Model stores") { diff --git a/src/lib/actions/uninstall/runtime-commands.ts b/src/lib/actions/uninstall/runtime-commands.ts index a0d3dc97b36..70b03f8daee 100644 --- a/src/lib/actions/uninstall/runtime-commands.ts +++ b/src/lib/actions/uninstall/runtime-commands.ts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import type { SpawnSyncOptions } from "node:child_process"; + import { createUninstallSandboxLifecycle, createUninstallSandboxObserver, @@ -12,6 +14,7 @@ import { } from "../../domain/uninstall/messaging"; import { isOllamaAuthProxyCommandLine } from "../../inference/ollama/process"; import { isModelRouterCommandLineForPort } from "../../onboard/model-router-process"; +import { MANAGED_STARTUP_RECEIPT_VOLUME_PREFIX } from "../../onboard/managed-startup/docker-receipt-transfer"; interface UninstallRuntimeCommands { env: NodeJS.ProcessEnv; @@ -21,6 +24,68 @@ interface UninstallRuntimeCommands { warn(message: string): void; } +interface ForceFreshDockerCleanupRuntime { + env: NodeJS.ProcessEnv; + error(message: string): void; + log(message: string): void; + runDocker(args: string[], options?: SpawnSyncOptions): RunResult; +} + +const MANAGED_STARTUP_RECEIPT_VOLUME_PATTERN = new RegExp( + `^${MANAGED_STARTUP_RECEIPT_VOLUME_PREFIX}-[0-9a-f]{32}$`, + "u", +); + +function nonEmptyLines(output: string): string[] { + return output + .split(/\r?\n/u) + .map((line) => line.trim()) + .filter(Boolean); +} + +export function removeForceFreshReceiptVolumes(runtime: ForceFreshDockerCleanupRuntime): boolean { + const inventory = runtime.runDocker(["volume", "ls", "--format", "{{.Name}}"], { + env: runtime.env, + }); + if (inventory.status !== 0) { + runtime.error( + "Could not inventory managed-startup receipt volumes during force-fresh cleanup.", + ); + return false; + } + const volumes = nonEmptyLines(inventory.stdout).filter((name) => + MANAGED_STARTUP_RECEIPT_VOLUME_PATTERN.test(name), + ); + for (const volume of volumes) { + const removed = runtime.runDocker(["volume", "rm", "-f", volume], { + env: runtime.env, + stdio: "ignore", + }); + if (removed.status !== 0) { + runtime.error(`Managed-startup receipt volume '${volume}' could not be removed.`); + return false; + } + runtime.log(`Removed managed-startup receipt volume ${volume}`); + } + const remaining = runtime.runDocker(["volume", "ls", "--format", "{{.Name}}"], { + env: runtime.env, + }); + if (remaining.status !== 0) { + runtime.error("Could not verify managed-startup receipt volume cleanup."); + return false; + } + const retained = nonEmptyLines(remaining.stdout).find((name) => + MANAGED_STARTUP_RECEIPT_VOLUME_PATTERN.test(name), + ); + if (retained) { + runtime.error( + `Managed-startup receipt volume '${retained}' remains after force-fresh cleanup.`, + ); + return false; + } + return true; +} + export async function deleteSelectedGatewaySandbox( runtime: UninstallRuntimeCommands, gatewayName: string, From 54686ef28563bd391c7a13c1a015a7e4afe8d28a Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 18 Sep 2026 09:10:43 -0700 Subject: [PATCH 24/39] fix(uninstall): require receipt volume ownership Signed-off-by: Aaron Erickson --- install.sh | 4 +- .../docker-engine-27-receipt-transfer-e2e.ts | 2 +- scripts/install.sh | 46 ++++++---- .../uninstall/run-plan-docker-scope.test.ts | 27 ++++-- src/lib/actions/uninstall/runtime-commands.ts | 66 ++++++++++---- .../docker-receipt-transfer.test.ts | 4 + .../docker-receipt-transfer.ts | 6 +- .../install-force-fresh.test.ts | 90 ++++++++++++++++--- 8 files changed, 189 insertions(+), 56 deletions(-) diff --git a/install.sh b/install.sh index b43a979df54..706b1c7ff11 100755 --- a/install.sh +++ b/install.sh @@ -313,7 +313,7 @@ bootstrap_usage() { printf " Use only with NEMOCLAW_AGENT=hermes, no registered sandboxes, no local model profile,\n" printf " and the build, cloud, or routed NVIDIA hosted provider\n" printf " --fresh Discard any failed/interrupted onboarding session and start over\n" - printf " --force-fresh-install Destroy all NemoClaw and OpenShell state, then reinstall\n" + printf " --force-fresh-install Destroy all NemoClaw and OpenShell state, then reinstall (macOS only)\n" printf " --force-station-install Validate an unrecognized Station GB300 release profile without onboarding\n" printf " --version, -v Print installer version and exit\n" printf " --help, -h Show this help message and exit\n\n" @@ -327,7 +327,7 @@ bootstrap_usage() { printf " Use only with NEMOCLAW_AGENT=hermes, no registered sandboxes, no local model profile,\n" printf " and the build, cloud, or routed NVIDIA hosted provider\n" printf " NEMOCLAW_FRESH=1 Same as --fresh\n" - printf " NEMOCLAW_FORCE_FRESH_INSTALL=1 Same as --force-fresh-install\n" + printf " NEMOCLAW_FORCE_FRESH_INSTALL=1 Same as --force-fresh-install (macOS only)\n" printf " NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 Same as --yes-i-accept-third-party-software\n" printf " NEMOCLAW_NO_EXPRESS=1 Skip express install prompt on supported platforms\n" printf " NEMOCLAW_SANDBOX_NAME Sandbox name to create/use\n" diff --git a/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts b/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts index f650a089fa7..b2579f77234 100644 --- a/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts +++ b/scripts/checks/docker-engine-27-receipt-transfer-e2e.ts @@ -407,7 +407,7 @@ async function verifyDockerEngine27ReceiptTransfer(daemonName: string): Promise< let failedSeed = ""; let failedVolume = ""; const failingDockerRun = (args: readonly string[]): CommandResult => { - if (args[0] === "volume" && args[1] === "create") failedVolume = String(args[2] ?? ""); + if (args[0] === "volume" && args[1] === "create") failedVolume = String(args.at(-1) ?? ""); if (args[0] === "create") failedSeed = seedName(args); return innerDocker(args); }; diff --git a/scripts/install.sh b/scripts/install.sh index 5bff331b2fd..e92bc8f3d3d 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -1056,7 +1056,7 @@ usage() { printf " Use only with NEMOCLAW_AGENT=hermes, no registered sandboxes, no local model profile,\n" printf " and the build, cloud, or routed NVIDIA hosted provider\n" printf " --fresh Discard any failed/interrupted onboarding session and start over\n" - printf " --force-fresh-install Destroy all NemoClaw and OpenShell state, then reinstall\n" + printf " --force-fresh-install Destroy all NemoClaw and OpenShell state, then reinstall (macOS only)\n" printf " --station-deepseek Use DeepSeek V4 Flash for DGX Station express install (interactive terminal required)\n" printf " --force-station-install Validate an unrecognized Station GB300 release profile without onboarding\n" printf " --version, -v Print installer version and exit\n" @@ -1070,7 +1070,7 @@ usage() { printf " and the build, cloud, or routed NVIDIA hosted provider\n" printf " NEMOCLAW_NON_INTERACTIVE_SUDO_MODE=prompt Allow sudo prompts during non-interactive onboarding\n" printf " NEMOCLAW_FRESH=1 Same as --fresh\n" - printf " NEMOCLAW_FORCE_FRESH_INSTALL=1 Same as --force-fresh-install\n" + printf " NEMOCLAW_FORCE_FRESH_INSTALL=1 Same as --force-fresh-install (macOS only)\n" printf " NEMOCLAW_NO_EXPRESS=1 Skip the Express prompt on detected platforms\n" printf " NEMOCLAW_SANDBOX_NAME Sandbox name to create/use\n" printf " HF_TOKEN Optional Hugging Face read token for managed-vLLM downloads\n" @@ -3569,6 +3569,7 @@ run_preupgrade_backup() { } force_fresh_install_has_existing_state() { + local container_inventory receipt_volume_inventory volume_name if [[ -e "$(nemoclaw_state_root)" ]] \ || [[ -e "${HOME}/.config/nemoclaw" ]] \ || [[ -e "${HOME}/.config/openshell" ]] \ @@ -3586,11 +3587,20 @@ force_fresh_install_has_existing_state() { && brew list --formula nvidia/openshell/openshell >/dev/null 2>&1; then return 0 fi - if command_exists docker \ - && docker info >/dev/null 2>&1 \ - && docker ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract 2>/dev/null \ - | grep -q .; then - return 0 + if command_exists docker; then + docker info >/dev/null 2>&1 || return 2 + container_inventory="$( + docker ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract 2>/dev/null + )" || return 2 + [[ -z "$container_inventory" ]] || return 0 + receipt_volume_inventory="$( + docker volume ls \ + --filter label=io.nvidia.nemoclaw.managed-startup.receipt=1 \ + --format '{{.Name}}' 2>/dev/null + )" || return 2 + while IFS= read -r volume_name; do + [[ "$volume_name" =~ ^nemoclaw-managed-startup-receipt-volume-[0-9a-f]{32}$ ]] && return 0 + done <<<"$receipt_volume_inventory" fi return 1 } @@ -3661,16 +3671,20 @@ remove_macos_openshell_for_force_fresh_install() { } run_force_fresh_install_reset() { - local source_root + local source_root existing_state_status=0 warn "Force-fresh install selected. NemoClaw will destroy all NemoClaw/OpenShell sandboxes, gateways, credentials, configuration, and recovery state on this host. Model caches are kept." - if force_fresh_install_has_existing_state; then - source_root="$(force_fresh_install_source_root)" - prepare_force_fresh_uninstaller "$source_root" - run_force_fresh_uninstaller "$source_root" \ - || error "Force-fresh cleanup stopped because the authoritative whole-host uninstall did not complete. The installer preserved remaining state for a safe retry." - else - info "No existing NemoClaw or OpenShell installation was found; continuing with a clean install." - fi + force_fresh_install_has_existing_state || existing_state_status=$? + case "$existing_state_status" in + 0) + source_root="$(force_fresh_install_source_root)" + prepare_force_fresh_uninstaller "$source_root" + run_force_fresh_uninstaller "$source_root" \ + || error "Force-fresh cleanup stopped because the authoritative whole-host uninstall did not complete. The installer preserved remaining state for a safe retry." + ;; + 1) info "No existing NemoClaw or OpenShell installation was found; continuing with a clean install." ;; + 2) error "Docker is installed but unavailable. Start the selected local Docker or Colima daemon, then rerun --force-fresh-install. No cleanup started." ;; + *) error "Could not inspect existing NemoClaw or OpenShell state. No cleanup started." ;; + esac remove_macos_openshell_for_force_fresh_install FRESH=1 export NEMOCLAW_FRESH=1 diff --git a/src/lib/actions/uninstall/run-plan-docker-scope.test.ts b/src/lib/actions/uninstall/run-plan-docker-scope.test.ts index cf1ecc11305..e93ad8f3cb6 100644 --- a/src/lib/actions/uninstall/run-plan-docker-scope.test.ts +++ b/src/lib/actions/uninstall/run-plan-docker-scope.test.ts @@ -9,6 +9,7 @@ import { type UninstallRunDeps, type UninstallRunOptions, } from "./run-plan"; +import { MANAGED_STARTUP_RECEIPT_VOLUME_LABEL } from "../../onboard/managed-startup/docker-receipt-transfer"; function ok(stdout = ""): RunResult { return { status: 0, stdout, stderr: "" }; @@ -156,20 +157,31 @@ describe("uninstall Docker resource scope", () => { ["removes", 0, 0], ["fails closed on", 1, 1], ] as const)( - "%s retained managed-startup receipt volumes during force-fresh cleanup", + "%s labelled receipt volumes and preserves unlabelled matches during force-fresh cleanup", async (_scenario, removalStatus, expectedExitCode) => { const owned = `nemoclaw-managed-startup-receipt-volume-${"a".repeat(32)}`; - const unrelated = "nemoclaw-managed-startup-receipt-volume-user-data"; + const unlabelled = `nemoclaw-managed-startup-receipt-volume-${"b".repeat(32)}`; const calls: string[][] = []; - let ownedPresent = true; + const volumeLabels = new Map([ + [owned, "1"], + [unlabelled, null], + ]); const routes: Record RunResult> = { info: () => ok(), "ps -a --format {{.ID}} {{.Image}} {{.Names}}": () => ok(), "images --format {{.ID}} {{.Repository}}:{{.Tag}}": () => ok(), - "volume ls --format {{.Name}}": () => - ok(`${ownedPresent ? `${owned}\n` : ""}${unrelated}\n`), + [`volume ls --filter label=${MANAGED_STARTUP_RECEIPT_VOLUME_LABEL}=1 --format {{.Name}}`]: + () => + ok( + [...volumeLabels] + .filter(([, label]) => label === "1") + .map(([name]) => name) + .join("\n"), + ), + [`volume inspect --format {{json .Labels}} ${owned}`]: () => + ok(JSON.stringify({ [MANAGED_STARTUP_RECEIPT_VOLUME_LABEL]: "1" })), [`volume rm -f ${owned}`]: () => { - ownedPresent = removalStatus === 0 ? false : ownedPresent; + removalStatus === 0 ? volumeLabels.delete(owned) : false; return { status: removalStatus, stdout: removalStatus === 0 ? owned : "", stderr: "" }; }, "volume inspect openshell-cluster-nemoclaw": () => ({ @@ -211,7 +223,8 @@ describe("uninstall Docker resource scope", () => { expect(result.exitCode).toBe(expectedExitCode); expect(calls).toContainEqual(["volume", "rm", "-f", owned]); - expect(calls).not.toContainEqual(["volume", "rm", "-f", unrelated]); + expect(calls).not.toContainEqual(["volume", "rm", "-f", unlabelled]); + expect(volumeLabels.has(unlabelled)).toBe(true); }, ); }); diff --git a/src/lib/actions/uninstall/runtime-commands.ts b/src/lib/actions/uninstall/runtime-commands.ts index 70b03f8daee..7155eb6e048 100644 --- a/src/lib/actions/uninstall/runtime-commands.ts +++ b/src/lib/actions/uninstall/runtime-commands.ts @@ -14,7 +14,10 @@ import { } from "../../domain/uninstall/messaging"; import { isOllamaAuthProxyCommandLine } from "../../inference/ollama/process"; import { isModelRouterCommandLineForPort } from "../../onboard/model-router-process"; -import { MANAGED_STARTUP_RECEIPT_VOLUME_PREFIX } from "../../onboard/managed-startup/docker-receipt-transfer"; +import { + MANAGED_STARTUP_RECEIPT_VOLUME_LABEL, + MANAGED_STARTUP_RECEIPT_VOLUME_PREFIX, +} from "../../onboard/managed-startup/docker-receipt-transfer"; interface UninstallRuntimeCommands { env: NodeJS.ProcessEnv; @@ -43,19 +46,57 @@ function nonEmptyLines(output: string): string[] { .filter(Boolean); } -export function removeForceFreshReceiptVolumes(runtime: ForceFreshDockerCleanupRuntime): boolean { - const inventory = runtime.runDocker(["volume", "ls", "--format", "{{.Name}}"], { - env: runtime.env, - }); +function ownedForceFreshReceiptVolumes(runtime: ForceFreshDockerCleanupRuntime): string[] | null { + const inventory = runtime.runDocker( + [ + "volume", + "ls", + "--filter", + `label=${MANAGED_STARTUP_RECEIPT_VOLUME_LABEL}=1`, + "--format", + "{{.Name}}", + ], + { env: runtime.env }, + ); if (inventory.status !== 0) { runtime.error( "Could not inventory managed-startup receipt volumes during force-fresh cleanup.", ); - return false; + return null; } const volumes = nonEmptyLines(inventory.stdout).filter((name) => MANAGED_STARTUP_RECEIPT_VOLUME_PATTERN.test(name), ); + for (const volume of volumes) { + const inspected = runtime.runDocker( + ["volume", "inspect", "--format", "{{json .Labels}}", volume], + { env: runtime.env }, + ); + if (inspected.status !== 0) { + runtime.error(`Could not inspect managed-startup receipt volume '${volume}'.`); + return null; + } + try { + const labels: unknown = JSON.parse(inspected.stdout); + if ( + typeof labels !== "object" || + labels === null || + Reflect.get(labels, MANAGED_STARTUP_RECEIPT_VOLUME_LABEL) !== "1" + ) { + runtime.error(`Managed-startup receipt volume '${volume}' has invalid ownership metadata.`); + return null; + } + } catch { + runtime.error(`Managed-startup receipt volume '${volume}' has invalid ownership metadata.`); + return null; + } + } + return volumes; +} + +export function removeForceFreshReceiptVolumes(runtime: ForceFreshDockerCleanupRuntime): boolean { + const volumes = ownedForceFreshReceiptVolumes(runtime); + if (volumes === null) return false; for (const volume of volumes) { const removed = runtime.runDocker(["volume", "rm", "-f", volume], { env: runtime.env, @@ -67,16 +108,9 @@ export function removeForceFreshReceiptVolumes(runtime: ForceFreshDockerCleanupR } runtime.log(`Removed managed-startup receipt volume ${volume}`); } - const remaining = runtime.runDocker(["volume", "ls", "--format", "{{.Name}}"], { - env: runtime.env, - }); - if (remaining.status !== 0) { - runtime.error("Could not verify managed-startup receipt volume cleanup."); - return false; - } - const retained = nonEmptyLines(remaining.stdout).find((name) => - MANAGED_STARTUP_RECEIPT_VOLUME_PATTERN.test(name), - ); + const remaining = ownedForceFreshReceiptVolumes(runtime); + if (remaining === null) return false; + const [retained] = remaining; if (retained) { runtime.error( `Managed-startup receipt volume '${retained}' remains after force-fresh cleanup.`, diff --git a/src/lib/onboard/managed-startup/docker-receipt-transfer.test.ts b/src/lib/onboard/managed-startup/docker-receipt-transfer.test.ts index 7c48d801cef..5d820116eba 100644 --- a/src/lib/onboard/managed-startup/docker-receipt-transfer.test.ts +++ b/src/lib/onboard/managed-startup/docker-receipt-transfer.test.ts @@ -10,6 +10,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { cleanupDockerDaemonReceiptBestEffort, dockerDaemonReceiptMount, + MANAGED_STARTUP_RECEIPT_VOLUME_LABEL, transferDockerReceiptToDaemon, } from "./docker-receipt-transfer"; @@ -51,6 +52,9 @@ describe("Docker daemon receipt transfer", () => { }); expect(calls.map((args) => args[0])).toEqual(["volume", "create", "cp", "rm"]); + expect(calls[0]).toEqual( + expect.arrayContaining(["--label", `${MANAGED_STARTUP_RECEIPT_VOLUME_LABEL}=1`]), + ); const seedArgs = calls[1] ?? []; expect(seedArgs).toEqual( expect.arrayContaining([ diff --git a/src/lib/onboard/managed-startup/docker-receipt-transfer.ts b/src/lib/onboard/managed-startup/docker-receipt-transfer.ts index a3e00f283da..907d6dac0e3 100644 --- a/src/lib/onboard/managed-startup/docker-receipt-transfer.ts +++ b/src/lib/onboard/managed-startup/docker-receipt-transfer.ts @@ -11,6 +11,7 @@ import { cleanupTempDir } from "../temp-files"; const RECEIPT_VOLUME_DIRECTORY = "/run/nemoclaw/managed-startup-receipt-transfer"; const RECEIPT_SEED_PREFIX = "nemoclaw-managed-startup-receipt-seed"; export const MANAGED_STARTUP_RECEIPT_VOLUME_PREFIX = "nemoclaw-managed-startup-receipt-volume"; +export const MANAGED_STARTUP_RECEIPT_VOLUME_LABEL = "io.nvidia.nemoclaw.managed-startup.receipt"; export type DockerDaemonReceipt = { readonly hostPath: string; @@ -105,7 +106,10 @@ export function transferDockerReceiptToDaemon( let volumeCreated = false; let seedCreated = false; try { - const volume = dockerRun(["volume", "create", volumeName], options.dockerOptions); + const volume = dockerRun( + ["volume", "create", "--label", `${MANAGED_STARTUP_RECEIPT_VOLUME_LABEL}=1`, volumeName], + options.dockerOptions, + ); if (!hasZeroDockerExitStatus(volume)) { throw new Error(`Could not create managed-startup receipt volume: ${commandDetail(volume)}`); } diff --git a/test/installer-integration/install-force-fresh.test.ts b/test/installer-integration/install-force-fresh.test.ts index d896d164ff5..2c4c5a58f1f 100644 --- a/test/installer-integration/install-force-fresh.test.ts +++ b/test/installer-integration/install-force-fresh.test.ts @@ -34,16 +34,19 @@ function callPayloadFunction(command: string, env: Record { - const result = spawnSync("bash", [INSTALLER, "--help"], { +it.each([ + ["public bootstrap", INSTALLER], + ["versioned payload", INSTALLER_PAYLOAD], +])("documents the macOS-only destructive option in the %s help", (_surface, installer) => { + const result = spawnSync("bash", [installer, "--help"], { cwd: path.join(import.meta.dirname, "../.."), encoding: "utf-8", }); expect(result.status).toBe(0); const output = `${result.stdout}${result.stderr}`; - expect(output).toContain("--force-fresh-install"); - expect(output).toContain("NEMOCLAW_FORCE_FRESH_INSTALL=1"); + expect(output).toMatch(/--force-fresh-install .*macOS only/u); + expect(output).toMatch(/NEMOCLAW_FORCE_FRESH_INSTALL=1 .*macOS only/u); }); it("detects a Homebrew-only OpenShell installation", () => { @@ -95,6 +98,7 @@ it("does not treat an unlabeled prefix-matching Docker container as owned state" case "$*" in info) exit 0 ;; "ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract") exit 0 ;; + "volume ls --filter label=io.nvidia.nemoclaw.managed-startup.receipt=1 --format {{.Name}}") exit 0 ;; *) exit 1 ;; esac `, @@ -108,6 +112,56 @@ esac expect(result.status).not.toBe(0); }); +it("detects labelled Docker receipt-volume-only state", () => { + const { root: tmp, binDir: fakeBin } = installerCheckout("nemoclaw-force-fresh-receipt-detect-"); + writeExecutable( + path.join(fakeBin, "docker"), + `#!/usr/bin/env bash +case "$*" in + info) exit 0 ;; + "ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract") exit 0 ;; + "volume ls --filter label=io.nvidia.nemoclaw.managed-startup.receipt=1 --format {{.Name}}") + printf 'nemoclaw-managed-startup-receipt-volume-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n' + ;; + *) exit 1 ;; +esac +`, + ); + + const result = callPayloadFunction("force_fresh_install_has_existing_state", { + HOME: tmp, + PATH: `${fakeBin}:${TEST_SYSTEM_PATH}`, + }); + + expect(result.status).toBe(0); +}); + +it("stops before cleanup when Docker is installed but unavailable", () => { + const { root: tmp, binDir: fakeBin } = installerCheckout( + "nemoclaw-force-fresh-docker-unavailable-", + ); + const cleanupMarker = path.join(tmp, "cleanup-started"); + writeExecutable(path.join(fakeBin, "docker"), "#!/usr/bin/env bash\nexit 1\n"); + + const result = callPayloadFunction( + ` + warn() { :; } + remove_macos_openshell_for_force_fresh_install() { touch "$CLEANUP_MARKER"; } + prepare_force_fresh_uninstaller() { touch "$CLEANUP_MARKER"; } + run_force_fresh_install_reset + `, + { + CLEANUP_MARKER: cleanupMarker, + HOME: tmp, + PATH: `${fakeBin}:${TEST_SYSTEM_PATH}`, + }, + ); + + expect(result.status).not.toBe(0); + expect(`${result.stdout}${result.stderr}`).toContain("Docker is installed but unavailable"); + expect(fs.existsSync(cleanupMarker)).toBe(false); +}); + it("runs destructive cleanup through the staged candidate CLI", () => { const { root: tmp, binDir: fakeBin } = installerCheckout("nemoclaw-force-fresh-cli-"); const logPath = path.join(tmp, "node.log"); @@ -290,8 +344,14 @@ it("bounds staged uninstaller preparation before destructive cleanup", () => { expect(fs.existsSync(cleanupMarker)).toBe(false); }, 15_000); -it("routes the public force-fresh flag through reset before CLI installation", () => { - const run = runInstallerSourcedBody(` +it.each([ + ["flag", "--force-fresh-install", {}], + ["environment variable", "", { NEMOCLAW_FORCE_FRESH_INSTALL: "1" }], +] as const)( + "routes the public force-fresh %s through reset before CLI installation", + (_source, forceFreshArg, extraEnv) => { + const run = runInstallerSourcedBody( + ` set -e order="" record() { order="\${order}$1 "; } @@ -321,15 +381,19 @@ it("routes the public force-fresh flag through reset before CLI installation", ( finalize_install() { :; } clear_station_resume_after_completed_onboarding() { :; } uname() { printf 'Darwin'; } - main --force-fresh-install --non-interactive --yes-i-accept-third-party-software + main ${forceFreshArg} --non-interactive --yes-i-accept-third-party-software + record "fresh:$FRESH:$NEMOCLAW_FORCE_FRESH_INSTALL" printf '%s' "$order" - `); - onTestFinished(run.remove); + `, + { extraEnv: { ...extraEnv } }, + ); + onTestFinished(run.remove); - expect(run.result.status, run.output).toBe(0); - expect(run.result.stdout.trim()).toBe("reset install"); - expect(run.result.stdout).not.toContain("legacy-backup"); -}); + expect(run.result.status, run.output).toBe(0); + expect(run.result.stdout.trim()).toBe("reset install fresh:1:1"); + expect(run.result.stdout).not.toContain("legacy-backup"); + }, +); it("continues force-fresh installation when Homebrew fails after the pinned runtime lands", () => { const result = callPayloadFunction(` From af68e983537a07ffad11514d091ee4e38a91f363 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 18 Sep 2026 09:29:44 -0700 Subject: [PATCH 25/39] test(onboard): parse labelled receipt volume name Signed-off-by: Aaron Erickson --- .../docker-shared-state-rollback-authority.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/onboard/managed-bootstrap/docker-shared-state-rollback-authority.test.ts b/src/lib/onboard/managed-bootstrap/docker-shared-state-rollback-authority.test.ts index 9b9069d6b0d..a4ac6454c33 100644 --- a/src/lib/onboard/managed-bootstrap/docker-shared-state-rollback-authority.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-shared-state-rollback-authority.test.ts @@ -279,7 +279,7 @@ describe("Docker managed-bootstrap shared-state rollback authority", () => { expect(volumeCreates).toHaveLength(1); expect(rollbackMount).toMatch( new RegExp( - `^type=volume,src=${String(volumeCreates[0]?.[2])},dst=${path.posix.dirname(MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY)},readonly$`, + `^type=volume,src=${String(volumeCreates[0]?.at(-1))},dst=${path.posix.dirname(MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY)},readonly$`, "u", ), ); From 53a32c097a8e2a0bbc841ab9453a00554fc644a2 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 18 Sep 2026 10:16:59 -0700 Subject: [PATCH 26/39] fix(uninstall): verify managed OpenShell binaries Signed-off-by: Aaron Erickson --- scripts/install-openshell.sh | 23 +++++ scripts/install.sh | 6 +- .../run-plan-homebrew-openshell.test.ts | 16 ++++ src/lib/actions/uninstall/run-plan.ts | 86 ++++++++++++++++++- .../install-force-fresh.test.ts | 19 +++- 5 files changed, 144 insertions(+), 6 deletions(-) diff --git a/scripts/install-openshell.sh b/scripts/install-openshell.sh index 6688ff81da4..dd23e559c7b 100755 --- a/scripts/install-openshell.sh +++ b/scripts/install-openshell.sh @@ -1031,6 +1031,26 @@ install_bins() { fi } +write_managed_user_install_manifest() { + local dir="$1" binary digest manifest + manifest="$tmpdir/nemoclaw-openshell-managed-v1" + : >"$manifest" + for binary in openshell openshell-gateway openshell-sandbox openshell-driver-vm; do + case "$binary" in + openshell) ;; + *) [ -x "$tmpdir/$binary" ] || continue ;; + esac + [ -x "$dir/$binary" ] || continue + digest="$($SHA_CMD "$dir/$binary" | awk '{print $1}')" \ + || fail "Could not hash the installed $binary binary." + [[ "$digest" =~ ^[a-f0-9]{64}$ ]] \ + || fail "The installed $binary binary returned an invalid SHA-256 digest." + printf '%s %s\n' "$digest" "$binary" >>"$manifest" + done + [ -s "$manifest" ] || fail "The managed OpenShell install manifest is empty." + install -m 600 "$manifest" "$dir/.nemoclaw-openshell-managed-v1" +} + if [ -w "$target_dir" ]; then install_bins "$target_dir" elif [ "${NEMOCLAW_NON_INTERACTIVE:-}" = "1" ] || [ ! -t 0 ]; then @@ -1057,5 +1077,8 @@ fi required_driver_bins_installed_in_dir "$target_dir" \ || fail "OpenShell release '$RELEASE_TAG' did not install the required Docker-driver binaries." require_openshell_messaging_features "$target_dir/openshell" +if [ "$target_dir" = "${XDG_BIN_HOME:-$HOME/.local/bin}" ]; then + write_managed_user_install_manifest "$target_dir" +fi info "$("$target_dir/openshell" --version 2>&1 || echo openshell) installed" diff --git a/scripts/install.sh b/scripts/install.sh index e92bc8f3d3d..33b08151601 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -3659,15 +3659,15 @@ run_force_fresh_uninstaller() { remove_macos_openshell_for_force_fresh_install() { local formula="nvidia/openshell/openshell" openshell_path="" - openshell_path="$(command -v openshell 2>/dev/null || true)" if command_exists brew && brew list --formula "$formula" >/dev/null 2>&1; then brew services stop "$formula" >/dev/null 2>&1 || true brew uninstall --force "$formula" \ || error "Homebrew could not remove OpenShell during the force-fresh install. Rerun the same command after Homebrew is healthy." - elif [[ "$openshell_path" == /opt/homebrew/* || "$openshell_path" == /usr/local/* ]]; then - error "The force-fresh installer found a system OpenShell binary that Homebrew did not identify as ${formula}. Remove that installation explicitly, then rerun." fi hash -r 2>/dev/null || true + openshell_path="$(command -v openshell 2>/dev/null || true)" + [[ -z "$openshell_path" ]] \ + || error "The force-fresh installer found a remaining OpenShell executable at ${openshell_path}. Remove or reconcile that installation explicitly, then rerun." } run_force_fresh_install_reset() { diff --git a/src/lib/actions/uninstall/run-plan-homebrew-openshell.test.ts b/src/lib/actions/uninstall/run-plan-homebrew-openshell.test.ts index 89ad21a6b41..dc2a58f26a0 100644 --- a/src/lib/actions/uninstall/run-plan-homebrew-openshell.test.ts +++ b/src/lib/actions/uninstall/run-plan-homebrew-openshell.test.ts @@ -44,6 +44,7 @@ async function uninstallOpenShell(options: { brewAvailable: boolean; brewStatus: number | null; forceFreshReset?: boolean; + managedUserLocal?: boolean; platform?: NodeJS.Platform; }) { const home = "/tmp/nemoclaw-uninstall-test"; @@ -64,6 +65,7 @@ async function uninstallOpenShell(options: { env: { HOME: home } as NodeJS.ProcessEnv, existsSync: (target) => existing.has(String(target)), hasPortableRuntimeCleanup: () => false, + isManagedOpenShellBinary: () => options.managedUserLocal ?? false, isTty: true, log: (line) => logs.push(line), platform: options.platform ?? "darwin", @@ -107,6 +109,7 @@ it("lets force-fresh remove only managed user-local OpenShell binaries", async ( brewAvailable: true, brewStatus: 0, forceFreshReset: true, + managedUserLocal: true, }); const userLocal = executablePaths.filter((target) => target.includes("/.local/bin/")); const system = executablePaths.filter((target) => target.startsWith("/usr/local/bin/")); @@ -116,6 +119,19 @@ it("lets force-fresh remove only managed user-local OpenShell binaries", async ( expect(new Set(remaining)).toEqual(new Set(system)); }); +it("retains unverified user-local OpenShell binaries during force-fresh cleanup", async () => { + const { executablePaths, remaining, removed, result } = await uninstallOpenShell({ + brewAvailable: true, + brewStatus: 0, + forceFreshReset: true, + managedUserLocal: false, + }); + + expect(result.exitCode).toBe(0); + expect(removed).toEqual([]); + expect(new Set(remaining)).toEqual(new Set(executablePaths)); +}); + it.each([ { label: "Homebrew is unavailable", diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index 52d33ee2e6a..e2ec48e6544 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import type { SpawnSyncOptions } from "node:child_process"; -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -174,6 +174,7 @@ export interface UninstallRunDeps { fs?: FileSystemDeps; getTrustedActiveOpenShellGatewayUserServiceIdentity?: typeof getTrustedActiveOpenShellGatewayUserServiceIdentity; isPortFree?: (port: number) => boolean; + isManagedOpenShellBinary?: (target: string, userBin: string) => boolean; isTty?: boolean; kill?: (pid: number, signal?: NodeJS.Signals | number) => boolean; log?: (message: string) => void; @@ -536,6 +537,7 @@ interface UninstallRuntime { existsSync: (target: string) => boolean; getTrustedActiveOpenShellGatewayUserServiceIdentity: typeof getTrustedActiveOpenShellGatewayUserServiceIdentity; isPortFree: ((port: number) => boolean) | undefined; + isManagedOpenShellBinary: (target: string, userBin: string) => boolean; isTty: boolean; kill: (pid: number, signal?: NodeJS.Signals | number) => boolean; log: (message: string) => void; @@ -588,6 +590,8 @@ function buildRuntime(deps: UninstallRunDeps): UninstallRuntime { deps.getTrustedActiveOpenShellGatewayUserServiceIdentity ?? getTrustedActiveOpenShellGatewayUserServiceIdentity, isPortFree: deps.isPortFree, + isManagedOpenShellBinary: + deps.isManagedOpenShellBinary ?? defaultManagedOpenShellBinaryOwnership, // Side-effect-free TTY check + EAGAIN-tolerant reader; the // process.stdin/non-blocking-fd hazard is documented in core/stdin.ts. isTty: deps.isTty ?? isStdinTty(), @@ -856,6 +860,71 @@ function reportRetainedMacOsOpenShell(runtime: UninstallRuntime): void { ); } +const MANAGED_OPENSHELL_INSTALL_MANIFEST = ".nemoclaw-openshell-managed-v1"; +const MANAGED_OPENSHELL_BINARY_NAMES = new Set([ + "openshell", + "openshell-gateway", + "openshell-sandbox", + "openshell-driver-vm", +]); + +function managedOpenShellManifest(userBin: string): ReadonlyMap | null { + let opened: OpenRegularFile | null = null; + try { + opened = openRegularFileNoFollow(path.join(userBin, MANAGED_OPENSHELL_INSTALL_MANIFEST)); + const owner = process.getuid?.(); + const stat = opened.stat(); + if (owner === undefined || stat.uid !== owner || (stat.mode & 0o022) !== 0) return null; + const entries = new Map(); + for (const line of opened.readUtf8(2_048).trim().split(/\r?\n/u)) { + const match = /^([a-f0-9]{64}) (openshell(?:-gateway|-sandbox|-driver-vm)?)$/u.exec(line); + const digest = match?.[1]; + const binary = match?.[2]; + if (!digest || !binary || entries.has(binary)) return null; + entries.set(binary, digest); + } + return entries.size > 0 ? entries : null; + } catch { + return null; + } finally { + opened?.close(); + } +} + +function regularFileSha256(target: string): { digest: string; uid: number } | null { + let opened: OpenRegularFile | null = null; + try { + opened = openRegularFileNoFollow(target); + const stat = opened.stat(); + return { + digest: createHash("sha256") + .update(opened.readBytes(256 * 1024 * 1024)) + .digest("hex"), + uid: stat.uid, + }; + } catch { + return null; + } finally { + opened?.close(); + } +} + +function defaultManagedOpenShellBinaryOwnership(target: string, userBin: string): boolean { + const binary = path.basename(target); + if ( + path.dirname(path.resolve(target)) !== path.resolve(userBin) || + !MANAGED_OPENSHELL_BINARY_NAMES.has(binary) + ) { + return false; + } + const expected = managedOpenShellManifest(userBin)?.get(binary); + const actual = regularFileSha256(target); + const owner = process.getuid?.(); + return Boolean( + expected && actual && owner !== undefined && actual.uid === owner && actual.digest === expected, + ); +} + function removeForceFreshUserLocalOpenShell( paths: UninstallPaths, runtime: UninstallRuntime, @@ -863,9 +932,22 @@ function removeForceFreshUserLocalOpenShell( const userBin = path.resolve( runtime.env.XDG_BIN_HOME || path.join(runtime.env.HOME || os.homedir(), ".local", "bin"), ); + let removed = 0; + let retained = false; try { for (const target of paths.openshellInstallPaths) { - if (path.dirname(path.resolve(target)) === userBin) removePath(target, runtime); + if (path.dirname(path.resolve(target)) !== userBin || !runtime.existsSync(target)) continue; + if (!runtime.isManagedOpenShellBinary(target, userBin)) { + retained = true; + runtime.warn( + `Leaving ${target} in place because its managed OpenShell install manifest is absent or does not match.`, + ); + continue; + } + if (removePath(target, runtime)) removed += 1; + } + if (removed > 0 && !retained) { + removePath(path.join(userBin, MANAGED_OPENSHELL_INSTALL_MANIFEST), runtime); } return true; } catch (error) { diff --git a/test/installer-integration/install-force-fresh.test.ts b/test/installer-integration/install-force-fresh.test.ts index 2c4c5a58f1f..596913459dd 100644 --- a/test/installer-integration/install-force-fresh.test.ts +++ b/test/installer-integration/install-force-fresh.test.ts @@ -222,7 +222,7 @@ exit 0 const result = callPayloadFunction("remove_macos_openshell_for_force_fresh_install", { FORCE_FRESH_LOG: logPath, HOME: tmp, - PATH: `${localBin}:${fakeBin}:${TEST_SYSTEM_PATH}`, + PATH: `${fakeBin}:${TEST_SYSTEM_PATH}`, }); expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); @@ -257,6 +257,23 @@ exit 7 expect(`${result.stdout}${result.stderr}`).toContain("Homebrew could not remove OpenShell"); }); +it("rejects a remaining OpenShell executable outside managed and Homebrew paths", () => { + const { root: tmp, binDir: fakeBin } = installerCheckout("nemoclaw-force-fresh-foreign-bin-"); + const foreignBin = path.join(tmp, "foreign", "bin"); + fs.mkdirSync(foreignBin, { recursive: true }); + writeExecutable(path.join(foreignBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n"); + + const result = callPayloadFunction("remove_macos_openshell_for_force_fresh_install", { + HOME: tmp, + PATH: `${foreignBin}:${fakeBin}:${TEST_SYSTEM_PATH}`, + }); + + expect(result.status).not.toBe(0); + expect(`${result.stdout}${result.stderr}`).toContain( + `remaining OpenShell executable at ${path.join(foreignBin, "openshell")}`, + ); +}); + it("runs cleanup before selecting fresh onboarding", () => { const result = callPayloadFunction(` warn() { :; } From 3c5214b1ddbbfd7a9b5e9b9afa678095c8e7af63 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 18 Sep 2026 10:24:28 -0700 Subject: [PATCH 27/39] test(installer): trust managed ownership template Signed-off-by: Aaron Erickson --- scripts/checks/extract-installer-pins.mts | 2 ++ test/install/installer-homebrew-formula-reuse-trust.test.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/checks/extract-installer-pins.mts b/scripts/checks/extract-installer-pins.mts index b39deabb002..66bdd62e5fe 100644 --- a/scripts/checks/extract-installer-pins.mts +++ b/scripts/checks/extract-installer-pins.mts @@ -522,6 +522,8 @@ const TRUSTED_OPENSHELL_RELEASES: readonly OpenShellReleaseTrust[] = [ "2b6ad3e0730d3220da05d13b88fdba4458de46840bad57942ecad26a5d606017", // Exact #11251 template after immutable stable-override validation. "24cb9e67b855e8a69df32aae992f4756ef2b29bcdc7846ef57bcfeacb3c1a9a3", + // Exact #12076 template with digest-backed managed user-local install ownership. + "9e5f1b67cb1d3bad223607967c639725badafb1ffd4dbc4cef33f2a1d9c04c1f", ], manifests: [ { diff --git a/test/install/installer-homebrew-formula-reuse-trust.test.ts b/test/install/installer-homebrew-formula-reuse-trust.test.ts index 954725a4a02..090fa9f974f 100644 --- a/test/install/installer-homebrew-formula-reuse-trust.test.ts +++ b/test/install/installer-homebrew-formula-reuse-trust.test.ts @@ -14,7 +14,7 @@ const INSTALLER_SOURCE = fs.readFileSync( "utf8", ); const TRUSTED_V00116_TEMPLATE_DIGEST = - "24cb9e67b855e8a69df32aae992f4756ef2b29bcdc7846ef57bcfeacb3c1a9a3"; + "9e5f1b67cb1d3bad223607967c639725badafb1ffd4dbc4cef33f2a1d9c04c1f"; const tempDirs: string[] = []; afterEach(() => { From 4a4f73b1fd5886927df2bba520e8d82412d41812 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 18 Sep 2026 10:32:38 -0700 Subject: [PATCH 28/39] fix(installer): record managed OpenShell ownership Signed-off-by: Aaron Erickson --- scripts/checks/extract-installer-pins.mts | 2 -- scripts/install-openshell.sh | 23 ------------- scripts/install.sh | 32 +++++++++++++++++++ src/lib/actions/uninstall/run-plan.ts | 2 +- ...aller-homebrew-formula-reuse-trust.test.ts | 2 +- .../install-force-fresh.test.ts | 23 +++++++++++++ 6 files changed, 57 insertions(+), 27 deletions(-) diff --git a/scripts/checks/extract-installer-pins.mts b/scripts/checks/extract-installer-pins.mts index 66bdd62e5fe..b39deabb002 100644 --- a/scripts/checks/extract-installer-pins.mts +++ b/scripts/checks/extract-installer-pins.mts @@ -522,8 +522,6 @@ const TRUSTED_OPENSHELL_RELEASES: readonly OpenShellReleaseTrust[] = [ "2b6ad3e0730d3220da05d13b88fdba4458de46840bad57942ecad26a5d606017", // Exact #11251 template after immutable stable-override validation. "24cb9e67b855e8a69df32aae992f4756ef2b29bcdc7846ef57bcfeacb3c1a9a3", - // Exact #12076 template with digest-backed managed user-local install ownership. - "9e5f1b67cb1d3bad223607967c639725badafb1ffd4dbc4cef33f2a1d9c04c1f", ], manifests: [ { diff --git a/scripts/install-openshell.sh b/scripts/install-openshell.sh index dd23e559c7b..6688ff81da4 100755 --- a/scripts/install-openshell.sh +++ b/scripts/install-openshell.sh @@ -1031,26 +1031,6 @@ install_bins() { fi } -write_managed_user_install_manifest() { - local dir="$1" binary digest manifest - manifest="$tmpdir/nemoclaw-openshell-managed-v1" - : >"$manifest" - for binary in openshell openshell-gateway openshell-sandbox openshell-driver-vm; do - case "$binary" in - openshell) ;; - *) [ -x "$tmpdir/$binary" ] || continue ;; - esac - [ -x "$dir/$binary" ] || continue - digest="$($SHA_CMD "$dir/$binary" | awk '{print $1}')" \ - || fail "Could not hash the installed $binary binary." - [[ "$digest" =~ ^[a-f0-9]{64}$ ]] \ - || fail "The installed $binary binary returned an invalid SHA-256 digest." - printf '%s %s\n' "$digest" "$binary" >>"$manifest" - done - [ -s "$manifest" ] || fail "The managed OpenShell install manifest is empty." - install -m 600 "$manifest" "$dir/.nemoclaw-openshell-managed-v1" -} - if [ -w "$target_dir" ]; then install_bins "$target_dir" elif [ "${NEMOCLAW_NON_INTERACTIVE:-}" = "1" ] || [ ! -t 0 ]; then @@ -1077,8 +1057,5 @@ fi required_driver_bins_installed_in_dir "$target_dir" \ || fail "OpenShell release '$RELEASE_TAG' did not install the required Docker-driver binaries." require_openshell_messaging_features "$target_dir/openshell" -if [ "$target_dir" = "${XDG_BIN_HOME:-$HOME/.local/bin}" ]; then - write_managed_user_install_manifest "$target_dir" -fi info "$("$target_dir/openshell" --version 2>&1 || echo openshell) installed" diff --git a/scripts/install.sh b/scripts/install.sh index 33b08151601..50a4f34e11d 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -2343,6 +2343,37 @@ NODE # over their own openshell version) # Both modes defer when NEMOCLAW_DEFER_OPENSHELL_INSTALL=1 so the pre-upgrade # backup flow can run before any version bump. +record_managed_user_local_openshell_install() { + local user_bin manifest temp_manifest binary binary_path digest + local -a sha256_command + user_bin="${XDG_BIN_HOME:-${HOME}/.local/bin}" + manifest="${user_bin}/.nemoclaw-openshell-managed-v1" + if command_exists sha256sum; then + sha256_command=(sha256sum) + elif command_exists shasum; then + sha256_command=(shasum -a 256) + else + error "Could not record managed OpenShell ownership because no SHA-256 command is available." + fi + temp_manifest="$(mktemp "${manifest}.tmp.XXXXXX")" \ + || error "Could not create the managed OpenShell install manifest." + _cleanup_files+=("$temp_manifest") + chmod 600 "$temp_manifest" \ + || error "Could not secure the managed OpenShell install manifest." + for binary in openshell openshell-gateway; do + binary_path="${user_bin}/${binary}" + [[ -f "$binary_path" && ! -L "$binary_path" && -x "$binary_path" ]] \ + || error "The verified standalone OpenShell install did not provide ${binary_path}." + digest="$("${sha256_command[@]}" "$binary_path" | awk '{print $1}')" \ + || error "Could not hash the installed ${binary} binary." + [[ "$digest" =~ ^[a-f0-9]{64}$ ]] \ + || error "The installed ${binary} binary returned an invalid SHA-256 digest." + printf '%s %s\n' "$digest" "$binary" >>"$temp_manifest" + done + mv -f -- "$temp_manifest" "$manifest" \ + || error "Could not publish the managed OpenShell install manifest." +} + maybe_install_openshell_during_install() { local mode="${1:-force}" local explicit_openshell_bin="${NEMOCLAW_OPENSHELL_BIN:-}" @@ -2398,6 +2429,7 @@ maybe_install_openshell_during_install() { standalone) prefer_user_local_openshell verified-install \ || error "The verified standalone OpenShell installation did not provide trusted executable user-local CLI and gateway binaries. The installer stopped before gateway recovery." + record_managed_user_local_openshell_install warn "Homebrew is not installed; using the verified standalone OpenShell gateway without reboot persistence." ;; *) diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index e2ec48e6544..b0e845e62ef 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -876,7 +876,7 @@ function managedOpenShellManifest(userBin: string): ReadonlyMap const stat = opened.stat(); if (owner === undefined || stat.uid !== owner || (stat.mode & 0o022) !== 0) return null; const entries = new Map(); - for (const line of opened.readUtf8(2_048).trim().split(/\r?\n/u)) { + for (const line of opened.readBytes(2_048).toString("utf8").trim().split(/\r?\n/u)) { const match = /^([a-f0-9]{64}) (openshell(?:-gateway|-sandbox|-driver-vm)?)$/u.exec(line); const digest = match?.[1]; const binary = match?.[2]; diff --git a/test/install/installer-homebrew-formula-reuse-trust.test.ts b/test/install/installer-homebrew-formula-reuse-trust.test.ts index 090fa9f974f..954725a4a02 100644 --- a/test/install/installer-homebrew-formula-reuse-trust.test.ts +++ b/test/install/installer-homebrew-formula-reuse-trust.test.ts @@ -14,7 +14,7 @@ const INSTALLER_SOURCE = fs.readFileSync( "utf8", ); const TRUSTED_V00116_TEMPLATE_DIGEST = - "9e5f1b67cb1d3bad223607967c639725badafb1ffd4dbc4cef33f2a1d9c04c1f"; + "24cb9e67b855e8a69df32aae992f4756ef2b29bcdc7846ef57bcfeacb3c1a9a3"; const tempDirs: string[] = []; afterEach(() => { diff --git a/test/installer-integration/install-force-fresh.test.ts b/test/installer-integration/install-force-fresh.test.ts index 596913459dd..b2fd87baed5 100644 --- a/test/installer-integration/install-force-fresh.test.ts +++ b/test/installer-integration/install-force-fresh.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -187,6 +188,28 @@ printf 'destroy=%s args=%s\n' "\${NEMOCLAW_UNINSTALL_DESTROY_USER_DATA:-}" "$*" ); }); +it("records digest ownership after a verified managed user-local OpenShell install", () => { + const { root: tmp } = installerCheckout("nemoclaw-force-fresh-openshell-manifest-"); + const localBin = path.join(tmp, ".local", "bin"); + fs.mkdirSync(localBin, { recursive: true }); + const openshellSource = "#!/usr/bin/env bash\nprintf 'openshell 0.0.116\\n'\n"; + const gatewaySource = "#!/usr/bin/env bash\nexit 0\n"; + writeExecutable(path.join(localBin, "openshell"), openshellSource); + writeExecutable(path.join(localBin, "openshell-gateway"), gatewaySource); + + const result = callPayloadFunction("record_managed_user_local_openshell_install", { + HOME: tmp, + XDG_BIN_HOME: localBin, + }); + const manifest = fs.readFileSync(path.join(localBin, ".nemoclaw-openshell-managed-v1"), "utf8"); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(manifest.trim().split("\n")).toEqual([ + `${createHash("sha256").update(openshellSource).digest("hex")} openshell`, + `${createHash("sha256").update(gatewaySource).digest("hex")} openshell-gateway`, + ]); +}); + it("stops when the staged candidate uninstaller fails", () => { const { root: tmp, binDir: fakeBin } = installerCheckout("nemoclaw-force-fresh-failure-"); const sourceRoot = path.join(tmp, "candidate"); From 19f1eabd82d7e5d1b8e4944b92c2b9d11da0f041 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 18 Sep 2026 11:22:45 -0700 Subject: [PATCH 29/39] fix(installer): block unowned legacy receipts --- scripts/install.sh | 34 +++++++++--- .../install-force-fresh.test.ts | 55 ++++++++++++++++++- 2 files changed, 78 insertions(+), 11 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index 50a4f34e11d..4d4764469c0 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -3601,7 +3601,9 @@ run_preupgrade_backup() { } force_fresh_install_has_existing_state() { - local container_inventory receipt_volume_inventory volume_name + local container_inventory receipt_volume_inventory volume_name volume_label volume_suffix + local existing_state=0 managed_docker_state=0 + _FORCE_FRESH_UNVERIFIED_RECEIPT_VOLUME="" if [[ -e "$(nemoclaw_state_root)" ]] \ || [[ -e "${HOME}/.config/nemoclaw" ]] \ || [[ -e "${HOME}/.config/openshell" ]] \ @@ -3613,27 +3615,42 @@ force_fresh_install_has_existing_state() { || [[ -e "${HOME}/.local/bin/openshell-driver-vm" ]] \ || command_exists nemoclaw \ || command_exists openshell; then - return 0 + existing_state=1 fi if command_exists brew \ && brew list --formula nvidia/openshell/openshell >/dev/null 2>&1; then - return 0 + existing_state=1 fi if command_exists docker; then docker info >/dev/null 2>&1 || return 2 container_inventory="$( docker ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract 2>/dev/null )" || return 2 - [[ -z "$container_inventory" ]] || return 0 + [[ -z "$container_inventory" ]] || managed_docker_state=1 receipt_volume_inventory="$( - docker volume ls \ - --filter label=io.nvidia.nemoclaw.managed-startup.receipt=1 \ - --format '{{.Name}}' 2>/dev/null + docker volume ls --format '{{.Name}}' 2>/dev/null )" || return 2 while IFS= read -r volume_name; do - [[ "$volume_name" =~ ^nemoclaw-managed-startup-receipt-volume-[0-9a-f]{32}$ ]] && return 0 + case "$volume_name" in + nemoclaw-managed-startup-receipt-volume-*) ;; + *) continue ;; + esac + volume_suffix="${volume_name#nemoclaw-managed-startup-receipt-volume-}" + [[ "${#volume_suffix}" -eq 32 && "$volume_suffix" =~ ^[0-9a-f]+$ ]] || continue + volume_label="$( + docker volume inspect \ + --format '{{ index .Labels "io.nvidia.nemoclaw.managed-startup.receipt" }}' \ + "$volume_name" 2>/dev/null + )" || return 2 + if [[ "$volume_label" == "1" ]]; then + managed_docker_state=1 + else + _FORCE_FRESH_UNVERIFIED_RECEIPT_VOLUME="$volume_name" + return 3 + fi done <<<"$receipt_volume_inventory" fi + ((existing_state == 0 && managed_docker_state == 0)) || return 0 return 1 } @@ -3715,6 +3732,7 @@ run_force_fresh_install_reset() { ;; 1) info "No existing NemoClaw or OpenShell installation was found; continuing with a clean install." ;; 2) error "Docker is installed but unavailable. Start the selected local Docker or Colima daemon, then rerun --force-fresh-install. No cleanup started." ;; + 3) error "Force-fresh cleanup found pre-label receipt volume ${_FORCE_FRESH_UNVERIFIED_RECEIPT_VOLUME}. NemoClaw cannot prove ownership of this legacy volume. Inspect it, remove it only after confirming it belongs to the interrupted NemoClaw install, then rerun. No cleanup started." ;; *) error "Could not inspect existing NemoClaw or OpenShell state. No cleanup started." ;; esac remove_macos_openshell_for_force_fresh_install diff --git a/test/installer-integration/install-force-fresh.test.ts b/test/installer-integration/install-force-fresh.test.ts index b2fd87baed5..b9cefb6c210 100644 --- a/test/installer-integration/install-force-fresh.test.ts +++ b/test/installer-integration/install-force-fresh.test.ts @@ -77,6 +77,7 @@ it("detects Docker-only state only through an authoritative managed-image label" case "$*" in info) exit 0 ;; "ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract") printf '0123456789ab\n' ;; + "volume ls --format {{.Name}}") exit 0 ;; *) exit 1 ;; esac `, @@ -99,7 +100,7 @@ it("does not treat an unlabeled prefix-matching Docker container as owned state" case "$*" in info) exit 0 ;; "ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract") exit 0 ;; - "volume ls --filter label=io.nvidia.nemoclaw.managed-startup.receipt=1 --format {{.Name}}") exit 0 ;; + "volume ls --format {{.Name}}") exit 0 ;; *) exit 1 ;; esac `, @@ -121,10 +122,16 @@ it("detects labelled Docker receipt-volume-only state", () => { case "$*" in info) exit 0 ;; "ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract") exit 0 ;; - "volume ls --filter label=io.nvidia.nemoclaw.managed-startup.receipt=1 --format {{.Name}}") + "volume ls --format {{.Name}}") printf 'nemoclaw-managed-startup-receipt-volume-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n' ;; - *) exit 1 ;; + *) + if [[ "$1" == "volume" && "$2" == "inspect" ]]; then + printf '1\n' + exit 0 + fi + exit 1 + ;; esac `, ); @@ -137,6 +144,48 @@ esac expect(result.status).toBe(0); }); +it("stops before cleanup for a pre-label receipt-volume-only state", () => { + const { root: tmp, binDir: fakeBin } = installerCheckout("nemoclaw-force-fresh-legacy-receipt-"); + const cleanupMarker = path.join(tmp, "cleanup-started"); + const legacyVolume = "nemoclaw-managed-startup-receipt-volume-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + writeExecutable( + path.join(fakeBin, "docker"), + `#!/usr/bin/env bash +case "$*" in + info) exit 0 ;; + "ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract") exit 0 ;; + "volume ls --format {{.Name}}") printf '%s\n' "$LEGACY_VOLUME" ;; + *) + if [[ "$1" == "volume" && "$2" == "inspect" ]]; then + printf '\n' + exit 0 + fi + exit 1 + ;; +esac +`, + ); + + const result = callPayloadFunction( + ` + warn() { :; } + remove_macos_openshell_for_force_fresh_install() { touch "$CLEANUP_MARKER"; } + prepare_force_fresh_uninstaller() { touch "$CLEANUP_MARKER"; } + run_force_fresh_install_reset + `, + { + CLEANUP_MARKER: cleanupMarker, + HOME: tmp, + LEGACY_VOLUME: legacyVolume, + PATH: `${fakeBin}:${TEST_SYSTEM_PATH}`, + }, + ); + + expect(result.status).not.toBe(0); + expect(`${result.stdout}${result.stderr}`).toContain(`pre-label receipt volume ${legacyVolume}`); + expect(fs.existsSync(cleanupMarker)).toBe(false); +}); + it("stops before cleanup when Docker is installed but unavailable", () => { const { root: tmp, binDir: fakeBin } = installerCheckout( "nemoclaw-force-fresh-docker-unavailable-", From b20ed4f0480ce8378291bceff4277bc08237ad2d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 18 Sep 2026 11:32:43 -0700 Subject: [PATCH 30/39] fix(uninstall): fail closed on cleanup residue --- scripts/install.sh | 10 +-- .../uninstall/run-plan-docker-scope.test.ts | 61 ++++++++++++++++++- src/lib/actions/uninstall/run-plan.ts | 36 ++++++++--- .../docker-receipt-transfer.test.ts | 7 ++- .../install-force-fresh.test.ts | 34 ++++++----- 5 files changed, 117 insertions(+), 31 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index 4d4764469c0..33f28e448ce 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -3707,16 +3707,18 @@ run_force_fresh_uninstaller() { } remove_macos_openshell_for_force_fresh_install() { - local formula="nvidia/openshell/openshell" openshell_path="" + local formula="nvidia/openshell/openshell" binary="" openshell_path="" if command_exists brew && brew list --formula "$formula" >/dev/null 2>&1; then brew services stop "$formula" >/dev/null 2>&1 || true brew uninstall --force "$formula" \ || error "Homebrew could not remove OpenShell during the force-fresh install. Rerun the same command after Homebrew is healthy." fi hash -r 2>/dev/null || true - openshell_path="$(command -v openshell 2>/dev/null || true)" - [[ -z "$openshell_path" ]] \ - || error "The force-fresh installer found a remaining OpenShell executable at ${openshell_path}. Remove or reconcile that installation explicitly, then rerun." + for binary in openshell openshell-gateway openshell-sandbox openshell-driver-vm; do + openshell_path="$(command -v "$binary" 2>/dev/null || true)" + [[ -z "$openshell_path" ]] \ + || error "The force-fresh installer found a remaining OpenShell executable at ${openshell_path}. Remove or reconcile that installation explicitly, then rerun." + done } run_force_fresh_install_reset() { diff --git a/src/lib/actions/uninstall/run-plan-docker-scope.test.ts b/src/lib/actions/uninstall/run-plan-docker-scope.test.ts index e93ad8f3cb6..e97f402e60b 100644 --- a/src/lib/actions/uninstall/run-plan-docker-scope.test.ts +++ b/src/lib/actions/uninstall/run-plan-docker-scope.test.ts @@ -181,7 +181,10 @@ describe("uninstall Docker resource scope", () => { [`volume inspect --format {{json .Labels}} ${owned}`]: () => ok(JSON.stringify({ [MANAGED_STARTUP_RECEIPT_VOLUME_LABEL]: "1" })), [`volume rm -f ${owned}`]: () => { - removalStatus === 0 ? volumeLabels.delete(owned) : false; + const successfulRemoval: Partial boolean>> = { + 0: () => volumeLabels.delete(owned), + }; + successfulRemoval[removalStatus]?.(); return { status: removalStatus, stdout: removalStatus === 0 ? owned : "", stderr: "" }; }, "volume inspect openshell-cluster-nemoclaw": () => ({ @@ -227,4 +230,60 @@ describe("uninstall Docker resource scope", () => { expect(volumeLabels.has(unlabelled)).toBe(true); }, ); + + it.each(["container", "planned volume"] as const)( + "fails force-fresh cleanup when a required Docker %s cannot be removed", + async (resource) => { + const containerInventory: Record = { + container: "owned-id nemoclaw-sandbox:test openshell-owned", + "planned volume": "", + }; + const plannedVolumeInspection: Record = { + container: { status: 1, stdout: "", stderr: "not found" }, + "planned volume": ok(), + }; + const routes: Record = { + info: ok(), + "ps -a --format {{.ID}} {{.Image}} {{.Names}}": ok(containerInventory[resource]), + "images --format {{.ID}} {{.Repository}}:{{.Tag}}": ok(), + "rm -f owned-id": { status: 1, stdout: "", stderr: "busy" }, + "volume inspect openshell-cluster-nemoclaw": plannedVolumeInspection[resource], + "volume rm -f openshell-cluster-nemoclaw": { + status: 1, + stdout: "", + stderr: "busy", + }, + }; + const runDocker = vi.fn((args: string[]): RunResult => { + return routes[args.join(" ")] ?? ok(); + }); + + const result = await runUninstallPlan( + { + assumeYes: true, + deleteModels: false, + destroyUserData: true, + forceFreshReset: true, + keepOpenShell: true, + }, + { + commandExists: () => true, + env: { HOME: "/tmp/nemoclaw-force-fresh-docker-failure" } as NodeJS.ProcessEnv, + existsSync: () => false, + hasPortableRuntimeCleanup: () => false, + isTty: false, + kill: () => true, + log: () => undefined, + rmSync: vi.fn(), + run: (command, args) => + command === "openshell" && args.join(" ") === "gateway list -o json" + ? ok(JSON.stringify([{ name: "nemoclaw" }])) + : ok(), + runDocker, + }, + ); + + expect(result.exitCode).toBe(1); + }, + ); }); diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index b0e845e62ef..dbdb3be059b 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -2360,10 +2360,14 @@ function stopBedrockRuntimeAdapterForUninstall( throw new IncompleteBedrockRuntimeAdapterCleanupError(); } -function removeDockerContainers(runtime: UninstallRuntime, gatewayName?: string): void { +function removeDockerContainers(runtime: UninstallRuntime, gatewayName?: string): boolean { const result = runtime.runDocker(["ps", "-a", "--format", "{{.ID}} {{.Image}} {{.Names}}"], { env: runtime.env, }); + if (result.status !== 0) { + runtime.warn("Failed to inventory Docker containers"); + return false; + } const ids = splitNonEmptyLines(result.stdout) .filter((line) => { const fields = dockerInventoryFields(line, 3); @@ -2394,13 +2398,18 @@ function removeDockerContainers(runtime: UninstallRuntime, gatewayName?: string) .map((line) => line.split(/\s+/)[0]); if (ids.length === 0) { runtime.log(`No ${runtimeBranding(runtime).display}/OpenShell Docker containers found`); - return; + return true; } + let removedAll = true; for (const id of [...new Set(ids)]) { if (runtime.runDocker(["rm", "-f", id], { env: runtime.env, stdio: "ignore" }).status === 0) runtime.log(`Removed Docker container ${id}`); - else runtime.warn(`Failed to remove Docker container ${id}`); + else { + runtime.warn(`Failed to remove Docker container ${id}`); + removedAll = false; + } } + return removedAll; } function removeDockerImages(runtime: UninstallRuntime): void { @@ -2450,18 +2459,22 @@ function isOwnedDockerImageRepository(imageRef: string): boolean { ); } -function removeDockerVolume(name: string, runtime: UninstallRuntime): void { +function removeDockerVolume(name: string, runtime: UninstallRuntime): boolean { if ( runtime.runDocker(["volume", "inspect", name], { env: runtime.env, stdio: "ignore" }).status !== 0 ) - return; + return true; if ( runtime.runDocker(["volume", "rm", "-f", name], { env: runtime.env, stdio: "ignore" }) .status === 0 ) runtime.log(`Removed Docker volume ${name}`); - else runtime.warn(`Failed to remove Docker volume ${name}`); + else { + runtime.warn(`Failed to remove Docker volume ${name}`); + return false; + } + return true; } function executeDockerResourceStep( @@ -2484,7 +2497,7 @@ function executeDockerResourceStep( ); return false; } - removeDockerContainers( + const removedContainers = removeDockerContainers( runtime, scopedToSelectedGateway ? options.gatewayName || resolveGatewayName(GATEWAY_PORT) : undefined, ); @@ -2493,8 +2506,15 @@ function executeDockerResourceStep( } else { removeDockerImages(runtime); } - for (const volumeName of volumeNames) removeDockerVolume(volumeName, runtime); + let removedVolumes = true; + for (const volumeName of volumeNames) { + if (!removeDockerVolume(volumeName, runtime)) removedVolumes = false; + } if (!options.forceFreshReset) return true; + if (!removedContainers || !removedVolumes) { + runtime.error("Force-fresh cleanup could not remove every required Docker resource."); + return false; + } if (scopedToSelectedGateway) { runtime.error( "Force-fresh cleanup preserved receipt volumes because another gateway environment remains.", diff --git a/src/lib/onboard/managed-startup/docker-receipt-transfer.test.ts b/src/lib/onboard/managed-startup/docker-receipt-transfer.test.ts index 5d820116eba..7ca4dd5c468 100644 --- a/src/lib/onboard/managed-startup/docker-receipt-transfer.test.ts +++ b/src/lib/onboard/managed-startup/docker-receipt-transfer.test.ts @@ -52,9 +52,10 @@ describe("Docker daemon receipt transfer", () => { }); expect(calls.map((args) => args[0])).toEqual(["volume", "create", "cp", "rm"]); - expect(calls[0]).toEqual( - expect.arrayContaining(["--label", `${MANAGED_STARTUP_RECEIPT_VOLUME_LABEL}=1`]), - ); + const volumeArgs = calls[0] ?? []; + const labelIndex = volumeArgs.indexOf("--label"); + expect(labelIndex).toBeGreaterThanOrEqual(0); + expect(volumeArgs[labelIndex + 1]).toBe(`${MANAGED_STARTUP_RECEIPT_VOLUME_LABEL}=1`); const seedArgs = calls[1] ?? []; expect(seedArgs).toEqual( expect.arrayContaining([ diff --git a/test/installer-integration/install-force-fresh.test.ts b/test/installer-integration/install-force-fresh.test.ts index b9cefb6c210..613fd0e0528 100644 --- a/test/installer-integration/install-force-fresh.test.ts +++ b/test/installer-integration/install-force-fresh.test.ts @@ -186,11 +186,12 @@ esac expect(fs.existsSync(cleanupMarker)).toBe(false); }); -it("stops before cleanup when Docker is installed but unavailable", () => { +it("stops before cleanup when local state exists but Docker is unavailable", () => { const { root: tmp, binDir: fakeBin } = installerCheckout( "nemoclaw-force-fresh-docker-unavailable-", ); const cleanupMarker = path.join(tmp, "cleanup-started"); + fs.mkdirSync(path.join(tmp, ".config", "nemoclaw"), { recursive: true }); writeExecutable(path.join(fakeBin, "docker"), "#!/usr/bin/env bash\nexit 1\n"); const result = callPayloadFunction( @@ -329,22 +330,25 @@ exit 7 expect(`${result.stdout}${result.stderr}`).toContain("Homebrew could not remove OpenShell"); }); -it("rejects a remaining OpenShell executable outside managed and Homebrew paths", () => { - const { root: tmp, binDir: fakeBin } = installerCheckout("nemoclaw-force-fresh-foreign-bin-"); - const foreignBin = path.join(tmp, "foreign", "bin"); - fs.mkdirSync(foreignBin, { recursive: true }); - writeExecutable(path.join(foreignBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n"); +it.each(["openshell", "openshell-gateway", "openshell-sandbox", "openshell-driver-vm"])( + "rejects a remaining %s executable outside managed and Homebrew paths", + (binary) => { + const { root: tmp, binDir: fakeBin } = installerCheckout("nemoclaw-force-fresh-foreign-bin-"); + const foreignBin = path.join(tmp, "foreign", "bin"); + fs.mkdirSync(foreignBin, { recursive: true }); + writeExecutable(path.join(foreignBin, binary), "#!/usr/bin/env bash\nexit 0\n"); - const result = callPayloadFunction("remove_macos_openshell_for_force_fresh_install", { - HOME: tmp, - PATH: `${foreignBin}:${fakeBin}:${TEST_SYSTEM_PATH}`, - }); + const result = callPayloadFunction("remove_macos_openshell_for_force_fresh_install", { + HOME: tmp, + PATH: `${foreignBin}:${fakeBin}:${TEST_SYSTEM_PATH}`, + }); - expect(result.status).not.toBe(0); - expect(`${result.stdout}${result.stderr}`).toContain( - `remaining OpenShell executable at ${path.join(foreignBin, "openshell")}`, - ); -}); + expect(result.status).not.toBe(0); + expect(`${result.stdout}${result.stderr}`).toContain( + `remaining OpenShell executable at ${path.join(foreignBin, binary)}`, + ); + }, +); it("runs cleanup before selecting fresh onboarding", () => { const result = callPayloadFunction(` From f40660cdd56bbcaa86cee5b3898c399eb12a7184 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 18 Sep 2026 11:58:58 -0700 Subject: [PATCH 31/39] fix(uninstall): prove Docker volume absence --- .../uninstall/run-plan-docker-scope.test.ts | 44 +++++++++++++------ src/lib/actions/uninstall/run-plan.ts | 30 ++++++++++--- 2 files changed, 55 insertions(+), 19 deletions(-) diff --git a/src/lib/actions/uninstall/run-plan-docker-scope.test.ts b/src/lib/actions/uninstall/run-plan-docker-scope.test.ts index e97f402e60b..b2a2e50db05 100644 --- a/src/lib/actions/uninstall/run-plan-docker-scope.test.ts +++ b/src/lib/actions/uninstall/run-plan-docker-scope.test.ts @@ -190,7 +190,7 @@ describe("uninstall Docker resource scope", () => { "volume inspect openshell-cluster-nemoclaw": () => ({ status: 1, stdout: "", - stderr: "Error: no such volume", + stderr: "Error response from daemon: get openshell-cluster-nemoclaw: no such volume", }), }; const runDocker = vi.fn((args: string[]) => { @@ -231,23 +231,38 @@ describe("uninstall Docker resource scope", () => { }, ); - it.each(["container", "planned volume"] as const)( - "fails force-fresh cleanup when a required Docker %s cannot be removed", - async (resource) => { - const containerInventory: Record = { - container: "owned-id nemoclaw-sandbox:test openshell-owned", - "planned volume": "", - }; - const plannedVolumeInspection: Record = { - container: { status: 1, stdout: "", stderr: "not found" }, - "planned volume": ok(), - }; + it.each([ + { + containerInventory: "owned-id nemoclaw-sandbox:test openshell-owned", + failureCommand: ["rm", "-f", "owned-id"], + plannedVolumeInspection: { + status: 1, + stdout: "", + stderr: "Error response from daemon: get openshell-cluster-nemoclaw: no such volume", + }, + scenario: "container removal fails", + }, + { + containerInventory: "", + failureCommand: ["volume", "rm", "-f", "openshell-cluster-nemoclaw"], + plannedVolumeInspection: ok(), + scenario: "planned-volume removal fails", + }, + { + containerInventory: "", + failureCommand: ["volume", "inspect", "openshell-cluster-nemoclaw"], + plannedVolumeInspection: { status: 1, stdout: "", stderr: "permission denied" }, + scenario: "planned-volume inspection is inconclusive", + }, + ] as const)( + "fails force-fresh cleanup when $scenario", + async ({ containerInventory, failureCommand, plannedVolumeInspection }) => { const routes: Record = { info: ok(), - "ps -a --format {{.ID}} {{.Image}} {{.Names}}": ok(containerInventory[resource]), + "ps -a --format {{.ID}} {{.Image}} {{.Names}}": ok(containerInventory), "images --format {{.ID}} {{.Repository}}:{{.Tag}}": ok(), "rm -f owned-id": { status: 1, stdout: "", stderr: "busy" }, - "volume inspect openshell-cluster-nemoclaw": plannedVolumeInspection[resource], + "volume inspect openshell-cluster-nemoclaw": plannedVolumeInspection, "volume rm -f openshell-cluster-nemoclaw": { status: 1, stdout: "", @@ -284,6 +299,7 @@ describe("uninstall Docker resource scope", () => { ); expect(result.exitCode).toBe(1); + expect(runDocker).toHaveBeenCalledWith(failureCommand, expect.any(Object)); }, ); }); diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index dbdb3be059b..3b0a400f5db 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -2459,12 +2459,32 @@ function isOwnedDockerImageRepository(imageRef: string): boolean { ); } +function dockerVolumeInspectionProvesAbsence(name: string, result: RunResult): boolean { + if (result.status !== 1 || result.error || result.signal) return false; + const escapedName = name.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const detail = `${result.stderr} ${result.stdout}`.trim(); + return [ + new RegExp( + `^(?:Error response from daemon:\\s*)?(?:No such volume|No such object):?\\s*${escapedName}$`, + "iu", + ), + new RegExp( + `^(?:Error response from daemon:\\s*)?get\\s+${escapedName}:\\s*no such volume$`, + "iu", + ), + ].some((pattern) => pattern.test(detail)); +} + function removeDockerVolume(name: string, runtime: UninstallRuntime): boolean { - if ( - runtime.runDocker(["volume", "inspect", name], { env: runtime.env, stdio: "ignore" }).status !== - 0 - ) - return true; + const inspection = runtime.runDocker(["volume", "inspect", name], { + env: runtime.env, + stdio: "ignore", + }); + if (inspection.status !== 0) { + if (dockerVolumeInspectionProvesAbsence(name, inspection)) return true; + runtime.warn(`Failed to inspect Docker volume ${name}`); + return false; + } if ( runtime.runDocker(["volume", "rm", "-f", name], { env: runtime.env, stdio: "ignore" }) .status === 0 From f3ba4ed558128f9377f772a9cf4961be3d504cfc Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 18 Sep 2026 12:46:51 -0700 Subject: [PATCH 32/39] fix(uninstall): reconcile labelled containers --- scripts/install.sh | 2 +- .../uninstall/run-plan-docker-scope.test.ts | 53 ++++++++++++++++++ src/lib/actions/uninstall/run-plan.ts | 54 ++++++++++++++++++- .../install-force-fresh.test.ts | 8 +-- 4 files changed, 110 insertions(+), 7 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index 33f28e448ce..d3dadba4122 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -3624,7 +3624,7 @@ force_fresh_install_has_existing_state() { if command_exists docker; then docker info >/dev/null 2>&1 || return 2 container_inventory="$( - docker ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract 2>/dev/null + docker ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract=1 2>/dev/null )" || return 2 [[ -z "$container_inventory" ]] || managed_docker_state=1 receipt_volume_inventory="$( diff --git a/src/lib/actions/uninstall/run-plan-docker-scope.test.ts b/src/lib/actions/uninstall/run-plan-docker-scope.test.ts index b2a2e50db05..c7d1f734b9a 100644 --- a/src/lib/actions/uninstall/run-plan-docker-scope.test.ts +++ b/src/lib/actions/uninstall/run-plan-docker-scope.test.ts @@ -302,4 +302,57 @@ describe("uninstall Docker resource scope", () => { expect(runDocker).toHaveBeenCalledWith(failureCommand, expect.any(Object)); }, ); + + it("removes and verifies a label-owned force-fresh container with unrelated name and image", async () => { + const ownedId = "0123456789ab"; + const routes: Record = { + info: ok(), + "ps -a --format {{.ID}} {{.Image}} {{.Names}}": ok( + `${ownedId} registry.example.com/unrelated:latest arbitrary-name`, + ), + "ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract=1": ok(ownedId), + "images --format {{.ID}} {{.Repository}}:{{.Tag}}": ok(), + [`rm -f ${ownedId}`]: ok(ownedId), + [`container inspect ${ownedId}`]: { + status: 1, + stdout: "", + stderr: `Error: No such object: ${ownedId}`, + }, + "volume inspect openshell-cluster-nemoclaw": { + status: 1, + stdout: "", + stderr: "Error response from daemon: get openshell-cluster-nemoclaw: no such volume", + }, + }; + const runDocker = vi.fn((args: string[]): RunResult => routes[args.join(" ")] ?? ok()); + + const result = await runUninstallPlan( + { + assumeYes: true, + deleteModels: false, + destroyUserData: true, + forceFreshReset: true, + keepOpenShell: true, + }, + { + commandExists: () => true, + env: { HOME: "/tmp/nemoclaw-force-fresh-labelled-container" } as NodeJS.ProcessEnv, + existsSync: () => false, + hasPortableRuntimeCleanup: () => false, + isTty: false, + kill: () => true, + log: () => undefined, + rmSync: vi.fn(), + run: (command, args) => + command === "openshell" && args.join(" ") === "gateway list -o json" + ? ok(JSON.stringify([{ name: "nemoclaw" }])) + : ok(), + runDocker, + }, + ); + + expect(result.exitCode).toBe(0); + expect(runDocker).toHaveBeenCalledWith(["rm", "-f", ownedId], expect.any(Object)); + expect(runDocker).toHaveBeenCalledWith(["container", "inspect", ownedId], expect.any(Object)); + }); }); diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index 3b0a400f5db..19c16afad61 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -228,6 +228,7 @@ const OPENSHELL_COMMAND_MISSING_ERROR = const INTERRUPTED_ONBOARD_SESSION_MAX_BYTES = 1024 * 1024; export const MANAGED_INFERENCE_CONTAINER_NAME_PATTERN = /^(?:nemoclaw-vllm|nemoclaw-vllm-worker|nemoclaw-llama-cpp|nemoclaw-vllm-cluster-rank-[0-9]+)$/; +const MANAGED_IMAGE_CONTRACT_CONTAINER_LABEL = "io.nvidia.nemoclaw.managed-image.contract=1"; function defaultCommandExists(command: string, env: NodeJS.ProcessEnv): boolean { if (!command || command.includes("\0")) return false; @@ -2412,6 +2413,51 @@ function removeDockerContainers(runtime: UninstallRuntime, gatewayName?: string) return removedAll; } +function dockerContainerInspectionProvesAbsence(id: string, result: RunResult): boolean { + if (result.status !== 1 || result.error || result.signal) return false; + const escapedId = id.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const detail = `${result.stderr} ${result.stdout}`.trim(); + return new RegExp( + `^(?:(?:Error response from daemon|Error):\\s*)?(?:No such container|No such object):?\\s*${escapedId}$`, + "iu", + ).test(detail); +} + +function removeForceFreshManagedImageContainers(runtime: UninstallRuntime): boolean { + const inventory = runtime.runDocker( + ["ps", "-aq", "--filter", `label=${MANAGED_IMAGE_CONTRACT_CONTAINER_LABEL}`], + { env: runtime.env }, + ); + if (inventory.status !== 0) { + runtime.warn("Failed to inventory managed-image Docker containers"); + return false; + } + const ids = [...new Set(splitNonEmptyLines(inventory.stdout))]; + let removedAll = true; + for (const id of ids) { + if (!/^[0-9a-f]{12,64}$/u.test(id)) { + runtime.warn("Managed-image Docker container inventory returned an invalid ID"); + removedAll = false; + continue; + } + const removal = runtime.runDocker(["rm", "-f", id], { + env: runtime.env, + stdio: "ignore", + }); + const inspection = runtime.runDocker(["container", "inspect", id], { + env: runtime.env, + stdio: "ignore", + }); + if (removal.status === 0 && dockerContainerInspectionProvesAbsence(id, inspection)) { + runtime.log(`Removed managed-image Docker container ${id}`); + continue; + } + runtime.warn(`Failed to remove and verify managed-image Docker container ${id}`); + removedAll = false; + } + return removedAll; +} + function removeDockerImages(runtime: UninstallRuntime): void { const result = runtime.runDocker(["images", "--format", "{{.ID}} {{.Repository}}:{{.Tag}}"], { env: runtime.env, @@ -2465,7 +2511,7 @@ function dockerVolumeInspectionProvesAbsence(name: string, result: RunResult): b const detail = `${result.stderr} ${result.stdout}`.trim(); return [ new RegExp( - `^(?:Error response from daemon:\\s*)?(?:No such volume|No such object):?\\s*${escapedName}$`, + `^(?:(?:Error response from daemon|Error):\\s*)?(?:No such volume|No such object):?\\s*${escapedName}$`, "iu", ), new RegExp( @@ -2521,6 +2567,10 @@ function executeDockerResourceStep( runtime, scopedToSelectedGateway ? options.gatewayName || resolveGatewayName(GATEWAY_PORT) : undefined, ); + const removedManagedImageContainers = + !options.forceFreshReset || scopedToSelectedGateway + ? true + : removeForceFreshManagedImageContainers(runtime); if (scopedToSelectedGateway) { runtime.log("Sibling gateways remain; kept shared Docker images."); } else { @@ -2531,7 +2581,7 @@ function executeDockerResourceStep( if (!removeDockerVolume(volumeName, runtime)) removedVolumes = false; } if (!options.forceFreshReset) return true; - if (!removedContainers || !removedVolumes) { + if (!removedContainers || !removedManagedImageContainers || !removedVolumes) { runtime.error("Force-fresh cleanup could not remove every required Docker resource."); return false; } diff --git a/test/installer-integration/install-force-fresh.test.ts b/test/installer-integration/install-force-fresh.test.ts index 613fd0e0528..0ba286819e2 100644 --- a/test/installer-integration/install-force-fresh.test.ts +++ b/test/installer-integration/install-force-fresh.test.ts @@ -76,7 +76,7 @@ it("detects Docker-only state only through an authoritative managed-image label" `#!/usr/bin/env bash case "$*" in info) exit 0 ;; - "ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract") printf '0123456789ab\n' ;; + "ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract=1") printf '0123456789ab\n' ;; "volume ls --format {{.Name}}") exit 0 ;; *) exit 1 ;; esac @@ -99,7 +99,7 @@ it("does not treat an unlabeled prefix-matching Docker container as owned state" `#!/usr/bin/env bash case "$*" in info) exit 0 ;; - "ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract") exit 0 ;; + "ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract=1") exit 0 ;; "volume ls --format {{.Name}}") exit 0 ;; *) exit 1 ;; esac @@ -121,7 +121,7 @@ it("detects labelled Docker receipt-volume-only state", () => { `#!/usr/bin/env bash case "$*" in info) exit 0 ;; - "ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract") exit 0 ;; + "ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract=1") exit 0 ;; "volume ls --format {{.Name}}") printf 'nemoclaw-managed-startup-receipt-volume-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n' ;; @@ -153,7 +153,7 @@ it("stops before cleanup for a pre-label receipt-volume-only state", () => { `#!/usr/bin/env bash case "$*" in info) exit 0 ;; - "ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract") exit 0 ;; + "ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract=1") exit 0 ;; "volume ls --format {{.Name}}") printf '%s\n' "$LEGACY_VOLUME" ;; *) if [[ "$1" == "volume" && "$2" == "inspect" ]]; then From efebb60ca3fde0a3403262d55aa6ee5ebe7aa06f Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 18 Sep 2026 12:57:52 -0700 Subject: [PATCH 33/39] fix(uninstall): capture Docker absence proof --- src/lib/actions/uninstall/run-plan-docker-scope.test.ts | 9 ++++++++- src/lib/actions/uninstall/run-plan.ts | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/lib/actions/uninstall/run-plan-docker-scope.test.ts b/src/lib/actions/uninstall/run-plan-docker-scope.test.ts index c7d1f734b9a..1dcda8e4c77 100644 --- a/src/lib/actions/uninstall/run-plan-docker-scope.test.ts +++ b/src/lib/actions/uninstall/run-plan-docker-scope.test.ts @@ -300,6 +300,10 @@ describe("uninstall Docker resource scope", () => { expect(result.exitCode).toBe(1); expect(runDocker).toHaveBeenCalledWith(failureCommand, expect.any(Object)); + expect(runDocker).toHaveBeenCalledWith( + ["volume", "inspect", "openshell-cluster-nemoclaw"], + expect.objectContaining({ stdio: ["ignore", "pipe", "pipe"] }), + ); }, ); @@ -353,6 +357,9 @@ describe("uninstall Docker resource scope", () => { expect(result.exitCode).toBe(0); expect(runDocker).toHaveBeenCalledWith(["rm", "-f", ownedId], expect.any(Object)); - expect(runDocker).toHaveBeenCalledWith(["container", "inspect", ownedId], expect.any(Object)); + expect(runDocker).toHaveBeenCalledWith( + ["container", "inspect", ownedId], + expect.objectContaining({ stdio: ["ignore", "pipe", "pipe"] }), + ); }); }); diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index 19c16afad61..ce16dd9e427 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -2446,7 +2446,7 @@ function removeForceFreshManagedImageContainers(runtime: UninstallRuntime): bool }); const inspection = runtime.runDocker(["container", "inspect", id], { env: runtime.env, - stdio: "ignore", + stdio: ["ignore", "pipe", "pipe"], }); if (removal.status === 0 && dockerContainerInspectionProvesAbsence(id, inspection)) { runtime.log(`Removed managed-image Docker container ${id}`); @@ -2524,7 +2524,7 @@ function dockerVolumeInspectionProvesAbsence(name: string, result: RunResult): b function removeDockerVolume(name: string, runtime: UninstallRuntime): boolean { const inspection = runtime.runDocker(["volume", "inspect", name], { env: runtime.env, - stdio: "ignore", + stdio: ["ignore", "pipe", "pipe"], }); if (inspection.status !== 0) { if (dockerVolumeInspectionProvesAbsence(name, inspection)) return true; From 5da6600587cfa091ede0b7ce11cf71a7568f83a0 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 18 Sep 2026 13:30:27 -0700 Subject: [PATCH 34/39] fix(installer): preflight force-fresh ownership --- scripts/install.sh | 65 ++++++++++++++++-- .../install-force-fresh.test.ts | 68 +++++++++++++++++++ 2 files changed, 126 insertions(+), 7 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index b469fb281e4..81c975eb3d2 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -2391,8 +2391,16 @@ record_managed_user_local_openshell_install() { _cleanup_files+=("$temp_manifest") chmod 600 "$temp_manifest" \ || error "Could not secure the managed OpenShell install manifest." - for binary in openshell openshell-gateway; do + for binary in openshell openshell-gateway openshell-sandbox openshell-driver-vm; do binary_path="${user_bin}/${binary}" + if [[ ! -e "$binary_path" ]]; then + case "$binary" in + openshell | openshell-gateway) + error "The verified standalone OpenShell install did not provide ${binary_path}." + ;; + *) continue ;; + esac + fi [[ -f "$binary_path" && ! -L "$binary_path" && -x "$binary_path" ]] \ || error "The verified standalone OpenShell install did not provide ${binary_path}." digest="$("${sha256_command[@]}" "$binary_path" | awk '{print $1}')" \ @@ -3632,18 +3640,19 @@ run_preupgrade_backup() { } force_fresh_install_has_existing_state() { - local container_inventory receipt_volume_inventory volume_name volume_label volume_suffix + local container_inventory receipt_volume_inventory user_bin volume_name volume_label volume_suffix local existing_state=0 managed_docker_state=0 _FORCE_FRESH_UNVERIFIED_RECEIPT_VOLUME="" + user_bin="${XDG_BIN_HOME:-${HOME}/.local/bin}" if [[ -e "$(nemoclaw_state_root)" ]] \ || [[ -e "${HOME}/.config/nemoclaw" ]] \ || [[ -e "${HOME}/.config/openshell" ]] \ || [[ -e "${HOME}/.local/state/nemoclaw" ]] \ - || [[ -e "${HOME}/.local/bin/nemoclaw" ]] \ - || [[ -e "${HOME}/.local/bin/openshell" ]] \ - || [[ -e "${HOME}/.local/bin/openshell-gateway" ]] \ - || [[ -e "${HOME}/.local/bin/openshell-sandbox" ]] \ - || [[ -e "${HOME}/.local/bin/openshell-driver-vm" ]] \ + || [[ -e "${user_bin}/nemoclaw" ]] \ + || [[ -e "${user_bin}/openshell" ]] \ + || [[ -e "${user_bin}/openshell-gateway" ]] \ + || [[ -e "${user_bin}/openshell-sandbox" ]] \ + || [[ -e "${user_bin}/openshell-driver-vm" ]] \ || command_exists nemoclaw \ || command_exists openshell; then existing_state=1 @@ -3737,6 +3746,47 @@ run_force_fresh_uninstaller() { --yes --destroy-user-data --force-fresh-reset --all-gateway-ports } +force_fresh_managed_user_local_openshell_binary() { + local binary="$1" binary_path="$2" user_bin="$3" + local manifest="${user_bin}/.nemoclaw-openshell-managed-v1" digest="" + [[ "$binary_path" == "${user_bin}/${binary}" ]] || return 1 + [[ -f "$binary_path" && ! -L "$binary_path" && -x "$binary_path" && -O "$binary_path" ]] \ + || return 1 + [[ -f "$manifest" && ! -L "$manifest" && -O "$manifest" ]] || return 1 + if command_exists sha256sum; then + digest="$(sha256sum "$binary_path" | awk '{print $1}')" || return 1 + elif command_exists shasum; then + digest="$(shasum -a 256 "$binary_path" | awk '{print $1}')" || return 1 + else + return 1 + fi + [[ "$digest" =~ ^[a-f0-9]{64}$ ]] || return 1 + grep -Fqx -- "${digest} ${binary}" "$manifest" +} + +preflight_macos_openshell_for_force_fresh_install() { + local formula="nvidia/openshell/openshell" binary="" binary_path="" + local user_bin="${XDG_BIN_HOME:-${HOME}/.local/bin}" homebrew_bin="" + if command_exists brew \ + && brew list --formula "$formula" >/dev/null 2>&1; then + homebrew_bin="$(brew --prefix 2>/dev/null)/bin" \ + || error "Homebrew could not report its installation prefix before force-fresh cleanup. No cleanup started." + [[ "$homebrew_bin" != "/bin" ]] \ + || error "Homebrew returned an empty installation prefix before force-fresh cleanup. No cleanup started." + fi + for binary in openshell openshell-gateway openshell-sandbox openshell-driver-vm; do + binary_path="${user_bin}/${binary}" + if [[ -e "$binary_path" ]]; then + force_fresh_managed_user_local_openshell_binary "$binary" "$binary_path" "$user_bin" \ + || error "The force-fresh installer found an unverified OpenShell executable at ${binary_path}. Remove or reconcile it explicitly, then rerun. No cleanup started." + fi + binary_path="$(command -v "$binary" 2>/dev/null || true)" + [[ -z "$binary_path" || "$binary_path" == "${user_bin}/${binary}" ]] && continue + [[ -n "$homebrew_bin" && "$binary_path" == "${homebrew_bin}/${binary}" ]] && continue + error "The force-fresh installer found an unverified OpenShell executable at ${binary_path}. Remove or reconcile it explicitly, then rerun. No cleanup started." + done +} + remove_macos_openshell_for_force_fresh_install() { local formula="nvidia/openshell/openshell" binary="" openshell_path="" if command_exists brew && brew list --formula "$formula" >/dev/null 2>&1; then @@ -3755,6 +3805,7 @@ remove_macos_openshell_for_force_fresh_install() { run_force_fresh_install_reset() { local source_root existing_state_status=0 warn "Force-fresh install selected. NemoClaw will destroy all NemoClaw/OpenShell sandboxes, gateways, credentials, configuration, and recovery state on this host. Model caches are kept." + preflight_macos_openshell_for_force_fresh_install force_fresh_install_has_existing_state || existing_state_status=$? case "$existing_state_status" in 0) diff --git a/test/installer-integration/install-force-fresh.test.ts b/test/installer-integration/install-force-fresh.test.ts index 0ba286819e2..b7511458f97 100644 --- a/test/installer-integration/install-force-fresh.test.ts +++ b/test/installer-integration/install-force-fresh.test.ts @@ -244,8 +244,12 @@ it("records digest ownership after a verified managed user-local OpenShell insta fs.mkdirSync(localBin, { recursive: true }); const openshellSource = "#!/usr/bin/env bash\nprintf 'openshell 0.0.116\\n'\n"; const gatewaySource = "#!/usr/bin/env bash\nexit 0\n"; + const sandboxSource = "#!/usr/bin/env bash\nprintf 'sandbox\\n'\n"; + const driverSource = "#!/usr/bin/env bash\nprintf 'driver\\n'\n"; writeExecutable(path.join(localBin, "openshell"), openshellSource); writeExecutable(path.join(localBin, "openshell-gateway"), gatewaySource); + writeExecutable(path.join(localBin, "openshell-sandbox"), sandboxSource); + writeExecutable(path.join(localBin, "openshell-driver-vm"), driverSource); const result = callPayloadFunction("record_managed_user_local_openshell_install", { HOME: tmp, @@ -257,6 +261,8 @@ it("records digest ownership after a verified managed user-local OpenShell insta expect(manifest.trim().split("\n")).toEqual([ `${createHash("sha256").update(openshellSource).digest("hex")} openshell`, `${createHash("sha256").update(gatewaySource).digest("hex")} openshell-gateway`, + `${createHash("sha256").update(sandboxSource).digest("hex")} openshell-sandbox`, + `${createHash("sha256").update(driverSource).digest("hex")} openshell-driver-vm`, ]); }); @@ -350,6 +356,37 @@ it.each(["openshell", "openshell-gateway", "openshell-sandbox", "openshell-drive }, ); +it("rejects a foreign OpenShell executable before staged cleanup begins", () => { + const { root: tmp, binDir: fakeBin } = installerCheckout( + "nemoclaw-force-fresh-foreign-preflight-", + ); + const foreignBin = path.join(tmp, "foreign", "bin"); + const cleanupMarker = path.join(tmp, "cleanup-started"); + fs.mkdirSync(foreignBin, { recursive: true }); + writeExecutable(path.join(foreignBin, "openshell-sandbox"), "#!/usr/bin/env bash\nexit 0\n"); + + const result = callPayloadFunction( + ` + warn() { :; } + prepare_force_fresh_uninstaller() { touch "$CLEANUP_MARKER"; } + run_force_fresh_uninstaller() { touch "$CLEANUP_MARKER"; } + remove_macos_openshell_for_force_fresh_install() { touch "$CLEANUP_MARKER"; } + run_force_fresh_install_reset + `, + { + CLEANUP_MARKER: cleanupMarker, + HOME: tmp, + PATH: `${foreignBin}:${fakeBin}:${TEST_SYSTEM_PATH}`, + }, + ); + + expect(result.status).not.toBe(0); + expect(`${result.stdout}${result.stderr}`).toContain( + `unverified OpenShell executable at ${path.join(foreignBin, "openshell-sandbox")}`, + ); + expect(fs.existsSync(cleanupMarker)).toBe(false); +}); + it("runs cleanup before selecting fresh onboarding", () => { const result = callPayloadFunction(` warn() { :; } @@ -390,6 +427,37 @@ it.each(["openshell-gateway", "openshell-sandbox", "openshell-driver-vm"])( }, ); +it("routes a managed helper from XDG_BIN_HOME through the staged uninstaller", () => { + const { root: tmp } = installerCheckout("nemoclaw-force-fresh-xdg-helper-detect-"); + const userBin = path.join(tmp, "xdg-bin"); + const helperSource = "#!/usr/bin/env bash\nprintf 'sandbox\\n'\n"; + fs.mkdirSync(userBin, { recursive: true }); + writeExecutable(path.join(userBin, "openshell-sandbox"), helperSource); + fs.writeFileSync( + path.join(userBin, ".nemoclaw-openshell-managed-v1"), + `${createHash("sha256").update(helperSource).digest("hex")} openshell-sandbox\n`, + { mode: 0o600 }, + ); + + const result = callPayloadFunction( + ` + warn() { :; } + force_fresh_install_source_root() { printf '/tmp/staged-candidate'; } + prepare_force_fresh_uninstaller() { printf 'prepare:%s\\n' "$1"; } + run_force_fresh_uninstaller() { printf 'uninstall:%s\\n' "$1"; } + remove_macos_openshell_for_force_fresh_install() { :; } + run_force_fresh_install_reset + `, + { HOME: tmp, XDG_BIN_HOME: userBin }, + ); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual([ + "prepare:/tmp/staged-candidate", + "uninstall:/tmp/staged-candidate", + ]); +}); + it("stops before package removal when managed uninstall rejects partial state", () => { const result = callPayloadFunction(` warn() { :; } From 1fbfd62992b6655289541823b4a81f17fae54913 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 18 Sep 2026 14:08:41 -0700 Subject: [PATCH 35/39] fix(installer): fail closed on cleanup ownership --- scripts/install.sh | 49 ++++++----- .../uninstall/run-plan-docker-scope.test.ts | 75 ++++------------- src/lib/actions/uninstall/run-plan.ts | 70 ++++------------ .../install-force-fresh.test.ts | 84 ++++++++++++++++--- 4 files changed, 130 insertions(+), 148 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index 81c975eb3d2..3858df5cb5d 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -3640,8 +3640,10 @@ run_preupgrade_backup() { } force_fresh_install_has_existing_state() { - local container_inventory receipt_volume_inventory user_bin volume_name volume_label volume_suffix + local container_id container_image container_inventory container_label_inventory container_name + local receipt_volume_inventory user_bin volume_name volume_label volume_suffix local existing_state=0 managed_docker_state=0 + _FORCE_FRESH_UNVERIFIED_DOCKER_CONTAINER="" _FORCE_FRESH_UNVERIFIED_RECEIPT_VOLUME="" user_bin="${XDG_BIN_HOME:-${HOME}/.local/bin}" if [[ -e "$(nemoclaw_state_root)" ]] \ @@ -3663,10 +3665,25 @@ force_fresh_install_has_existing_state() { fi if command_exists docker; then docker info >/dev/null 2>&1 || return 2 - container_inventory="$( + container_label_inventory="$( docker ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract=1 2>/dev/null )" || return 2 - [[ -z "$container_inventory" ]] || managed_docker_state=1 + _FORCE_FRESH_UNVERIFIED_DOCKER_CONTAINER="$(printf '%s\n' "$container_label_inventory" | sed -n '1p')" + container_inventory="$( + docker ps -a --format '{{.ID}} {{.Image}} {{.Names}}' 2>/dev/null + )" || return 2 + while read -r container_id container_image container_name; do + [[ -n "$container_id" ]] || continue + case "$container_name" in + openshell-* | nemoclaw-*) _FORCE_FRESH_UNVERIFIED_DOCKER_CONTAINER="$container_id" ;; + esac + case "$container_image" in + nemoclaw-* | openshell/* | ghcr.io/nvidia/nemoclaw | ghcr.io/nvidia/nemoclaw:* | ghcr.io/nvidia/nemoclaw@* | ghcr.io/nvidia/nemoclaw/* | ghcr.io/nvidia/nemoclaw-*) + _FORCE_FRESH_UNVERIFIED_DOCKER_CONTAINER="$container_id" + ;; + esac + [[ -z "$_FORCE_FRESH_UNVERIFIED_DOCKER_CONTAINER" ]] || break + done <<<"$container_inventory" receipt_volume_inventory="$( docker volume ls --format '{{.Name}}' 2>/dev/null )" || return 2 @@ -3690,6 +3707,9 @@ force_fresh_install_has_existing_state() { fi done <<<"$receipt_volume_inventory" fi + if ((existing_state == 0)) && [[ -n "$_FORCE_FRESH_UNVERIFIED_DOCKER_CONTAINER" ]]; then + return 4 + fi ((existing_state == 0 && managed_docker_state == 0)) || return 0 return 1 } @@ -3746,24 +3766,6 @@ run_force_fresh_uninstaller() { --yes --destroy-user-data --force-fresh-reset --all-gateway-ports } -force_fresh_managed_user_local_openshell_binary() { - local binary="$1" binary_path="$2" user_bin="$3" - local manifest="${user_bin}/.nemoclaw-openshell-managed-v1" digest="" - [[ "$binary_path" == "${user_bin}/${binary}" ]] || return 1 - [[ -f "$binary_path" && ! -L "$binary_path" && -x "$binary_path" && -O "$binary_path" ]] \ - || return 1 - [[ -f "$manifest" && ! -L "$manifest" && -O "$manifest" ]] || return 1 - if command_exists sha256sum; then - digest="$(sha256sum "$binary_path" | awk '{print $1}')" || return 1 - elif command_exists shasum; then - digest="$(shasum -a 256 "$binary_path" | awk '{print $1}')" || return 1 - else - return 1 - fi - [[ "$digest" =~ ^[a-f0-9]{64}$ ]] || return 1 - grep -Fqx -- "${digest} ${binary}" "$manifest" -} - preflight_macos_openshell_for_force_fresh_install() { local formula="nvidia/openshell/openshell" binary="" binary_path="" local user_bin="${XDG_BIN_HOME:-${HOME}/.local/bin}" homebrew_bin="" @@ -3777,8 +3779,8 @@ preflight_macos_openshell_for_force_fresh_install() { for binary in openshell openshell-gateway openshell-sandbox openshell-driver-vm; do binary_path="${user_bin}/${binary}" if [[ -e "$binary_path" ]]; then - force_fresh_managed_user_local_openshell_binary "$binary" "$binary_path" "$user_bin" \ - || error "The force-fresh installer found an unverified OpenShell executable at ${binary_path}. Remove or reconcile it explicitly, then rerun. No cleanup started." + [[ -f "$binary_path" && ! -L "$binary_path" && -x "$binary_path" && -O "$binary_path" ]] \ + || error "The force-fresh installer found an unsafe OpenShell executable at ${binary_path}. Remove or reconcile it explicitly, then rerun. No cleanup started." fi binary_path="$(command -v "$binary" 2>/dev/null || true)" [[ -z "$binary_path" || "$binary_path" == "${user_bin}/${binary}" ]] && continue @@ -3817,6 +3819,7 @@ run_force_fresh_install_reset() { 1) info "No existing NemoClaw or OpenShell installation was found; continuing with a clean install." ;; 2) error "Docker is installed but unavailable. Start the selected local Docker or Colima daemon, then rerun --force-fresh-install. No cleanup started." ;; 3) error "Force-fresh cleanup found pre-label receipt volume ${_FORCE_FRESH_UNVERIFIED_RECEIPT_VOLUME}. NemoClaw cannot prove ownership of this legacy volume. Inspect it, remove it only after confirming it belongs to the interrupted NemoClaw install, then rerun. No cleanup started." ;; + 4) error "Force-fresh cleanup found unverified Docker container ${_FORCE_FRESH_UNVERIFIED_DOCKER_CONTAINER}. NemoClaw cannot prove this Docker-only resource belongs to the current installation. Inspect and reconcile it explicitly, then rerun. No cleanup started." ;; *) error "Could not inspect existing NemoClaw or OpenShell state. No cleanup started." ;; esac remove_macos_openshell_for_force_fresh_install diff --git a/src/lib/actions/uninstall/run-plan-docker-scope.test.ts b/src/lib/actions/uninstall/run-plan-docker-scope.test.ts index 1dcda8e4c77..fb942fbbd78 100644 --- a/src/lib/actions/uninstall/run-plan-docker-scope.test.ts +++ b/src/lib/actions/uninstall/run-plan-docker-scope.test.ts @@ -235,6 +235,7 @@ describe("uninstall Docker resource scope", () => { { containerInventory: "owned-id nemoclaw-sandbox:test openshell-owned", failureCommand: ["rm", "-f", "owned-id"], + imageInventory: "", plannedVolumeInspection: { status: 1, stdout: "", @@ -245,23 +246,37 @@ describe("uninstall Docker resource scope", () => { { containerInventory: "", failureCommand: ["volume", "rm", "-f", "openshell-cluster-nemoclaw"], + imageInventory: "", plannedVolumeInspection: ok(), scenario: "planned-volume removal fails", }, { containerInventory: "", failureCommand: ["volume", "inspect", "openshell-cluster-nemoclaw"], + imageInventory: "", plannedVolumeInspection: { status: 1, stdout: "", stderr: "permission denied" }, scenario: "planned-volume inspection is inconclusive", }, + { + containerInventory: "", + failureCommand: ["rmi", "-f", "owned-image"], + imageInventory: "owned-image ghcr.io/nvidia/nemoclaw/openclaw-sandbox:test", + plannedVolumeInspection: { + status: 1, + stdout: "", + stderr: "Error response from daemon: get openshell-cluster-nemoclaw: no such volume", + }, + scenario: "owned image removal fails", + }, ] as const)( "fails force-fresh cleanup when $scenario", - async ({ containerInventory, failureCommand, plannedVolumeInspection }) => { + async ({ containerInventory, failureCommand, imageInventory, plannedVolumeInspection }) => { const routes: Record = { info: ok(), "ps -a --format {{.ID}} {{.Image}} {{.Names}}": ok(containerInventory), - "images --format {{.ID}} {{.Repository}}:{{.Tag}}": ok(), + "images --format {{.ID}} {{.Repository}}:{{.Tag}}": ok(imageInventory), "rm -f owned-id": { status: 1, stdout: "", stderr: "busy" }, + "rmi -f owned-image": { status: 1, stdout: "", stderr: "busy" }, "volume inspect openshell-cluster-nemoclaw": plannedVolumeInspection, "volume rm -f openshell-cluster-nemoclaw": { status: 1, @@ -306,60 +321,4 @@ describe("uninstall Docker resource scope", () => { ); }, ); - - it("removes and verifies a label-owned force-fresh container with unrelated name and image", async () => { - const ownedId = "0123456789ab"; - const routes: Record = { - info: ok(), - "ps -a --format {{.ID}} {{.Image}} {{.Names}}": ok( - `${ownedId} registry.example.com/unrelated:latest arbitrary-name`, - ), - "ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract=1": ok(ownedId), - "images --format {{.ID}} {{.Repository}}:{{.Tag}}": ok(), - [`rm -f ${ownedId}`]: ok(ownedId), - [`container inspect ${ownedId}`]: { - status: 1, - stdout: "", - stderr: `Error: No such object: ${ownedId}`, - }, - "volume inspect openshell-cluster-nemoclaw": { - status: 1, - stdout: "", - stderr: "Error response from daemon: get openshell-cluster-nemoclaw: no such volume", - }, - }; - const runDocker = vi.fn((args: string[]): RunResult => routes[args.join(" ")] ?? ok()); - - const result = await runUninstallPlan( - { - assumeYes: true, - deleteModels: false, - destroyUserData: true, - forceFreshReset: true, - keepOpenShell: true, - }, - { - commandExists: () => true, - env: { HOME: "/tmp/nemoclaw-force-fresh-labelled-container" } as NodeJS.ProcessEnv, - existsSync: () => false, - hasPortableRuntimeCleanup: () => false, - isTty: false, - kill: () => true, - log: () => undefined, - rmSync: vi.fn(), - run: (command, args) => - command === "openshell" && args.join(" ") === "gateway list -o json" - ? ok(JSON.stringify([{ name: "nemoclaw" }])) - : ok(), - runDocker, - }, - ); - - expect(result.exitCode).toBe(0); - expect(runDocker).toHaveBeenCalledWith(["rm", "-f", ownedId], expect.any(Object)); - expect(runDocker).toHaveBeenCalledWith( - ["container", "inspect", ownedId], - expect.objectContaining({ stdio: ["ignore", "pipe", "pipe"] }), - ); - }); }); diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index ce16dd9e427..bbae04f9f89 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -228,7 +228,6 @@ const OPENSHELL_COMMAND_MISSING_ERROR = const INTERRUPTED_ONBOARD_SESSION_MAX_BYTES = 1024 * 1024; export const MANAGED_INFERENCE_CONTAINER_NAME_PATTERN = /^(?:nemoclaw-vllm|nemoclaw-vllm-worker|nemoclaw-llama-cpp|nemoclaw-vllm-cluster-rank-[0-9]+)$/; -const MANAGED_IMAGE_CONTRACT_CONTAINER_LABEL = "io.nvidia.nemoclaw.managed-image.contract=1"; function defaultCommandExists(command: string, env: NodeJS.ProcessEnv): boolean { if (!command || command.includes("\0")) return false; @@ -2413,55 +2412,14 @@ function removeDockerContainers(runtime: UninstallRuntime, gatewayName?: string) return removedAll; } -function dockerContainerInspectionProvesAbsence(id: string, result: RunResult): boolean { - if (result.status !== 1 || result.error || result.signal) return false; - const escapedId = id.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); - const detail = `${result.stderr} ${result.stdout}`.trim(); - return new RegExp( - `^(?:(?:Error response from daemon|Error):\\s*)?(?:No such container|No such object):?\\s*${escapedId}$`, - "iu", - ).test(detail); -} - -function removeForceFreshManagedImageContainers(runtime: UninstallRuntime): boolean { - const inventory = runtime.runDocker( - ["ps", "-aq", "--filter", `label=${MANAGED_IMAGE_CONTRACT_CONTAINER_LABEL}`], - { env: runtime.env }, - ); - if (inventory.status !== 0) { - runtime.warn("Failed to inventory managed-image Docker containers"); - return false; - } - const ids = [...new Set(splitNonEmptyLines(inventory.stdout))]; - let removedAll = true; - for (const id of ids) { - if (!/^[0-9a-f]{12,64}$/u.test(id)) { - runtime.warn("Managed-image Docker container inventory returned an invalid ID"); - removedAll = false; - continue; - } - const removal = runtime.runDocker(["rm", "-f", id], { - env: runtime.env, - stdio: "ignore", - }); - const inspection = runtime.runDocker(["container", "inspect", id], { - env: runtime.env, - stdio: ["ignore", "pipe", "pipe"], - }); - if (removal.status === 0 && dockerContainerInspectionProvesAbsence(id, inspection)) { - runtime.log(`Removed managed-image Docker container ${id}`); - continue; - } - runtime.warn(`Failed to remove and verify managed-image Docker container ${id}`); - removedAll = false; - } - return removedAll; -} - -function removeDockerImages(runtime: UninstallRuntime): void { +function removeDockerImages(runtime: UninstallRuntime): boolean { const result = runtime.runDocker(["images", "--format", "{{.ID}} {{.Repository}}:{{.Tag}}"], { env: runtime.env, }); + if (result.status !== 0) { + runtime.warn("Failed to inventory Docker images"); + return false; + } const ids = splitNonEmptyLines(result.stdout) // `openclaw` is deliberately absent: NemoClaw builds no image under that // name, so the term only ever selected the separate OpenClaw project's @@ -2471,13 +2429,18 @@ function removeDockerImages(runtime: UninstallRuntime): void { .map((line) => line.split(/\s+/)[0]); if (ids.length === 0) { runtime.log(`No ${runtimeBranding(runtime).display}/OpenShell Docker images found`); - return; + return true; } + let removedAll = true; for (const id of [...new Set(ids)]) { if (runtime.runDocker(["rmi", "-f", id], { env: runtime.env, stdio: "ignore" }).status === 0) runtime.log(`Removed Docker image ${id}`); - else runtime.warn(`Failed to remove Docker image ${id}`); + else { + runtime.warn(`Failed to remove Docker image ${id}`); + removedAll = false; + } } + return removedAll; } function dockerInventoryFields(line: string, expectedFields: number): string[] { @@ -2567,21 +2530,18 @@ function executeDockerResourceStep( runtime, scopedToSelectedGateway ? options.gatewayName || resolveGatewayName(GATEWAY_PORT) : undefined, ); - const removedManagedImageContainers = - !options.forceFreshReset || scopedToSelectedGateway - ? true - : removeForceFreshManagedImageContainers(runtime); + let removedImages = true; if (scopedToSelectedGateway) { runtime.log("Sibling gateways remain; kept shared Docker images."); } else { - removeDockerImages(runtime); + removedImages = removeDockerImages(runtime); } let removedVolumes = true; for (const volumeName of volumeNames) { if (!removeDockerVolume(volumeName, runtime)) removedVolumes = false; } if (!options.forceFreshReset) return true; - if (!removedContainers || !removedManagedImageContainers || !removedVolumes) { + if (!removedContainers || !removedImages || !removedVolumes) { runtime.error("Force-fresh cleanup could not remove every required Docker resource."); return false; } diff --git a/test/installer-integration/install-force-fresh.test.ts b/test/installer-integration/install-force-fresh.test.ts index b7511458f97..a8bfced9978 100644 --- a/test/installer-integration/install-force-fresh.test.ts +++ b/test/installer-integration/install-force-fresh.test.ts @@ -69,49 +69,80 @@ exit 1 expect(result.stdout).toBe("present"); }); -it("detects Docker-only state only through an authoritative managed-image label", () => { +it("stops before cleanup for a label-only Docker container without trusted state", () => { const { root: tmp, binDir: fakeBin } = installerCheckout("nemoclaw-force-fresh-detect-docker-"); + const cleanupMarker = path.join(tmp, "cleanup-started"); writeExecutable( path.join(fakeBin, "docker"), `#!/usr/bin/env bash case "$*" in info) exit 0 ;; "ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract=1") printf '0123456789ab\n' ;; + "ps -a --format {{.ID}} {{.Image}} {{.Names}}") + printf '0123456789ab registry.example.com/unrelated:latest arbitrary-name\n' + ;; "volume ls --format {{.Name}}") exit 0 ;; *) exit 1 ;; esac `, ); - const result = callPayloadFunction("force_fresh_install_has_existing_state && printf 'present'", { - HOME: tmp, - PATH: `${fakeBin}:${TEST_SYSTEM_PATH}`, - }); + const result = callPayloadFunction( + ` + warn() { :; } + prepare_force_fresh_uninstaller() { touch "$CLEANUP_MARKER"; } + run_force_fresh_uninstaller() { touch "$CLEANUP_MARKER"; } + remove_macos_openshell_for_force_fresh_install() { touch "$CLEANUP_MARKER"; } + run_force_fresh_install_reset + `, + { + CLEANUP_MARKER: cleanupMarker, + HOME: tmp, + PATH: `${fakeBin}:${TEST_SYSTEM_PATH}`, + }, + ); - expect(result.status).toBe(0); - expect(result.stdout).toBe("present"); + expect(result.status).not.toBe(0); + expect(`${result.stdout}${result.stderr}`).toContain("unverified Docker container 0123456789ab"); + expect(fs.existsSync(cleanupMarker)).toBe(false); }); -it("does not treat an unlabeled prefix-matching Docker container as owned state", () => { +it("stops before cleanup for a convention-matching Docker-only container", () => { const { root: tmp, binDir: fakeBin } = installerCheckout("nemoclaw-force-fresh-ignore-docker-"); + const cleanupMarker = path.join(tmp, "cleanup-started"); writeExecutable( path.join(fakeBin, "docker"), `#!/usr/bin/env bash case "$*" in info) exit 0 ;; "ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract=1") exit 0 ;; + "ps -a --format {{.ID}} {{.Image}} {{.Names}}") + printf 'fedcba987654 redis:7 openshell-orphan\n' + ;; "volume ls --format {{.Name}}") exit 0 ;; *) exit 1 ;; esac `, ); - const result = callPayloadFunction("force_fresh_install_has_existing_state", { - HOME: tmp, - PATH: `${fakeBin}:${TEST_SYSTEM_PATH}`, - }); + const result = callPayloadFunction( + ` + warn() { :; } + prepare_force_fresh_uninstaller() { touch "$CLEANUP_MARKER"; } + run_force_fresh_uninstaller() { touch "$CLEANUP_MARKER"; } + remove_macos_openshell_for_force_fresh_install() { touch "$CLEANUP_MARKER"; } + run_force_fresh_install_reset + `, + { + CLEANUP_MARKER: cleanupMarker, + HOME: tmp, + PATH: `${fakeBin}:${TEST_SYSTEM_PATH}`, + }, + ); expect(result.status).not.toBe(0); + expect(`${result.stdout}${result.stderr}`).toContain("unverified Docker container fedcba987654"); + expect(fs.existsSync(cleanupMarker)).toBe(false); }); it("detects labelled Docker receipt-volume-only state", () => { @@ -122,6 +153,7 @@ it("detects labelled Docker receipt-volume-only state", () => { case "$*" in info) exit 0 ;; "ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract=1") exit 0 ;; + "ps -a --format {{.ID}} {{.Image}} {{.Names}}") exit 0 ;; "volume ls --format {{.Name}}") printf 'nemoclaw-managed-startup-receipt-volume-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n' ;; @@ -154,6 +186,7 @@ it("stops before cleanup for a pre-label receipt-volume-only state", () => { case "$*" in info) exit 0 ;; "ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract=1") exit 0 ;; + "ps -a --format {{.ID}} {{.Image}} {{.Names}}") exit 0 ;; "volume ls --format {{.Name}}") printf '%s\n' "$LEGACY_VOLUME" ;; *) if [[ "$1" == "volume" && "$2" == "inspect" ]]; then @@ -458,6 +491,33 @@ it("routes a managed helper from XDG_BIN_HOME through the staged uninstaller", ( ]); }); +it("lets the staged uninstaller reject a malformed ownership manifest before package removal", () => { + const { root: tmp } = installerCheckout("nemoclaw-force-fresh-malformed-manifest-"); + const userBin = path.join(tmp, "xdg-bin"); + const packageMarker = path.join(tmp, "package-removal-started"); + fs.mkdirSync(userBin, { recursive: true }); + writeExecutable(path.join(userBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n"); + fs.writeFileSync(path.join(userBin, ".nemoclaw-openshell-managed-v1"), "malformed\n", { + mode: 0o600, + }); + + const result = callPayloadFunction( + ` + warn() { :; } + force_fresh_install_source_root() { printf '/tmp/staged-candidate'; } + prepare_force_fresh_uninstaller() { :; } + run_force_fresh_uninstaller() { printf 'canonical-ownership-rejected\\n'; return 9; } + remove_macos_openshell_for_force_fresh_install() { touch "$PACKAGE_MARKER"; } + run_force_fresh_install_reset + `, + { HOME: tmp, PACKAGE_MARKER: packageMarker, XDG_BIN_HOME: userBin }, + ); + + expect(result.status).not.toBe(0); + expect(result.stdout).toContain("canonical-ownership-rejected"); + expect(fs.existsSync(packageMarker)).toBe(false); +}); + it("stops before package removal when managed uninstall rejects partial state", () => { const result = callPayloadFunction(` warn() { :; } From 3f55ca7753a7876ade8f967a01dec15fff6484a5 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 18 Sep 2026 14:47:08 -0700 Subject: [PATCH 36/39] fix(installer): centralize force-fresh ownership --- docs/reference/commands.mdx | 4 +- install.sh | 4 +- scripts/install.sh | 11 ++- .../internal/uninstall/run-plan.test.ts | 22 +++++ src/commands/internal/uninstall/run-plan.ts | 12 ++- .../uninstall/run-plan-docker-scope.test.ts | 81 ++++++++++++------- .../run-plan-homebrew-openshell.test.ts | 42 +++++++++- src/lib/actions/uninstall/run-plan.ts | 38 ++++++++- .../install-force-fresh.test.ts | 80 +++++++++++++----- 9 files changed, 233 insertions(+), 61 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 90373ae041e..029afc94cd8 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -110,7 +110,7 @@ The hosted installer accepts options after `bash -s --`. These options control i ### `--force-fresh-install` -On macOS, remove every NemoClaw- and OpenShell-managed sandbox, gateway, credential, configuration file, and recovery record before reinstalling and starting fresh onboarding. This destructive reset keeps downloaded model caches, but the removed state cannot be recovered without an external backup. Unlike `--fresh`, it removes the existing managed installation instead of only requesting fresh onboarding state. +On Apple silicon macOS, remove every NemoClaw- and OpenShell-managed sandbox, gateway, credential, configuration file, and recovery record before reinstalling and starting fresh onboarding. This destructive reset keeps downloaded model caches, but the removed state cannot be recovered without an external backup. Unlike `--fresh`, it removes the existing managed installation instead of only requesting fresh onboarding state. Use the hosted installer so the selected candidate is staged outside the state it removes: @@ -119,7 +119,7 @@ curl -fsSL https://www.nvidia.com/nemoclaw.sh | \ bash -s -- --force-fresh-install ``` -The equivalent environment variable is `NEMOCLAW_FORCE_FRESH_INSTALL=1`. The installer rejects this option on non-macOS hosts and stops without deleting remaining state if the authoritative whole-host uninstall cannot complete. +The equivalent environment variable is `NEMOCLAW_FORCE_FRESH_INSTALL=1`. The installer rejects this option outside Apple silicon macOS and stops without deleting remaining state if the authoritative whole-host uninstall cannot complete. ### `--local-model-runtime=vllm` diff --git a/install.sh b/install.sh index 706b1c7ff11..e5f29a9f640 100755 --- a/install.sh +++ b/install.sh @@ -313,7 +313,7 @@ bootstrap_usage() { printf " Use only with NEMOCLAW_AGENT=hermes, no registered sandboxes, no local model profile,\n" printf " and the build, cloud, or routed NVIDIA hosted provider\n" printf " --fresh Discard any failed/interrupted onboarding session and start over\n" - printf " --force-fresh-install Destroy all NemoClaw and OpenShell state, then reinstall (macOS only)\n" + printf " --force-fresh-install Destroy all NemoClaw and OpenShell state, then reinstall (Apple silicon macOS only)\n" printf " --force-station-install Validate an unrecognized Station GB300 release profile without onboarding\n" printf " --version, -v Print installer version and exit\n" printf " --help, -h Show this help message and exit\n\n" @@ -327,7 +327,7 @@ bootstrap_usage() { printf " Use only with NEMOCLAW_AGENT=hermes, no registered sandboxes, no local model profile,\n" printf " and the build, cloud, or routed NVIDIA hosted provider\n" printf " NEMOCLAW_FRESH=1 Same as --fresh\n" - printf " NEMOCLAW_FORCE_FRESH_INSTALL=1 Same as --force-fresh-install (macOS only)\n" + printf " NEMOCLAW_FORCE_FRESH_INSTALL=1 Same as --force-fresh-install (Apple silicon macOS only)\n" printf " NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 Same as --yes-i-accept-third-party-software\n" printf " NEMOCLAW_NO_EXPRESS=1 Skip express install prompt on supported platforms\n" printf " NEMOCLAW_SANDBOX_NAME Sandbox name to create/use\n" diff --git a/scripts/install.sh b/scripts/install.sh index 3858df5cb5d..f1fd6df298c 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -1087,7 +1087,7 @@ usage() { printf " Use only with NEMOCLAW_AGENT=hermes, no registered sandboxes, no local model profile,\n" printf " and the build, cloud, or routed NVIDIA hosted provider\n" printf " --fresh Discard any failed/interrupted onboarding session and start over\n" - printf " --force-fresh-install Destroy all NemoClaw and OpenShell state, then reinstall (macOS only)\n" + printf " --force-fresh-install Destroy all NemoClaw and OpenShell state, then reinstall (Apple silicon macOS only)\n" printf " --station-deepseek Use DeepSeek V4 Flash for DGX Station express install (interactive terminal required)\n" printf " --force-station-install Validate an unrecognized Station GB300 release profile without onboarding\n" printf " --version, -v Print installer version and exit\n" @@ -1101,7 +1101,7 @@ usage() { printf " and the build, cloud, or routed NVIDIA hosted provider\n" printf " NEMOCLAW_NON_INTERACTIVE_SUDO_MODE=prompt Allow sudo prompts during non-interactive onboarding\n" printf " NEMOCLAW_FRESH=1 Same as --fresh\n" - printf " NEMOCLAW_FORCE_FRESH_INSTALL=1 Same as --force-fresh-install (macOS only)\n" + printf " NEMOCLAW_FORCE_FRESH_INSTALL=1 Same as --force-fresh-install (Apple silicon macOS only)\n" printf " NEMOCLAW_NO_EXPRESS=1 Skip the Express prompt on detected platforms\n" printf " NEMOCLAW_SANDBOX_NAME Sandbox name to create/use\n" printf " HF_TOKEN Optional Hugging Face read token for managed-vLLM downloads\n" @@ -3761,6 +3761,9 @@ run_force_fresh_uninstaller() { node_bin="$(command -v node 2>/dev/null || true)" [[ -n "$node_bin" && -x "$node_bin" ]] \ || error "Node.js is required for the force-fresh uninstaller." + "$node_bin" "${source_root}/bin/nemoclaw.js" internal uninstall run-plan \ + --force-fresh-ownership-preflight \ + || error "Force-fresh cleanup stopped because the staged canonical ownership preflight rejected a user-local OpenShell executable. Reconcile the reported binary, then rerun. No cleanup started." NEMOCLAW_UNINSTALL_DESTROY_USER_DATA=1 "$node_bin" \ "${source_root}/bin/nemoclaw.js" internal uninstall run-plan \ --yes --destroy-user-data --force-fresh-reset --all-gateway-ports @@ -3831,8 +3834,8 @@ run_force_fresh_install_reset() { validate_force_fresh_install_platform() { truthy_env "${FORCE_FRESH_INSTALL:-}" || return 0 - [[ "$(uname -s)" == "Darwin" ]] \ - || error "--force-fresh-install currently supports macOS only." + [[ "$(uname -s)" == "Darwin" && "$(uname -m)" == "arm64" ]] \ + || error "--force-fresh-install currently supports Apple silicon macOS only." } # Return nonzero when OpenShell is absent or its version command fails. diff --git a/src/commands/internal/uninstall/run-plan.test.ts b/src/commands/internal/uninstall/run-plan.test.ts index 89492bcd8fc..e732d34f9f4 100644 --- a/src/commands/internal/uninstall/run-plan.test.ts +++ b/src/commands/internal/uninstall/run-plan.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it, vi } from "vitest"; import type { Config } from "@oclif/core"; const mocks = vi.hoisted(() => ({ + preflightForceFreshUserLocalOpenShellOwnership: vi.fn(), runUninstallPlan: vi.fn(), runUninstallPlanProduction: vi.fn(), })); @@ -19,6 +20,27 @@ vi.mock("../../../lib/actions/uninstall/all-gateway-ports", () => ({ import InternalUninstallRunPlanCommand from "./run-plan"; describe("internal uninstall command", () => { + it("runs the hidden force-fresh ownership preflight without starting uninstall", async () => { + const previousExitCode = process.exitCode; + process.exitCode = undefined; + mocks.preflightForceFreshUserLocalOpenShellOwnership.mockReturnValue(false); + const instance = new InternalUninstallRunPlanCommand([], {} as Config); + Object.defineProperty(instance, "parse", { + value: vi.fn().mockResolvedValue({ + flags: { "force-fresh-ownership-preflight": true }, + }), + }); + + try { + await instance.run(); + expect(mocks.preflightForceFreshUserLocalOpenShellOwnership).toHaveBeenCalledOnce(); + expect(mocks.runUninstallPlanProduction).not.toHaveBeenCalled(); + expect(process.exitCode).toBe(1); + } finally { + process.exitCode = previousExitCode; + } + }); + it("awaits the production uninstall transaction before applying its exit result", async () => { const previousExitCode = process.exitCode; let complete!: (value: { exitCode: number }) => void; diff --git a/src/commands/internal/uninstall/run-plan.ts b/src/commands/internal/uninstall/run-plan.ts index 3a91633e8fe..9e4a4fb6041 100644 --- a/src/commands/internal/uninstall/run-plan.ts +++ b/src/commands/internal/uninstall/run-plan.ts @@ -7,7 +7,10 @@ import { runUninstallAllGatewayPorts, } from "../../../lib/actions/uninstall/all-gateway-ports"; import { backupAllUnderPortableHostFence } from "../../../lib/actions/maintenance"; -import { runUninstallPlanProduction } from "../../../lib/actions/uninstall/run-plan"; +import { + preflightForceFreshUserLocalOpenShellOwnership, + runUninstallPlanProduction, +} from "../../../lib/actions/uninstall/run-plan"; import { CLI_DISPLAY_NAME, CLI_NAME } from "../../../lib/cli/branding"; import { NemoClawCommand } from "../../../lib/cli/nemoclaw-oclif-command"; import { GATEWAY_PORT } from "../../../lib/core/ports"; @@ -30,6 +33,7 @@ export default class InternalUninstallRunPlanCommand extends NemoClawCommand { "Uninstall every gateway port on this host, not only the port NEMOCLAW_GATEWAY_PORT selects", }), "all-gateway-ports-child": Flags.boolean({ hidden: true }), + "force-fresh-ownership-preflight": Flags.boolean({ hidden: true }), "force-fresh-reset": Flags.boolean({ hidden: true }), "keep-openshell": Flags.boolean({ description: "Leave the openshell binary installed" }), "delete-models": Flags.boolean({ @@ -48,6 +52,12 @@ export default class InternalUninstallRunPlanCommand extends NemoClawCommand { public async run(): Promise { const { flags } = await this.parse(InternalUninstallRunPlanCommand); + if (flags["force-fresh-ownership-preflight"]) { + this.applyExitResult({ + exitCode: preflightForceFreshUserLocalOpenShellOwnership() ? 0 : 1, + }); + return; + } const options = { assumeYes: flags.yes ?? false, deleteModels: flags["delete-models"] ?? false, diff --git a/src/lib/actions/uninstall/run-plan-docker-scope.test.ts b/src/lib/actions/uninstall/run-plan-docker-scope.test.ts index fb942fbbd78..5120d106b50 100644 --- a/src/lib/actions/uninstall/run-plan-docker-scope.test.ts +++ b/src/lib/actions/uninstall/run-plan-docker-scope.test.ts @@ -233,50 +233,21 @@ describe("uninstall Docker resource scope", () => { it.each([ { - containerInventory: "owned-id nemoclaw-sandbox:test openshell-owned", - failureCommand: ["rm", "-f", "owned-id"], - imageInventory: "", - plannedVolumeInspection: { - status: 1, - stdout: "", - stderr: "Error response from daemon: get openshell-cluster-nemoclaw: no such volume", - }, - scenario: "container removal fails", - }, - { - containerInventory: "", failureCommand: ["volume", "rm", "-f", "openshell-cluster-nemoclaw"], - imageInventory: "", plannedVolumeInspection: ok(), scenario: "planned-volume removal fails", }, { - containerInventory: "", failureCommand: ["volume", "inspect", "openshell-cluster-nemoclaw"], - imageInventory: "", plannedVolumeInspection: { status: 1, stdout: "", stderr: "permission denied" }, scenario: "planned-volume inspection is inconclusive", }, - { - containerInventory: "", - failureCommand: ["rmi", "-f", "owned-image"], - imageInventory: "owned-image ghcr.io/nvidia/nemoclaw/openclaw-sandbox:test", - plannedVolumeInspection: { - status: 1, - stdout: "", - stderr: "Error response from daemon: get openshell-cluster-nemoclaw: no such volume", - }, - scenario: "owned image removal fails", - }, ] as const)( "fails force-fresh cleanup when $scenario", - async ({ containerInventory, failureCommand, imageInventory, plannedVolumeInspection }) => { + async ({ failureCommand, plannedVolumeInspection }) => { const routes: Record = { info: ok(), - "ps -a --format {{.ID}} {{.Image}} {{.Names}}": ok(containerInventory), - "images --format {{.ID}} {{.Repository}}:{{.Tag}}": ok(imageInventory), - "rm -f owned-id": { status: 1, stdout: "", stderr: "busy" }, - "rmi -f owned-image": { status: 1, stdout: "", stderr: "busy" }, + "ps -a --format {{.ID}} {{.Image}} {{.Names}}": ok(), "volume inspect openshell-cluster-nemoclaw": plannedVolumeInspection, "volume rm -f openshell-cluster-nemoclaw": { status: 1, @@ -321,4 +292,52 @@ describe("uninstall Docker resource scope", () => { ); }, ); + + it("preserves a convention-matching container and images during force-fresh cleanup", async () => { + const calls: string[][] = []; + const routes: Record = { + info: ok(), + "ps -a --format {{.ID}} {{.Image}} {{.Names}}": ok( + "foreign-id registry.example.com/unrelated:latest openshell-foreign", + ), + "volume inspect openshell-cluster-nemoclaw": { + status: 1, + stdout: "", + stderr: "Error response from daemon: get openshell-cluster-nemoclaw: no such volume", + }, + }; + const runDocker = vi.fn((args: string[]): RunResult => { + calls.push(args); + return routes[args.join(" ")] ?? ok(); + }); + + const result = await runUninstallPlan( + { + assumeYes: true, + deleteModels: false, + destroyUserData: true, + forceFreshReset: true, + keepOpenShell: true, + }, + { + commandExists: () => true, + env: { HOME: "/tmp/nemoclaw-force-fresh-foreign-container" } as NodeJS.ProcessEnv, + existsSync: () => false, + hasPortableRuntimeCleanup: () => false, + isTty: false, + kill: () => true, + log: () => undefined, + rmSync: vi.fn(), + run: (command, args) => + command === "openshell" && args.join(" ") === "gateway list -o json" + ? ok(JSON.stringify([{ name: "nemoclaw" }])) + : ok(), + runDocker, + }, + ); + + expect(result.exitCode).toBe(1); + expect(calls).not.toContainEqual(["rm", "-f", "foreign-id"]); + expect(calls.some((args) => args[0] === "rmi")).toBe(false); + }); }); diff --git a/src/lib/actions/uninstall/run-plan-homebrew-openshell.test.ts b/src/lib/actions/uninstall/run-plan-homebrew-openshell.test.ts index dc2a58f26a0..6a859dd4e30 100644 --- a/src/lib/actions/uninstall/run-plan-homebrew-openshell.test.ts +++ b/src/lib/actions/uninstall/run-plan-homebrew-openshell.test.ts @@ -1,9 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { expect, it, vi } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { expect, it, onTestFinished, vi } from "vitest"; import { + preflightForceFreshUserLocalOpenShellOwnership, type RunResult, runUninstallPlan as runUninstallPlanBase, type UninstallRunDeps, @@ -21,6 +25,42 @@ function ok(stdout = ""): RunResult { return { status: 0, stdout, stderr: "" }; } +it("rejects an unverified user-local OpenShell binary before cleanup", () => { + const home = "/tmp/nemoclaw-force-fresh-ownership-preflight"; + const target = `${home}/.local/bin/openshell`; + const errors: string[] = []; + + expect( + preflightForceFreshUserLocalOpenShellOwnership({ + env: { HOME: home } as NodeJS.ProcessEnv, + error: (message) => errors.push(message), + existsSync: (candidate) => candidate === target, + isManagedOpenShellBinary: () => false, + }), + ).toBe(false); + expect(errors).toEqual([ + `Force-fresh ownership preflight rejected ${target}: its managed OpenShell install manifest is absent or does not match. No cleanup started.`, + ]); +}); + +it("rejects a malformed managed-install manifest through the canonical parser", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-ownership-preflight-")); + onTestFinished(() => fs.rmSync(home, { force: true, recursive: true })); + const userBin = path.join(home, "bin"); + fs.mkdirSync(userBin); + fs.writeFileSync(path.join(userBin, "openshell"), "#!/bin/sh\n", { mode: 0o755 }); + fs.writeFileSync(path.join(userBin, ".nemoclaw-openshell-managed-v1"), "malformed\n", { + mode: 0o600, + }); + + expect( + preflightForceFreshUserLocalOpenShellOwnership({ + env: { HOME: home, XDG_BIN_HOME: userBin } as NodeJS.ProcessEnv, + error: () => undefined, + }), + ).toBe(false); +}); + async function runUninstallPlan(deps: UninstallRunDeps, forceFreshReset = false) { return await runUninstallPlanBase( { assumeYes: true, deleteModels: false, forceFreshReset, keepOpenShell: false }, diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index bbae04f9f89..918edc7b193 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -925,6 +925,25 @@ function defaultManagedOpenShellBinaryOwnership(target: string, userBin: string) ); } +export function preflightForceFreshUserLocalOpenShellOwnership( + deps: UninstallRunDeps = {}, +): boolean { + const runtime = buildRuntime(deps); + const userBin = path.resolve( + runtime.env.XDG_BIN_HOME || path.join(runtime.env.HOME || os.homedir(), ".local", "bin"), + ); + let accepted = true; + for (const binary of MANAGED_OPENSHELL_BINARY_NAMES) { + const target = path.join(userBin, binary); + if (!runtime.existsSync(target) || runtime.isManagedOpenShellBinary(target, userBin)) continue; + runtime.error( + `Force-fresh ownership preflight rejected ${target}: its managed OpenShell install manifest is absent or does not match. No cleanup started.`, + ); + accepted = false; + } + return accepted; +} + function removeForceFreshUserLocalOpenShell( paths: UninstallPaths, runtime: UninstallRuntime, @@ -2360,7 +2379,11 @@ function stopBedrockRuntimeAdapterForUninstall( throw new IncompleteBedrockRuntimeAdapterCleanupError(); } -function removeDockerContainers(runtime: UninstallRuntime, gatewayName?: string): boolean { +function removeDockerContainers( + runtime: UninstallRuntime, + gatewayName?: string, + allowConventionRemoval = true, +): boolean { const result = runtime.runDocker(["ps", "-a", "--format", "{{.ID}} {{.Image}} {{.Names}}"], { env: runtime.env, }); @@ -2400,6 +2423,12 @@ function removeDockerContainers(runtime: UninstallRuntime, gatewayName?: string) runtime.log(`No ${runtimeBranding(runtime).display}/OpenShell Docker containers found`); return true; } + if (!allowConventionRemoval) { + runtime.warn( + `Force-fresh cleanup preserved convention-matching Docker container ${ids[0]} because no trusted host state binds that container ID to this installation.`, + ); + return false; + } let removedAll = true; for (const id of [...new Set(ids)]) { if (runtime.runDocker(["rm", "-f", id], { env: runtime.env, stdio: "ignore" }).status === 0) @@ -2529,10 +2558,17 @@ function executeDockerResourceStep( const removedContainers = removeDockerContainers( runtime, scopedToSelectedGateway ? options.gatewayName || resolveGatewayName(GATEWAY_PORT) : undefined, + !options.forceFreshReset, ); + if (options.forceFreshReset && !removedContainers) { + runtime.error("Force-fresh cleanup found a Docker container without verified ownership."); + return false; + } let removedImages = true; if (scopedToSelectedGateway) { runtime.log("Sibling gateways remain; kept shared Docker images."); + } else if (options.forceFreshReset) { + runtime.log("Kept Docker images because repository naming is not force-fresh ownership proof."); } else { removedImages = removeDockerImages(runtime); } diff --git a/test/installer-integration/install-force-fresh.test.ts b/test/installer-integration/install-force-fresh.test.ts index a8bfced9978..bae2ac53717 100644 --- a/test/installer-integration/install-force-fresh.test.ts +++ b/test/installer-integration/install-force-fresh.test.ts @@ -38,17 +38,20 @@ function callPayloadFunction(command: string, env: Record { - const result = spawnSync("bash", [installer, "--help"], { - cwd: path.join(import.meta.dirname, "../.."), - encoding: "utf-8", - }); +])( + "documents the Apple silicon macOS-only destructive option in the %s help", + (_surface, installer) => { + const result = spawnSync("bash", [installer, "--help"], { + cwd: path.join(import.meta.dirname, "../.."), + encoding: "utf-8", + }); - expect(result.status).toBe(0); - const output = `${result.stdout}${result.stderr}`; - expect(output).toMatch(/--force-fresh-install .*macOS only/u); - expect(output).toMatch(/NEMOCLAW_FORCE_FRESH_INSTALL=1 .*macOS only/u); -}); + expect(result.status).toBe(0); + const output = `${result.stdout}${result.stderr}`; + expect(output).toMatch(/--force-fresh-install .*Apple silicon macOS only/u); + expect(output).toMatch(/NEMOCLAW_FORCE_FRESH_INSTALL=1 .*Apple silicon macOS only/u); + }, +); it("detects a Homebrew-only OpenShell installation", () => { const { root: tmp, binDir: fakeBin } = installerCheckout("nemoclaw-force-fresh-detect-"); @@ -255,7 +258,7 @@ it("runs destructive cleanup through the staged candidate CLI", () => { writeExecutable( path.join(fakeBin, "node"), `#!/usr/bin/env bash -printf 'destroy=%s args=%s\n' "\${NEMOCLAW_UNINSTALL_DESTROY_USER_DATA:-}" "$*" > "$FORCE_FRESH_LOG" +printf 'destroy=%s args=%s\n' "\${NEMOCLAW_UNINSTALL_DESTROY_USER_DATA:-}" "$*" >> "$FORCE_FRESH_LOG" `, ); @@ -267,7 +270,8 @@ printf 'destroy=%s args=%s\n' "\${NEMOCLAW_UNINSTALL_DESTROY_USER_DATA:-}" "$*" expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); expect(fs.readFileSync(logPath, "utf-8")).toBe( - `destroy=1 args=${path.join(sourceRoot, "bin", "nemoclaw.js")} internal uninstall run-plan --yes --destroy-user-data --force-fresh-reset --all-gateway-ports\n`, + `destroy= args=${path.join(sourceRoot, "bin", "nemoclaw.js")} internal uninstall run-plan --force-fresh-ownership-preflight\n` + + `destroy=1 args=${path.join(sourceRoot, "bin", "nemoclaw.js")} internal uninstall run-plan --yes --destroy-user-data --force-fresh-reset --all-gateway-ports\n`, ); }); @@ -492,29 +496,53 @@ it("routes a managed helper from XDG_BIN_HOME through the staged uninstaller", ( }); it("lets the staged uninstaller reject a malformed ownership manifest before package removal", () => { - const { root: tmp } = installerCheckout("nemoclaw-force-fresh-malformed-manifest-"); + const { root: tmp, binDir: fakeBin } = installerCheckout( + "nemoclaw-force-fresh-malformed-manifest-", + ); const userBin = path.join(tmp, "xdg-bin"); const packageMarker = path.join(tmp, "package-removal-started"); + const preflightLog = path.join(tmp, "preflight.log"); + const sourceRoot = path.join(tmp, "candidate"); + fs.mkdirSync(path.join(sourceRoot, "bin"), { recursive: true }); + fs.writeFileSync(path.join(sourceRoot, "bin", "nemoclaw.js"), "// staged candidate\n"); fs.mkdirSync(userBin, { recursive: true }); writeExecutable(path.join(userBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n"); fs.writeFileSync(path.join(userBin, ".nemoclaw-openshell-managed-v1"), "malformed\n", { mode: 0o600, }); + writeExecutable( + path.join(fakeBin, "node"), + `#!/usr/bin/env bash +printf '%s\n' "$*" > "$PREFLIGHT_LOG" +case "$*" in + *--force-fresh-ownership-preflight) exit 9 ;; + *) touch "$PACKAGE_MARKER"; exit 0 ;; +esac +`, + ); const result = callPayloadFunction( ` warn() { :; } - force_fresh_install_source_root() { printf '/tmp/staged-candidate'; } + force_fresh_install_source_root() { printf '%s' "$SOURCE_ROOT"; } prepare_force_fresh_uninstaller() { :; } - run_force_fresh_uninstaller() { printf 'canonical-ownership-rejected\\n'; return 9; } remove_macos_openshell_for_force_fresh_install() { touch "$PACKAGE_MARKER"; } run_force_fresh_install_reset `, - { HOME: tmp, PACKAGE_MARKER: packageMarker, XDG_BIN_HOME: userBin }, + { + HOME: tmp, + PACKAGE_MARKER: packageMarker, + PATH: `${fakeBin}:${TEST_SYSTEM_PATH}`, + PREFLIGHT_LOG: preflightLog, + SOURCE_ROOT: sourceRoot, + XDG_BIN_HOME: userBin, + }, ); expect(result.status).not.toBe(0); - expect(result.stdout).toContain("canonical-ownership-rejected"); + expect(fs.readFileSync(preflightLog, "utf8")).toContain( + "internal uninstall run-plan --force-fresh-ownership-preflight", + ); expect(fs.existsSync(packageMarker)).toBe(false); }); @@ -601,7 +629,7 @@ it.each([ warn() { :; } finalize_install() { :; } clear_station_resume_after_completed_onboarding() { :; } - uname() { printf 'Darwin'; } + uname() { [[ "$1" == "-m" ]] && printf 'arm64' || printf 'Darwin'; } main ${forceFreshArg} --non-interactive --yes-i-accept-third-party-software record "fresh:$FRESH:$NEMOCLAW_FORCE_FRESH_INSTALL" printf '%s' "$order" @@ -653,7 +681,21 @@ it("rejects force-fresh installation outside macOS", () => { expect(result.status).not.toBe(0); expect(`${result.stdout}${result.stderr}`).toContain( - "--force-fresh-install currently supports macOS only", + "--force-fresh-install currently supports Apple silicon macOS only", + ); +}); + +it("rejects force-fresh installation on Intel macOS", () => { + const result = callPayloadFunction(` + error() { printf '%s' "$*" >&2; return 1; } + uname() { [[ "$1" == "-s" ]] && printf 'Darwin' || printf 'x86_64'; } + FORCE_FRESH_INSTALL=1 + validate_force_fresh_install_platform + `); + + expect(result.status).not.toBe(0); + expect(`${result.stdout}${result.stderr}`).toContain( + "--force-fresh-install currently supports Apple silicon macOS only", ); }); From ca641521fd9f1db6b352658356e4bd62ed2681f6 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 18 Sep 2026 15:04:35 -0700 Subject: [PATCH 37/39] test(uninstall): assert Docker inventory --- src/lib/actions/uninstall/run-plan-docker-scope.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/actions/uninstall/run-plan-docker-scope.test.ts b/src/lib/actions/uninstall/run-plan-docker-scope.test.ts index 5120d106b50..c8b84ab81ec 100644 --- a/src/lib/actions/uninstall/run-plan-docker-scope.test.ts +++ b/src/lib/actions/uninstall/run-plan-docker-scope.test.ts @@ -337,6 +337,7 @@ describe("uninstall Docker resource scope", () => { ); expect(result.exitCode).toBe(1); + expect(calls).toContainEqual(["ps", "-a", "--format", "{{.ID}} {{.Image}} {{.Names}}"]); expect(calls).not.toContainEqual(["rm", "-f", "foreign-id"]); expect(calls.some((args) => args[0] === "rmi")).toBe(false); }); From d31a653952f124395723cb6ca2c4a14b4418f0ed Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 18 Sep 2026 15:45:25 -0700 Subject: [PATCH 38/39] fix(installer): bound destructive cleanup --- scripts/install.sh | 55 +++++--- .../uninstall/run-plan-docker-scope.test.ts | 125 +++++++----------- src/lib/actions/uninstall/runtime-commands.ts | 19 +-- .../install-force-fresh.test.ts | 115 +++++++++++++--- 4 files changed, 182 insertions(+), 132 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index f1fd6df298c..4d42204c6ac 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -1386,6 +1386,7 @@ spin() { } FORCE_FRESH_PREPARE_TIMEOUT_SECONDS=900 +FORCE_FRESH_UNINSTALL_TIMEOUT_SECONDS=900 installer_command_group_is_alive() { local command_pid="$1" @@ -1439,7 +1440,11 @@ run_bounded_installer_command() ( ((pending_signal == 0)) || cleanup_bounded_installer_command "$pending_signal" while installer_command_group_is_alive "$command_pid"; do if ((ticks >= timeout_seconds * 10)); then - printf '[ERROR] Timed out during %s after %s seconds. Existing state was preserved; force-fresh cleanup did not start. Retry when package installation is healthy.\n' "$label" "$timeout_seconds" >&2 + if [[ "${NEMOCLAW_FORCE_FRESH_CLEANUP_ACTIVE:-}" == "1" ]]; then + printf '[ERROR] Timed out during %s after %s seconds. Force-fresh cleanup is incomplete; remaining state was preserved for retry and OpenShell package removal did not start.\n' "$label" "$timeout_seconds" >&2 + else + printf '[ERROR] Timed out during %s after %s seconds. Existing state was preserved; force-fresh cleanup did not start. Retry when package installation is healthy.\n' "$label" "$timeout_seconds" >&2 + fi return 124 fi sleep 0.1 @@ -3641,8 +3646,8 @@ run_preupgrade_backup() { force_fresh_install_has_existing_state() { local container_id container_image container_inventory container_label_inventory container_name - local receipt_volume_inventory user_bin volume_name volume_label volume_suffix - local existing_state=0 managed_docker_state=0 + local receipt_volume_inventory user_bin volume_name volume_suffix + local existing_state=0 _FORCE_FRESH_UNVERIFIED_DOCKER_CONTAINER="" _FORCE_FRESH_UNVERIFIED_RECEIPT_VOLUME="" user_bin="${XDG_BIN_HOME:-${HOME}/.local/bin}" @@ -3694,23 +3699,14 @@ force_fresh_install_has_existing_state() { esac volume_suffix="${volume_name#nemoclaw-managed-startup-receipt-volume-}" [[ "${#volume_suffix}" -eq 32 && "$volume_suffix" =~ ^[0-9a-f]+$ ]] || continue - volume_label="$( - docker volume inspect \ - --format '{{ index .Labels "io.nvidia.nemoclaw.managed-startup.receipt" }}' \ - "$volume_name" 2>/dev/null - )" || return 2 - if [[ "$volume_label" == "1" ]]; then - managed_docker_state=1 - else - _FORCE_FRESH_UNVERIFIED_RECEIPT_VOLUME="$volume_name" - return 3 - fi + _FORCE_FRESH_UNVERIFIED_RECEIPT_VOLUME="$volume_name" + return 3 done <<<"$receipt_volume_inventory" fi if ((existing_state == 0)) && [[ -n "$_FORCE_FRESH_UNVERIFIED_DOCKER_CONTAINER" ]]; then return 4 fi - ((existing_state == 0 && managed_docker_state == 0)) || return 0 + ((existing_state == 0)) || return 0 return 1 } @@ -3757,16 +3753,33 @@ prepare_force_fresh_uninstaller() { } run_force_fresh_uninstaller() { - local source_root="$1" node_bin + local source_root="$1" node_bin preflight_status=0 cleanup_status=0 node_bin="$(command -v node 2>/dev/null || true)" [[ -n "$node_bin" && -x "$node_bin" ]] \ || error "Node.js is required for the force-fresh uninstaller." - "$node_bin" "${source_root}/bin/nemoclaw.js" internal uninstall run-plan \ + run_bounded_installer_command \ + "Validating force-fresh OpenShell ownership" \ + "$FORCE_FRESH_UNINSTALL_TIMEOUT_SECONDS" \ + "$node_bin" "${source_root}/bin/nemoclaw.js" internal uninstall run-plan \ --force-fresh-ownership-preflight \ - || error "Force-fresh cleanup stopped because the staged canonical ownership preflight rejected a user-local OpenShell executable. Reconcile the reported binary, then rerun. No cleanup started." - NEMOCLAW_UNINSTALL_DESTROY_USER_DATA=1 "$node_bin" \ + || preflight_status=$? + if ((preflight_status == 124)); then + error "The staged canonical ownership preflight timed out. No cleanup started; retry when local process execution is healthy." + elif ((preflight_status != 0)); then + error "Force-fresh cleanup stopped because the staged canonical ownership preflight rejected a user-local OpenShell executable. Reconcile the reported binary, then rerun. No cleanup started." + fi + NEMOCLAW_FORCE_FRESH_CLEANUP_ACTIVE=1 run_bounded_installer_command \ + "Running authoritative force-fresh cleanup" \ + "$FORCE_FRESH_UNINSTALL_TIMEOUT_SECONDS" \ + env NEMOCLAW_UNINSTALL_DESTROY_USER_DATA=1 "$node_bin" \ "${source_root}/bin/nemoclaw.js" internal uninstall run-plan \ - --yes --destroy-user-data --force-fresh-reset --all-gateway-ports + --yes --destroy-user-data --force-fresh-reset --all-gateway-ports \ + || cleanup_status=$? + if ((cleanup_status == 124)); then + error "Authoritative force-fresh cleanup timed out and is incomplete. Inspect the remaining state, then rerun; OpenShell package removal did not start." + elif ((cleanup_status != 0)); then + return "$cleanup_status" + fi } preflight_macos_openshell_for_force_fresh_install() { @@ -3821,7 +3834,7 @@ run_force_fresh_install_reset() { ;; 1) info "No existing NemoClaw or OpenShell installation was found; continuing with a clean install." ;; 2) error "Docker is installed but unavailable. Start the selected local Docker or Colima daemon, then rerun --force-fresh-install. No cleanup started." ;; - 3) error "Force-fresh cleanup found pre-label receipt volume ${_FORCE_FRESH_UNVERIFIED_RECEIPT_VOLUME}. NemoClaw cannot prove ownership of this legacy volume. Inspect it, remove it only after confirming it belongs to the interrupted NemoClaw install, then rerun. No cleanup started." ;; + 3) error "Force-fresh cleanup found unverified receipt volume ${_FORCE_FRESH_UNVERIFIED_RECEIPT_VOLUME}. Docker names and mutable labels are not ownership proof. Inspect it, remove it only after confirming it belongs to the interrupted NemoClaw install, then rerun. No cleanup started." ;; 4) error "Force-fresh cleanup found unverified Docker container ${_FORCE_FRESH_UNVERIFIED_DOCKER_CONTAINER}. NemoClaw cannot prove this Docker-only resource belongs to the current installation. Inspect and reconcile it explicitly, then rerun. No cleanup started." ;; *) error "Could not inspect existing NemoClaw or OpenShell state. No cleanup started." ;; esac diff --git a/src/lib/actions/uninstall/run-plan-docker-scope.test.ts b/src/lib/actions/uninstall/run-plan-docker-scope.test.ts index c8b84ab81ec..1aace960460 100644 --- a/src/lib/actions/uninstall/run-plan-docker-scope.test.ts +++ b/src/lib/actions/uninstall/run-plan-docker-scope.test.ts @@ -153,83 +153,58 @@ describe("uninstall Docker resource scope", () => { expect(calls).toContainEqual(["rmi", "-f", "i-openshell"]); }); - it.each([ - ["removes", 0, 0], - ["fails closed on", 1, 1], - ] as const)( - "%s labelled receipt volumes and preserves unlabelled matches during force-fresh cleanup", - async (_scenario, removalStatus, expectedExitCode) => { - const owned = `nemoclaw-managed-startup-receipt-volume-${"a".repeat(32)}`; - const unlabelled = `nemoclaw-managed-startup-receipt-volume-${"b".repeat(32)}`; - const calls: string[][] = []; - const volumeLabels = new Map([ - [owned, "1"], - [unlabelled, null], - ]); - const routes: Record RunResult> = { - info: () => ok(), - "ps -a --format {{.ID}} {{.Image}} {{.Names}}": () => ok(), - "images --format {{.ID}} {{.Repository}}:{{.Tag}}": () => ok(), - [`volume ls --filter label=${MANAGED_STARTUP_RECEIPT_VOLUME_LABEL}=1 --format {{.Name}}`]: - () => - ok( - [...volumeLabels] - .filter(([, label]) => label === "1") - .map(([name]) => name) - .join("\n"), - ), - [`volume inspect --format {{json .Labels}} ${owned}`]: () => - ok(JSON.stringify({ [MANAGED_STARTUP_RECEIPT_VOLUME_LABEL]: "1" })), - [`volume rm -f ${owned}`]: () => { - const successfulRemoval: Partial boolean>> = { - 0: () => volumeLabels.delete(owned), - }; - successfulRemoval[removalStatus]?.(); - return { status: removalStatus, stdout: removalStatus === 0 ? owned : "", stderr: "" }; - }, - "volume inspect openshell-cluster-nemoclaw": () => ({ - status: 1, - stdout: "", - stderr: "Error response from daemon: get openshell-cluster-nemoclaw: no such volume", - }), - }; - const runDocker = vi.fn((args: string[]) => { - calls.push(args); - const command = args.join(" "); - return (routes[command] ?? (() => ok()))(); - }); + it("preserves labelled and unlabelled receipt-volume matches during force-fresh cleanup", async () => { + const owned = `nemoclaw-managed-startup-receipt-volume-${"a".repeat(32)}`; + const unlabelled = `nemoclaw-managed-startup-receipt-volume-${"b".repeat(32)}`; + const calls: string[][] = []; + const routes: Record RunResult> = { + info: () => ok(), + "ps -a --format {{.ID}} {{.Image}} {{.Names}}": () => ok(), + [`volume ls --filter label=${MANAGED_STARTUP_RECEIPT_VOLUME_LABEL}=1 --format {{.Name}}`]: + () => ok(owned), + [`volume inspect --format {{json .Labels}} ${owned}`]: () => + ok(JSON.stringify({ [MANAGED_STARTUP_RECEIPT_VOLUME_LABEL]: "1" })), + "volume inspect openshell-cluster-nemoclaw": () => ({ + status: 1, + stdout: "", + stderr: "Error response from daemon: get openshell-cluster-nemoclaw: no such volume", + }), + }; + const runDocker = vi.fn((args: string[]) => { + calls.push(args); + const command = args.join(" "); + return (routes[command] ?? (() => ok()))(); + }); - const result = await runUninstallPlan( - { - assumeYes: true, - deleteModels: false, - destroyUserData: true, - forceFreshReset: true, - keepOpenShell: true, - }, - { - commandExists: () => true, - env: { HOME: "/tmp/nemoclaw-force-fresh-receipts" } as NodeJS.ProcessEnv, - existsSync: () => false, - hasPortableRuntimeCleanup: () => false, - isTty: false, - kill: () => true, - log: () => undefined, - rmSync: vi.fn(), - run: (command, args) => - command === "openshell" && args.join(" ") === "gateway list -o json" - ? ok(JSON.stringify([{ name: "nemoclaw" }])) - : ok(), - runDocker, - }, - ); + const result = await runUninstallPlan( + { + assumeYes: true, + deleteModels: false, + destroyUserData: true, + forceFreshReset: true, + keepOpenShell: true, + }, + { + commandExists: () => true, + env: { HOME: "/tmp/nemoclaw-force-fresh-receipts" } as NodeJS.ProcessEnv, + existsSync: () => false, + hasPortableRuntimeCleanup: () => false, + isTty: false, + kill: () => true, + log: () => undefined, + rmSync: vi.fn(), + run: (command, args) => + command === "openshell" && args.join(" ") === "gateway list -o json" + ? ok(JSON.stringify([{ name: "nemoclaw" }])) + : ok(), + runDocker, + }, + ); - expect(result.exitCode).toBe(expectedExitCode); - expect(calls).toContainEqual(["volume", "rm", "-f", owned]); - expect(calls).not.toContainEqual(["volume", "rm", "-f", unlabelled]); - expect(volumeLabels.has(unlabelled)).toBe(true); - }, - ); + expect(result.exitCode).toBe(1); + expect(calls).not.toContainEqual(["volume", "rm", "-f", owned]); + expect(calls).not.toContainEqual(["volume", "rm", "-f", unlabelled]); + }); it.each([ { diff --git a/src/lib/actions/uninstall/runtime-commands.ts b/src/lib/actions/uninstall/runtime-commands.ts index 7155eb6e048..28c09213863 100644 --- a/src/lib/actions/uninstall/runtime-commands.ts +++ b/src/lib/actions/uninstall/runtime-commands.ts @@ -97,23 +97,10 @@ function ownedForceFreshReceiptVolumes(runtime: ForceFreshDockerCleanupRuntime): export function removeForceFreshReceiptVolumes(runtime: ForceFreshDockerCleanupRuntime): boolean { const volumes = ownedForceFreshReceiptVolumes(runtime); if (volumes === null) return false; - for (const volume of volumes) { - const removed = runtime.runDocker(["volume", "rm", "-f", volume], { - env: runtime.env, - stdio: "ignore", - }); - if (removed.status !== 0) { - runtime.error(`Managed-startup receipt volume '${volume}' could not be removed.`); - return false; - } - runtime.log(`Removed managed-startup receipt volume ${volume}`); - } - const remaining = ownedForceFreshReceiptVolumes(runtime); - if (remaining === null) return false; - const [retained] = remaining; - if (retained) { + const [unverified] = volumes; + if (unverified) { runtime.error( - `Managed-startup receipt volume '${retained}' remains after force-fresh cleanup.`, + `Preserved managed-startup receipt volume '${unverified}' because its Docker name and mutable label are not trusted ownership proof.`, ); return false; } diff --git a/test/installer-integration/install-force-fresh.test.ts b/test/installer-integration/install-force-fresh.test.ts index bae2ac53717..dc4bfac85c5 100644 --- a/test/installer-integration/install-force-fresh.test.ts +++ b/test/installer-integration/install-force-fresh.test.ts @@ -35,6 +35,15 @@ function callPayloadFunction(command: string, env: Record { +it("stops before cleanup for a labelled receipt-volume-only state", () => { const { root: tmp, binDir: fakeBin } = installerCheckout("nemoclaw-force-fresh-receipt-detect-"); + const cleanupMarker = path.join(tmp, "cleanup-started"); + const receiptVolume = "nemoclaw-managed-startup-receipt-volume-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; writeExecutable( path.join(fakeBin, "docker"), `#!/usr/bin/env bash @@ -157,29 +168,35 @@ case "$*" in info) exit 0 ;; "ps -aq --filter label=io.nvidia.nemoclaw.managed-image.contract=1") exit 0 ;; "ps -a --format {{.ID}} {{.Image}} {{.Names}}") exit 0 ;; - "volume ls --format {{.Name}}") - printf 'nemoclaw-managed-startup-receipt-volume-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n' - ;; - *) - if [[ "$1" == "volume" && "$2" == "inspect" ]]; then - printf '1\n' - exit 0 - fi - exit 1 - ;; + "volume ls --format {{.Name}}") printf '%s\n' "$RECEIPT_VOLUME" ;; + *) exit 1 ;; esac `, ); - const result = callPayloadFunction("force_fresh_install_has_existing_state", { - HOME: tmp, - PATH: `${fakeBin}:${TEST_SYSTEM_PATH}`, - }); + const result = callPayloadFunction( + ` + warn() { :; } + remove_macos_openshell_for_force_fresh_install() { touch "$CLEANUP_MARKER"; } + prepare_force_fresh_uninstaller() { touch "$CLEANUP_MARKER"; } + run_force_fresh_install_reset + `, + { + CLEANUP_MARKER: cleanupMarker, + HOME: tmp, + PATH: `${fakeBin}:${TEST_SYSTEM_PATH}`, + RECEIPT_VOLUME: receiptVolume, + }, + ); - expect(result.status).toBe(0); + expect(result.status).not.toBe(0); + expect(`${result.stdout}${result.stderr}`).toContain( + `unverified receipt volume ${receiptVolume}`, + ); + expect(fs.existsSync(cleanupMarker)).toBe(false); }); -it("stops before cleanup for a pre-label receipt-volume-only state", () => { +it("stops before cleanup for an unlabelled receipt-volume-only state", () => { const { root: tmp, binDir: fakeBin } = installerCheckout("nemoclaw-force-fresh-legacy-receipt-"); const cleanupMarker = path.join(tmp, "cleanup-started"); const legacyVolume = "nemoclaw-managed-startup-receipt-volume-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; @@ -218,7 +235,7 @@ esac ); expect(result.status).not.toBe(0); - expect(`${result.stdout}${result.stderr}`).toContain(`pre-label receipt volume ${legacyVolume}`); + expect(`${result.stdout}${result.stderr}`).toContain(`unverified receipt volume ${legacyVolume}`); expect(fs.existsSync(cleanupMarker)).toBe(false); }); @@ -568,6 +585,7 @@ it("stops before package removal when managed uninstall rejects partial state", it("bounds staged uninstaller preparation before destructive cleanup", () => { const { root: tmp } = installerCheckout("nemoclaw-force-fresh-timeout-"); const cleanupMarker = path.join(tmp, "cleanup-started"); + const descendantPidFile = path.join(tmp, "descendant.pid"); const startedAt = performance.now(); const result = callPayloadFunction( ` @@ -575,12 +593,18 @@ it("bounds staged uninstaller preparation before destructive cleanup", () => { FORCE_FRESH_PREPARE_TIMEOUT_SECONDS=1 force_fresh_install_has_existing_state() { return 0; } force_fresh_install_source_root() { printf '/tmp/staged-candidate'; } - bash() { trap '' TERM; while :; do sleep 1; done; } + bash() { + (trap '' TERM; while :; do sleep 1; done) & + child_pid=$! + printf '%s' "$child_pid" > "$DESCENDANT_PID_FILE" + trap '' TERM + wait "$child_pid" + } run_force_fresh_uninstaller() { touch "$CLEANUP_MARKER"; } remove_macos_openshell_for_force_fresh_install() { touch "$CLEANUP_MARKER"; } run_force_fresh_install_reset `, - { CLEANUP_MARKER: cleanupMarker, HOME: tmp }, + { CLEANUP_MARKER: cleanupMarker, DESCENDANT_PID_FILE: descendantPidFile, HOME: tmp }, ); const elapsedMs = performance.now() - startedAt; @@ -590,9 +614,60 @@ it("bounds staged uninstaller preparation before destructive cleanup", () => { expect(`${result.stdout}${result.stderr}`).toContain( "Force-fresh uninstaller preparation timed out before cleanup", ); + expect(processExists(Number(fs.readFileSync(descendantPidFile, "utf8")))).toBe(false); expect(fs.existsSync(cleanupMarker)).toBe(false); }, 15_000); +it("bounds and reaps timed-out authoritative cleanup before package removal", () => { + const { root: tmp, binDir: fakeBin } = installerCheckout( + "nemoclaw-force-fresh-uninstall-timeout-", + ); + const sourceRoot = path.join(tmp, "candidate"); + const descendantPidFile = path.join(tmp, "cleanup-descendant.pid"); + const packageMarker = path.join(tmp, "package-removal-started"); + fs.mkdirSync(path.join(sourceRoot, "bin"), { recursive: true }); + fs.writeFileSync(path.join(sourceRoot, "bin", "nemoclaw.js"), "// staged candidate\n"); + writeExecutable( + path.join(fakeBin, "node"), + `#!/usr/bin/env bash +case "$*" in + *--force-fresh-ownership-preflight) exit 0 ;; +esac +(trap '' TERM; while :; do sleep 1; done) & +child_pid=$! +printf '%s' "$child_pid" > "$DESCENDANT_PID_FILE" +trap '' TERM +wait "$child_pid" +`, + ); + + const result = callPayloadFunction( + ` + warn() { :; } + FORCE_FRESH_UNINSTALL_TIMEOUT_SECONDS=1 + force_fresh_install_has_existing_state() { return 0; } + force_fresh_install_source_root() { printf '%s' "$SOURCE_ROOT"; } + prepare_force_fresh_uninstaller() { :; } + remove_macos_openshell_for_force_fresh_install() { touch "$PACKAGE_MARKER"; } + run_force_fresh_install_reset + `, + { + DESCENDANT_PID_FILE: descendantPidFile, + HOME: tmp, + PACKAGE_MARKER: packageMarker, + PATH: `${fakeBin}:${TEST_SYSTEM_PATH}`, + SOURCE_ROOT: sourceRoot, + }, + ); + + expect(result.status).not.toBe(0); + expect(`${result.stdout}${result.stderr}`).toContain( + "Authoritative force-fresh cleanup timed out and is incomplete", + ); + expect(processExists(Number(fs.readFileSync(descendantPidFile, "utf8")))).toBe(false); + expect(fs.existsSync(packageMarker)).toBe(false); +}, 15_000); + it.each([ ["flag", "--force-fresh-install", {}], ["environment variable", "", { NEMOCLAW_FORCE_FRESH_INSTALL: "1" }], From a9bd2f8bd5749c00764430a24f8943d1267c2666 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 18 Sep 2026 16:04:26 -0700 Subject: [PATCH 39/39] fix(uninstall): inventory all receipt volumes --- .../uninstall/run-plan-docker-scope.test.ts | 23 +++++---- src/lib/actions/uninstall/runtime-commands.ts | 50 +++---------------- 2 files changed, 20 insertions(+), 53 deletions(-) diff --git a/src/lib/actions/uninstall/run-plan-docker-scope.test.ts b/src/lib/actions/uninstall/run-plan-docker-scope.test.ts index 1aace960460..68fd04ec773 100644 --- a/src/lib/actions/uninstall/run-plan-docker-scope.test.ts +++ b/src/lib/actions/uninstall/run-plan-docker-scope.test.ts @@ -9,7 +9,6 @@ import { type UninstallRunDeps, type UninstallRunOptions, } from "./run-plan"; -import { MANAGED_STARTUP_RECEIPT_VOLUME_LABEL } from "../../onboard/managed-startup/docker-receipt-transfer"; function ok(stdout = ""): RunResult { return { status: 0, stdout, stderr: "" }; @@ -153,17 +152,21 @@ describe("uninstall Docker resource scope", () => { expect(calls).toContainEqual(["rmi", "-f", "i-openshell"]); }); - it("preserves labelled and unlabelled receipt-volume matches during force-fresh cleanup", async () => { - const owned = `nemoclaw-managed-startup-receipt-volume-${"a".repeat(32)}`; - const unlabelled = `nemoclaw-managed-startup-receipt-volume-${"b".repeat(32)}`; + it.each([ + { + kind: "labelled", + volume: `nemoclaw-managed-startup-receipt-volume-${"a".repeat(32)}`, + }, + { + kind: "unlabelled", + volume: `nemoclaw-managed-startup-receipt-volume-${"b".repeat(32)}`, + }, + ])("preserves an $kind receipt-volume match during force-fresh cleanup", async ({ volume }) => { const calls: string[][] = []; const routes: Record RunResult> = { info: () => ok(), "ps -a --format {{.ID}} {{.Image}} {{.Names}}": () => ok(), - [`volume ls --filter label=${MANAGED_STARTUP_RECEIPT_VOLUME_LABEL}=1 --format {{.Name}}`]: - () => ok(owned), - [`volume inspect --format {{json .Labels}} ${owned}`]: () => - ok(JSON.stringify({ [MANAGED_STARTUP_RECEIPT_VOLUME_LABEL]: "1" })), + "volume ls --format {{.Name}}": () => ok(volume), "volume inspect openshell-cluster-nemoclaw": () => ({ status: 1, stdout: "", @@ -202,8 +205,8 @@ describe("uninstall Docker resource scope", () => { ); expect(result.exitCode).toBe(1); - expect(calls).not.toContainEqual(["volume", "rm", "-f", owned]); - expect(calls).not.toContainEqual(["volume", "rm", "-f", unlabelled]); + expect(calls).toContainEqual(["volume", "ls", "--format", "{{.Name}}"]); + expect(calls).not.toContainEqual(["volume", "rm", "-f", volume]); }); it.each([ diff --git a/src/lib/actions/uninstall/runtime-commands.ts b/src/lib/actions/uninstall/runtime-commands.ts index 28c09213863..81846e679a6 100644 --- a/src/lib/actions/uninstall/runtime-commands.ts +++ b/src/lib/actions/uninstall/runtime-commands.ts @@ -14,10 +14,7 @@ import { } from "../../domain/uninstall/messaging"; import { isOllamaAuthProxyCommandLine } from "../../inference/ollama/process"; import { isModelRouterCommandLineForPort } from "../../onboard/model-router-process"; -import { - MANAGED_STARTUP_RECEIPT_VOLUME_LABEL, - MANAGED_STARTUP_RECEIPT_VOLUME_PREFIX, -} from "../../onboard/managed-startup/docker-receipt-transfer"; +import { MANAGED_STARTUP_RECEIPT_VOLUME_PREFIX } from "../../onboard/managed-startup/docker-receipt-transfer"; interface UninstallRuntimeCommands { env: NodeJS.ProcessEnv; @@ -46,56 +43,23 @@ function nonEmptyLines(output: string): string[] { .filter(Boolean); } -function ownedForceFreshReceiptVolumes(runtime: ForceFreshDockerCleanupRuntime): string[] | null { - const inventory = runtime.runDocker( - [ - "volume", - "ls", - "--filter", - `label=${MANAGED_STARTUP_RECEIPT_VOLUME_LABEL}=1`, - "--format", - "{{.Name}}", - ], - { env: runtime.env }, - ); +function forceFreshReceiptVolumeMatches(runtime: ForceFreshDockerCleanupRuntime): string[] | null { + const inventory = runtime.runDocker(["volume", "ls", "--format", "{{.Name}}"], { + env: runtime.env, + }); if (inventory.status !== 0) { runtime.error( "Could not inventory managed-startup receipt volumes during force-fresh cleanup.", ); return null; } - const volumes = nonEmptyLines(inventory.stdout).filter((name) => + return nonEmptyLines(inventory.stdout).filter((name) => MANAGED_STARTUP_RECEIPT_VOLUME_PATTERN.test(name), ); - for (const volume of volumes) { - const inspected = runtime.runDocker( - ["volume", "inspect", "--format", "{{json .Labels}}", volume], - { env: runtime.env }, - ); - if (inspected.status !== 0) { - runtime.error(`Could not inspect managed-startup receipt volume '${volume}'.`); - return null; - } - try { - const labels: unknown = JSON.parse(inspected.stdout); - if ( - typeof labels !== "object" || - labels === null || - Reflect.get(labels, MANAGED_STARTUP_RECEIPT_VOLUME_LABEL) !== "1" - ) { - runtime.error(`Managed-startup receipt volume '${volume}' has invalid ownership metadata.`); - return null; - } - } catch { - runtime.error(`Managed-startup receipt volume '${volume}' has invalid ownership metadata.`); - return null; - } - } - return volumes; } export function removeForceFreshReceiptVolumes(runtime: ForceFreshDockerCleanupRuntime): boolean { - const volumes = ownedForceFreshReceiptVolumes(runtime); + const volumes = forceFreshReceiptVolumeMatches(runtime); if (volumes === null) return false; const [unverified] = volumes; if (unverified) {