From 14b38bda3a5aa2303cd858ccd3f783cfbac7a15a Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 11 Jun 2026 08:55:18 -0400 Subject: [PATCH 1/4] test(e2e): add launchable smoke Vitest coverage --- .github/workflows/e2e-vitest-scenarios.yaml | 69 +++ .../live/launchable-smoke.test.ts | 460 ++++++++++++++++++ 2 files changed, 529 insertions(+) create mode 100644 test/e2e-scenario/live/launchable-smoke.test.ts diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 07bb8c54f54..6b37377a95c 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -207,6 +207,75 @@ jobs: if-no-files-found: ignore retention-days: 14 + launchable-smoke-vitest: + if: ${{ inputs.scenarios == '' }} + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/launchable-smoke + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-launchable" + NEMOCLAW_RECREATE_SANDBOX: "1" + SKIP_DOCKER_PULL: "1" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Authenticate to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + shell: bash + run: | + set -euo pipefail + if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then + echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls." + exit 0 + fi + login_succeeded=0 + for attempt in 1 2 3; do + if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then + login_succeeded=1 + break + fi + if [[ "$attempt" -lt 3 ]]; then + echo "::warning::Docker Hub login attempt ${attempt} failed; retrying." + sleep 5 + fi + done + if [[ "$login_succeeded" -ne 1 ]]; then + echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls." + fi + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Run launchable smoke live test + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + run: | + set -euo pipefail + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/launchable-smoke.test.ts \ + --silent=false --reporter=default + + - name: Upload launchable smoke artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-launchable-smoke + path: e2e-artifacts/vitest/launchable-smoke/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + onboard-negative-paths-vitest: runs-on: ubuntu-latest timeout-minutes: 15 diff --git a/test/e2e-scenario/live/launchable-smoke.test.ts b/test/e2e-scenario/live/launchable-smoke.test.ts new file mode 100644 index 00000000000..c5f39c1c542 --- /dev/null +++ b/test/e2e-scenario/live/launchable-smoke.test.ts @@ -0,0 +1,460 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { randomUUID } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import type { ArtifactSink } from "../fixtures/artifacts.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; + +// Migrated coverage for test/e2e/test-launchable-smoke.sh (Refs #2599/#5098). +// This is intentionally a single live Vitest test instead of a new fixture +// family: the contract is the real Ubuntu launchable path, so the test invokes +// scripts/brev-launchable-ci-cpu.sh via sudo, then proves the launchable-built +// CLI can onboard, route inference.local, and run an OpenClaw agent turn. +// Legacy shell deletion/nightly lane retirement is deferred to #5098 Phase 11. + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const LAUNCHABLE_SCRIPT = path.join(REPO_ROOT, "scripts", "brev-launchable-ci-cpu.sh"); +const SENTINEL = "/var/run/nemoclaw-launchable-ready"; +const MODEL = "nvidia/nemotron-3-super-120b-a12b"; +const DEFAULT_SANDBOX_NAME = `e2e-launchable-${randomUUID().slice(0, 8)}`; +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? DEFAULT_SANDBOX_NAME; +const TEST_TIMEOUT_MS = 30 * 60_000; +const INSTALL_TIMEOUT_MS = 30 * 60_000; +const ONBOARD_TIMEOUT_MS = 15 * 60_000; +const INFERENCE_TIMEOUT_MS = 2 * 60_000; + +type ChatCompletion = { + choices?: Array<{ + message?: { content?: unknown; reasoning_content?: unknown; reasoning?: unknown }; + }>; +}; + +type AgentJsonDoc = { + payloads?: Array<{ text?: unknown }>; + result?: { payloads?: Array<{ text?: unknown }> }; +}; + +function runEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + ...extra, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + }; +} + +async function runBash( + host: HostCliClient, + script: string, + options: { + artifactName: string; + cwd?: string; + env?: NodeJS.ProcessEnv; + redactionValues?: string[]; + timeoutMs?: number; + }, +): Promise { + return host.command("bash", ["-lc", script], { + artifactName: options.artifactName, + cwd: options.cwd, + env: options.env ?? runEnv(), + redactionValues: options.redactionValues, + timeoutMs: options.timeoutMs, + }); +} + +function expectExitZero(result: ShellProbeResult, label: string): void { + expect(result.exitCode, `${label}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`).toBe(0); +} + +function parseChatContent(raw: string): string { + const response = JSON.parse(raw) as ChatCompletion; + const message = response.choices?.[0]?.message; + const content = message?.content ?? message?.reasoning_content ?? message?.reasoning ?? ""; + return typeof content === "string" ? content.trim() : ""; +} + +function parseAgentJsonDocs(raw: string): AgentJsonDoc[] { + try { + const parsed = JSON.parse(raw) as AgentJsonDoc | AgentJsonDoc[]; + return Array.isArray(parsed) ? parsed : [parsed]; + } catch { + // Fall through to a raw decoder-style scan. `openclaw agent --json` has + // emitted both single JSON documents and log-prefixed streams across + // versions, so match the legacy helper's permissive extraction shape. + } + + const docs: AgentJsonDoc[] = []; + for (let index = 0; index < raw.length; index += 1) { + if (raw[index] !== "{") continue; + for (let end = index + 1; end <= raw.length; end += 1) { + try { + const parsed = JSON.parse(raw.slice(index, end)) as AgentJsonDoc | AgentJsonDoc[]; + docs.push(...(Array.isArray(parsed) ? parsed : [parsed])); + index = end - 1; + break; + } catch { + // Keep extending the candidate slice until it becomes valid JSON. + } + } + } + return docs; +} + +function parseAgentText(raw: string): string { + return parseAgentJsonDocs(raw) + .flatMap((doc) => doc.payloads ?? doc.result?.payloads ?? []) + .map((payload) => payload.text) + .filter((text): text is string => typeof text === "string") + .join("\n"); +} + +async function preseedLaunchableClone( + host: HostCliClient, + cloneDir: string, + artifacts: ArtifactSink, +): Promise { + await artifacts.writeJson("launchable-clone.json", { cloneDir, ref: "main" }); + const result = await runBash( + host, + [ + `rm -rf ${JSON.stringify(cloneDir)}`, + `git clone --local --no-hardlinks ${JSON.stringify(REPO_ROOT)} ${JSON.stringify(cloneDir)}`, + `git -C ${JSON.stringify(cloneDir)} checkout -B main HEAD`, + `git -C ${JSON.stringify(cloneDir)} remote set-url origin ${JSON.stringify(cloneDir)}`, + ].join(" && "), + { + artifactName: "phase-0-preseed-launchable-clone", + env: runEnv(), + timeoutMs: 120_000, + }, + ); + expectExitZero(result, "preseed launchable clone"); +} + +async function cleanupLaunchableState(host: HostCliClient, cloneDir: string): Promise { + await runBash( + host, + [ + `if command -v nemoclaw >/dev/null 2>&1; then nemoclaw ${JSON.stringify(SANDBOX_NAME)} destroy --yes 2>/dev/null || true; fi`, + `if command -v openshell >/dev/null 2>&1; then openshell sandbox delete ${JSON.stringify(SANDBOX_NAME)} 2>/dev/null || true; fi`, + "if command -v openshell >/dev/null 2>&1; then openshell gateway destroy -g nemoclaw 2>/dev/null || true; fi", + `sudo rm -rf ${JSON.stringify(cloneDir)} 2>/dev/null || rm -rf ${JSON.stringify(cloneDir)} || true`, + ].join("\n"), + { + artifactName: "cleanup-launchable-state", + env: runEnv({ PATH: `/usr/local/bin:${process.env.PATH ?? ""}` }), + timeoutMs: 180_000, + }, + ); +} + +async function expectPongFromSandboxInference( + sandboxExec: (command: string[], artifactName: string) => Promise, +): Promise { + const payload = JSON.stringify({ + model: MODEL, + messages: [{ role: "user", content: "Reply with exactly one word: PONG" }], + max_tokens: 100, + }); + + let lastContent = ""; + let lastResult: ShellProbeResult | undefined; + for (let attempt = 1; attempt <= 3; attempt += 1) { + lastResult = await sandboxExec( + [ + "curl", + "-s", + "--max-time", + "60", + "https://inference.local/v1/chat/completions", + "-H", + "Content-Type: application/json", + "-d", + payload, + ], + `phase-6-sandbox-inference-attempt-${attempt}`, + ); + if (lastResult.stdout.trim()) { + try { + lastContent = parseChatContent(lastResult.stdout); + } catch { + lastContent = lastResult.stdout.slice(0, 200); + } + if (/PONG/i.test(lastContent)) return; + } + if (attempt < 3) await new Promise((resolve) => setTimeout(resolve, 5_000)); + } + + throw new Error( + `sandbox inference.local expected PONG after 3 attempts; last content='${lastContent}'; ` + + `stdout='${lastResult?.stdout.slice(0, 300) ?? ""}'; stderr='${lastResult?.stderr.slice(0, 300) ?? ""}'`, + ); +} + +const runLaunchableSmokeTest = shouldRunLiveE2EScenarios() ? test : test.skip; + +runLaunchableSmokeTest( + "launchable smoke: bootstrap, onboard, sandbox health, live inference, cleanup", + { timeout: TEST_TIMEOUT_MS }, + async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { + validateSandboxName(SANDBOX_NAME); + + await artifacts.writeJson("scenario.json", { + id: "launchable-smoke", + runner: "vitest", + boundary: "ubuntu-launchable-install-flow", + migratedFrom: "test/e2e/test-launchable-smoke.sh", + refs: ["#2599", "#5098"], + phases: [ + "preseed-launchable-clone", + "prerequisites", + "brev-launchable-ci-cpu", + "install-artifacts", + "onboard", + "sandbox-health", + "live-inference", + "cleanup", + ], + workflowRetirement: "deferred to #5098 Phase 11", + }); + + const apiKey = secrets.required("NVIDIA_API_KEY"); + expect(apiKey.startsWith("nvapi-"), "NVIDIA_API_KEY must start with nvapi-").toBe(true); + + expect(fs.existsSync(LAUNCHABLE_SCRIPT), `${LAUNCHABLE_SCRIPT} missing`).toBe(true); + + const sudo = await host.command("sudo", ["-n", "true"], { + artifactName: "prereq-passwordless-sudo", + env: runEnv(), + timeoutMs: 30_000, + }); + if (sudo.exitCode !== 0) skip("passwordless sudo is required for launchable smoke"); + + const dockerInfo = await host.command("docker", ["info"], { + artifactName: "prereq-docker-info", + env: runEnv(), + timeoutMs: 30_000, + }); + expectExitZero(dockerInfo, "Docker is running"); + + const network = await host.command( + "curl", + ["-sf", "--max-time", "10", "https://integrate.api.nvidia.com/v1/models"], + { artifactName: "prereq-integrate-api-models", env: runEnv(), timeoutMs: 30_000 }, + ); + expectExitZero(network, "integrate.api.nvidia.com reachable"); + + const cloneDir = path.join(os.tmpdir(), `NemoClaw-launchable-vitest-${randomUUID()}`); + cleanup.add(`remove launchable clone ${cloneDir}`, async () => + cleanupLaunchableState(host, cloneDir), + ); + await cleanupLaunchableState(host, cloneDir); + await preseedLaunchableClone(host, cloneDir, artifacts); + + const installLog = artifacts.pathFor("launch-plugin.log"); + const install = await host.command("sudo", ["-E", "bash", LAUNCHABLE_SCRIPT], { + artifactName: "phase-2-brev-launchable-ci-cpu", + env: runEnv({ + LAUNCH_LOG: installLog, + NEMOCLAW_CLONE_DIR: cloneDir, + NEMOCLAW_REF: "main", + SKIP_DOCKER_PULL: process.env.SKIP_DOCKER_PULL ?? "1", + }), + timeoutMs: INSTALL_TIMEOUT_MS, + }); + expectExitZero(install, "brev-launchable-ci-cpu.sh completed"); + + const pathEnv = runEnv({ PATH: `/usr/local/bin:${process.env.PATH ?? ""}` }); + + const nemoclawHelp = await runBash(host, "command -v nemoclaw && nemoclaw --help >/dev/null", { + artifactName: "phase-3-nemoclaw-help", + env: pathEnv, + timeoutMs: 30_000, + }); + expectExitZero(nemoclawHelp, "nemoclaw is on PATH and --help works"); + + const openshellVersion = await runBash(host, "command -v openshell && openshell --version", { + artifactName: "phase-3-openshell-version", + env: pathEnv, + timeoutMs: 30_000, + }); + expectExitZero(openshellVersion, "openshell is on PATH and --version works"); + + const nodeVersion = await host.command( + "node", + [ + "-p", + "JSON.stringify({version: process.version, major: Number(process.versions.node.split('.')[0])})", + ], + { artifactName: "phase-3-node-version", env: pathEnv, timeoutMs: 30_000 }, + ); + expectExitZero(nodeVersion, "node version probe"); + const node = JSON.parse(nodeVersion.stdout) as { version: string; major: number }; + await artifacts.writeJson("node-version.json", node); + expect( + node.major, + `Node.js too old after launchable install: ${node.version}`, + ).toBeGreaterThanOrEqual(20); + + const dockerAfterInstall = await host.command("docker", ["info"], { + artifactName: "phase-3-docker-info-after-install", + env: pathEnv, + timeoutMs: 30_000, + }); + expectExitZero(dockerAfterInstall, "Docker running after install"); + expect(fs.existsSync(SENTINEL), `${SENTINEL} missing`).toBe(true); + expect(fs.existsSync(path.join(cloneDir, ".git")), `${cloneDir}/.git missing`).toBe(true); + expect(fs.existsSync(path.join(cloneDir, "dist")), `${cloneDir}/dist missing`).toBe(true); + expect( + fs.existsSync(path.join(cloneDir, "nemoclaw", "dist")), + `${cloneDir}/nemoclaw/dist missing`, + ).toBe(true); + + const onboard = await host.command("nemoclaw", ["onboard", "--non-interactive"], { + artifactName: "phase-4-onboard", + cwd: cloneDir, + env: runEnv({ + PATH: `/usr/local/bin:${process.env.PATH ?? ""}`, + NVIDIA_API_KEY: apiKey, + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + NEMOCLAW_RECREATE_SANDBOX: "1", + }), + redactionValues: [apiKey], + timeoutMs: ONBOARD_TIMEOUT_MS, + }); + expectExitZero(onboard, "nemoclaw onboard --non-interactive"); + + const list = await host.command("nemoclaw", ["list"], { + artifactName: "phase-5-nemoclaw-list", + cwd: cloneDir, + env: pathEnv, + timeoutMs: 60_000, + }); + expectExitZero(list, "nemoclaw list"); + expect(list.stdout).toContain(SANDBOX_NAME); + + const status = await host.command("nemoclaw", [SANDBOX_NAME, "status"], { + artifactName: "phase-5-nemoclaw-status", + cwd: cloneDir, + env: pathEnv, + timeoutMs: 60_000, + }); + expectExitZero(status, `nemoclaw ${SANDBOX_NAME} status`); + + const inferenceConfig = await host.command("openshell", ["inference", "get"], { + artifactName: "phase-5-openshell-inference-get", + env: pathEnv, + timeoutMs: 30_000, + }); + expectExitZero(inferenceConfig, "openshell inference get"); + expect(inferenceConfig.stdout).toMatch(/nvidia-prod/i); + + const gatewayContainer = await runBash( + host, + "docker ps --format '{{.Names}}' | grep -E 'nemoclaw|openshell'", + { artifactName: "phase-5-gateway-container", env: pathEnv, timeoutMs: 30_000 }, + ); + const gatewayContainerNames = gatewayContainer.stdout.trim(); + await artifacts.writeJson("gateway-container.json", { + confirmed: gatewayContainerNames.length > 0, + stdout: gatewayContainer.stdout, + }); + expect(gatewayContainerNames, "expected a NemoClaw/OpenShell gateway container").not.toBe(""); + + const directPayload = JSON.stringify({ + model: MODEL, + messages: [{ role: "user", content: "Reply with exactly one word: PONG" }], + max_tokens: 100, + }); + const direct = await host.command( + "curl", + [ + "-s", + "--max-time", + "30", + "-X", + "POST", + "https://integrate.api.nvidia.com/v1/chat/completions", + "-H", + "Content-Type: application/json", + "-H", + `Authorization: Bearer ${apiKey}`, + "-d", + directPayload, + ], + { + artifactName: "phase-6-direct-nvidia-chat", + env: pathEnv, + redactionValues: [apiKey], + timeoutMs: INFERENCE_TIMEOUT_MS, + }, + ); + expectExitZero(direct, "direct NVIDIA Endpoints chat completion"); + expect(parseChatContent(direct.stdout)).toMatch(/PONG/i); + + const sandboxExec = (command: string[], artifactName: string) => + sandbox.exec(SANDBOX_NAME, command, { + artifactName, + env: pathEnv, + timeoutMs: INFERENCE_TIMEOUT_MS, + }); + await expectPongFromSandboxInference(sandboxExec); + + const sessionId = `e2e-launchable-${Date.now()}-${randomUUID()}`; + const agent = await sandboxExec( + [ + "openclaw", + "agent", + "--agent", + "main", + "--json", + "--thinking", + "off", + "--session-id", + sessionId, + "-m", + "What is 6 multiplied by 7? Reply with only the integer, no extra words.", + ], + "phase-6-openclaw-agent", + ); + expect( + agent.exitCode, + `openclaw agent failed; rc=${agent.exitCode}; stdout='${agent.stdout.slice(0, 300)}'; stderr='${agent.stderr.slice(0, 300)}'`, + ).toBe(0); + const agentReply = parseAgentText(agent.stdout); + expect( + /(^|[^0-9])42([^0-9]|$)/.test(agentReply), + `expected agent reply to contain 42; rc=${agent.exitCode}; reply='${agentReply.slice(0, 200)}'; stdout='${agent.stdout.slice(0, 300)}'; stderr='${agent.stderr.slice(0, 300)}'`, + ).toBe(true); + + const destroy = await host.command("nemoclaw", [SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "phase-7-nemoclaw-destroy", + cwd: cloneDir, + env: pathEnv, + timeoutMs: 120_000, + }); + expectExitZero(destroy, `destroy ${SANDBOX_NAME}`); + await sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { + artifactName: "phase-7-openshell-gateway-destroy", + env: pathEnv, + timeoutMs: 60_000, + }); + + const registryFile = path.join(os.homedir(), ".nemoclaw", "sandboxes.json"); + if (fs.existsSync(registryFile)) { + expect(fs.readFileSync(registryFile, "utf8")).not.toContain(`"${SANDBOX_NAME}"`); + } + + await cleanupLaunchableState(host, cloneDir); + }, +); From 24a4b9f86283b64ec0ded5e7a971627463b584db Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 11 Jun 2026 10:40:44 -0400 Subject: [PATCH 2/4] ci(e2e): allow selective Vitest job dispatch --- .github/workflows/e2e-vitest-scenarios.yaml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 6b37377a95c..41eddc5e065 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -11,12 +11,17 @@ on: required: false default: "" type: string + jobs: + description: "Optional comma-separated free-standing live Vitest job ids. Empty runs all enabled jobs." + required: false + default: "" + type: string permissions: contents: read concurrency: - group: e2e-vitest-scenarios-${{ github.ref }}-${{ inputs.scenarios || 'supported' }} + group: e2e-vitest-scenarios-${{ github.ref }}-${{ inputs.scenarios || 'supported' }}-${{ inputs.jobs || 'all-jobs' }} cancel-in-progress: false jobs: @@ -42,6 +47,7 @@ jobs: name: Generate Vitest scenario matrix env: SCENARIOS: ${{ inputs.scenarios }} + JOBS: ${{ inputs.jobs }} run: | set -euo pipefail args=(--emit-live-matrix) @@ -52,6 +58,10 @@ jobs: fi args+=(--scenarios "${SCENARIOS}") fi + if [ -n "${JOBS}" ] && [[ ! "${JOBS}" =~ ^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$ ]]; then + echo "::error::Invalid jobs input: ${JOBS}" >&2 + exit 1 + fi matrix="$(npx tsx test/e2e-scenario/scenarios/run.ts "${args[@]}")" echo "matrix=${matrix}" >> "$GITHUB_OUTPUT" MATRIX_JSON="${matrix}" python - <<'PY' >> "$GITHUB_STEP_SUMMARY" @@ -69,6 +79,7 @@ jobs: live-scenarios: needs: generate-matrix + if: ${{ inputs.jobs == '' }} runs-on: ${{ matrix.runner }} timeout-minutes: 45 strategy: @@ -169,6 +180,7 @@ jobs: # because the matrix above only runs registry-scenarios.test.ts. Modeled on # #5049's free-standing pattern. openshell-version-pin-vitest: + if: ${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',openshell-version-pin-vitest,') }} runs-on: ubuntu-latest timeout-minutes: 15 env: @@ -208,7 +220,7 @@ jobs: retention-days: 14 launchable-smoke-vitest: - if: ${{ inputs.scenarios == '' }} + if: ${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',launchable-smoke-vitest,') }} runs-on: ubuntu-latest timeout-minutes: 30 env: @@ -277,6 +289,7 @@ jobs: retention-days: 14 onboard-negative-paths-vitest: + if: ${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',onboard-negative-paths-vitest,') }} runs-on: ubuntu-latest timeout-minutes: 15 env: @@ -323,7 +336,7 @@ jobs: # protocol/history contract. The retained legacy bash lane remains the # source for full closeout until a later PR proves replacement and deletes it. openclaw-tui-chat-correlation-vitest: - if: ${{ inputs.scenarios == '' }} + if: ${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',openclaw-tui-chat-correlation-vitest,') }} runs-on: ubuntu-latest timeout-minutes: 75 env: From ccc9c7cd76b896c543589762f723f4e0cbf13bcf Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 11 Jun 2026 12:32:53 -0400 Subject: [PATCH 3/4] ci(e2e): align launchable-smoke-vitest dispatch --- .github/workflows/e2e-vitest-scenarios.yaml | 73 ++++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 65e06c7eda4..537b6f051bb 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -40,7 +40,7 @@ jobs: SCENARIOS: ${{ inputs.scenarios }} run: | set -euo pipefail - allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery" + allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,launchable-smoke-vitest" if [ -n "${JOBS}" ] && [ -n "${SCENARIOS}" ]; then echo "::error::Use either scenarios or jobs, not both." >&2 exit 1 @@ -298,6 +298,76 @@ jobs: if-no-files-found: ignore retention-days: 14 + launchable-smoke-vitest: + needs: validate-jobs + if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',launchable-smoke-vitest,') }} + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/launchable-smoke + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-launchable" + NEMOCLAW_RECREATE_SANDBOX: "1" + SKIP_DOCKER_PULL: "1" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Authenticate to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + shell: bash + run: | + set -euo pipefail + if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then + echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls." + exit 0 + fi + login_succeeded=0 + for attempt in 1 2 3; do + if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then + login_succeeded=1 + break + fi + if [[ "$attempt" -lt 3 ]]; then + echo "::warning::Docker Hub login attempt ${attempt} failed; retrying." + sleep 5 + fi + done + if [[ "$login_succeeded" -ne 1 ]]; then + echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls." + fi + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Run launchable smoke live test + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + run: | + set -euo pipefail + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/launchable-smoke.test.ts \ + --silent=false --reporter=default + + - name: Upload launchable smoke artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-launchable-smoke + path: e2e-artifacts/vitest/launchable-smoke/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + # Focused coverage slice for the #2603/#3145 OpenClaw websocket # protocol/history contract. The retained legacy bash lane remains the # source for full closeout until a later PR proves replacement and deletes it. @@ -481,6 +551,7 @@ jobs: live-scenarios, openshell-version-pin-vitest, onboard-negative-paths-vitest, + launchable-smoke-vitest, openclaw-tui-chat-correlation-vitest, gateway-guard-recovery, ] From 505189f6cd9df238de36c5c192caa995f6da7648 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 11 Jun 2026 18:03:14 -0400 Subject: [PATCH 4/4] test(e2e): stabilize launchable smoke dispatch --- .github/workflows/e2e-vitest-scenarios.yaml | 3 +- .../live/launchable-smoke.test.ts | 58 ++++++++++++++----- 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 4d324b35614..08d76852737 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -93,7 +93,7 @@ jobs: SCENARIOS: ${{ inputs.scenarios }} run: | set -euo pipefail - allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,credential-migration-vitest,hermes-e2e-vitest,network-policy-vitest,token-rotation-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery" + allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,credential-migration-vitest,hermes-e2e-vitest,network-policy-vitest,token-rotation-vitest,launchable-smoke-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery" args=(--emit-live-matrix) matrix="" hermes_selected=false @@ -648,6 +648,7 @@ jobs: NEMOCLAW_RUN_E2E_SCENARIOS: "1" NEMOCLAW_SANDBOX_NAME: "e2e-launchable" NEMOCLAW_RECREATE_SANDBOX: "1" + NEMOCLAW_MODEL: minimaxai/minimax-m2.7 SKIP_DOCKER_PULL: "1" steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 diff --git a/test/e2e-scenario/live/launchable-smoke.test.ts b/test/e2e-scenario/live/launchable-smoke.test.ts index c5f39c1c542..466ac355a6c 100644 --- a/test/e2e-scenario/live/launchable-smoke.test.ts +++ b/test/e2e-scenario/live/launchable-smoke.test.ts @@ -13,6 +13,7 @@ import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { isTransientProviderValidationFailure } from "./network-policy-transient-provider.ts"; // Migrated coverage for test/e2e/test-launchable-smoke.sh (Refs #2599/#5098). // This is intentionally a single live Vitest test instead of a new fixture @@ -24,13 +25,14 @@ import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const LAUNCHABLE_SCRIPT = path.join(REPO_ROOT, "scripts", "brev-launchable-ci-cpu.sh"); const SENTINEL = "/var/run/nemoclaw-launchable-ready"; -const MODEL = "nvidia/nemotron-3-super-120b-a12b"; +const MODEL = process.env.NEMOCLAW_MODEL ?? "nvidia/nemotron-3-super-120b-a12b"; const DEFAULT_SANDBOX_NAME = `e2e-launchable-${randomUUID().slice(0, 8)}`; const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? DEFAULT_SANDBOX_NAME; const TEST_TIMEOUT_MS = 30 * 60_000; const INSTALL_TIMEOUT_MS = 30 * 60_000; const ONBOARD_TIMEOUT_MS = 15 * 60_000; const INFERENCE_TIMEOUT_MS = 2 * 60_000; +const ONBOARD_ATTEMPTS = 3; type ChatCompletion = { choices?: Array<{ @@ -158,6 +160,10 @@ async function cleanupLaunchableState(host: HostCliClient, cloneDir: string): Pr ); } +async function sleep(ms: number): Promise { + await new Promise((resolve) => setTimeout(resolve, ms)); +} + async function expectPongFromSandboxInference( sandboxExec: (command: string[], artifactName: string) => Promise, ): Promise { @@ -192,7 +198,7 @@ async function expectPongFromSandboxInference( } if (/PONG/i.test(lastContent)) return; } - if (attempt < 3) await new Promise((resolve) => setTimeout(resolve, 5_000)); + if (attempt < 3) await sleep(5_000); } throw new Error( @@ -320,19 +326,41 @@ runLaunchableSmokeTest( `${cloneDir}/nemoclaw/dist missing`, ).toBe(true); - const onboard = await host.command("nemoclaw", ["onboard", "--non-interactive"], { - artifactName: "phase-4-onboard", - cwd: cloneDir, - env: runEnv({ - PATH: `/usr/local/bin:${process.env.PATH ?? ""}`, - NVIDIA_API_KEY: apiKey, - NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, - NEMOCLAW_RECREATE_SANDBOX: "1", - }), - redactionValues: [apiKey], - timeoutMs: ONBOARD_TIMEOUT_MS, - }); - expectExitZero(onboard, "nemoclaw onboard --non-interactive"); + let onboard: ShellProbeResult | undefined; + for (let attempt = 1; attempt <= ONBOARD_ATTEMPTS; attempt += 1) { + onboard = await host.command("nemoclaw", ["onboard", "--non-interactive"], { + artifactName: attempt === 1 ? "phase-4-onboard" : `phase-4-onboard-attempt-${attempt}`, + cwd: cloneDir, + env: runEnv({ + PATH: `/usr/local/bin:${process.env.PATH ?? ""}`, + NVIDIA_API_KEY: apiKey, + NEMOCLAW_MODEL: MODEL, + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + NEMOCLAW_RECREATE_SANDBOX: "1", + }), + redactionValues: [apiKey], + timeoutMs: ONBOARD_TIMEOUT_MS, + }); + if (onboard.exitCode === 0) break; + if (isTransientProviderValidationFailure(onboard) && attempt < ONBOARD_ATTEMPTS) { + await sleep(30_000 * attempt); + continue; + } + if (isTransientProviderValidationFailure(onboard) && process.env.GITHUB_ACTIONS === "true") { + await artifacts.writeJson("transient-provider-validation.skip.json", { + reason: "transient NVIDIA Endpoints validation failure during launchable onboard", + attempts: ONBOARD_ATTEMPTS, + sourceBoundary: "external NVIDIA Endpoints provider availability", + removalCondition: + "remove once CI endpoint validation is stable for a release cycle or covered by a hermetic provider-validation fixture", + }); + skip( + `NVIDIA Endpoints validation hit a transient upstream/rate-limit failure after ${ONBOARD_ATTEMPTS} attempts`, + ); + } + break; + } + expectExitZero(onboard as ShellProbeResult, "nemoclaw onboard --non-interactive"); const list = await host.command("nemoclaw", ["list"], { artifactName: "phase-5-nemoclaw-list",