diff --git a/test/deepagents-code-tui-startup-check.test.ts b/test/deepagents-code-tui-startup-check.test.ts new file mode 100644 index 00000000000..04491ea0881 --- /dev/null +++ b/test/deepagents-code-tui-startup-check.test.ts @@ -0,0 +1,351 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync, type SpawnSyncReturns, spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +import { CONTEXT_PATTERNS, TOKEN_PREFIX_PATTERNS } from "../src/lib/security/secret-patterns.ts"; + +const tuiStartupCheckPath = path.join( + process.cwd(), + "test", + "e2e", + "e2e-cloud-experimental", + "checks", + "10-deepagents-code-tui-startup.sh", +); +const tuiStartupCheckSource = fs + .readFileSync(tuiStartupCheckPath, "utf8") + .replace('\nif [[ "${BASH_SOURCE[0]}" == "$0" ]]; then\n main "$@"\nfi\n', "\n"); + +function runTuiStartupCheckHelper(snippet: string, env: NodeJS.ProcessEnv = {}): string { + return execFileSync("bash", ["-s"], { + encoding: "utf8", + env: { ...process.env, ...env }, + input: `${tuiStartupCheckSource}\n${snippet}\n`, + }); +} + +function runTuiStartupCheckHelperResult( + snippet: string, + env: NodeJS.ProcessEnv = {}, +): SpawnSyncReturns { + return spawnSync("bash", ["-s"], { + encoding: "utf8", + env: { ...process.env, ...env }, + input: `${tuiStartupCheckSource}\n${snippet}\n`, + }); +} + +function fingerprint(pattern: RegExp): string { + return `${pattern.source}::${pattern.flags}`; +} + +function secretFixture(...parts: string[]): string { + return parts.join(""); +} + +describe("Deep Agents Code TUI startup check helpers", () => { + it("rejects unsafe TUI startup timeout values before sandbox execution", () => { + const validate = (timeout: string) => + runTuiStartupCheckHelper( + 'if is_positive_integer "$TUI_TIMEOUT"; then printf valid; else printf invalid; fi', + { DEEPAGENTS_TUI_TIMEOUT: timeout }, + ); + + expect(validate("90")).toBe("valid"); + expect(validate("0")).toBe("invalid"); + expect(validate("1; touch /tmp/nemoclaw-tui-timeout-injection")).toBe("invalid"); + }); + + it("matches prompt-shaped TUI readiness text without accepting banner-only startup text", () => { + const readiness = (capture: string) => + runTuiStartupCheckHelper( + 'if printf "%s" "$CAPTURE" | is_tui_ready_capture; then printf ready; else printf not-ready; fi', + { CAPTURE: capture }, + ); + + expect(readiness("Deep Agents Code starting...\nLoading tools...")).toBe("not-ready"); + expect(readiness("Press Enter to continue")).toBe("not-ready"); + expect(readiness("What would you like to do next?")).toBe("ready"); + expect(readiness("Enter your task, then press Enter")).toBe("ready"); + expect(readiness("How can I help with the codebase today?")).toBe("ready"); + }); + + it("does not treat generic TUI exit status 1 as a clean Ctrl-C exit", () => { + const assertExit = (exitCode: string) => + runTuiStartupCheckHelper( + [ + "PASSED=0", + "FAILED=0", + 'capture="$(mktemp)"', + 'printf "NEMOCLAW_TUI_EXIT_CAPTURED:%s\\n" "$EXIT_CODE" >"$capture"', + 'assert_clean_exit_code "$capture" 2>/dev/null', + 'rm -f -- "$capture"', + 'printf "passed=%s failed=%s" "$PASSED" "$FAILED"', + ].join("; "), + { EXIT_CODE: exitCode }, + ); + + expect(assertExit("0")).toBe( + "10-deepagents-code-tui-startup: OK (dcode TUI exited cleanly after Ctrl-C (exit 0))\npassed=1 failed=0", + ); + expect(assertExit("130")).toBe( + "10-deepagents-code-tui-startup: OK (dcode TUI exited cleanly after Ctrl-C (exit 130))\npassed=1 failed=0", + ); + expect(assertExit("1")).toBe("passed=0 failed=1"); + }); + + it("detects and redacts every canonical secret family in TUI startup artifacts", () => { + const detectsSecret = (token: string) => + runTuiStartupCheckHelper( + 'if printf "%s" "$TOKEN" | contains_secret; then printf secret; else printf clean; fi', + { TOKEN: token }, + ); + const redactsSecret = (token: string) => + runTuiStartupCheckHelper('printf "%s" "$TOKEN" | redact_secrets', { TOKEN: token }); + const canonicalSamples = new Map([ + [fingerprint(TOKEN_PREFIX_PATTERNS[0]), { name: "nvapi", sample: "nvapi-abcdefghijklmnop" }], + [fingerprint(TOKEN_PREFIX_PATTERNS[1]), { name: "nvcf", sample: "nvcf-abcdefghijklmnopq" }], + [fingerprint(TOKEN_PREFIX_PATTERNS[2]), { name: "ghp", sample: "ghp_abcdefghijklmnopqr" }], + [ + fingerprint(TOKEN_PREFIX_PATTERNS[3]), + { name: "github_pat", sample: "github_pat_abcdefghijklmnopqrstuvwxyz0123" }, + ], + [fingerprint(TOKEN_PREFIX_PATTERNS[4]), { name: "sk_proj", sample: "sk-proj-abcdefghij" }], + [fingerprint(TOKEN_PREFIX_PATTERNS[5]), { name: "sk_ant", sample: "sk-ant-abcdefghijk" }], + [ + fingerprint(TOKEN_PREFIX_PATTERNS[6]), + { name: "sk", sample: "sk-abcdefghijklmnopqrstuvwx" }, + ], + [ + fingerprint(TOKEN_PREFIX_PATTERNS[7]), + { name: "xoxb", sample: secretFixture("xox", "b", "-", "1234567890") }, + ], + [fingerprint(TOKEN_PREFIX_PATTERNS[8]), { name: "akia", sample: "AKIAABCDEFGHIJKLMNOP" }], + [fingerprint(TOKEN_PREFIX_PATTERNS[8]), { name: "asia", sample: "ASIAABCDEFGHIJKLMNOP" }], + [fingerprint(TOKEN_PREFIX_PATTERNS[9]), { name: "hf", sample: "hf_abcdefghijklmnopq" }], + [fingerprint(TOKEN_PREFIX_PATTERNS[10]), { name: "glpat", sample: "glpat-abcdefghijklmn" }], + [fingerprint(TOKEN_PREFIX_PATTERNS[11]), { name: "gsk", sample: "gsk_abcdefghijklmnop" }], + [fingerprint(TOKEN_PREFIX_PATTERNS[12]), { name: "pypi", sample: "pypi-abcdefghijklmnop" }], + [ + fingerprint(TOKEN_PREFIX_PATTERNS[13]), + { name: "telegram_bot", sample: "bot123456789:AbcDefGhiJklMnoPqrStuVwxYz012345678" }, + ], + [ + fingerprint(TOKEN_PREFIX_PATTERNS[14]), + { name: "telegram", sample: "123456789:AbcDefGhiJklMnoPqrStuVwxYz012345678" }, + ], + [ + fingerprint(TOKEN_PREFIX_PATTERNS[15]), + { + name: "discord", + sample: "ABCDEFGHIJKLMNOPQRSTUVWX.Abcdef.ZZZZZZZZZZZZZZZZZZZZZZZZZZZ", + }, + ], + [ + fingerprint(CONTEXT_PATTERNS[0]), + { + name: "bearer_context", + sample: "Authorization: Bearer abcdefghijklmnopqrst", + rawSecret: "abcdefghijklmnopqrst", + }, + ], + [ + fingerprint(CONTEXT_PATTERNS[1]), + { + name: "api_key_context", + sample: "API_KEY=abcdefghijklmnopqrst", + rawSecret: "abcdefghijklmnopqrst", + }, + ], + ]); + const extraSamples = [ + { name: "akia", sample: "AKIAABCDEFGHIJKLMNOP" }, + { name: "xoxp", sample: secretFixture("xox", "p", "-", "1234567890") }, + { name: "xoxa", sample: secretFixture("xox", "a", "-", "1234567890") }, + { name: "xoxs", sample: secretFixture("xox", "s", "-", "1234567890") }, + { + name: "xapp", + sample: secretFixture("x", "app", "-", "1", "-", "A1B2C3", "-", "12345", "-", "abcde"), + }, + { + name: "token_context", + sample: "TOKEN=abcdefghijklmnopqrst", + rawSecret: "abcdefghijklmnopqrst", + }, + { + name: "secret_context", + sample: "SECRET=abcdefghijklmnopqrst", + rawSecret: "abcdefghijklmnopqrst", + }, + { + name: "password_context", + sample: "PASSWORD=abcdefghijklmnopqrst", + rawSecret: "abcdefghijklmnopqrst", + }, + { + name: "credential_context", + sample: "CREDENTIAL=abcdefghijklmnopqrst", + rawSecret: "abcdefghijklmnopqrst", + }, + { + name: "suffix_key_context", + sample: "SERVICE_KEY=abcdefghijklmnopqrst", + rawSecret: "abcdefghijklmnopqrst", + }, + ]; + + const canonicalFingerprints = [...TOKEN_PREFIX_PATTERNS, ...CONTEXT_PATTERNS].map(fingerprint); + expect([...canonicalSamples.keys()]).toEqual(canonicalFingerprints); + + for (const { name, sample, rawSecret } of [...canonicalSamples.values(), ...extraSamples]) { + expect(detectsSecret(sample), `${name} should be detected`).toBe("secret"); + const redacted = redactsSecret(sample); + expect(redacted, `${name} should include a redaction marker`).toContain("[REDACTED_SECRET]"); + expect(redacted, `${name} should not retain the raw secret`).not.toContain( + rawSecret ?? sample, + ); + } + expect(detectsSecret("plain startup text")).toBe("clean"); + }); + + it("removes raw TUI startup artifacts after writing the sanitized capture", () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dcode-tui-")); + const rawCapture = path.join(tempDir, "raw.log"); + const expectLog = path.join(tempDir, "expect.log"); + const combinedCapture = path.join(tempDir, "combined.log"); + const sanitizedCapture = path.join(tempDir, "sanitized.log"); + + try { + const output = runTuiStartupCheckHelper( + [ + 'raw_capture_file="$RAW_CAPTURE"', + 'expect_log_file="$EXPECT_LOG"', + 'combined_capture_file="$COMBINED_CAPTURE"', + 'plain_capture_file="$SANITIZED_CAPTURE"', + 'SENSITIVE_CAPTURE_FILES=("$raw_capture_file" "$expect_log_file" "$combined_capture_file")', + 'printf "%s\\n" "$SECRET_TOKEN" >"$raw_capture_file"', + 'printf "%s\\n" "expect output" >"$expect_log_file"', + 'cat "$raw_capture_file" "$expect_log_file" >"$combined_capture_file"', + 'strip_terminal_control_sequences <"$combined_capture_file" >"$plain_capture_file"', + "cleanup_sensitive_captures", + 'for artifact in "$raw_capture_file" "$expect_log_file" "$combined_capture_file"; do if [ -e "$artifact" ]; then printf "leaked:%s" "$artifact"; exit 1; fi; done', + 'if [ -e "$plain_capture_file" ]; then printf sanitized; else printf missing; fi', + ].join("; "), + { + COMBINED_CAPTURE: combinedCapture, + EXPECT_LOG: expectLog, + RAW_CAPTURE: rawCapture, + SANITIZED_CAPTURE: sanitizedCapture, + SECRET_TOKEN: `sk-${"A".repeat(20)}`, + }, + ); + + expect(output).toBe("sanitized"); + expect(fs.existsSync(rawCapture)).toBe(false); + expect(fs.existsSync(expectLog)).toBe(false); + expect(fs.existsSync(combinedCapture)).toBe(false); + expect(fs.existsSync(sanitizedCapture)).toBe(true); + } finally { + fs.rmSync(tempDir, { force: true, recursive: true }); + } + }); + + it("redacts retained TUI startup capture when a secret scan fails", () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dcode-tui-redact-")); + const rawCapture = path.join(tempDir, "raw.log"); + const expectLog = path.join(tempDir, "expect.log"); + const combinedCapture = path.join(tempDir, "combined.log"); + const sanitizedCapture = path.join(tempDir, "sanitized.log"); + const prefixedToken = `sk-${"A".repeat(20)}`; + const bearerToken = "b".repeat(20); + const apiKey = "c".repeat(20); + + try { + const result = runTuiStartupCheckHelperResult( + [ + 'raw_capture_file="$RAW_CAPTURE"', + 'expect_log_file="$EXPECT_LOG"', + 'combined_capture_file="$COMBINED_CAPTURE"', + 'plain_capture_file="$SANITIZED_CAPTURE"', + 'SENSITIVE_CAPTURE_FILES=("$raw_capture_file" "$expect_log_file" "$combined_capture_file")', + 'printf "%s\\nAuthorization: Bearer %s\\nAPI_KEY=%s\\n" "$PREFIXED_TOKEN" "$BEARER_TOKEN" "$API_KEY_VALUE" >"$raw_capture_file"', + 'printf "%s\\n" "expect output" >"$expect_log_file"', + 'cat "$raw_capture_file" "$expect_log_file" >"$combined_capture_file"', + 'strip_terminal_control_sequences <"$combined_capture_file" >"$plain_capture_file"', + "secret_detected=0", + 'if contains_secret <"$plain_capture_file"; then secret_detected=1; redact_secrets_in_file "$plain_capture_file"; fi', + "cleanup_sensitive_captures", + 'if [ "$secret_detected" -eq 1 ]; then fail_test "secret-shaped value found in sanitized TUI capture"; fi', + 'printf "secret_detected=%s failed=%s\\n" "$secret_detected" "$FAILED"', + '[ "$FAILED" -eq 0 ]', + ].join("; "), + { + API_KEY_VALUE: apiKey, + BEARER_TOKEN: bearerToken, + COMBINED_CAPTURE: combinedCapture, + EXPECT_LOG: expectLog, + PREFIXED_TOKEN: prefixedToken, + RAW_CAPTURE: rawCapture, + SANITIZED_CAPTURE: sanitizedCapture, + }, + ); + + const output = `${result.stdout}\n${result.stderr}`; + const sanitizedText = fs.readFileSync(sanitizedCapture, "utf8"); + expect(result.status).not.toBe(0); + expect(output).toContain("secret_detected=1 failed=1"); + expect(output).not.toContain(prefixedToken); + expect(output).not.toContain(bearerToken); + expect(output).not.toContain(apiKey); + expect(sanitizedText).not.toContain(prefixedToken); + expect(sanitizedText).not.toContain(bearerToken); + expect(sanitizedText).not.toContain(apiKey); + expect(sanitizedText).toContain("Authorization: Bearer [REDACTED_SECRET]"); + expect(sanitizedText).toContain("API_KEY=[REDACTED_SECRET]"); + expect(fs.existsSync(rawCapture)).toBe(false); + expect(fs.existsSync(expectLog)).toBe(false); + expect(fs.existsSync(combinedCapture)).toBe(false); + } finally { + fs.rmSync(tempDir, { force: true, recursive: true }); + } + }); + + it("truncates retained TUI startup capture when redaction fails", () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dcode-tui-redact-fail-")); + const sanitizedCapture = path.join(tempDir, "sanitized.log"); + const prefixedToken = `sk-${"A".repeat(20)}`; + + try { + const result = runTuiStartupCheckHelperResult( + [ + 'plain_capture_file="$SANITIZED_CAPTURE"', + 'printf "%s\\n" "$PREFIXED_TOKEN" >"$plain_capture_file"', + "redact_secrets() { return 1; }", + 'if ! redact_secrets_in_file "$plain_capture_file"; then printf "redaction_failed=1\\n"; fi', + 'if [ -e "$plain_capture_file" ] && grep -Fq "$PREFIXED_TOKEN" "$plain_capture_file"; then printf "leaked=1\\n"; fi', + 'printf "failed=%s\\n" "$FAILED"', + ].join("; "), + { + PREFIXED_TOKEN: prefixedToken, + SANITIZED_CAPTURE: sanitizedCapture, + }, + ); + + const output = `${result.stdout}\n${result.stderr}`; + const sanitizedText = fs.readFileSync(sanitizedCapture, "utf8"); + expect(result.status).toBe(0); + expect(output).toContain("redaction_failed=1"); + expect(output).toContain("failed=1"); + expect(output).not.toContain(prefixedToken); + expect(output).not.toContain("leaked=1"); + expect(sanitizedText).toBe("[redaction failed; sanitized capture unavailable]\n"); + } finally { + fs.rmSync(tempDir, { force: true, recursive: true }); + } + }); +}); diff --git a/test/e2e-scenario/live/cloud-experimental-check-list.ts b/test/e2e-scenario/live/cloud-experimental-check-list.ts index 0d6c59fbb09..f2ac3bb33c3 100644 --- a/test/e2e-scenario/live/cloud-experimental-check-list.ts +++ b/test/e2e-scenario/live/cloud-experimental-check-list.ts @@ -6,6 +6,7 @@ export const DEEPAGENTS_CLOUD_EXPERIMENTAL_CHECKS = [ "test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh", "test/e2e/e2e-cloud-experimental/checks/08-deepagents-code-secret-boundary.sh", "test/e2e/e2e-cloud-experimental/checks/09-deepagents-code-tavily-opt-in.sh", + "test/e2e/e2e-cloud-experimental/checks/10-deepagents-code-tui-startup.sh", ] as const; export function cloudExperimentalChecksForOnboarding( diff --git a/test/e2e-scenario/live/cloud-experimental-checks.ts b/test/e2e-scenario/live/cloud-experimental-checks.ts index c5b1b20cf6a..349ce20df71 100644 --- a/test/e2e-scenario/live/cloud-experimental-checks.ts +++ b/test/e2e-scenario/live/cloud-experimental-checks.ts @@ -12,6 +12,44 @@ import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const REQUIRED_CHECK_SKIP_PATTERN = /(^|\n).*\bSKIP\b/i; +export type CloudExperimentalChecksEvidence = { + scenarioId: string; + sandboxName: string; + checkScripts: readonly string[]; + terminalConnectHint?: { + agent: string; + interactiveCommand: string; + statusLine: string; + source: string; + }; +}; + +const DEEPAGENTS_CODE_ONBOARDING = "cloud-langchain-deepagents-code"; +const DEEPAGENTS_CODE_TUI_CHECK = + "test/e2e/e2e-cloud-experimental/checks/10-deepagents-code-tui-startup.sh"; +const DEEPAGENTS_CODE_CONNECT_HINT = { + agent: "langchain-deepagents-code", + interactiveCommand: "dcode", + statusLine: "Interactive: dcode", + source: "agents/langchain-deepagents-code/manifest.yaml:runtime.interactive_command", +}; + +export function buildCloudExperimentalChecksEvidence( + scenarioId: string, + sandboxName: string, + checkScripts: readonly string[], +): CloudExperimentalChecksEvidence { + return { + scenarioId, + sandboxName, + checkScripts, + ...(scenarioId === DEEPAGENTS_CODE_ONBOARDING && + checkScripts.includes(DEEPAGENTS_CODE_TUI_CHECK) + ? { terminalConnectHint: DEEPAGENTS_CODE_CONNECT_HINT } + : {}), + }; +} + export function buildCloudExperimentalCommandEnv( sandboxName: string, apiKey: string, @@ -74,11 +112,10 @@ export async function runE2eCloudExperimentalChecks( context: Pick, ): Promise { const apiKey = context.secrets.optional("NVIDIA_INFERENCE_API_KEY") ?? ""; - await context.artifacts.writeJson("e2e-cloud-experimental-checks.json", { - scenarioId, - sandboxName, - checkScripts, - }); + await context.artifacts.writeJson( + "e2e-cloud-experimental-checks.json", + buildCloudExperimentalChecksEvidence(scenarioId, sandboxName, checkScripts), + ); await Promise.resolve( checkScripts.length > 0 ? assertDeepAgentsRuntimeObserved(sandboxName, context) : undefined, ); diff --git a/test/e2e-scenario/support-tests/platform-parity-cloud-experimental.test.ts b/test/e2e-scenario/support-tests/platform-parity-cloud-experimental.test.ts index 7a97979be86..86a0b8a895f 100644 --- a/test/e2e-scenario/support-tests/platform-parity-cloud-experimental.test.ts +++ b/test/e2e-scenario/support-tests/platform-parity-cloud-experimental.test.ts @@ -2,12 +2,15 @@ // SPDX-License-Identifier: Apache-2.0 import { spawnSync } from "node:child_process"; +import fs from "node:fs"; import path from "node:path"; import { describe, expect, it } from "vitest"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { DEEPAGENTS_CLOUD_EXPERIMENTAL_CHECKS } from "../live/cloud-experimental-check-list.ts"; import { assertRequiredCloudExperimentalResult, + buildCloudExperimentalChecksEvidence, buildCloudExperimentalCommandEnv, } from "../live/cloud-experimental-checks.ts"; @@ -62,6 +65,43 @@ describe("P0-E cloud-experimental parity guardrails", () => { ); }); + it("registers executable Deep Agents cloud-experimental checks", () => { + expect(DEEPAGENTS_CLOUD_EXPERIMENTAL_CHECKS).toEqual([ + "test/e2e/e2e-cloud-experimental/checks/05-deepagents-code-landlock-readonly.sh", + "test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh", + "test/e2e/e2e-cloud-experimental/checks/08-deepagents-code-secret-boundary.sh", + "test/e2e/e2e-cloud-experimental/checks/09-deepagents-code-tavily-opt-in.sh", + "test/e2e/e2e-cloud-experimental/checks/10-deepagents-code-tui-startup.sh", + ]); + + for (const scriptPath of DEEPAGENTS_CLOUD_EXPERIMENTAL_CHECKS) { + const mode = fs.statSync(path.join(process.cwd(), scriptPath)).mode; + expect(mode & 0o111, `${scriptPath} must be executable`).not.toBe(0); + } + }); + + it("documents Deep Agents check scripts in generated launch/QA evidence", () => { + const evidence = buildCloudExperimentalChecksEvidence( + "cloud-langchain-deepagents-code", + "deepagents-sandbox", + DEEPAGENTS_CLOUD_EXPERIMENTAL_CHECKS, + ); + + expect(evidence).toMatchObject({ + scenarioId: "cloud-langchain-deepagents-code", + sandboxName: "deepagents-sandbox", + }); + expect(evidence.checkScripts).toContain( + "test/e2e/e2e-cloud-experimental/checks/10-deepagents-code-tui-startup.sh", + ); + expect(evidence.terminalConnectHint).toEqual({ + agent: "langchain-deepagents-code", + interactiveCommand: "dcode", + statusLine: "Interactive: dcode", + source: "agents/langchain-deepagents-code/manifest.yaml:runtime.interactive_command", + }); + }); + it("builds a minimal cloud-experimental child environment", () => { const env = buildCloudExperimentalCommandEnv("deepagents-sandbox", "secret-key", { HOME: "/home/runner", diff --git a/test/e2e/e2e-cloud-experimental/checks/10-deepagents-code-tui-startup.sh b/test/e2e/e2e-cloud-experimental/checks/10-deepagents-code-tui-startup.sh new file mode 100755 index 00000000000..a3c390c9331 --- /dev/null +++ b/test/e2e/e2e-cloud-experimental/checks/10-deepagents-code-tui-startup.sh @@ -0,0 +1,287 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Case: Deep Agents Code interactive TUI startup (#5620). +# +# This live check runs against a real Deep Agents Code sandbox. It proves the +# interactive `dcode` TUI starts in a PTY, reaches a prompt-like startup state, +# exits after Ctrl-C, and leaves only sanitized, secret-free capture artifacts. +# +# shellcheck disable=SC2016 +# expect(1) Tcl: $env(...) and {...} are Tcl/sh expansion, not bash expansion. + +set -euo pipefail + +SANDBOX_NAME="${SANDBOX_NAME:-${NEMOCLAW_SANDBOX_NAME:-e2e-cloud-onboard}}" +PREFIX="10-deepagents-code-tui-startup" +TUI_TIMEOUT="${DEEPAGENTS_TUI_TIMEOUT:-90}" +# Shell-only live check fallback for remote e2e hosts; Vitest parity coverage in +# test/deepagents-code-tui-startup-check.test.ts pins this to secret-patterns.ts. +SECRET_PATTERN='(?:nvapi-[A-Za-z0-9_-]{10,}|nvcf-[A-Za-z0-9_-]{10,}|ghp_[A-Za-z0-9_-]{10,}|github_pat_[A-Za-z0-9_]{30,}|sk-proj-[A-Za-z0-9_-]{10,}|sk-ant-[A-Za-z0-9_-]{10,}|sk-[A-Za-z0-9_-]{20,}|(?:xox[bpas]|xapp)-[A-Za-z0-9-]{10,}|A(?:K|S)IA[A-Z0-9]{16}|hf_[A-Za-z0-9]{10,}|glpat-[A-Za-z0-9_-]{10,}|gsk_[A-Za-z0-9]{10,}|pypi-[A-Za-z0-9_-]{10,}|\bbot[0-9]{8,10}:[A-Za-z0-9_-]{35}\b|\b[0-9]{8,10}:[A-Za-z0-9_-]{35}\b|\b[A-Za-z0-9]{24}\.[A-Za-z0-9_-]{6}\.[A-Za-z0-9_-]{27,}\b)' +CONTEXT_SECRET_VALUE_PATTERN='[A-Za-z0-9_.+\/=-]{10,}' +# Upstream dcode does not expose a stable machine-readable TUI ready marker. +# Keep this localized heuristic prompt-shaped; do not match banner-only text. +TUI_READY_PATTERN='(what would you like|what do you want|enter (your )?(task|message|prompt)|describe (the )?(task|change)|how can i help)' +SENSITIVE_CAPTURE_FILES=() + +ok() { printf '%s\n' "${PREFIX}: OK ($*)"; } +info() { printf '%s\n' "${PREFIX}: $*"; } +fail_test() { + printf '%s\n' "${PREFIX}: FAIL: $1" >&2 + FAILED=$((FAILED + 1)) +} +pass() { + ok "$1" + PASSED=$((PASSED + 1)) +} + +sandbox_exec() { + openshell sandbox exec --name "$SANDBOX_NAME" -- bash -c "$1" 2>&1 +} + +is_positive_integer() { + [[ "$1" =~ ^[1-9][0-9]*$ ]] +} + +contains_secret() { + NEMOCLAW_TOKEN_SECRET_PATTERN="$SECRET_PATTERN" \ + NEMOCLAW_CONTEXT_SECRET_VALUE_PATTERN="$CONTEXT_SECRET_VALUE_PATTERN" \ + perl -0ne ' + BEGIN { + $token_pattern = $ENV{"NEMOCLAW_TOKEN_SECRET_PATTERN"}; + $context_value_pattern = $ENV{"NEMOCLAW_CONTEXT_SECRET_VALUE_PATTERN"}; + } + if ( + /$token_pattern/ || + /Bearer\s+$context_value_pattern/i || + /(?:_KEY|API_KEY|SECRET|TOKEN|PASSWORD|CREDENTIAL)[=: ]["'\''"]?$context_value_pattern/i + ) { + $found = 1; + } + END { exit($found ? 0 : 1) } + ' +} + +redact_secrets() { + NEMOCLAW_TOKEN_SECRET_PATTERN="$SECRET_PATTERN" \ + NEMOCLAW_CONTEXT_SECRET_VALUE_PATTERN="$CONTEXT_SECRET_VALUE_PATTERN" \ + perl -0pe ' + BEGIN { + $token_pattern = $ENV{"NEMOCLAW_TOKEN_SECRET_PATTERN"}; + $context_value_pattern = $ENV{"NEMOCLAW_CONTEXT_SECRET_VALUE_PATTERN"}; + } + s/$token_pattern/[REDACTED_SECRET]/g; + s/(Bearer\s+)$context_value_pattern/${1}[REDACTED_SECRET]/gi; + s/((?:_KEY|API_KEY|SECRET|TOKEN|PASSWORD|CREDENTIAL)[=: ]["'\''"]?)$context_value_pattern/${1}[REDACTED_SECRET]/gi; + ' +} + +redact_secrets_in_file() { + local target_file="$1" + local redacted_file + redacted_file="${target_file}.redacted.$$" + if redact_secrets <"$target_file" >"$redacted_file"; then + mv -- "$redacted_file" "$target_file" + else + rm -f -- "$redacted_file" + if ! printf '%s\n' "[redaction failed; sanitized capture unavailable]" >"$target_file"; then + rm -f -- "$target_file" + fi + fail_test "unable to redact sanitized TUI capture" + return 1 + fi +} + +is_tui_ready_capture() { + grep -Eiq "$TUI_READY_PATTERN" +} + +strip_terminal_control_sequences() { + perl -pe 's/\x1b\][^\a]*(?:\a|\x1b\\)//g; s/\x1b\[[0-9;?]*[ -\/]*[@-~]//g; s/\r/\n/g' +} + +cleanup_sensitive_captures() { + local artifact + for artifact in "${SENSITIVE_CAPTURE_FILES[@]}"; do + [ -n "$artifact" ] && rm -f -- "$artifact" + done +} + +make_capture_dir() { + if [ -n "${DEEPAGENTS_TUI_CAPTURE_DIR:-}" ]; then + mkdir -p "$DEEPAGENTS_TUI_CAPTURE_DIR" + printf '%s\n' "$DEEPAGENTS_TUI_CAPTURE_DIR" + else + mktemp -d "${TMPDIR:-/tmp}/${PREFIX}.XXXXXX" + fi +} + +run_tui_expect() { + local raw_capture_file="$1" + env \ + NEMOCLAW_TUI_CAPTURE="$raw_capture_file" \ + NEMOCLAW_TUI_READY_PATTERN="$TUI_READY_PATTERN" \ + NEMOCLAW_TUI_SANDBOX_NAME="$SANDBOX_NAME" \ + NEMOCLAW_TUI_TIMEOUT="$TUI_TIMEOUT" \ + expect <<'EXPECT' +set timeout $env(NEMOCLAW_TUI_TIMEOUT) +set sandbox $env(NEMOCLAW_TUI_SANDBOX_NAME) +set capture $env(NEMOCLAW_TUI_CAPTURE) +set ready_pattern $env(NEMOCLAW_TUI_READY_PATTERN) +log_file -a $capture + +set cmd [list openshell sandbox exec --name $sandbox --tty -- sh -lc {export TERM=xterm-256color; cd /sandbox; dcode; status=$?; printf "\nNEMOCLAW_TUI_EXIT:%s\n" "$status"}] +spawn {*}$cmd +expect { + -nocase -re $ready_pattern { + puts "\nNEMOCLAW_TUI_READY" + send -- "\003" + } + timeout { + puts "\nNEMOCLAW_TUI_TIMEOUT" + send -- "\003" + exit 20 + } + eof { + puts "\nNEMOCLAW_TUI_EOF_BEFORE_READY" + exit 21 + } +} + +set timeout 20 +expect { + -re {NEMOCLAW_TUI_EXIT:([0-9]+)} { + puts "\nNEMOCLAW_TUI_EXIT_CAPTURED:$expect_out(1,string)" + exit 0 + } + timeout { + puts "\nNEMOCLAW_TUI_EXIT_TIMEOUT" + send -- "\003" + exit 22 + } + eof { + puts "\nNEMOCLAW_TUI_EOF_BEFORE_EXIT" + exit 23 + } +} +EXPECT +} + +assert_clean_exit_code() { + local plain_capture_file="$1" + local exit_code + exit_code="$(sed -n 's/.*NEMOCLAW_TUI_EXIT_CAPTURED:\([0-9]\+\).*/\1/p' "$plain_capture_file" | tail -n1)" + if [ -z "$exit_code" ]; then + fail_test "TUI capture did not include an exit-status marker" + return + fi + case "$exit_code" in + 0 | 130) pass "dcode TUI exited cleanly after Ctrl-C (exit ${exit_code})" ;; + *) fail_test "dcode TUI exited with unexpected status ${exit_code}" ;; + esac +} + +PASSED=0 +FAILED=0 + +main() { + trap cleanup_sensitive_captures EXIT + + if ! is_positive_integer "$TUI_TIMEOUT"; then + fail_test "DEEPAGENTS_TUI_TIMEOUT must be a positive integer" + printf '%s\n' "${PREFIX}: $PASSED passed, $FAILED failed" + exit 1 + fi + + if ! command -v expect >/dev/null 2>&1; then + fail_test "expect is required for the Deep Agents Code TUI startup check" + printf '%s\n' "${PREFIX}: $PASSED passed, $FAILED failed" + exit 1 + fi + if ! command -v perl >/dev/null 2>&1; then + fail_test "perl is required to sanitize and redact Deep Agents Code TUI captures" + printf '%s\n' "${PREFIX}: $PASSED passed, $FAILED failed" + exit 1 + fi + + local probe_output + if ! probe_output="$(sandbox_exec 'if test -d /sandbox/.deepagents && command -v dcode >/dev/null 2>&1; then printf "NEMOCLAW_DCODE_PROBE:deepagents\n"; else printf "NEMOCLAW_DCODE_PROBE:other\n"; fi')"; then + fail_test "unable to probe sandbox '${SANDBOX_NAME}' for Deep Agents Code markers" + printf '%s\n' "${PREFIX}: $PASSED passed, $FAILED failed" + exit 1 + fi + case "$probe_output" in + *NEMOCLAW_DCODE_PROBE:deepagents*) ;; + *NEMOCLAW_DCODE_PROBE:other*) + info "SKIP: sandbox '${SANDBOX_NAME}' is not a Deep Agents Code sandbox" + exit 0 + ;; + *) + fail_test "unexpected sandbox probe output for '${SANDBOX_NAME}'" + printf '%s\n' "${PREFIX}: $PASSED passed, $FAILED failed" + exit 1 + ;; + esac + + local capture_dir raw_capture_file expect_log_file combined_capture_file plain_capture_file + capture_dir="$(make_capture_dir)" + raw_capture_file="${capture_dir}/${PREFIX}.raw.log" + expect_log_file="${capture_dir}/${PREFIX}.expect.log" + combined_capture_file="${capture_dir}/${PREFIX}.combined.log" + plain_capture_file="${capture_dir}/${PREFIX}.sanitized.log" + SENSITIVE_CAPTURE_FILES=("$raw_capture_file" "$expect_log_file" "$combined_capture_file") + : >"$raw_capture_file" + : >"$expect_log_file" + + info "Running Deep Agents Code TUI startup check in sandbox: $SANDBOX_NAME" + info "Capture directory: $capture_dir" + + set +e + run_tui_expect "$raw_capture_file" >"$expect_log_file" 2>&1 + local expect_rc + expect_rc=$? + set -e + + cat "$raw_capture_file" "$expect_log_file" >"$combined_capture_file" + strip_terminal_control_sequences <"$combined_capture_file" >"$plain_capture_file" + local secret_detected=0 + if contains_secret <"$plain_capture_file"; then + secret_detected=1 + if ! redact_secrets_in_file "$plain_capture_file"; then + : + fi + if [ -e "$plain_capture_file" ] && contains_secret <"$plain_capture_file"; then + fail_test "secret-shaped value remained after redacting sanitized TUI capture" + fi + fi + cleanup_sensitive_captures + + if [ "$expect_rc" -eq 0 ]; then + pass "finite expect harness reached startup and observed exit" + else + fail_test "finite expect harness exited ${expect_rc}" + fi + + if grep -q "NEMOCLAW_TUI_READY" "$plain_capture_file" && is_tui_ready_capture <"$plain_capture_file"; then + pass "dcode TUI rendered a usable startup prompt signature" + else + fail_test "dcode TUI prompt-ready marker missing from capture" + fi + + assert_clean_exit_code "$plain_capture_file" + + if [ "$secret_detected" -eq 1 ]; then + fail_test "secret-shaped value found in sanitized TUI capture" + else + pass "sanitized TUI capture does not contain secret-shaped values" + fi + + printf '%s\n' "${PREFIX}: $PASSED passed, $FAILED failed" + info "sanitized capture: ${plain_capture_file}" + [ "$FAILED" -eq 0 ] || exit 1 +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/test/langchain-deepagents-code-image.test.ts b/test/langchain-deepagents-code-image.test.ts index 9f3168b6b71..605256e24eb 100644 --- a/test/langchain-deepagents-code-image.test.ts +++ b/test/langchain-deepagents-code-image.test.ts @@ -24,6 +24,14 @@ const headlessCheckPath = path.join( "checks", "07-deepagents-code-headless-inference.sh", ); +const tuiStartupCheckPath = path.join( + process.cwd(), + "test", + "e2e", + "e2e-cloud-experimental", + "checks", + "10-deepagents-code-tui-startup.sh", +); const DCODE_CANONICAL_PATH = "/usr/local/bin:/opt/venv/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin"; @@ -109,12 +117,19 @@ function makeStartScriptFixture(tempDir: string): { } { const envFile = path.join(tempDir, "proxy-env.sh"); const scriptPath = path.join(tempDir, "start.sh"); - const fixture = readAgentFile("start.sh") + const original = readAgentFile("start.sh"); + expect(original).toContain("local target=/tmp/nemoclaw-proxy-env.sh"); + expect(original).toContain('tmp="$(mktemp /tmp/nemoclaw-proxy-env.XXXXXX)"'); + const fixture = original .replace("local target=/tmp/nemoclaw-proxy-env.sh", `local target="${envFile}"`) .replace( 'tmp="$(mktemp /tmp/nemoclaw-proxy-env.XXXXXX)"', `tmp="$(mktemp "${tempDir}/nemoclaw-proxy-env.XXXXXX")"`, ); + expect(fixture).toContain(`local target="${envFile}"`); + expect(fixture).toContain(`tmp="$(mktemp "${tempDir}/nemoclaw-proxy-env.XXXXXX")"`); + expect(fixture).not.toContain("local target=/tmp/nemoclaw-proxy-env.sh"); + expect(fixture).not.toContain('tmp="$(mktemp /tmp/nemoclaw-proxy-env.XXXXXX)"'); fs.writeFileSync(scriptPath, fixture, "utf8"); fs.chmodSync(scriptPath, 0o755); return { envFile, scriptPath }; @@ -365,6 +380,7 @@ describe("LangChain Deep Agents Code image contracts", () => { ), "utf8", ); + const tuiStartupCheck = fs.readFileSync(tuiStartupCheckPath, "utf8"); expect(landlockCheck).toContain("test -d /sandbox/.deepagents && command -v dcode"); expect(landlockCheck).toContain("touch /sandbox/.deepagents/deepagents-landlock-test"); @@ -429,6 +445,31 @@ describe("LangChain Deep Agents Code image contracts", () => { expect(secretBoundaryCheck).toContain("assert_no_rejected_interval_audit_logs"); expect(secretBoundaryCheck).toContain("assert_no_rejected_interval_network_logs"); expect(secretBoundaryCheck).toContain("sha256sum ${DEEPAGENTS_ENV_FILE@Q}"); + expect(tuiStartupCheck).toContain("Case: Deep Agents Code interactive TUI startup"); + expect(tuiStartupCheck).toContain("test -d /sandbox/.deepagents && command -v dcode"); + expect(tuiStartupCheck).toContain("expect <<'EXPECT'"); + expect(tuiStartupCheck).toContain( + "set cmd [list openshell sandbox exec --name $sandbox --tty -- sh -lc", + ); + expect(tuiStartupCheck).toContain("spawn {*}$cmd"); + expect(tuiStartupCheck).not.toContain("-nocase -re {(deep agents|"); + expect(tuiStartupCheck).toContain("NEMOCLAW_DCODE_PROBE:deepagents"); + expect(tuiStartupCheck).toContain("NEMOCLAW_DCODE_PROBE:other"); + expect(tuiStartupCheck).toContain("unable to probe sandbox"); + expect(tuiStartupCheck).toContain("unexpected sandbox probe output"); + expect(tuiStartupCheck).toContain("cd /sandbox; dcode"); + expect(tuiStartupCheck).toContain("NEMOCLAW_TUI_READY"); + expect(tuiStartupCheck).toContain("NEMOCLAW_TUI_EXIT_CAPTURED"); + expect(tuiStartupCheck).toContain("DEEPAGENTS_TUI_TIMEOUT must be a positive integer"); + expect(tuiStartupCheck).toContain("strip_terminal_control_sequences"); + expect(tuiStartupCheck).toContain("is_tui_ready_capture"); + expect(tuiStartupCheck).toContain("redact_secrets_in_file"); + expect(tuiStartupCheck).toContain("trap cleanup_sensitive_captures EXIT"); + expect(tuiStartupCheck).toContain("cleanup_sensitive_captures"); + expect(tuiStartupCheck).toContain("${PREFIX}.sanitized.log"); + expect(tuiStartupCheck).toContain("secret-shaped value found in sanitized TUI capture"); + expect(tuiStartupCheck).toContain("nvapi-"); + expect(tuiStartupCheck).toContain("sk-"); const tavilyOptInCheck = fs.readFileSync( path.join( process.cwd(), @@ -450,6 +491,7 @@ describe("LangChain Deep Agents Code image contracts", () => { "test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh", "test/e2e/e2e-cloud-experimental/checks/08-deepagents-code-secret-boundary.sh", "test/e2e/e2e-cloud-experimental/checks/09-deepagents-code-tavily-opt-in.sh", + "test/e2e/e2e-cloud-experimental/checks/10-deepagents-code-tui-startup.sh", ]); });