test(vitest): improve failure and developer feedback

AGENTS.md

@@ -50,6 +50,10 @@ Package-specific guides:
 | Run all tests for broad changes | `npm test` |
 | Render behavior-oriented test tree | `npm run test:spec` |
 | Run fast source tests | `npm run test:fast` |
+| Run tests affected by current changes | `npm run test:changed` |
+| Watch focused source tests | `npm run test:watch` |
+| Shuffle focused tests without coverage | `npm run test:shuffle` |
+| Diagnose async leaks or shutdown hangs | `npm run test:diagnose:leaks` |
 | Run integration tests | `npm run test:integration` |
 | Run package contracts | `npm run test:package` |
 | Run E2E support tests | `npx vitest run --project e2e-support` |
@@ -95,6 +99,8 @@ When writing tests:
 - Import CLI source from ordinary tests. Put genuine compiled-artifact assertions under `test/package-contract/`.
 - Keep project globs disjoint; `npm run test:projects:check` derives membership from Vitest and rejects overlap.
 - Deterministic projects clear mock calls, restore `vi.spyOn`, and undo `vi.stubEnv` and `vi.stubGlobal` before each test. Create those spies and stubs in `beforeEach` or the test body unless a documented import-time stub must run before module evaluation. Restore direct environment or global mutations yourself, and reset mock implementations explicitly when needed. Live E2E and automatic `mockReset` are intentionally excluded.
+- Use `npm run test:changed` or `npm run test:watch` for focused CLI, plugin, and E2E-support feedback. Add only concrete opaque-input mappings to `test/helpers/vitest-watch-triggers.ts` when the import graph cannot see a YAML, Python, shell, generated, or workflow dependency.
+- Use `npm run test:shuffle -- --sequence.seed=<seed>` to replay a printed test-order seed. Use `npm run test:diagnose:leaks` for async-resource or shutdown-hang diagnostics; both commands keep coverage disabled, and leak diagnostics can accompany exit code 0 when assertions pass.
 - Write behavior-oriented titles, put local issue references in a final `(#1234)` suffix, and use `npm run test:spec` for the hierarchical specification view.
 - Mock external dependencies; don't call real NVIDIA APIs in unit tests
 - E2E tests run on ephemeral Brev cloud instances

CONTRIBUTING.md

@@ -201,6 +201,10 @@ These are the primary npm scripts for day-to-day development:
 | `npm test` | Build package artifacts and run every non-live Vitest project for broad changes |
 | `npm run test:spec` | Run every non-live test with hierarchical behavior-oriented output |
 | `npm run test:fast` | Clean `dist/` and run source CLI, plugin, and E2E-support tests |
+| `npm run test:changed` | Run tests affected by staged, unstaged, or untracked changes in the CLI, plugin, and E2E-support projects |
+| `npm run test:watch` | Watch the CLI, plugin, and E2E-support projects and rerun affected tests |
+| `npm run test:shuffle` | Shuffle test order in the focused source projects without collecting coverage |
+| `npm run test:diagnose:leaks` | Report async-resource leaks and diagnose a Vitest process that hangs during shutdown |
 | `npm run test:integration` | Clean-build the CLI and run root integration and installer tests |
 | `npm run test:package` | Clean-build CLI/plugin artifacts and run compiled-package contracts |
 | `npm run test:live-e2e` | Opt into live E2E scenarios (mutates real external state) |
@@ -222,6 +226,33 @@ npx vitest run --project e2e-support
 This project is fast and does not run live targets. Live E2E remains opt-in through
 `npm run test:live-e2e` or the applicable GitHub Actions workflow.
 
+### Focused Vitest Feedback
+
+Use `npm run test:changed` for the staged, unstaged, and untracked changes in the current checkout,
+or keep `npm run test:watch` running while editing. Both commands select only the source-backed
+`cli`, `plugin`, and `e2e-support` projects. Watch mode also maps the repository's current opaque
+YAML, Python, shell, generated, and workflow inputs to the concrete contract tests that read or
+execute them outside Vitest's import graph. Add a narrow mapping in
+`test/helpers/vitest-watch-triggers.ts` when a new opaque input needs the same treatment.
+
+Use `npm run test:shuffle` to expose order dependencies in those focused projects. The command
+shuffles tests within files and leaves coverage disabled. Vitest prints the chosen seed at the
+start of the run. Replay that order by appending the printed value:
+
+```bash
+npm run test:shuffle -- --sequence.seed=6692
+```
+
+Use `npm run test:diagnose:leaks` when a test file leaves an async resource active or Vitest hangs
+during shutdown. It enables Vitest's async-leak detector and hanging-process reporter while
+keeping coverage disabled. This is a diagnostic command: inspect its leak output even when all
+assertions pass, because reported async leaks do not independently change a successful test exit
+code.
+
+Vitest chooses the environment-appropriate reporter for ordinary local runs. In CI, console logs
+from passing tests stay hidden while logs attached to failures are replayed; GitHub Actions still
+receives test annotations.
+
 ### Test State Isolation
 
 The `cli`, `integration`, `installer-integration`, `package-contract`, `plugin`, and `e2e-support`

package.json

@@ -26,6 +26,10 @@
     "test": "npm run clean:cli && npm --prefix nemoclaw run clean && npm run build:cli && npm --prefix nemoclaw run build && vitest run --project cli --project integration --project installer-integration --project package-contract --project plugin --project e2e-support",
     "test:spec": "npm test -- --reporter=tree",
     "test:fast": "npm run clean:cli && vitest run --project cli --project plugin --project e2e-support",
+    "test:changed": "vitest run --changed --project cli --project plugin --project e2e-support",
+    "test:watch": "vitest watch --project cli --project plugin --project e2e-support",
+    "test:shuffle": "vitest run --project cli --project plugin --project e2e-support --sequence.shuffle.tests --coverage=false",
+    "test:diagnose:leaks": "vitest run --project cli --project plugin --project e2e-support --detectAsyncLeaks --coverage=false --reporter=default --reporter=hanging-process",
     "test:integration": "npm run clean:cli && npm run build:cli && vitest run --project integration --project installer-integration",
     "test:package": "npm run clean:cli && npm --prefix nemoclaw run clean && npm run build:cli && npm --prefix nemoclaw run build && vitest run --project package-contract",
     "test:coverage:cli": "npm run clean:cli && npm run build:cli && tsx scripts/check-dist-sourcemaps.ts dist && vitest run --project cli --project integration --coverage --coverage.reporter=text-summary --coverage.reporter=json-summary --coverage.reportsDirectory=coverage/cli --coverage.include=\"bin/**/*.js\" --coverage.include=\"src/**/*.ts\" --coverage.exclude=\"test/**/*.js\" --coverage.exclude=\"test/**/*.ts\" && tsx scripts/check-coverage-ratchet.ts coverage/cli/coverage-summary.json ci/coverage-threshold-cli.json \"CLI coverage\"",

test/helpers/vitest-feedback.ts

@@ -0,0 +1,15 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+export type VitestFeedback = {
+  isCi: boolean;
+  silent: false | "passed-only";
+};
+
+export function resolveVitestFeedback(
+  environment: NodeJS.ProcessEnv = process.env,
+): VitestFeedback {
+  const isCi =
+    environment.GITHUB_ACTIONS === "true" || environment.CI === "true" || environment.CI === "1";
+  return { isCi, silent: isCi ? "passed-only" : false };
+}

test/helpers/vitest-npm-script.ts

@@ -0,0 +1,47 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { spawnSync } from "node:child_process";
+import fs from "node:fs";
+import path from "node:path";
+
+import type { OwnedTestResources } from "./owned-test-resources";
+
+export function runVitestNpmScript(
+  resources: OwnedTestResources,
+  script: string,
+  extraArguments: string[] = [],
+): string {
+  const npmCli = process.env.npm_execpath;
+  if (!npmCli) throw new Error("npm_execpath is required to exercise npm script dispatch");
+  const fixtureRoot = resources.temporaryDirectory("nemoclaw-vitest-feedback-");
+  const fakeBin = path.join(fixtureRoot, "bin");
+  const commandLog = path.join(fixtureRoot, "commands.log");
+  const scriptShell = path.join(fixtureRoot, "script-shell");
+  fs.mkdirSync(fakeBin);
+  fs.writeFileSync(
+    path.join(fakeBin, "vitest"),
+    '#!/bin/sh\nprintf \'vitest %s\\n\' "$*" > "$COMMAND_LOG"\n',
+    { mode: 0o755 },
+  );
+  fs.writeFileSync(
+    scriptShell,
+    `#!/bin/sh\nPATH="$FAKE_BIN:${path.dirname(process.execPath)}:/usr/bin:/bin"\nexport PATH\nexec /bin/sh "$@"\n`,
+    { mode: 0o755 },
+  );
+
+  const result = spawnSync(process.execPath, [npmCli, "run", script, ...extraArguments], {
+    cwd: process.cwd(),
+    encoding: "utf8",
+    env: {
+      ...process.env,
+      COMMAND_LOG: commandLog,
+      FAKE_BIN: fakeBin,
+      npm_config_script_shell: scriptShell,
+    },
+  });
+  if (result.status !== 0) {
+    throw new Error(result.stderr || result.stdout || `${script} exited ${result.status}`);
+  }
+  return fs.readFileSync(commandLog, "utf8").trim();
+}

test/helpers/vitest-watch-triggers.ts

@@ -0,0 +1,103 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+export type VitestWatchTriggerPattern = {
+  pattern: RegExp;
+  testsToRun: (file: string, match: RegExpMatchArray) => string[];
+};
+
+const E2E_WORKFLOW_CONTRACTS = [
+  "test/e2e/support/channels-add-remove-workflow-boundary.test.ts",
+  "test/e2e/support/dcode-profile-import-gate-workflow-boundary.test.ts",
+  "test/e2e/support/dockerhub-auth-workflow-boundary.test.ts",
+  "test/e2e/support/e2e-host-dependency-workflow-boundary.test.ts",
+  "test/e2e/support/e2e-operations-workflow-boundary.test.ts",
+  "test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts",
+  "test/e2e/support/e2e-workflow.test.ts",
+  "test/e2e/support/e2e-workflow-trace.test.ts",
+  "test/e2e/support/gateway-guard-workflow-boundary.test.ts",
+  "test/e2e/support/hermes-dashboard-workflow-boundary.test.ts",
+  "test/e2e/support/hermes-workflow-boundary.test.ts",
+  "test/e2e/support/inference-switch-workflow-boundary.test.ts",
+  "test/e2e/support/jetson-workflow-boundary.test.ts",
+  "test/e2e/support/mcp-workflow-boundary.test.ts",
+  "test/e2e/support/mcp-workflow-compatibility.test.ts",
+  "test/e2e/support/openclaw-discord-workflow-boundary.test.ts",
+  "test/e2e/support/openclaw-plugin-runtime-exdev-workflow-boundary.test.ts",
+  "test/e2e/support/openclaw-slack-workflow-boundary.test.ts",
+  "test/e2e/support/openshell-gateway-auth-contract-workflow-boundary.test.ts",
+  "test/e2e/support/openshell-gateway-upgrade-workflow-boundary.test.ts",
+  "test/e2e/support/overlayfs-autofix-workflow-boundary.test.ts",
+  "test/e2e/support/prepare-e2e-workflow-boundary.test.ts",
+  "test/e2e/support/rlimit-connect-workflow-boundary.test.ts",
+  "test/e2e/support/sandbox-images-workflow-boundary.test.ts",
+  "test/e2e/support/sandbox-operations-workflow-boundary.test.ts",
+  "test/e2e/support/security-posture-workflow-boundary.test.ts",
+  "test/e2e/support/shared-e2e-workflow-boundary.test.ts",
+  "test/e2e/support/spark-install-workflow-boundary.test.ts",
+  "test/e2e/support/tunnel-lifecycle-workflow-boundary.test.ts",
+  "test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts",
+  "test/e2e/support/workflow-plan.test.ts",
+] as const;
+
+function runTests(...tests: string[]): () => string[] {
+  return () => [...tests];
+}
+
+export const vitestWatchTriggerPatterns: VitestWatchTriggerPattern[] = [
+  {
+    pattern: /(?:^|\/)agents\/hermes\/policy-additions\.yaml$/,
+    testsToRun: runTests(
+      "src/lib/onboard/initial-policy-real-policy.test.ts",
+      "src/lib/onboard/initial-policy.test.ts",
+    ),
+  },
+  {
+    pattern: /(?:^|\/)src\/lib\/messaging\/channels\/[^/]+\/policy\/(?:hermes|openclaw)\.yaml$/,
+    testsToRun: runTests("src/lib/messaging/channels/policy.test.ts"),
+  },
+  {
+    pattern: /(?:^|\/)nemoclaw-blueprint\/policies\/presets\/local-inference\.yaml$/,
+    testsToRun: runTests(
+      "src/lib/onboard/inference-providers/compatible-endpoint-gateway-route.test.ts",
+    ),
+  },
+  {
+    pattern: /(?:^|\/)agents\/hermes\/(?:mcp-config-transaction|runtime-config-guard)\.py$/,
+    testsToRun: runTests("src/lib/actions/sandbox/gateway-restart-hermes-drift.test.ts"),
+  },
+  {
+    pattern: /(?:^|\/)test\/e2e\/lib\/ci-compatible-inference\.sh$/,
+    testsToRun: runTests("test/e2e/support/hosted-inference.test.ts"),
+  },
+  {
+    pattern: /(?:^|\/)scripts\/e2e\/sanitize-trace-timing\.py$/,
+    testsToRun: runTests(
+      "test/e2e/support/e2e-scorecard.test.ts",
+      "test/e2e/support/sanitize-trace-timing.test.ts",
+    ),
+  },
+  {
+    pattern: /(?:^|\/)test\/e2e\/manifests\/[^/]+\.yaml$/,
+    testsToRun: runTests("test/e2e/support/e2e-manifests.test.ts"),
+  },
+  {
+    pattern: /(?:^|\/)test\/e2e\/docs\/parity-inventory\.generated\.json$/,
+    testsToRun: runTests("test/e2e/support/e2e-migration-policy.test.ts"),
+  },
+  {
+    pattern: /(?:^|\/)\.github\/workflows\/e2e\.yaml$/,
+    testsToRun: runTests(...E2E_WORKFLOW_CONTRACTS),
+  },
+];
+
+export function resolveVitestWatchTests(file: string): string[] {
+  const normalized = file.replaceAll("\\", "/");
+  const tests = new Set<string>();
+  for (const trigger of vitestWatchTriggerPatterns) {
+    const match = trigger.pattern.exec(normalized);
+    if (!match) continue;
+    for (const test of trigger.testsToRun(normalized, match)) tests.add(test);
+  }
+  return [...tests];
+}

test/vitest-developer-feedback.test.ts

@@ -0,0 +1,60 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect } from "vitest";
+
+import rootVitestConfig from "../vitest.config";
+import { test as it } from "./helpers/owned-test-resources";
+import { resolveVitestFeedback } from "./helpers/vitest-feedback";
+import { runVitestNpmScript } from "./helpers/vitest-npm-script";
+
+type RootTestOptions = {
+  reporters?: unknown;
+  silent?: boolean | "passed-only";
+};
+
+const focusedProjects = "--project cli --project plugin --project e2e-support";
+
+describe("Vitest developer feedback", () => {
+  it("lets Vitest select reporters and preserves failed-test logs in CI (#6692)", () => {
+    const testOptions = rootVitestConfig.test as RootTestOptions;
+
+    expect(testOptions).not.toHaveProperty("reporters");
+    expect(resolveVitestFeedback({})).toEqual({ isCi: false, silent: false });
+    expect(resolveVitestFeedback({ CI: "0" })).toEqual({ isCi: false, silent: false });
+    expect(resolveVitestFeedback({ CI: "1" })).toEqual({
+      isCi: true,
+      silent: "passed-only",
+    });
+    expect(resolveVitestFeedback({ CI: "true" })).toEqual({
+      isCi: true,
+      silent: "passed-only",
+    });
+    expect(resolveVitestFeedback({ GITHUB_ACTIONS: "true" })).toEqual({
+      isCi: true,
+      silent: "passed-only",
+    });
+    expect(testOptions.silent).toBe(resolveVitestFeedback().silent);
+  });
+
+  it("runs changed and watch feedback on the focused source projects (#6692)", ({ resources }) => {
+    expect(runVitestNpmScript(resources, "test:changed")).toBe(
+      `vitest run --changed ${focusedProjects}`,
+    );
+    expect(runVitestNpmScript(resources, "test:watch")).toBe(`vitest watch ${focusedProjects}`);
+  });
+
+  it("passes a reproducible seed to test-only shuffle diagnostics outside coverage (#6692)", ({
+    resources,
+  }) => {
+    expect(runVitestNpmScript(resources, "test:shuffle", ["--", "--sequence.seed=6692"])).toBe(
+      `vitest run ${focusedProjects} --sequence.shuffle.tests --coverage=false --sequence.seed=6692`,
+    );
+  });
+
+  it("runs opt-in async-leak diagnostics outside coverage (#6692)", ({ resources }) => {
+    expect(runVitestNpmScript(resources, "test:diagnose:leaks")).toBe(
+      `vitest run ${focusedProjects} --detectAsyncLeaks --coverage=false --reporter=default --reporter=hanging-process`,
+    );
+  });
+});