Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
287 changes: 209 additions & 78 deletions .github/workflows/e2e.yaml

Large diffs are not rendered by default.

27 changes: 27 additions & 0 deletions .github/workflows/main.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,30 @@ jobs:
- name: Run plugin coverage
uses: ./.github/actions/ci-plugin-coverage

e2e-support:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Checkout
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
persist-credentials: false

- name: Setup Node.js
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
with:
node-version: "22"
cache: npm

- name: Install E2E support dependencies
run: npm ci --ignore-scripts

- name: Build CLI artifacts for E2E support
run: npm run build:cli

- name: Run E2E support tests
run: npx vitest run --project e2e-support

test-e2e-ollama-proxy:
runs-on: ubuntu-latest
timeout-minutes: 5
Expand All @@ -176,6 +200,7 @@ jobs:
- real-openclaw-dist-harness
- cli-tests
- plugin-tests
- e2e-support
- test-e2e-ollama-proxy
if: always()
runs-on: ubuntu-latest
Expand All @@ -189,6 +214,7 @@ jobs:
REAL_OPENCLAW_DIST_HARNESS_RESULT: ${{ needs['real-openclaw-dist-harness'].result }}
CLI_TESTS_RESULT: ${{ needs['cli-tests'].result }}
PLUGIN_TESTS_RESULT: ${{ needs['plugin-tests'].result }}
E2E_SUPPORT_RESULT: ${{ needs['e2e-support'].result }}
E2E_PROXY_RESULT: ${{ needs['test-e2e-ollama-proxy'].result }}
run: |
set -euo pipefail
Expand All @@ -208,6 +234,7 @@ jobs:
require_success "real-openclaw-dist-harness" "$REAL_OPENCLAW_DIST_HARNESS_RESULT"
require_success "cli-tests" "$CLI_TESTS_RESULT"
require_success "plugin-tests" "$PLUGIN_TESTS_RESULT"
require_success "e2e-support" "$E2E_SUPPORT_RESULT"
require_success "test-e2e-ollama-proxy" "$E2E_PROXY_RESULT"

sandbox-images-and-e2e:
Expand Down
134 changes: 134 additions & 0 deletions .github/workflows/post-merge-e2e-risk-gate-shadow.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

name: E2E / Post-merge Risk Gate Shadow

on:
push:
branches: [main]

permissions:
# Required to dispatch the separate E2E workflow; no other Actions mutation is performed.
actions: write
checks: write
contents: read

concurrency:
group: e2e-post-merge-risk-gate-${{ github.sha }}
cancel-in-progress: false

jobs:
shadow:
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' }}
runs-on: ubuntu-latest
timeout-minutes: 120
steps:
- name: Checkout trusted controller
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ github.event.after }}
fetch-depth: 0
persist-credentials: false

- name: Setup Node
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0
with:
node-version: "22"
cache: npm

- name: Install trusted controller dependencies
run: npm ci --ignore-scripts

- id: workspace
name: Create private controller workspace
shell: bash
run: |
set -euo pipefail
work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-e2e-risk-gate.XXXXXX")"
chmod 700 "$work_dir"
printf 'work_dir=%s\n' "$work_dir" >> "$GITHUB_OUTPUT"

- id: start
name: Build plan and dispatch exact-commit E2E
continue-on-error: true
env:
GITHUB_TOKEN: ${{ github.token }}
run: >-
node --experimental-strip-types tools/e2e-advisor/post-merge-risk-gate.mts
--mode start
--base "${{ github.event.before }}"
--commit "${{ github.event.after }}"
--work-dir "${{ steps.workspace.outputs.work_dir }}"

- name: Upload post-merge risk plan
if: ${{ always() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: post-merge-risk-plan-${{ github.event.after }}
path: ${{ steps.workspace.outputs.work_dir }}/post-merge-risk-plan.json
if-no-files-found: ignore
retention-days: 14

- name: Close shadow check after controller startup failure
if: ${{ always() && steps.start.outputs.check_id != '' && steps.start.outputs.dispatched != 'true' && steps.start.outputs.finalized != 'true' }}
env:
GITHUB_TOKEN: ${{ github.token }}
run: >-
node --experimental-strip-types tools/e2e-advisor/post-merge-risk-gate.mts
--mode abandon
--check-id "${{ steps.start.outputs.check_id }}"

- name: Propagate controller startup failure
if: ${{ steps.start.outcome == 'failure' }}
run: exit 1

- name: Wait for correlated E2E run
if: ${{ steps.start.outputs.dispatched == 'true' }}
continue-on-error: true
env:
GH_TOKEN: ${{ github.token }}
RUN_ID: ${{ steps.start.outputs.run_id }}
run: >-
timeout --signal=TERM --kill-after=30s 105m
gh run watch "$RUN_ID" --repo "$GITHUB_REPOSITORY" --exit-status

- name: Download correlated E2E evidence
if: ${{ always() && steps.start.outputs.dispatched == 'true' }}
continue-on-error: true
env:
GH_TOKEN: ${{ github.token }}
RUN_ID: ${{ steps.start.outputs.run_id }}
run: >-
gh run download "$RUN_ID" --repo "$GITHUB_REPOSITORY"
--dir "${{ steps.workspace.outputs.work_dir }}/evidence"

- id: finish
name: Complete exact-commit shadow check
if: ${{ always() && steps.start.outputs.dispatched == 'true' }}
continue-on-error: true
env:
GITHUB_TOKEN: ${{ github.token }}
run: >-
node --experimental-strip-types tools/e2e-advisor/post-merge-risk-gate.mts
--mode finish
--work-dir "${{ steps.workspace.outputs.work_dir }}"
--state-hash "${{ steps.start.outputs.state_hash }}"
--check-id "${{ steps.start.outputs.check_id }}"
--run-id "${{ steps.start.outputs.run_id }}"

- name: Close shadow check after completion failure
if: ${{ always() && steps.start.outputs.check_id != '' && steps.start.outputs.dispatched == 'true' && steps.finish.outcome == 'failure' }}
env:
GITHUB_TOKEN: ${{ github.token }}
run: >-
node --experimental-strip-types tools/e2e-advisor/post-merge-risk-gate.mts
--mode abandon
--check-id "${{ steps.start.outputs.check_id }}"

- name: Propagate shadow completion failure
if: ${{ steps.finish.outcome == 'failure' }}
run: exit 1

- name: Remove private controller workspace
if: ${{ always() && steps.workspace.outputs.work_dir != '' }}
run: rm -rf -- "${{ steps.workspace.outputs.work_dir }}"
30 changes: 30 additions & 0 deletions .github/workflows/pr.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,32 @@ jobs:
- name: Run plugin coverage
uses: ./.trusted-ci-actions/.github/actions/ci-plugin-coverage

e2e-support:
needs: changes
if: needs.changes.outputs.code == 'true'
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Checkout
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
persist-credentials: false

- name: Setup Node.js
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
with:
node-version: "22"
cache: npm

- name: Install E2E support dependencies
run: npm ci --ignore-scripts

- name: Build CLI artifacts for E2E support
run: npm run build:cli

- name: Run E2E support tests
run: npx vitest run --project e2e-support

test-e2e-ollama-proxy:
needs: changes
if: needs.changes.outputs.code == 'true'
Expand Down Expand Up @@ -335,6 +361,7 @@ jobs:
- installer-integration
- cli-tests
- plugin-tests
- e2e-support
- test-e2e-ollama-proxy
if: always()
runs-on: ubuntu-latest
Expand All @@ -350,6 +377,7 @@ jobs:
INSTALLER_INTEGRATION_RESULT: ${{ needs['installer-integration'].result }}
CLI_TESTS_RESULT: ${{ needs['cli-tests'].result }}
PLUGIN_TESTS_RESULT: ${{ needs['plugin-tests'].result }}
E2E_SUPPORT_RESULT: ${{ needs['e2e-support'].result }}
E2E_PROXY_RESULT: ${{ needs['test-e2e-ollama-proxy'].result }}
run: |
set -euo pipefail
Expand Down Expand Up @@ -384,6 +412,7 @@ jobs:
require_success "installer-integration" "$INSTALLER_INTEGRATION_RESULT"
require_success "cli-tests" "$CLI_TESTS_RESULT"
require_success "plugin-tests" "$PLUGIN_TESTS_RESULT"
require_success "e2e-support" "$E2E_SUPPORT_RESULT"
require_success "test-e2e-ollama-proxy" "$E2E_PROXY_RESULT"
else
require_success "docs-only-checks" "$DOCS_ONLY_RESULT"
Expand All @@ -392,6 +421,7 @@ jobs:
allow_success_or_skipped "installer-integration" "$INSTALLER_INTEGRATION_RESULT"
allow_success_or_skipped "cli-tests" "$CLI_TESTS_RESULT"
allow_success_or_skipped "plugin-tests" "$PLUGIN_TESTS_RESULT"
allow_success_or_skipped "e2e-support" "$E2E_SUPPORT_RESULT"
allow_success_or_skipped "test-e2e-ollama-proxy" "$E2E_PROXY_RESULT"
fi

Expand Down
4 changes: 3 additions & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ Package-specific guides:
| Run fast source tests | `npm run test:fast` |
| Run integration tests | `npm run test:integration` |
| Run package contracts | `npm run test:package` |
| Run E2E support tests | `npx vitest run --project e2e-support` |
| Run live E2E targets | `npm run test:live-e2e` |
| Run plugin tests | `cd nemoclaw && npm test` |
| Run repo-wide pre-commit and coverage checks | `npm run check` |
Expand Down Expand Up @@ -81,7 +82,8 @@ Tests are organized into disjoint Vitest projects defined in `vitest.config.ts`:
3. **`installer-integration`** — installer tests that spawn real `install.sh` processes
4. **`package-contract`** — `test/package-contract/**/*.test.ts` — the only non-live lane that imports compiled CLI/plugin artifacts
5. **`plugin`** — `nemoclaw/src/**/*.test.ts` — plugin unit tests co-located with source
6. **`e2e-support`** — fast tests for the E2E fixture/support layer
6. **`e2e-support`** — fast tests for the E2E fixture/support layer; this project runs in the
aggregate checks for code-changing PRs and code-changing pushes to `main`
7. **`e2e-live`** — opt-in live targets that mutate real external state
8. **`e2e-branch-validation`** — opt-in validation on an ephemeral Brev instance

Expand Down
11 changes: 11 additions & 0 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,17 @@ These are the primary npm scripts for day-to-day development:
| `npm run docs:preview:watch` | Publish branch-based Fern previews when docs files change |
| `npm run docs:deps` | Print the pinned Fern CLI version used by docs commands |

The `e2e-support` Vitest project is part of the aggregate checks for code-changing pull requests
and code-changing pushes to `main`. Run it directly when you change E2E fixtures, support helpers,
registries, or workflow boundary checks:

```bash
npx vitest run --project e2e-support
```

This project is fast and does not run live targets. Live E2E remains opt-in through
`npm run test:live-e2e` or the applicable GitHub Actions workflow.

### Test Titles as Behavioral Documentation

Write `describe` and `it` titles so the Vitest tree reads as behavioral documentation. Start test
Expand Down
78 changes: 78 additions & 0 deletions test/e2e-private-file.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { execFileSync, spawnSync } from "node:child_process";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { pathToFileURL } from "node:url";

import { describe, expect, it } from "vitest";
import {
readPrivateRegularFile,
writePrivateRegularFile,
} from "../tools/e2e-advisor/private-file.ts";

describe("private E2E controller files", () => {
it("writes private regular files without following links or truncating hardlink targets", () => {
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-private-file-"));
const regular = path.join(directory, "regular.json");
const target = path.join(directory, "target.json");
const symlink = path.join(directory, "symlink.json");
const hardlink = path.join(directory, "hardlink.json");
try {
writePrivateRegularFile(regular, "regular\n");
expect(readPrivateRegularFile(regular, { maxBytes: 64 })).toBe("regular\n");
expect(fs.statSync(regular).mode & 0o777).toBe(0o600);
writePrivateRegularFile(regular, "updated\n");
expect(readPrivateRegularFile(regular, { maxBytes: 64 })).toBe("updated\n");

fs.writeFileSync(target, "protected\n");
fs.symlinkSync(target, symlink);
fs.linkSync(target, hardlink);

expect(() => writePrivateRegularFile(symlink, "replaced\n")).toThrow();
expect(() => writePrivateRegularFile(hardlink, "replaced\n")).toThrow(/private regular/u);
expect(fs.readFileSync(target, "utf8")).toBe("protected\n");
} finally {
fs.rmSync(directory, { recursive: true, force: true });
}
});

it.skipIf(process.platform === "win32")("rejects FIFO paths without blocking", () => {
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-private-fifo-"));
const fifo = path.join(directory, "state.json");
try {
execFileSync("mkfifo", [fifo]);
const moduleUrl = pathToFileURL(path.resolve("tools/e2e-advisor/private-file.ts")).href;
const read = spawnSync(
process.execPath,
[
"--experimental-strip-types",
"--input-type=module",
"--eval",
`import { readPrivateRegularFile } from ${JSON.stringify(moduleUrl)}; readPrivateRegularFile(${JSON.stringify(fifo)}, { maxBytes: 64 });`,
],
{ encoding: "utf8", timeout: 2_000 },
);
const write = spawnSync(
process.execPath,
[
"--experimental-strip-types",
"--input-type=module",
"--eval",
`import { writePrivateRegularFile } from ${JSON.stringify(moduleUrl)}; writePrivateRegularFile(${JSON.stringify(fifo)}, "replaced\\n");`,
],
{ encoding: "utf8", timeout: 2_000 },
);

expect(read.error).toBeUndefined();
expect(read.status).not.toBe(0);
expect(write.error).toBeUndefined();
expect(write.status).not.toBe(0);
expect(fs.lstatSync(fifo).isFIFO()).toBe(true);
} finally {
fs.rmSync(directory, { recursive: true, force: true });
}
});
});
Loading
Loading