Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@ All notable changes to `@testsprite/testsprite-cli` are documented here. The for

## [Unreleased]

### Added

- **JUnit XML report export for batch `--wait` runs.** `test run --all` and batch `test rerun` (`--all` or multiple test ids) accept `--report junit --report-file <path>` to write a CI-friendly XML sidecar after polling completes. `--output json` is unchanged; the report is written even when the batch exits non-zero. `--dry-run` writes a canned sample without network calls.

## [0.2.0] - 2026-06-29

### Added
Expand Down
21 changes: 21 additions & 0 deletions DOCUMENTATION.md
Original file line number Diff line number Diff line change
Expand Up @@ -344,8 +344,18 @@ testsprite test run test_xxxxxxxx --target-url https://staging.example.com \

# Dry-run prints a canned queued response (no network, no credentials)
testsprite test run test_xxxxxxxx --dry-run --output json

# Batch BE run with JUnit XML for CI (sidecar; --output json unchanged)
testsprite test run --all --project proj_xxxxxxxx --wait \
--report junit --report-file ./results.xml --output json
Comment thread
coderabbitai[bot] marked this conversation as resolved.

# Optional custom suite name (default: testsprite:<projectId>)
testsprite test run --all --project proj_xxxxxxxx --wait \
--report junit --report-file ./results.xml --report-suite-name my-ci-suite --output json
```

Batch `--report` flags apply only to `test run --all --wait` (and batch `test rerun --wait`). `--report junit --report-file <path>` writes a JUnit XML sidecar after polling completes (atomic write); `--output json` is unchanged. Optional `--report-suite-name <name>` overrides the default `testsprite:<projectId>` suite name.

`--target-url` must be a publicly reachable URL — the CLI pre-flights it against local addresses (`localhost`, `127.x`, `::1`, `0.0.0.0`, `169.254.x`, RFC1918) and the backend resolves it via DNS. For testing against localhost, use the [TestSprite MCP plugin](https://www.testsprite.com/docs), which handles the local tunnel. The CLI auto-mints an idempotency key (printed to stderr under `--output json`, `--verbose`, or `--debug`); pass `--idempotency-key <uuid>` to control it explicitly.

#### `testsprite test rerun [test-id...]`
Expand All @@ -365,10 +375,20 @@ testsprite test rerun test_be_xxxx --skip-dependencies --output json
# Rerun every test in a project (batch)
testsprite test rerun --all --project proj_xxxxxxxx --wait --max-concurrency 4 --output json

# Batch rerun with JUnit XML for CI
testsprite test rerun --all --project proj_xxxxxxxx --wait \
--report junit --report-file ./results.xml --output json

# Optional custom suite name (default: testsprite:<projectId>)
testsprite test rerun --all --project proj_xxxxxxxx --wait \
--report junit --report-file ./results.xml --report-suite-name my-ci-suite --output json

# Several specific tests
testsprite test rerun test_aaaa test_bbbb --wait --output json
```

Batch `--report` flags apply only to batch `--wait` reruns (`--all` or multiple test ids). `--report junit --report-file <path>` writes a JUnit XML sidecar after polling completes (atomic write); `--output json` is unchanged. When `--project` is omitted, the CLI infers `projectId` from polled run rows for classname / default suite naming; if inference fails, pass `--project <id>` explicitly (required under `--dry-run`).

Flags:

- `--all` — rerun every test in the resolved project; requires `--project <id>`.
Expand All @@ -377,6 +397,7 @@ Flags:
- `--skip-dependencies` — backend only: rerun just the named test without expanding the producer/teardown closure.
- `--max-concurrency <n>` — with `--wait`, cap on in-flight polls during a batch rerun.
- `--idempotency-key <key>` — auto-minted when omitted (the minted key is printed to stderr under `--output json`, `--verbose`, or `--debug`).
- `--report junit --report-file <path>` — with batch `--wait`, write a JUnit XML sidecar after polling (atomic write). Optional `--report-suite-name <name>` overrides the default `testsprite:<projectId>` suite name. Requires `--wait`; not available on single-test reruns.

A batch rerun returns `accepted[]` (one `runId` per dispatched test) plus `deferred[]` for any test shed by the per-key run-rate limit; under `--wait`, a non-empty `deferred[]` exits 7 with a `nextAction` you can retry with a fresh idempotency key.

Expand Down
220 changes: 219 additions & 1 deletion src/commands/test.run.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* sleep injection is wired through `TestDeps.sleep` to avoid real delays.
*/

import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs';
import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { Command } from 'commander';
Expand Down Expand Up @@ -3624,3 +3624,221 @@ describe('dashboardUrl on run completion', () => {
).toBe(true);
});
});

describe('runTestRunAll — JUnit report export', () => {
const BATCH_FRESH_RESP: BatchRunFreshResponse = {
accepted: [
{ testId: 'test_be_01', runId: 'run_fresh_01', enqueuedAt: '2026-06-09T10:00:00.000Z' },
{ testId: 'test_be_02', runId: 'run_fresh_02', enqueuedAt: '2026-06-09T10:00:01.000Z' },
],
conflicts: [],
deferred: [],
skippedFrontend: [],
skippedIntegration: [],
};

function makeTerminalRun(runId: string, testId: string, status: string): RunResponse {
return {
runId,
testId,
projectId: 'project_be',
userId: 'user_1',
status: status as RunResponse['status'],
source: 'cli',
createdAt: '2026-06-09T10:00:00.000Z',
startedAt: '2026-06-09T10:00:01.000Z',
finishedAt: '2026-06-09T10:00:30.000Z',
codeVersion: 'v1',
targetUrl: 'https://api.example.com',
createdFrom: 'cli',
failedStepIndex: null,
failureKind: null,
error: null,
videoUrl: null,
stepSummary: {
total: 3,
completed: 3,
passedCount: status === 'passed' ? 3 : 0,
failedCount: 0,
},
};
}

it('--wait --report junit writes XML after polling', async () => {
const { credentialsPath } = makeCreds();
const dir = mkdtempSync(join(tmpdir(), 'junit-run-all-'));
const reportPath = join(dir, 'results.xml');
const fetchImpl = makeFetch((url, init) => {
if ((init.method ?? 'GET') === 'POST') return { body: BATCH_FRESH_RESP };
const runId = url.split('/runs/')[1]?.split('?')[0] ?? '';
if (runId === 'run_fresh_01')
return { body: makeTerminalRun('run_fresh_01', 'test_be_01', 'passed') };
if (runId === 'run_fresh_02')
return { body: makeTerminalRun('run_fresh_02', 'test_be_02', 'passed') };
return errorBody('NOT_FOUND');
});

await runTestRunAll(
{
profile: 'default',
output: 'json',
debug: false,
projectId: 'project_be',
wait: true,
timeoutSeconds: 60,
maxConcurrency: 5,
report: 'junit',
reportFile: reportPath,
},
{
credentialsPath,
fetchImpl,
stdout: () => undefined,
stderr: () => undefined,
sleep: instantSleep,
},
);

const xml = readFileSync(reportPath, 'utf8');
expect(xml).toContain('<testsuite name="testsprite:project_be"');
expect(xml).toContain('name="test_be_01"');
expect(xml).toContain('name="test_be_02"');
expect(xml).toContain('failures="0"');
});

it('--wait --report junit writes XML even when batch exits 1', async () => {
const { credentialsPath } = makeCreds();
const dir = mkdtempSync(join(tmpdir(), 'junit-run-fail-'));
const reportPath = join(dir, 'results.xml');
const fetchImpl = makeFetch((url, init) => {
if ((init.method ?? 'GET') === 'POST') return { body: BATCH_FRESH_RESP };
const runId = url.split('/runs/')[1]?.split('?')[0] ?? '';
if (runId === 'run_fresh_01')
return { body: makeTerminalRun('run_fresh_01', 'test_be_01', 'passed') };
if (runId === 'run_fresh_02')
return { body: makeTerminalRun('run_fresh_02', 'test_be_02', 'failed') };
return errorBody('NOT_FOUND');
});

await expect(
runTestRunAll(
{
profile: 'default',
output: 'json',
debug: false,
projectId: 'project_be',
wait: true,
timeoutSeconds: 60,
maxConcurrency: 5,
report: 'junit',
reportFile: reportPath,
},
{
credentialsPath,
fetchImpl,
stdout: () => undefined,
stderr: () => undefined,
sleep: instantSleep,
},
),
).rejects.toMatchObject({ exitCode: 1 });

const xml = readFileSync(reportPath, 'utf8');
expect(xml).toContain('failures="1"');
expect(xml).toContain('name="test_be_02"');
});

it('rejects --report without --wait', async () => {
await expect(
runTestRunAll(
{
profile: 'default',
output: 'json',
debug: false,
projectId: 'project_be',
wait: false,
timeoutSeconds: 60,
maxConcurrency: 5,
report: 'junit',
reportFile: './results.xml',
},
{},
),
).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 });
});

it('rejects --report-suite-name without --report', async () => {
await expect(
runTestRunAll(
{
profile: 'default',
output: 'json',
debug: false,
projectId: 'project_be',
wait: true,
timeoutSeconds: 60,
maxConcurrency: 5,
reportSuiteName: 'orphan-suite',
},
{},
),
).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 });
});

it('--dry-run --report junit writes canned sample XML', async () => {
const dir = mkdtempSync(join(tmpdir(), 'junit-run-dry-'));
const reportPath = join(dir, 'results.xml');

await runTestRunAll(
{
profile: 'default',
output: 'json',
debug: false,
dryRun: true,
projectId: 'project_be',
wait: true,
timeoutSeconds: 60,
maxConcurrency: 5,
report: 'junit',
reportFile: reportPath,
},
{
stdout: () => undefined,
stderr: () => undefined,
},
);

const xml = readFileSync(reportPath, 'utf8');
expect(xml).toContain('name="test_fresh_wave_01"');
expect(xml).toContain('failures="1"');
});

it('--dry-run --report junit --report-suite-name overrides canned suite name', async () => {
const dir = mkdtempSync(join(tmpdir(), 'junit-run-dry-suite-'));
const reportPath = join(dir, 'results.xml');

await runTestRunAll(
{
profile: 'default',
output: 'json',
debug: false,
dryRun: true,
projectId: 'project_be',
wait: true,
timeoutSeconds: 60,
maxConcurrency: 5,
report: 'junit',
reportFile: reportPath,
reportSuiteName: 'ci-checkout-suite',
},
{
stdout: () => undefined,
stderr: () => undefined,
},
);

const xml = readFileSync(reportPath, 'utf8');
expect(xml).toContain('<testsuite name="ci-checkout-suite"');
expect(xml).not.toContain('testsprite:project_be');
});
});
Loading
Loading