diff --git a/CHANGELOG.md b/CHANGELOG.md index 91e0d49..a4ab9b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to `@testsprite/testsprite-cli` are documented here. The for ## [Unreleased] +### Added + +- **JUnit XML report export for batch `--wait` runs.** `test run --all` and batch `test rerun` (`--all` or multiple test ids) accept `--report junit --report-file ` to write a CI-friendly XML sidecar after polling completes. `--output json` is unchanged; the report is written even when the batch exits non-zero. `--dry-run` writes a canned sample without network calls. + ## [0.2.0] - 2026-06-29 ### Added diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 700671b..6693980 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -344,8 +344,18 @@ testsprite test run test_xxxxxxxx --target-url https://staging.example.com \ # Dry-run prints a canned queued response (no network, no credentials) testsprite test run test_xxxxxxxx --dry-run --output json + +# Batch BE run with JUnit XML for CI (sidecar; --output json unchanged) +testsprite test run --all --project proj_xxxxxxxx --wait \ + --report junit --report-file ./results.xml --output json + +# Optional custom suite name (default: testsprite:) +testsprite test run --all --project proj_xxxxxxxx --wait \ + --report junit --report-file ./results.xml --report-suite-name my-ci-suite --output json ``` +Batch `--report` flags apply only to `test run --all --wait` (and batch `test rerun --wait`). `--report junit --report-file ` writes a JUnit XML sidecar after polling completes (atomic write); `--output json` is unchanged. Optional `--report-suite-name ` overrides the default `testsprite:` suite name. + `--target-url` must be a publicly reachable URL — the CLI pre-flights it against local addresses (`localhost`, `127.x`, `::1`, `0.0.0.0`, `169.254.x`, RFC1918) and the backend resolves it via DNS. For testing against localhost, use the [TestSprite MCP plugin](https://www.testsprite.com/docs), which handles the local tunnel. The CLI auto-mints an idempotency key (printed to stderr under `--output json`, `--verbose`, or `--debug`); pass `--idempotency-key ` to control it explicitly. #### `testsprite test rerun [test-id...]` @@ -365,10 +375,20 @@ testsprite test rerun test_be_xxxx --skip-dependencies --output json # Rerun every test in a project (batch) testsprite test rerun --all --project proj_xxxxxxxx --wait --max-concurrency 4 --output json +# Batch rerun with JUnit XML for CI +testsprite test rerun --all --project proj_xxxxxxxx --wait \ + --report junit --report-file ./results.xml --output json + +# Optional custom suite name (default: testsprite:) +testsprite test rerun --all --project proj_xxxxxxxx --wait \ + --report junit --report-file ./results.xml --report-suite-name my-ci-suite --output json + # Several specific tests testsprite test rerun test_aaaa test_bbbb --wait --output json ``` +Batch `--report` flags apply only to batch `--wait` reruns (`--all` or multiple test ids). `--report junit --report-file ` writes a JUnit XML sidecar after polling completes (atomic write); `--output json` is unchanged. When `--project` is omitted, the CLI infers `projectId` from polled run rows for classname / default suite naming; if inference fails, pass `--project ` explicitly (required under `--dry-run`). + Flags: - `--all` — rerun every test in the resolved project; requires `--project `. @@ -377,6 +397,7 @@ Flags: - `--skip-dependencies` — backend only: rerun just the named test without expanding the producer/teardown closure. - `--max-concurrency ` — with `--wait`, cap on in-flight polls during a batch rerun. - `--idempotency-key ` — auto-minted when omitted (the minted key is printed to stderr under `--output json`, `--verbose`, or `--debug`). +- `--report junit --report-file ` — with batch `--wait`, write a JUnit XML sidecar after polling (atomic write). Optional `--report-suite-name ` overrides the default `testsprite:` suite name. Requires `--wait`; not available on single-test reruns. A batch rerun returns `accepted[]` (one `runId` per dispatched test) plus `deferred[]` for any test shed by the per-key run-rate limit; under `--wait`, a non-empty `deferred[]` exits 7 with a `nextAction` you can retry with a fresh idempotency key. diff --git a/src/commands/test.run.spec.ts b/src/commands/test.run.spec.ts index 0a10df5..f33d08a 100644 --- a/src/commands/test.run.spec.ts +++ b/src/commands/test.run.spec.ts @@ -5,7 +5,7 @@ * sleep injection is wired through `TestDeps.sleep` to avoid real delays. */ -import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { Command } from 'commander'; @@ -3624,3 +3624,221 @@ describe('dashboardUrl on run completion', () => { ).toBe(true); }); }); + +describe('runTestRunAll — JUnit report export', () => { + const BATCH_FRESH_RESP: BatchRunFreshResponse = { + accepted: [ + { testId: 'test_be_01', runId: 'run_fresh_01', enqueuedAt: '2026-06-09T10:00:00.000Z' }, + { testId: 'test_be_02', runId: 'run_fresh_02', enqueuedAt: '2026-06-09T10:00:01.000Z' }, + ], + conflicts: [], + deferred: [], + skippedFrontend: [], + skippedIntegration: [], + }; + + function makeTerminalRun(runId: string, testId: string, status: string): RunResponse { + return { + runId, + testId, + projectId: 'project_be', + userId: 'user_1', + status: status as RunResponse['status'], + source: 'cli', + createdAt: '2026-06-09T10:00:00.000Z', + startedAt: '2026-06-09T10:00:01.000Z', + finishedAt: '2026-06-09T10:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://api.example.com', + createdFrom: 'cli', + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { + total: 3, + completed: 3, + passedCount: status === 'passed' ? 3 : 0, + failedCount: 0, + }, + }; + } + + it('--wait --report junit writes XML after polling', async () => { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'junit-run-all-')); + const reportPath = join(dir, 'results.xml'); + const fetchImpl = makeFetch((url, init) => { + if ((init.method ?? 'GET') === 'POST') return { body: BATCH_FRESH_RESP }; + const runId = url.split('/runs/')[1]?.split('?')[0] ?? ''; + if (runId === 'run_fresh_01') + return { body: makeTerminalRun('run_fresh_01', 'test_be_01', 'passed') }; + if (runId === 'run_fresh_02') + return { body: makeTerminalRun('run_fresh_02', 'test_be_02', 'passed') }; + return errorBody('NOT_FOUND'); + }); + + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + report: 'junit', + reportFile: reportPath, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: () => undefined, + sleep: instantSleep, + }, + ); + + const xml = readFileSync(reportPath, 'utf8'); + expect(xml).toContain(' { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'junit-run-fail-')); + const reportPath = join(dir, 'results.xml'); + const fetchImpl = makeFetch((url, init) => { + if ((init.method ?? 'GET') === 'POST') return { body: BATCH_FRESH_RESP }; + const runId = url.split('/runs/')[1]?.split('?')[0] ?? ''; + if (runId === 'run_fresh_01') + return { body: makeTerminalRun('run_fresh_01', 'test_be_01', 'passed') }; + if (runId === 'run_fresh_02') + return { body: makeTerminalRun('run_fresh_02', 'test_be_02', 'failed') }; + return errorBody('NOT_FOUND'); + }); + + await expect( + runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + report: 'junit', + reportFile: reportPath, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: () => undefined, + sleep: instantSleep, + }, + ), + ).rejects.toMatchObject({ exitCode: 1 }); + + const xml = readFileSync(reportPath, 'utf8'); + expect(xml).toContain('failures="1"'); + expect(xml).toContain('name="test_be_02"'); + }); + + it('rejects --report without --wait', async () => { + await expect( + runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: false, + timeoutSeconds: 60, + maxConcurrency: 5, + report: 'junit', + reportFile: './results.xml', + }, + {}, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); + + it('rejects --report-suite-name without --report', async () => { + await expect( + runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + reportSuiteName: 'orphan-suite', + }, + {}, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); + + it('--dry-run --report junit writes canned sample XML', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-run-dry-')); + const reportPath = join(dir, 'results.xml'); + + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + report: 'junit', + reportFile: reportPath, + }, + { + stdout: () => undefined, + stderr: () => undefined, + }, + ); + + const xml = readFileSync(reportPath, 'utf8'); + expect(xml).toContain('name="test_fresh_wave_01"'); + expect(xml).toContain('failures="1"'); + }); + + it('--dry-run --report junit --report-suite-name overrides canned suite name', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-run-dry-suite-')); + const reportPath = join(dir, 'results.xml'); + + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + report: 'junit', + reportFile: reportPath, + reportSuiteName: 'ci-checkout-suite', + }, + { + stdout: () => undefined, + stderr: () => undefined, + }, + ); + + const xml = readFileSync(reportPath, 'utf8'); + expect(xml).toContain('. */ + reportSuiteName?: string; } /** @@ -5071,6 +5086,32 @@ interface RunTestRunAllOptions extends CommonOptions { maxConcurrency: number; /** Caller-supplied idempotency token; auto-minted if absent. */ idempotencyKey?: string; + /** --report junit: write a JUnit XML sidecar after batch --wait completes. */ + report?: JUnitReportFormat; + /** --report-file: destination path for the JUnit XML artifact. */ + reportFile?: string; + /** --report-suite-name: optional override for the JUnit . */ + reportSuiteName?: string; +} + +async function writeBatchJUnitReportIfRequested( + opts: { + report?: JUnitReportFormat; + reportFile?: string; + reportSuiteName?: string; + projectId?: string; + }, + results: readonly JUnitTestResult[], +): Promise { + if (opts.report !== 'junit' || opts.reportFile === undefined) return; + const projectId = resolveBatchReportProjectId(opts, results); + const suiteName = opts.reportSuiteName ?? `testsprite:${projectId}`; + const xml = buildJUnitReport({ + suiteName, + classname: projectId, + results, + }); + await writeJUnitReportFile(opts.reportFile, xml); } /** @@ -5079,6 +5120,8 @@ interface RunTestRunAllOptions extends CommonOptions { interface CliBatchRunFreshResult { testId: string; runId: string | undefined; + /** Observed on polled runs; used for JUnit report naming when --project omitted. */ + projectId?: string; status: string; error?: { code: string; message: string; exitCode: number }; /** CLIENT-synthesized Portal deep link (projectId from opts, testId per item). */ @@ -5105,6 +5148,13 @@ export async function runTestRunAll( ) { throw localValidationError('max-concurrency', 'must be an integer between 1 and 100'); } + assertJUnitReportOptions({ + report: opts.report, + reportFile: opts.reportFile, + reportSuiteName: opts.reportSuiteName, + wait: opts.wait, + batchPath: true, + }); const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); const out = makeOutput(opts.output, deps); @@ -5128,6 +5178,12 @@ export async function runTestRunAll( idempotencyKey, ...(opts.wait ? { thenPoll: '/api/cli/v1/runs/?waitSeconds=25' } : {}), }; + if (opts.report === 'junit' && opts.reportFile !== undefined) { + await writeJUnitReportFile( + opts.reportFile, + sampleJUnitReportXml(opts.projectId, opts.reportSuiteName), + ); + } out.print(batchRunSample ?? envelope); return undefined; } @@ -5467,7 +5523,12 @@ export async function runTestRunAll( }, resolveAlternate, }); - return { testId: entry.testId, runId, status: finalRun.status }; + return { + testId: entry.testId, + runId, + projectId: finalRun.projectId, + status: finalRun.status, + }; } catch (err) { if (err instanceof TimeoutError) { return { @@ -5549,6 +5610,7 @@ export async function runTestRunAll( total: pollable.length, }, }; + await writeBatchJUnitReportIfRequested(opts, freshRunResults); out.print(jsonPayload); // Rate-deferred tests were never dispatched → the batch is incomplete (exit 7), @@ -5611,6 +5673,8 @@ export async function runTestRunAll( interface CliRerunResult { testId: string; runId: string; + /** Observed on polled runs; used for JUnit report naming when --project omitted. */ + projectId?: string; /** Terminal status, or 'timeout' for per-run deadline exceeded. */ status: string; /** Set when the test is a closure member (not the user's named test). */ @@ -5680,6 +5744,13 @@ export async function runTestRerun( } const isSingle = !opts.all && opts.testIds.length === 1; + assertJUnitReportOptions({ + report: opts.report, + reportFile: opts.reportFile, + reportSuiteName: opts.reportSuiteName, + wait: opts.wait, + batchPath: !isSingle, + }); // ------------------------------------------------------------------------- // Pre-flight: auto-heal + Free-tier hint (best-effort, non-blocking) @@ -5719,6 +5790,13 @@ export async function runTestRerun( idempotencyKey, ...(opts.wait ? { thenPoll: `/api/cli/v1/runs/?waitSeconds=25` } : {}), }; + if (opts.report === 'junit' && opts.reportFile !== undefined) { + const projectKey = resolveBatchReportProjectId(opts, []); + await writeJUnitReportFile( + opts.reportFile, + sampleJUnitReportXml(projectKey, opts.reportSuiteName), + ); + } out.print(findSample('POST', '/api/cli/v1/tests/batch/rerun')?.body() ?? envelope); } void client; @@ -6616,7 +6694,12 @@ export async function runTestRerun( }, resolveAlternate, }); - return { testId: entry.testId, runId: entry.runId, status: finalRun.status }; + return { + testId: entry.testId, + runId: entry.runId, + projectId: finalRun.projectId, + status: finalRun.status, + }; } catch (err) { if (err instanceof TimeoutError) { return { @@ -6704,6 +6787,7 @@ export async function runTestRerun( total: accepted.length, }, }; + await writeBatchJUnitReportIfRequested(opts, rerunResults); out.print(jsonPayload); // Determine exit code: timeout (deferred or any timeout) → 7; any fail → 1; all pass → 0 @@ -7428,11 +7512,21 @@ export function createTestCommand(deps: TestDeps = {}): Command { '--max-concurrency ', `with --all --wait, max in-flight polls at once (1-100, default: ${DEFAULT_BATCH_RUN_CONCURRENCY})`, ) + .option( + '--report ', + 'with --all --wait: write a JUnit XML sidecar report after polling (accepted: junit)', + ) + .option('--report-file ', 'output path for --report (atomic write)') + .option( + '--report-suite-name ', + 'optional JUnit override (default: testsprite:)', + ) .addHelpText( 'after', '\nDependency-aware fresh run (M4):\n' + ' testsprite test run --all --project run all BE tests in wave order\n' + ' testsprite test run --all --project --filter name-glob subset\n' + + ' testsprite test run --all --project --wait --report junit --report-file ./results.xml\n' + '\nBE tests can declare --produces/--needs at create time to drive wave ordering\n' + '(see `testsprite test create --help` for details).', ) @@ -7462,6 +7556,14 @@ export function createTestCommand(deps: TestDeps = {}): Command { '--filter only applies with --all (it narrows which project tests run). Remove --filter, or add --all --project .', ); } + const report = parseJUnitReportFormat(cmdOpts.report); + assertJUnitReportOptions({ + report, + reportFile: cmdOpts.reportFile, + reportSuiteName: cmdOpts.reportSuiteName, + wait: cmdOpts.wait === true, + batchPath: isAll, + }); if (isAll) { // --all path: wave-ordered fresh batch run. @@ -7492,6 +7594,9 @@ export function createTestCommand(deps: TestDeps = {}): Command { parseNumericFlag(cmdOpts.maxConcurrency, 'max-concurrency') ?? DEFAULT_BATCH_RUN_CONCURRENCY, idempotencyKey: cmdOpts.idempotencyKey, + report, + reportFile: cmdOpts.reportFile, + reportSuiteName: cmdOpts.reportSuiteName, }, deps, ); @@ -7599,6 +7704,15 @@ export function createTestCommand(deps: TestDeps = {}): Command { '--idempotency-key ', 'opaque key for safe retries (1–256 chars). Printed to stderr at --verbose if auto-generated.', ) + .option( + '--report ', + 'with batch --wait: write a JUnit XML sidecar report after polling (accepted: junit)', + ) + .option('--report-file ', 'output path for --report (atomic write)') + .option( + '--report-suite-name ', + 'optional JUnit override (default: testsprite:)', + ) .addHelpText( 'after', '\nNotes:\n' + @@ -7624,10 +7738,20 @@ export function createTestCommand(deps: TestDeps = {}): Command { // `--no-auto-heal`. There is no explicit `--auto-heal` flag, so // autoHealExplicit is always false in this design — the default-on value // is never a deliberate user choice to opt in. + const testIds = testIdsArg ?? []; + const isBatch = cmdOpts.all === true || testIds.length !== 1; + const report = parseJUnitReportFormat(cmdOpts.report); + assertJUnitReportOptions({ + report, + reportFile: cmdOpts.reportFile, + reportSuiteName: cmdOpts.reportSuiteName, + wait: cmdOpts.wait === true, + batchPath: isBatch, + }); await runTestRerun( { ...resolveCommonOptions(command), - testIds: testIdsArg ?? [], + testIds, all: cmdOpts.all === true, projectId: cmdOpts.project, skipTerminal: cmdOpts.skipTerminal === true, @@ -7642,6 +7766,9 @@ export function createTestCommand(deps: TestDeps = {}): Command { parseNumericFlag(cmdOpts.maxConcurrency, 'max-concurrency') ?? DEFAULT_BATCH_RUN_CONCURRENCY, idempotencyKey: cmdOpts.idempotencyKey, + report, + reportFile: cmdOpts.reportFile, + reportSuiteName: cmdOpts.reportSuiteName, }, deps, ); @@ -7665,6 +7792,9 @@ interface RunFlagOpts { project?: string; filter?: string; maxConcurrency?: string; + report?: string; + reportFile?: string; + reportSuiteName?: string; } interface WaitFlagOpts { @@ -7683,6 +7813,9 @@ interface RerunFlagOpts { skipDependencies?: boolean; maxConcurrency?: string; idempotencyKey?: string; + report?: string; + reportFile?: string; + reportSuiteName?: string; } interface UpdateFlagOpts { diff --git a/src/lib/dry-run/samples.test.ts b/src/lib/dry-run/samples.test.ts index e1bf8ff..a1838d6 100644 --- a/src/lib/dry-run/samples.test.ts +++ b/src/lib/dry-run/samples.test.ts @@ -1,5 +1,23 @@ import { describe, expect, it } from 'vitest'; -import { DRY_RUN_SAMPLE_ENTRIES, findSample } from './samples.js'; +import { DRY_RUN_SAMPLE_ENTRIES, findSample, sampleJUnitReportXml } from './samples.js'; + +describe('sampleJUnitReportXml', () => { + it('returns well-formed JUnit XML with canned batch ids', () => { + const xml = sampleJUnitReportXml('proj_dry'); + expect(xml).toContain(''); + expect(xml).toContain(' { + const xml = sampleJUnitReportXml('proj_dry', 'custom-ci-suite'); + expect(xml).toContain(' { it('resolves /me', () => { diff --git a/src/lib/dry-run/samples.ts b/src/lib/dry-run/samples.ts index 28fa1bb..4c7cbb0 100644 --- a/src/lib/dry-run/samples.ts +++ b/src/lib/dry-run/samples.ts @@ -27,6 +27,7 @@ import type { CliTestStep, } from '../../commands/test.js'; import type { MeResponse } from '../../commands/auth.js'; +import { buildJUnitReport } from '../junit-report.js'; import type { Page } from '../pagination.js'; import type { TriggerRunResponse, @@ -67,6 +68,37 @@ const SAMPLE_REQUEST_ID = 'req_dry-run'; export const SAMPLE_DRY_RUN_REQUEST_ID = SAMPLE_REQUEST_ID; +/** + * Canned JUnit XML for batch `--wait --report junit --dry-run`. Mirrors the + * fresh batch-run sample ids so agents can learn the sidecar shape offline. + */ +export function sampleJUnitReportXml( + projectId: string = SAMPLE_PROJECT_ID, + reportSuiteName?: string, +): string { + return buildJUnitReport({ + suiteName: reportSuiteName ?? `testsprite:${projectId}`, + classname: projectId, + results: [ + { + testId: SAMPLE_TEST_ID_FRESH_1, + runId: SAMPLE_BATCH_FRESH_RUN_ID_1, + status: 'passed', + }, + { + testId: SAMPLE_TEST_ID_FRESH_2, + runId: SAMPLE_BATCH_FRESH_RUN_ID_2, + status: 'failed', + error: { + code: 'ASSERTION', + message: 'Expected checkout heading to be visible', + exitCode: 1, + }, + }, + ], + }); +} + const me: MeResponse = { userId: SAMPLE_USER_ID, keyId: SAMPLE_KEY_ID, diff --git a/src/lib/junit-report.test.ts b/src/lib/junit-report.test.ts new file mode 100644 index 0000000..d504dce --- /dev/null +++ b/src/lib/junit-report.test.ts @@ -0,0 +1,330 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { ApiError } from './errors.js'; +import { + assertJUnitReportOptions, + buildJUnitReport, + escapeXml, + parseJUnitReportFormat, + resolveBatchReportProjectId, + writeJUnitReportFile, + type JUnitTestResult, +} from './junit-report.js'; + +function makeResult(overrides: Partial & { testId: string }): JUnitTestResult { + return { + status: 'passed', + ...overrides, + }; +} + +describe('escapeXml', () => { + it('escapes XML special characters', () => { + expect(escapeXml(`a&bd"e'f`)).toBe('a&b<c>d"e'f'); + }); + + it('leaves plain text unchanged', () => { + expect(escapeXml('test_abc123')).toBe('test_abc123'); + }); +}); + +describe('parseJUnitReportFormat', () => { + it('accepts junit', () => { + expect(parseJUnitReportFormat('junit')).toBe('junit'); + }); + + it('returns undefined for absent value', () => { + expect(parseJUnitReportFormat(undefined)).toBeUndefined(); + expect(parseJUnitReportFormat('')).toBeUndefined(); + }); + + it('rejects unknown formats', () => { + expect(() => parseJUnitReportFormat('html')).toThrowError(ApiError); + try { + parseJUnitReportFormat('html'); + } catch (err) { + expect((err as ApiError).code).toBe('VALIDATION_ERROR'); + expect((err as ApiError).exitCode).toBe(5); + } + }); +}); + +describe('assertJUnitReportOptions', () => { + it('allows absent report flags', () => { + expect(() => assertJUnitReportOptions({ wait: false, batchPath: true })).not.toThrow(); + }); + + it('rejects report-file without report', () => { + expect(() => + assertJUnitReportOptions({ reportFile: './out.xml', wait: true, batchPath: true }), + ).toThrowError(ApiError); + }); + + it('rejects report-suite-name without report', () => { + expect(() => + assertJUnitReportOptions({ + reportSuiteName: 'my-suite', + wait: true, + batchPath: true, + }), + ).toThrowError(ApiError); + }); + + it('rejects report on non-batch paths', () => { + expect(() => + assertJUnitReportOptions({ + report: 'junit', + reportFile: './out.xml', + wait: true, + batchPath: false, + }), + ).toThrowError(ApiError); + }); + + it('rejects report without wait', () => { + expect(() => + assertJUnitReportOptions({ + report: 'junit', + reportFile: './out.xml', + wait: false, + batchPath: true, + }), + ).toThrowError(ApiError); + }); + + it('rejects report without report-file', () => { + expect(() => + assertJUnitReportOptions({ report: 'junit', wait: true, batchPath: true }), + ).toThrowError(ApiError); + }); +}); + +describe('resolveBatchReportProjectId', () => { + it('prefers explicit projectId', () => { + expect(resolveBatchReportProjectId({ projectId: 'proj_a' }, [])).toBe('proj_a'); + }); + + it('infers from polled run rows', () => { + expect(resolveBatchReportProjectId({}, [{ projectId: 'proj_from_run' }])).toBe('proj_from_run'); + }); + + it('requires --project when the project cannot be inferred', () => { + expect(() => resolveBatchReportProjectId({}, [])).toThrowError(ApiError); + try { + resolveBatchReportProjectId({}, []); + } catch (err) { + expect((err as ApiError).code).toBe('VALIDATION_ERROR'); + expect((err as ApiError).exitCode).toBe(5); + } + }); +}); + +describe('buildJUnitReport', () => { + it('renders an empty suite', () => { + const xml = buildJUnitReport({ + suiteName: 'Dry suite', + classname: 'proj_empty', + results: [], + }); + expect(xml).toContain( + ''); + }); + + it('counts passed tests without child elements', () => { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [makeResult({ testId: 'test_a', status: 'passed' })], + }); + expect(xml).toContain(''); + expect(xml).not.toContain(' { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [ + makeResult({ + testId: 'test_fail', + status: 'failed', + runId: 'run_1', + }), + ], + }); + expect(xml).toContain(''); + expect(xml).toContain('runId: run_1'); + expect(xml).toContain('failures="1"'); + }); + + it('maps blocked and cancelled to failures', () => { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [ + makeResult({ testId: 't_blocked', status: 'blocked' }), + makeResult({ testId: 't_cancelled', status: 'cancelled' }), + ], + }); + expect(xml).toContain('failures="2"'); + expect(xml).toContain('type="blocked"'); + expect(xml).toContain('type="cancelled"'); + }); + + it('maps timeout to failure', () => { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [ + makeResult({ + testId: 't_timeout', + status: 'timeout', + error: { code: 'UNSUPPORTED', message: 'Timed out', exitCode: 7 }, + }), + ], + }); + expect(xml).toContain(''); + }); + + it('maps API error status to error elements', () => { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [ + makeResult({ + testId: 't_err', + status: 'error', + error: { code: 'NOT_FOUND', message: 'Run missing', exitCode: 4 }, + }), + ], + }); + expect(xml).toContain(''); + expect(xml).toContain('errors="1"'); + }); + + it('maps auth failures to error elements', () => { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [ + makeResult({ + testId: 't_auth', + status: 'failed', + error: { code: 'AUTH_INVALID', message: 'Bad key', exitCode: 3 }, + }), + ], + }); + expect(xml).toContain(''); + expect(xml).toContain('failures="0" errors="1"'); + }); + + it('escapes special characters in testcase names and messages', () => { + const xml = buildJUnitReport({ + suiteName: 'Suite "A"', + classname: 'proj<&>', + results: [ + makeResult({ + testId: 'test<1>', + status: 'failed', + error: { code: 'ASSERT', message: 'expected & "ok"', exitCode: 1 }, + }), + ], + }); + expect(xml).toContain('name="test<1>"'); + expect(xml).toContain('classname="proj<&>"'); + expect(xml).toContain('message="expected <true> & "ok""'); + }); + + it('aggregates mixed outcomes', () => { + const xml = buildJUnitReport({ + suiteName: 'Mixed', + classname: 'proj_mix', + results: [ + makeResult({ testId: 'p', status: 'passed' }), + makeResult({ testId: 'f', status: 'failed' }), + makeResult({ + testId: 'e', + status: 'error', + error: { code: 'INTERNAL', message: 'boom', exitCode: 10 }, + }), + ], + }); + expect(xml).toContain('tests="3" failures="1" errors="1" skipped="0"'); + }); +}); + +describe('writeJUnitReportFile', () => { + it('writes XML atomically to the target path', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-report-')); + const target = join(dir, 'results.xml'); + const xml = buildJUnitReport({ + suiteName: 'Suite', + classname: 'proj_write', + results: [makeResult({ testId: 't1', status: 'passed' })], + }); + + await writeJUnitReportFile(target, xml); + + expect(readFileSync(target, 'utf8')).toBe(xml); + rmSync(dir, { recursive: true, force: true }); + }); + + it('rejects a directory target', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-report-dir-')); + await expect(writeJUnitReportFile(dir, '')).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + }); + rmSync(dir, { recursive: true, force: true }); + }); + + it('rejects missing parent directory', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-report-parent-')); + const missing = join(dir, 'missing', 'out.xml'); + await expect(writeJUnitReportFile(missing, '')).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + }); + rmSync(dir, { recursive: true, force: true }); + }); + + it('overwrites an existing file', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-report-overwrite-')); + const target = join(dir, 'results.xml'); + writeFileSync(target, 'old', 'utf8'); + const xml = buildJUnitReport({ + suiteName: 'New', + classname: 'proj_new', + results: [], + }); + + await writeJUnitReportFile(target, xml); + + expect(readFileSync(target, 'utf8')).toBe(xml); + rmSync(dir, { recursive: true, force: true }); + }); + + it('rejects empty path', async () => { + await expect(writeJUnitReportFile('', '')).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + }); + }); + + it('rejects parent that is a file', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-report-file-parent-')); + const parentFile = join(dir, 'not-a-dir'); + writeFileSync(parentFile, 'x', 'utf8'); + const target = join(parentFile, 'out.xml'); + await expect(writeJUnitReportFile(target, '')).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + }); + rmSync(dir, { recursive: true, force: true }); + }); +}); diff --git a/src/lib/junit-report.ts b/src/lib/junit-report.ts new file mode 100644 index 0000000..3b11442 --- /dev/null +++ b/src/lib/junit-report.ts @@ -0,0 +1,267 @@ +import { createWriteStream } from 'node:fs'; +import { rename, stat, unlink } from 'node:fs/promises'; +import { basename, dirname, isAbsolute, join, resolve } from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { localValidationError, TransportError } from './errors.js'; + +export type JUnitReportFormat = 'junit'; + +/** Minimal testcase input shared by batch run and batch rerun poll results. */ +export interface JUnitTestResult { + testId: string; + runId?: string; + status: string; + /** Observed on polled runs; used for classname when --project is omitted. */ + projectId?: string; + error?: { code: string; message: string; exitCode?: number }; +} + +export interface JUnitReportBuildOptions { + suiteName: string; + classname: string; + results: readonly JUnitTestResult[]; +} + +export interface JUnitReportFlagOptions { + report?: JUnitReportFormat; + reportFile?: string; + reportSuiteName?: string; + wait: boolean; + /** True when the invocation is a batch path (run --all or rerun batch). */ + batchPath: boolean; +} + +const XML_DECL = ''; + +/** + * Parse `--report `. Only `junit` is accepted in v1. + */ +export function parseJUnitReportFormat(raw: string | undefined): JUnitReportFormat | undefined { + if (raw === undefined || raw === '') return undefined; + if (raw === 'junit') return 'junit'; + throw localValidationError('report', `unsupported report format "${raw}" — accepted: junit`); +} + +/** + * Validate `--report` / `--report-file` / `--report-suite-name` combinations. + * Report export is a sidecar artifact for batch `--wait` runs only. + */ +export function assertJUnitReportOptions(opts: JUnitReportFlagOptions): void { + if (opts.report === undefined) { + if (opts.reportFile !== undefined && opts.reportFile !== '') { + throw localValidationError('report-file', '--report-file requires --report junit'); + } + if (opts.reportSuiteName !== undefined && opts.reportSuiteName !== '') { + throw localValidationError( + 'report-suite-name', + '--report-suite-name requires --report junit', + ); + } + return; + } + + if (!opts.batchPath) { + throw localValidationError( + 'report', + '--report junit only applies to batch --wait runs (test run --all, or test rerun --all / multiple test ids)', + ); + } + if (!opts.wait) { + throw localValidationError( + 'report', + '--report junit requires --wait (the report is written after batch polling completes)', + ); + } + if (opts.reportFile === undefined || opts.reportFile === '') { + throw localValidationError('report-file', '--report junit requires --report-file '); + } +} + +/** + * Resolve the project id used for JUnit classname / default suite naming. + * Prefer explicit `--project`, then ids observed on polled run rows. + */ +export function resolveBatchReportProjectId( + opts: { projectId?: string }, + results: ReadonlyArray<{ projectId?: string }>, +): string { + if (opts.projectId) return opts.projectId; + const fromPoll = results.map(r => r.projectId).find((id): id is string => !!id); + if (fromPoll) return fromPoll; + throw localValidationError( + 'project', + '--report junit requires --project when the project cannot be inferred from run results', + ); +} + +/** + * Escape text for inclusion in XML element bodies and double-quoted attributes. + */ +export function escapeXml(text: string): string { + return text + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +} + +type JUnitOutcome = 'passed' | 'failure' | 'error' | 'skipped'; + +function classifyJUnitOutcome(status: string, error?: JUnitTestResult['error']): JUnitOutcome { + if (status === 'passed') return 'passed'; + if (status === 'skipped') return 'skipped'; + if (status === 'error' || error?.exitCode === 3) return 'error'; + return 'failure'; +} + +function failureMessage(result: JUnitTestResult): string { + if (result.error?.message) return result.error.message; + if (result.error?.code) return result.error.code; + return result.status; +} + +function renderTestcase(result: JUnitTestResult, classname: string): string { + const outcome = classifyJUnitOutcome(result.status, result.error); + const name = escapeXml(result.testId); + const cls = escapeXml(classname); + const lines = [` `]; + + if (outcome === 'failure') { + const message = escapeXml(failureMessage(result)); + const type = escapeXml(result.status); + const body = escapeXml( + [ + `status: ${result.status}`, + result.runId ? `runId: ${result.runId}` : undefined, + result.error?.code ? `code: ${result.error.code}` : undefined, + result.error?.message ? `message: ${result.error.message}` : undefined, + ] + .filter(Boolean) + .join('\n'), + ); + lines.push(` ${body}`); + } else if (outcome === 'error') { + const message = escapeXml(failureMessage(result)); + const type = escapeXml(result.error?.code ?? result.status); + const body = escapeXml( + [ + result.error?.code ? `code: ${result.error.code}` : undefined, + result.error?.message ? `message: ${result.error.message}` : undefined, + result.runId ? `runId: ${result.runId}` : undefined, + ] + .filter(Boolean) + .join('\n'), + ); + lines.push(` ${body}`); + } else if (outcome === 'skipped') { + lines.push(` `); + } + + lines.push(' '); + return lines.join('\n'); +} + +/** + * Build a JUnit XML document from batch poll results. Duration is `0` in v1 + * because batch poll envelopes do not carry per-run timing. + */ +export function buildJUnitReport(opts: JUnitReportBuildOptions): string { + const results = opts.results; + let failures = 0; + let errors = 0; + let skipped = 0; + + for (const result of results) { + const outcome = classifyJUnitOutcome(result.status, result.error); + if (outcome === 'failure') failures++; + else if (outcome === 'error') errors++; + else if (outcome === 'skipped') skipped++; + } + + const suiteName = escapeXml(opts.suiteName); + const testcases = results.map(r => renderTestcase(r, opts.classname)).join('\n'); + + return [ + XML_DECL, + '', + ` `, + testcases, + ' ', + '', + '', + ].join('\n'); +} + +async function assertReportFileParent(rawPath: string): Promise { + if (typeof rawPath !== 'string' || rawPath.length === 0) { + throw localValidationError('report-file', 'must be a non-empty file path'); + } + const resolved = isAbsolute(rawPath) ? rawPath : resolve(process.cwd(), rawPath); + if (resolved.endsWith('/') || resolved.endsWith('\\')) { + throw localValidationError('report-file', 'must point to a file, not a directory'); + } + + const parent = dirname(resolved); + let parentStat; + try { + parentStat = await stat(parent); + } catch { + throw localValidationError('report-file', `parent directory does not exist: ${parent}`); + } + if (!parentStat.isDirectory()) { + throw localValidationError('report-file', `parent path is not a directory: ${parent}`); + } + + let targetStat; + try { + targetStat = await stat(resolved); + } catch { + return resolved; + } + if (targetStat.isDirectory()) { + throw localValidationError('report-file', `must point to a file, not a directory: ${resolved}`); + } + return resolved; +} + +/** + * Atomically write JUnit XML to `--report-file` (temp sibling + rename). + */ +export async function writeJUnitReportFile(rawPath: string, xml: string): Promise { + const resolved = await assertReportFileParent(rawPath); + const parent = dirname(resolved); + const tmpPath = join(parent, `.${basename(resolved)}.tmp-${randomUUID()}`); + + await new Promise((resolvePromise, reject) => { + const stream = createWriteStream(tmpPath, { encoding: 'utf8' }); + let streamError: Error | null = null; + stream.on('error', err => { + streamError = err instanceof Error ? err : new Error(String(err)); + }); + stream.write(xml, err => { + if (err) { + streamError = err instanceof Error ? err : new Error(String(err)); + } + stream.end(() => { + if (streamError) { + unlink(tmpPath).catch(() => undefined); + reject( + new TransportError(`Failed to write --report-file ${resolved}: ${streamError.message}`), + ); + return; + } + rename(tmpPath, resolved) + .then(() => resolvePromise()) + .catch(renameErr => { + unlink(tmpPath).catch(() => undefined); + reject( + new TransportError( + `Failed to write --report-file ${resolved}: ${renameErr instanceof Error ? renameErr.message : String(renameErr)}`, + ), + ); + }); + }); + }); + }); +} diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 6fee8af..9e5e0d9 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -392,32 +392,40 @@ Exit codes: On failure/blocked/cancelled, run: testsprite test artifact get Options: - --all rerun all tests in the resolved project (requires - --project) (default: false) - --project project id (required with --all; returned by - \`testsprite project list\`) - --skip-terminal with --all: skip tests already in a terminal status - (passed|failed|blocked|cancelled) (default: false) - --status with --all: only dispatch tests whose status matches - one of these values (comma-separated; accepted: - draft|ready|queued|running|passed|failed|blocked|cancelled|unknown) - --filter with --all: only rerun tests whose name contains - this substring (case-insensitive) - --wait block until terminal status or --timeout elapses - (default: false) - --timeout with --wait, max seconds to wait (1–3600, default - 600) - --no-auto-heal opt out of AI heal-on-drift for this FE rerun - (default: auto-heal is ON). Costs 0.2 credits per - engage when a step has drifted. Ignored for backend - tests. - --skip-dependencies BE only: rerun only the named test without expanding - the producer/teardown closure (default: false) - --max-concurrency with --wait, max in-flight polls at once (1-100, - default: 50) - --idempotency-key opaque key for safe retries (1–256 chars). Printed - to stderr at --verbose if auto-generated. - -h, --help display help for command + --all rerun all tests in the resolved project (requires + --project) (default: false) + --project project id (required with --all; returned by + \`testsprite project list\`) + --skip-terminal with --all: skip tests already in a terminal + status (passed|failed|blocked|cancelled) + (default: false) + --status with --all: only dispatch tests whose status + matches one of these values (comma-separated; + accepted: + draft|ready|queued|running|passed|failed|blocked|cancelled|unknown) + --filter with --all: only rerun tests whose name contains + this substring (case-insensitive) + --wait block until terminal status or --timeout elapses + (default: false) + --timeout with --wait, max seconds to wait (1–3600, default + 600) + --no-auto-heal opt out of AI heal-on-drift for this FE rerun + (default: auto-heal is ON). Costs 0.2 credits per + engage when a step has drifted. Ignored for + backend tests. + --skip-dependencies BE only: rerun only the named test without + expanding the producer/teardown closure (default: + false) + --max-concurrency with --wait, max in-flight polls at once (1-100, + default: 50) + --idempotency-key opaque key for safe retries (1–256 chars). + Printed to stderr at --verbose if auto-generated. + --report with batch --wait: write a JUnit XML sidecar + report after polling (accepted: junit) + --report-file output path for --report (atomic write) + --report-suite-name optional JUnit override + (default: testsprite:) + -h, --help display help for command Notes: • rerun replays a saved run/script and is MORE LENIENT than a fresh \`test run\` @@ -490,28 +498,34 @@ Exit codes: On failure/blocked/cancelled, run: testsprite test artifact get Options: - --target-url override the project default env URL for this run - (http/https only, no localhost/private IPs) - --wait poll until terminal status or --timeout elapses - (default: false) - --timeout with --wait, max seconds to wait (1–3600, default - 600) - --idempotency-key opaque key for safe retries (1–256 chars). Printed - to stderr at --debug if auto-generated. - --all run all BE tests in the project (wave-ordered fresh - run; requires --project). Mutually exclusive with - . (default: false) - --project project id (required with --all; returned by - \`testsprite project list\`) - --filter with --all: only run tests whose name contains this - substring (case-insensitive) - --max-concurrency with --all --wait, max in-flight polls at once - (1-100, default: 50) - -h, --help display help for command + --target-url override the project default env URL for this run + (http/https only, no localhost/private IPs) + --wait poll until terminal status or --timeout elapses + (default: false) + --timeout with --wait, max seconds to wait (1–3600, default + 600) + --idempotency-key opaque key for safe retries (1–256 chars). + Printed to stderr at --debug if auto-generated. + --all run all BE tests in the project (wave-ordered + fresh run; requires --project). Mutually + exclusive with . (default: false) + --project project id (required with --all; returned by + \`testsprite project list\`) + --filter with --all: only run tests whose name contains + this substring (case-insensitive) + --max-concurrency with --all --wait, max in-flight polls at once + (1-100, default: 50) + --report with --all --wait: write a JUnit XML sidecar + report after polling (accepted: junit) + --report-file output path for --report (atomic write) + --report-suite-name optional JUnit override + (default: testsprite:) + -h, --help display help for command Dependency-aware fresh run (M4): testsprite test run --all --project run all BE tests in wave order testsprite test run --all --project --filter name-glob subset + testsprite test run --all --project --wait --report junit --report-file ./results.xml BE tests can declare --produces/--needs at create time to drive wave ordering (see \`testsprite test create --help\` for details).