diff --git a/docs/guide/reporters.md b/docs/guide/reporters.md index 36a6857d8c9c..1cdf85848c7f 100644 --- a/docs/guide/reporters.md +++ b/docs/guide/reporters.md @@ -576,6 +576,67 @@ export default defineConfig({ }) ``` +The GitHub Actions reporter automatically generates a [Job Summary](https://github.blog/news-insights/product-news/supercharging-github-actions-with-job-summaries/) with an overview of your test results. The summary includes test file and test case statistics, and highlights flaky tests that required retries. + +GitHub Actions Job Summary +GitHub Actions Job Summary + +The job summary is enabled by default and writes to the path specified by `$GITHUB_STEP_SUMMARY`. You can override it by using the `jobSummary.outputPath` option: + +```ts +export default defineConfig({ + test: { + reporters: [ + ['github-actions', { + jobSummary: { + outputPath: '/home/runner/jobs/summary/step', + }, + }], + ], + }, +}) +``` + +To disable the job summary: + +```ts +export default defineConfig({ + test: { + reporters: [ + ['github-actions', { jobSummary: { enabled: false } }], + ], + }, +}) +``` + +The flaky tests section of the summary includes permalink URLs that link test names directly to the relevant source lines on GitHub. These links are generated automatically using environment variables that GitHub Actions provides (`$GITHUB_REPOSITORY`, `$GITHUB_SHA`, and `$GITHUB_WORKSPACE`), so no configuration is needed in most cases. + +If you need to override these values — for example, when running in a container or a custom environment — you can customize them via the `fileLinks` option: + +- `repository`: the GitHub repository in `owner/repo` format. Defaults to `process.env.GITHUB_REPOSITORY`. +- `commitHash`: the commit SHA to use in permalink URLs. Defaults to `process.env.GITHUB_SHA`. +- `workspacePath`: the absolute path to the root of the repository on disk. Used to compute relative file paths for the permalink URLs. Defaults to `process.env.GITHUB_WORKSPACE`. + +All three values must be available for the links to be generated. + +```ts +export default defineConfig({ + test: { + reporters: [ + ['github-actions', { + jobSummary: { + fileLinks: { + repository: 'owner/repo', + commitHash: 'abcdefg', + workspacePath: '/home/runner/work/repo/', + }, + }, + }], + ], + }, +}) +``` + ### Blob Reporter Stores test results on the machine so they can be later merged using [`--merge-reports`](/guide/cli#merge-reports) command. diff --git a/docs/public/github-actions-job-summary-dark.png b/docs/public/github-actions-job-summary-dark.png new file mode 100644 index 000000000000..d1d40d1b1178 Binary files /dev/null and b/docs/public/github-actions-job-summary-dark.png differ diff --git a/docs/public/github-actions-job-summary-light.png b/docs/public/github-actions-job-summary-light.png new file mode 100644 index 000000000000..5c9bc078058d Binary files /dev/null and b/docs/public/github-actions-job-summary-light.png differ diff --git a/packages/vitest/src/node/reporters/github-actions.ts b/packages/vitest/src/node/reporters/github-actions.ts index 092f8aed1a14..e341b944dde7 100644 --- a/packages/vitest/src/node/reporters/github-actions.ts +++ b/packages/vitest/src/node/reporters/github-actions.ts @@ -4,9 +4,13 @@ import type { Vitest } from '../core' import type { TestProject } from '../project' import type { Reporter } from '../types/reporter' import type { TestCase, TestModule } from './reported-tasks' +import { writeFileSync } from 'node:fs' import { stripVTControlCharacters } from 'node:util' import { getFullName, getTasks } from '@vitest/runner/utils' +import { deepMerge } from '@vitest/utils/helpers' +import { relative } from 'pathe' import { capturePrintError } from '../printError' +import { noun } from './renderers/utils' export interface GithubActionsReporterOptions { onWritePath?: (path: string) => string @@ -14,14 +18,78 @@ export interface GithubActionsReporterOptions { * @default true */ displayAnnotations?: boolean + /** + * Configuration for the GitHub Actions Job Summary. + * + * When enabled, a markdown summary of test results is written to the path specified by `outputPath`. + */ + jobSummary?: Partial +} + +interface JobSummaryOptions { + /** + * Whether to generate the summary. + * + * @default true + */ + enabled: boolean + /** + * File path to write the summary to. + * + * @default process.env.GITHUB_STEP_SUMMARY + */ + outputPath: string | undefined + /** + * Configuration for generating permalink URLs to source files in the GitHub repository. + * + * When all three values are available (either from this config or the defaults picked from environment variables), test names in the summary will link to the relevant source lines. + */ + fileLinks: { + /** + * The GitHub repository in `owner/repo` format. + * + * @default process.env.GITHUB_REPOSITORY + */ + repository?: string | undefined + /** + * The commit SHA to use in permalink URLs. + * + * @default process.env.GITHUB_SHA + */ + commitHash?: string | undefined + /** + * The absolute path to the root of the repository on disk. + * + * This value is used to compute relative file paths for the permalink URLs. + * + * @default process.env.GITHUB_WORKSPACE + */ + workspacePath?: string | undefined + } +} + +type ResolvedOptions = Required + +const defaultOptions: ResolvedOptions = { + onWritePath: defaultOnWritePath, + displayAnnotations: true, + jobSummary: { + enabled: true, + outputPath: process.env.GITHUB_STEP_SUMMARY, + fileLinks: { + repository: process.env.GITHUB_REPOSITORY, + commitHash: process.env.GITHUB_SHA, + workspacePath: process.env.GITHUB_WORKSPACE, + }, + }, } export class GithubActionsReporter implements Reporter { ctx: Vitest = undefined! - options: GithubActionsReporterOptions + options: ResolvedOptions constructor(options: GithubActionsReporterOptions = {}) { - this.options = options + this.options = deepMerge(Object.create(null), defaultOptions, options) } onInit(ctx: Vitest): void { @@ -88,8 +156,6 @@ export class GithubActionsReporter implements Reporter { } } - const onWritePath = this.options.onWritePath ?? defaultOnWritePath - // format errors via `printError` for (const { project, title, error, file } of projectErrors) { const result = capturePrintError(error, this.ctx, { project, task: file }) @@ -100,7 +166,7 @@ export class GithubActionsReporter implements Reporter { const formatted = formatMessage({ command: 'error', properties: { - file: onWritePath(stack.file), + file: this.options.onWritePath(stack.file), title, line: String(stack.line), column: String(stack.column), @@ -109,6 +175,21 @@ export class GithubActionsReporter implements Reporter { }) this.ctx.logger.log(`\n${formatted}`) } + + if (this.options.jobSummary.enabled === true && this.options.jobSummary.outputPath) { + const summary = renderSummary(collectSummaryData(testModules), this.options.jobSummary.fileLinks) + + try { + writeFileSync( + this.options.jobSummary.outputPath, + summary, + { flag: 'a' }, + ) + } + catch (error) { + this.ctx.logger.warn('Could not write summary to `options.summary.outputPath`', error) + } + } } } @@ -165,3 +246,224 @@ function escapeProperty(s: string): string { .replace(/:/g, '%3A') .replace(/,/g, '%2C') } + +type SummaryTestsStats = Record<'failed' | 'passed' | 'expectedFail' | 'skipped' | 'todo', number> + +interface SummaryData { + fileStats: Pick + testsStats: SummaryTestsStats + flakyTests: Array<{ + path: { + relative: string + absolute: string + } + tests: Array<{ + testName: string + line: number | undefined + retries: { + allowed: number + count: number + ratio: number + } + }> + }> +} + +function collectSummaryData(testModules: ReadonlyArray): SummaryData { + const summaryData: SummaryData = { + fileStats: { + failed: 0, + passed: 0, + }, + testsStats: { + failed: 0, + passed: 0, + expectedFail: 0, + skipped: 0, + todo: 0, + }, + flakyTests: [], + } + + for (const module of testModules) { + const flakyTests: SummaryData['flakyTests'][number] = { + path: { relative: module.relativeModuleId, absolute: module.moduleId }, + tests: [], + } + + switch (module.task.result?.state) { + case 'fail': { + summaryData.fileStats.failed += 1 + break + } + case 'pass': { + summaryData.fileStats.passed += 1 + break + } + } + + for (const test of module.children.allTests()) { + switch (test.task.mode) { + case 'skip': { + summaryData.testsStats.skipped += 1 + break + } + case 'todo': { + summaryData.testsStats.todo += 1 + break + } + default: { + switch (test.task.result?.state) { + case 'fail': { + summaryData.testsStats.failed += 1 + break + } + case 'pass': { + if (test.task.fails) { + summaryData.testsStats.expectedFail += 1 + } + else { + summaryData.testsStats.passed += 1 + } + + break + } + } + } + } + + const diagnostic = test.diagnostic() + + if (diagnostic?.flaky) { + const retriesAllowed = typeof test.options.retry === 'number' + ? test.options.retry + : (test.options.retry?.count + // falling back to `retryCount` as this is used as the denominator to compute `retryRatio` + ?? diagnostic.retryCount) + const retriesRatio = diagnostic.retryCount / retriesAllowed + + flakyTests.tests.push({ + retries: { + allowed: retriesAllowed, + count: diagnostic.retryCount, + ratio: retriesRatio, + }, + line: test.task.location?.line, + testName: test.task.fullTestName, + }) + } + } + + if (flakyTests.tests.length > 0) { + flakyTests.tests.sort((a, b) => b.retries.ratio - a.retries.ratio) + + summaryData.flakyTests.push(flakyTests) + } + } + + return summaryData +} + +function createGitHubFileLinkCreator(fileLinks?: JobSummaryOptions['fileLinks']): (path: string, line?: number) => string | null { + const repository = fileLinks?.repository + const commitHash = fileLinks?.commitHash + const workspacePath = fileLinks?.workspacePath + + if (repository !== undefined && commitHash !== undefined && workspacePath !== undefined) { + return (path, line) => { + const lineFragment = line !== undefined ? `#L${line}` : '' + + return `https://github.com/${repository}/blob/${commitHash}/${relative(workspacePath, path)}${lineFragment}` + } + } + + return () => null +} + +function mdLink(text: string, url: string | null): string { + return url === null ? text : `[${text}](${url})` +} + +function renderStats({ fileStats, testsStats }: SummaryData): string { + const SEPARATOR_SYMBOL = ' · ' + + const fileInfoTotal = fileStats.failed + fileStats.passed + const primaryInfoTotal = testsStats.failed + testsStats.passed + testsStats.expectedFail + const secondaryInfoTotal = testsStats.skipped + testsStats.todo + + const fileInfo: string[] = [] + const primaryInfo: string[] = [] + const secondaryInfo: string[] = [] + + if (fileStats.failed > 0) { + fileInfo.push(`❌ **${fileStats.failed} ${noun(fileStats.failed, 'failure', 'failures')}**`) + } + + if (fileStats.passed > 0) { + fileInfo.push(`✅ **${fileStats.passed} ${noun(fileStats.passed, 'pass', 'passes')}**`) + } + + fileInfo.push(`${fileInfoTotal} total`) + + if (testsStats.failed > 0) { + primaryInfo.push(`❌ **${testsStats.failed} ${noun(testsStats.failed, 'failure', 'failures')}**`) + } + + if (testsStats.passed > 0) { + primaryInfo.push(`✅ **${testsStats.passed} ${noun(testsStats.passed, 'pass', 'passes')}**`) + } + + if (testsStats.expectedFail > 0) { + primaryInfo.push(`🔵 **${testsStats.expectedFail} expected ${noun(testsStats.expectedFail, 'failure', 'failures')}**`) + } + + primaryInfo.push(`${primaryInfoTotal} total`) + + if (testsStats.skipped > 0) { + secondaryInfo.push(`${testsStats.skipped} ${noun(testsStats.skipped, 'skip', 'skips')}`) + } + + if (testsStats.todo > 0) { + secondaryInfo.push(`${testsStats.todo} ${noun(testsStats.todo, 'todo', 'todos')}`) + } + + let output = `\n### Summary\n\n- **Test Files**: ${fileInfo.join(SEPARATOR_SYMBOL)}\n- **Test Results**: ${primaryInfo.join(SEPARATOR_SYMBOL)}\n` + + if (secondaryInfo.length > 0) { + secondaryInfo.push(`${secondaryInfoTotal} total`) + + output += `- **Other**: ${secondaryInfo.join(SEPARATOR_SYMBOL)}\n` + } + + return output +} + +const SUMMARY_HEADER = '## Vitest Test Report\n' + +function renderSummary(summaryData: SummaryData, fileLinks?: JobSummaryOptions['fileLinks']): string { + const fileLinkCreator = createGitHubFileLinkCreator(fileLinks) + + let summary = `${SUMMARY_HEADER}${renderStats(summaryData)}` + + if (summaryData.flakyTests.length > 0) { + summary += '\n### Flaky Tests\n\nThese tests passed only after one or more retries, indicating potential instability.\n' + + for (const flakyTests of summaryData.flakyTests) { + summary += `\n##### \`${flakyTests.path.relative}\` (${flakyTests.tests.length} flaky tests)\n` + + for (const flakyTest of flakyTests.tests) { + const retriesText = `passed on retry ${flakyTest.retries.count} out of ${flakyTest.retries.allowed}` + + summary += `\n- ${mdLink(`\`${flakyTest.testName}\``, fileLinkCreator(flakyTests.path.absolute, flakyTest.line))} (${flakyTest.retries.ratio >= 0.8 ? `**${retriesText}**` : retriesText})` + } + + summary += '\n' + } + } + + if (!summary.endsWith('\n')) { + summary += '\n' + } + + return summary +} diff --git a/packages/vitest/src/node/reporters/renderers/utils.ts b/packages/vitest/src/node/reporters/renderers/utils.ts index fdf7d7c76d0d..75fd91658b8b 100644 --- a/packages/vitest/src/node/reporters/renderers/utils.ts +++ b/packages/vitest/src/node/reporters/renderers/utils.ts @@ -284,3 +284,14 @@ export function truncateString(text: string, maxLength: number): string { function capitalize(text: T) { return `${text[0].toUpperCase()}${text.slice(1)}` as Capitalize } + +/** + * Returns the singular or plural form of a word based on the count. + */ +export function noun(count: number, singular: string, plural: string): string { + if (count === 1) { + return singular + } + + return plural +} diff --git a/test/cli/fixtures/reporters/github-actions/flaky/math.spec.ts b/test/cli/fixtures/reporters/github-actions/flaky/math.spec.ts new file mode 100644 index 000000000000..79e2abe1fcaa --- /dev/null +++ b/test/cli/fixtures/reporters/github-actions/flaky/math.spec.ts @@ -0,0 +1,39 @@ +import { test } from 'vitest' + +test('should add numbers correctly', { retry: 5 }, ({ expect, task }) => { + expect(task.result?.retryCount).toBe(0) +}) + +test('should subtract numbers correctly', { retry: 5 }, ({ expect, task }) => { + expect(task.result?.retryCount).toBe(1) +}) + +test('should multiply numbers correctly', { retry: 5 }, ({ expect, task }) => { + expect(task.result?.retryCount).toBe(5) +}) + +test('should divide numbers correctly', { retry: 5 }, ({ expect, task }) => { + expect(task.result?.retryCount).toBe(2) +}) + +test('should handle edge cases', { retry: 5 }, ({ expect, task }) => { + expect(task.result?.retryCount).toBe(4) +}) + +test('should validate input properly', { retry: 5 }, ({ expect, task }) => { + expect(task.result?.retryCount).toBe(4) +}) + +test.todo('should compute percentages') + +test.skip('should divide by zero', ({ expect }) => { + expect.unreachable() +}) + +test.fails('should work with linear equations', ({ expect }) => { + expect(true).toBe(false) +}) + +test('should compute square root of negative numbers', ({ expect }) => { + expect.unreachable() +}) diff --git a/test/cli/fixtures/reporters/github-actions/flaky/network.spec.ts b/test/cli/fixtures/reporters/github-actions/flaky/network.spec.ts new file mode 100644 index 000000000000..19ebd1526acd --- /dev/null +++ b/test/cli/fixtures/reporters/github-actions/flaky/network.spec.ts @@ -0,0 +1,15 @@ +import { describe, test } from 'vitest' + +describe('network', () => { + test('should fetch user data from API', { retry: 3 }, ({ expect, task }) => { + expect(task.result?.retryCount).toBe(2) + }) + + test('should handle network timeouts gracefully', { retry: 4 }, ({ expect, task }) => { + expect(task.result?.retryCount).toBe(4) + }) + + test('should retry failed requests', { retry: 3 }, ({ expect, task }) => { + expect(task.result?.retryCount).toBe(1) + }) +}) diff --git a/test/cli/test/reporters/github-actions.test.ts b/test/cli/test/reporters/github-actions.test.ts index 66dc815483ee..f4660f434056 100644 --- a/test/cli/test/reporters/github-actions.test.ts +++ b/test/cli/test/reporters/github-actions.test.ts @@ -1,3 +1,6 @@ +import { randomUUID } from 'node:crypto' +import { access, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' import { sep } from 'node:path' import { runVitest } from '#test-utils' import { resolve } from 'pathe' @@ -59,4 +62,150 @@ describe(GithubActionsReporter, () => { `) expect(stderr).toBe('') }) + + describe('summary', () => { + it('writes one when enabled', async ({ onTestFinished }) => { + const outputPath = resolve(tmpdir(), randomUUID()) + + onTestFinished(async () => { + await rm(outputPath).catch(() => { + console.error(`Could not remove ${outputPath}`) + }) + }) + + const workspacePath = resolve(import.meta.dirname, '..', '..', '..', '..') + + await runVitest({ + reporters: new GithubActionsReporter({ + jobSummary: { + outputPath, + fileLinks: { + commitHash: 'aaa', + repository: 'owner/repo', + workspacePath, + }, + }, + }), + root: './fixtures/reporters/github-actions', + }) + + const summary = await readFile(outputPath, 'utf8') + + expect(summary).toMatchInlineSnapshot(` + "## Vitest Test Report + + ### Summary + + - **Test Files**: ❌ **1 failure** · ✅ **1 pass** · 2 total + - **Test Results**: ❌ **1 failure** · ✅ **9 passes** · 🔵 **1 expected failure** · 11 total + - **Other**: 1 skip · 1 todo · 2 total + + ### Flaky Tests + + These tests passed only after one or more retries, indicating potential instability. + + ##### \`flaky/math.spec.ts\` (5 flaky tests) + + - [\`should multiply numbers correctly\`](https://github.com/owner/repo/blob/aaa/test/cli/fixtures/reporters/github-actions/flaky/math.spec.ts) (**passed on retry 5 out of 5**) + - [\`should handle edge cases\`](https://github.com/owner/repo/blob/aaa/test/cli/fixtures/reporters/github-actions/flaky/math.spec.ts) (**passed on retry 4 out of 5**) + - [\`should validate input properly\`](https://github.com/owner/repo/blob/aaa/test/cli/fixtures/reporters/github-actions/flaky/math.spec.ts) (**passed on retry 4 out of 5**) + - [\`should divide numbers correctly\`](https://github.com/owner/repo/blob/aaa/test/cli/fixtures/reporters/github-actions/flaky/math.spec.ts) (passed on retry 2 out of 5) + - [\`should subtract numbers correctly\`](https://github.com/owner/repo/blob/aaa/test/cli/fixtures/reporters/github-actions/flaky/math.spec.ts) (passed on retry 1 out of 5) + + ##### \`flaky/network.spec.ts\` (3 flaky tests) + + - [\`network > should handle network timeouts gracefully\`](https://github.com/owner/repo/blob/aaa/test/cli/fixtures/reporters/github-actions/flaky/network.spec.ts) (**passed on retry 4 out of 4**) + - [\`network > should fetch user data from API\`](https://github.com/owner/repo/blob/aaa/test/cli/fixtures/reporters/github-actions/flaky/network.spec.ts) (passed on retry 2 out of 3) + - [\`network > should retry failed requests\`](https://github.com/owner/repo/blob/aaa/test/cli/fixtures/reporters/github-actions/flaky/network.spec.ts) (passed on retry 1 out of 3) + " + `) + }) + + it.for([{ enabled: false }, { outputPath: undefined }] as const)('does not write one when disabled or without `outputPath`', async (options) => { + const outputPath = resolve(tmpdir(), randomUUID()) + + const workspacePath = resolve(import.meta.dirname, '..', '..', '..', '..') + + await runVitest({ + reporters: new GithubActionsReporter({ + jobSummary: { + outputPath, + ...options, + fileLinks: { + commitHash: 'aaa', + repository: 'owner/repo', + workspacePath, + }, + }, + }), + root: './fixtures/reporters/github-actions', + }) + + const summary = await access(outputPath).then(() => true).catch(() => false) + + expect(summary).toBe(false) + }) + + it.for([ + { commitHash: undefined }, + { repository: undefined }, + { workspacePath: undefined }, + ] as const)('writes one without links when one of `commitHash`, `repository` or `workspacePath` are not provided', async (options, { onTestFinished }) => { + const outputPath = resolve(tmpdir(), randomUUID()) + + onTestFinished(async () => { + await rm(outputPath).catch(() => { + console.error(`Could not remove ${outputPath}`) + }) + }) + + const workspacePath = resolve(import.meta.dirname, '..', '..', '..', '..') + + await runVitest({ + reporters: new GithubActionsReporter({ + jobSummary: { + outputPath, + fileLinks: { + commitHash: 'aaa', + repository: 'owner/repo', + workspacePath, + ...options, + }, + }, + }), + root: './fixtures/reporters/github-actions', + }) + + const summary = await readFile(outputPath, 'utf8') + + expect(summary).toMatchInlineSnapshot(` + "## Vitest Test Report + + ### Summary + + - **Test Files**: ❌ **1 failure** · ✅ **1 pass** · 2 total + - **Test Results**: ❌ **1 failure** · ✅ **9 passes** · 🔵 **1 expected failure** · 11 total + - **Other**: 1 skip · 1 todo · 2 total + + ### Flaky Tests + + These tests passed only after one or more retries, indicating potential instability. + + ##### \`flaky/math.spec.ts\` (5 flaky tests) + + - \`should multiply numbers correctly\` (**passed on retry 5 out of 5**) + - \`should handle edge cases\` (**passed on retry 4 out of 5**) + - \`should validate input properly\` (**passed on retry 4 out of 5**) + - \`should divide numbers correctly\` (passed on retry 2 out of 5) + - \`should subtract numbers correctly\` (passed on retry 1 out of 5) + + ##### \`flaky/network.spec.ts\` (3 flaky tests) + + - \`network > should handle network timeouts gracefully\` (**passed on retry 4 out of 4**) + - \`network > should fetch user data from API\` (passed on retry 2 out of 3) + - \`network > should retry failed requests\` (passed on retry 1 out of 3) + " + `) + }) + }) })