feat(reporters): add onTestCaseAttempt hook

docs/api/advanced/reporters.md

@@ -21,6 +21,7 @@ Vitest has its own test run lifecycle. These are represented by reporter's metho
           - [`onHookEnd(beforeEach)`](#onhookend)
           - [`onHookStart(afterEach)`](#onhookstart)
           - [`onHookEnd(afterEach)`](#onhookend)
+          - [`onTestCaseAttempt`](#ontestcaseattempt) <Version>5.1.0</Version>
         - [`onTestCaseResult`](#ontestcaseresult)
       - [`onHookStart(afterAll)`](#onhookstart)
       - [`onHookEnd(afterAll)`](#onhookend)
@@ -309,6 +310,28 @@ This method is called before the test starts to run or it was skipped. Note that
 Notice that it's possible to have [`testCase.result()`](/api/advanced/test-case#result) with `passed` or `failed` state already when `onTestCaseReady` is called. This can happen if test was running too fast and both `onTestCaseReady` and `onTestCaseResult` were scheduled to run in the same microtask.
 :::
 
+## onTestCaseAttempt <Version>5.1.0</Version> {#ontestcaseattempt}
+
+```ts
+function onTestCaseAttempt(testCase: TestCase, attempt: TestAttempt): Awaitable<void>
+```
+
+This method is called every time a run of the test finishes, including every [`retry`](/config/retry) and [`repeats`](/api/test#repeats) run. It is called after the `afterEach` hooks of that run, and before `onTestCaseResult` for the last run. Tests that are skipped without running, for example with `test.skip`, do not call it.
+
+The `attempt` argument has the same data as the matching entry in [`testCase.attempts()`](/api/advanced/test-case#attempts). Use it instead of reading the last item of `testCase.attempts()`, because `attempts()` can already include later runs by the time the hook is called, for example when running with `--merge-reports`.
+
+```ts
+import type { Reporter, TestAttempt, TestCase } from 'vitest/node'
+
+export default {
+  onTestCaseAttempt(testCase: TestCase, attempt: TestAttempt) {
+    if (attempt.state === 'failed') {
+      console.log(`${testCase.fullName} failed on retry ${attempt.retryIndex}`)
+    }
+  },
+} satisfies Reporter
+```
+
 ## onTestCaseResult
 
 ```ts

docs/api/advanced/test-case.md

@@ -327,7 +327,7 @@ For example, a test with `retry: 2` that passes on its third run returns:
 ```
 
 ::: info
-`attempts()` returns an empty array if the test has not run yet.
+`attempts()` returns an empty array if the test has not run yet. While the test is running, it only includes the runs that have finished. Use the [`onTestCaseAttempt`](/api/advanced/reporters#ontestcaseattempt) reporter hook to be notified when each run finishes.
 :::
 
 ## annotations

packages/vitest/src/node/reporters/reported-tasks.ts

@@ -8,6 +8,7 @@ import type {
   Suite as RunnerTestSuite,
   SerializableRetry,
   TaskMeta,
+  TaskResultAttempt,
   TestAnnotation,
   TestArtifact,
   TestBenchmark,
@@ -198,14 +199,7 @@ export class TestCase extends ReportedTaskImplementation {
    * Individual results for every retry and repeat attempt.
    */
   public attempts(): ReadonlyArray<TestAttempt> {
-    return (this.task.result?.attempts || []).map((attempt) => ({
-      state: attempt.state === 'pass' ? 'passed' : attempt.state === 'fail' ? 'failed' : 'skipped',
-      errors: attempt.errors as TestError[] | undefined,
-      duration: attempt.duration,
-      startTime: attempt.startTime,
-      retryIndex: attempt.retryIndex,
-      repeatIndex: attempt.repeatIndex,
-    }))
+    return (this.task.result?.attempts || []).map(toTestAttempt)
   }
 
   /**
@@ -804,6 +798,17 @@ function getReportedTask(
   return reportedTask
 }
 
+export function toTestAttempt(attempt: TaskResultAttempt): TestAttempt {
+  return {
+    state: attempt.state === 'pass' ? 'passed' : attempt.state === 'fail' ? 'failed' : 'skipped',
+    errors: attempt.errors as TestError[] | undefined,
+    duration: attempt.duration,
+    startTime: attempt.startTime,
+    retryIndex: attempt.retryIndex,
+    repeatIndex: attempt.repeatIndex,
+  }
+}
+
 function getSuiteState(task: RunnerTestSuite | RunnerTestFile): TestSuiteState {
   const mode = task.mode
   const state = task.result?.state

packages/vitest/src/node/test-run.ts

@@ -30,6 +30,7 @@ import { parseErrorStacktrace } from '@vitest/utils/source-map'
 import { extractSourcemapFromFile } from '@vitest/utils/source-map/node'
 import mime from 'mime/lite'
 import { basename, extname, resolve } from 'pathe'
+import { toTestAttempt } from './reporters/reported-tasks'
 
 export class TestRun {
   constructor(private vitest: Vitest) {}
@@ -271,6 +272,11 @@ export class TestRun {
       return await this.vitest.report('onTestCaseReady', entity)
     }
 
+    if (event === 'test-attempt-finished' && entity.type === 'test') {
+      assert(data?.attempt, 'Attempt must be provided for test-attempt-finished event')
+      return await this.vitest.report('onTestCaseAttempt', entity, toTestAttempt(data.attempt))
+    }
+
     if (event === 'test-finished' && entity.type === 'test') {
       return await this.vitest.report('onTestCaseResult', entity)
     }

packages/vitest/src/node/types/reporter.ts

@@ -12,6 +12,7 @@ import type { Vitest } from '../core'
 import type { TestProject } from '../project'
 import type {
   ReportedHookContext,
+  TestAttempt,
   TestCase,
   TestModule,
   TestSuite,
@@ -76,6 +77,11 @@ export interface Reporter {
    * The `result()` cannot be `pending`.
    */
   onTestCaseResult?: (testCase: TestCase) => Awaitable<void>
+  /**
+   * Called after each run of the test finishes, including every retry and repeat.
+   * Called before `onTestCaseResult` for the final attempt.
+   */
+  onTestCaseAttempt?: (testCase: TestCase, attempt: TestAttempt) => Awaitable<void>
 
   /**
    * Called when annotation is added via the `task.annotate` API.

packages/vitest/src/runtime/runner/run.ts

@@ -10,6 +10,8 @@ import type {
   Suite,
   SuiteHooks,
   Task,
+  TaskEventData,
+  TaskEventPack,
   TaskMeta,
   TaskResult,
   TaskResultAttempt,
@@ -456,7 +458,7 @@ async function callAroundEachHooks(
 }
 
 const packs = new Map<string, [TaskResult | undefined, TaskMeta]>()
-const eventsPacks: [string, TaskUpdateEvent, undefined][] = []
+const eventsPacks: TaskEventPack[] = []
 const pendingTasksUpdates: Promise<void>[] = []
 
 function sendTasksUpdate(runner: VitestRunner): void {
@@ -507,8 +509,13 @@ function throttle<T extends (...args: any[]) => void>(fn: T, ms: number): T {
 // throttle based on summary reporter's DURATION_UPDATE_INTERVAL_MS
 const sendTasksUpdateThrottled = throttle(sendTasksUpdate, 100)
 
-function updateTask(event: TaskUpdateEvent, task: Task, runner: VitestRunner): void {
-  eventsPacks.push([task.id, event, undefined])
+function updateTask(
+  event: TaskUpdateEvent,
+  task: Task,
+  runner: VitestRunner,
+  data?: TaskEventData,
+): void {
+  eventsPacks.push([task.id, event, data])
   packs.set(task.id, [task.result, task.meta])
   sendTasksUpdateThrottled(runner)
 }
@@ -609,14 +616,20 @@ async function runTest(test: Test, runner: VitestRunner): Promise<void> {
       const attemptErrorsStart = test.result.errors?.length ?? 0
       const recordAttempt = (state: TaskResultAttempt['state']) => {
         const errors = test.result!.errors?.slice(attemptErrorsStart)
-        test.result!.attempts!.push({
+        const attempt: TaskResultAttempt = {
           state,
           errors: errors?.length ? errors : undefined,
           duration: now() - attemptStart,
           startTime: attemptStartTime,
           retryIndex: retryCount,
           repeatIndex: repeatCount,
-        })
+        }
+        test.result!.attempts!.push(attempt)
+        updateTask('test-attempt-finished', test, runner, { attempt })
+        // tests that run once don't need a dedicated flush per attempt
+        if (retry || repeats) {
+          sendTasksUpdate(runner)
+        }
       }
       let beforeEachCleanups: unknown[] = []
       // fixtureCheckpoint is passed by callAroundEachHooks - it represents the count

packages/vitest/src/runtime/runner/types.ts

@@ -267,6 +267,7 @@ export type TaskResultPack = [
 export interface TaskEventData {
   annotation?: TestAnnotation | undefined
   artifact?: TestArtifact | undefined
+  attempt?: TaskResultAttempt | undefined
 }
 
 export type TaskEventPack = [
@@ -289,7 +290,11 @@ export type TaskUpdateEvent =
   | 'suite-failed-early'
   | 'test-prepare'
   | 'test-finished'
+  /**
+   * @deprecated Use `test-attempt-finished` instead.
+   */
   | 'test-retried'
+  | 'test-attempt-finished'
   | 'test-cancel'
   | 'suite-prepare'
   | 'suite-finished'

packages/vitest/src/utils/tasks.ts

@@ -133,6 +133,9 @@ export function convertTasksToEvents(
           task.artifacts.forEach((artifact) => {
             events.push([task.id, 'test-artifact', { artifact }])
           })
+          task.result?.attempts?.forEach((attempt) => {
+            events.push([task.id, 'test-attempt-finished', { attempt }])
+          })
           events.push([task.id, 'test-finished', undefined])
         }
       }

test/browser/specs/test-attempts.test.ts

@@ -0,0 +1,47 @@
+import { expect, test } from 'vitest'
+import { instances, runInlineBrowserTests } from './utils'
+
+test('reports every attempt through onTestCaseAttempt', async () => {
+  const events: string[] = []
+
+  const { stderr } = await runInlineBrowserTests(
+    {
+      'basic.test.ts': `
+        import { expect, test } from 'vitest'
+
+        let runs = 0
+        test('flaky', { retry: 1, repeats: 1 }, () => {
+          expect(++runs % 2).toBe(0)
+        })
+      `,
+    },
+    {
+      browser: {
+        instances: [instances[0]],
+      },
+      reporters: [
+        {
+          onTestCaseAttempt(testCase, attempt) {
+            events.push(
+              `attempt |${testCase.name}| [repeat ${attempt.repeatIndex}, retry ${attempt.retryIndex}, ${attempt.state}, attempts() ${testCase.attempts().length}]`,
+            )
+          },
+          onTestCaseResult(testCase) {
+            events.push(`result |${testCase.name}| ${testCase.result().state}`)
+          },
+        },
+      ],
+    },
+  )
+
+  expect(stderr).toBe('')
+  expect(events).toMatchInlineSnapshot(`
+    [
+      "attempt |flaky| [repeat 0, retry 0, failed, attempts() 1]",
+      "attempt |flaky| [repeat 0, retry 1, passed, attempts() 2]",
+      "attempt |flaky| [repeat 1, retry 0, failed, attempts() 3]",
+      "attempt |flaky| [repeat 1, retry 1, passed, attempts() 4]",
+      "result |flaky| passed",
+    ]
+  `)
+})