feat: introduce the new reporter API

docs/.vitepress/config.ts

@@ -353,6 +353,10 @@ export default ({ mode }: { mode: string }) => {
                 text: 'Running Tests',
                 link: '/advanced/guide/tests',
               },
+              {
+                text: 'Test Lifecycle',
+                link: '/advanced/guide/lifecycle',
+              },
               {
                 text: 'Extending Reporters',
                 link: '/advanced/reporters',

docs/advanced/guide/lifecycle.md

@@ -0,0 +1,3 @@
+# Test Lifecycle
+
+<!-- TODO: lifecyle diagram and reporter API -->

packages/vitest/src/node/core.ts

@@ -36,6 +36,7 @@ import { BlobReporter, readBlobs } from './reporters/blob'
 import { createBenchmarkReporters, createReporters } from './reporters/utils'
 import { VitestSpecifications } from './specifications'
 import { StateManager } from './state'
+import { TestRun } from './test-run'
 import { VitestWatcher } from './watcher'
 import { resolveBrowserWorkspace, resolveWorkspace } from './workspace/resolveWorkspace'
 
@@ -94,6 +95,7 @@ export class Vitest {
   /** @internal */ reporters: Reporter[] = undefined!
   /** @internal */ vitenode: ViteNodeServer = undefined!
   /** @internal */ runner: ViteNodeRunner = undefined!
+  /** @internal */ _testRun: TestRun = undefined!
 
   private isFirstRun = true
   private restartsCount = 0
@@ -213,6 +215,7 @@ export class Vitest {
     this._state = new StateManager()
     this._cache = new VitestCache(this.version)
     this._snapshot = new SnapshotManager({ ...resolved.snapshotOptions })
+    this._testRun = new TestRun(this)
 
     if (this.config.watch) {
       this.watcher.registerWatcher()
@@ -1159,6 +1162,13 @@ export class Vitest {
 
   /** @internal */
   async report<T extends keyof Reporter>(name: T, ...args: ArgumentsType<Reporter[T]>) {
+    if (name === 'onTaskUpdate') {
+      this._testRun.updated(
+        // @ts-expect-error let me go
+        ...args,
+      )
+    }
+
     await Promise.all(this.reporters.map(r => r[name]?.(
       // @ts-expect-error let me go
       ...args,

packages/vitest/src/node/reporters/reported-tasks.ts

@@ -5,7 +5,7 @@ import type {
   Suite as RunnerTestSuite,
   TaskMeta,
 } from '@vitest/runner'
-import type { TestError } from '@vitest/utils'
+import type { SerializedError, TestError } from '@vitest/utils'
 import type { TestProject } from '../project'
 
 class ReportedTaskImplementation {
@@ -122,12 +122,19 @@ export class TestCase extends ReportedTaskImplementation {
   }
 
   /**
-   * Test results. Will be `undefined` if test is skipped, not finished yet or was just collected.
+   * Test results.
+   * - **pending**: Test was collected, but didn't finish running yet.
+   * - **passed**: Test passed successfully
+   * - **failed**: Test failed to execute
+   * - **skipped**: Test was skipped during collection or dynamically with `ctx.skip()`.
    */
-  public result(): TestResult | undefined {
+  public result(): TestResult {
     const result = this.task.result
     if (!result || result.state === 'run' || result.state === 'queued') {
-      return undefined
+      return {
+        state: 'pending',
+        errors: undefined,
+      }
     }
     const state = result.state === 'fail'
       ? 'failed' as const
@@ -298,10 +305,33 @@ class TestCollection {
 
 export type { TestCollection }
 
+export interface TestSuiteStatistics {
+  total: number
+  completed: number
+  passed: number
+  failed: number
+  skipped: number
+  todo: number
+}
+
+function createStatistics() {
+  return {
+    total: 0,
+    completed: 0,
+    passed: 0,
+    failed: 0,
+    skipped: 0,
+    todo: 0,
+  }
+}
+
 abstract class SuiteImplementation extends ReportedTaskImplementation {
   /** @internal */
   declare public readonly task: RunnerTestSuite | RunnerTestFile
 
+  /** @internal */
+  public _statistic: TestSuiteStatistics = createStatistics()
+
   /**
    * Collection of suites and tests that are part of this suite.
    */
@@ -314,18 +344,41 @@ abstract class SuiteImplementation extends ReportedTaskImplementation {
   }
 
   /**
-   * Checks if the suite was skipped during collection.
+   * The number of tests in this suite with a specific state.
    */
-  public skipped(): boolean {
+  public statistics(): TestSuiteStatistics {
+    return { ...this._statistic }
+  }
+
+  /**
+   * Checks the running state of the suite.
+   */
+  public state(): TestSuiteState {
     const mode = this.task.mode
-    return mode === 'skip' || mode === 'todo'
+    const state = this.task.result?.state
+    if (mode === 'skip' || mode === 'todo' || state === 'skip' || state === 'todo') {
+      return 'skipped'
+    }
+    if (state === 'queued') {
+      return 'queued'
+    }
+    if (state == null || state === 'run' || state === 'only') {
+      return 'pending'
+    }
+    if (state === 'fail') {
+      return 'failed'
+    }
+    if (state === 'pass') {
+      return 'passed'
+    }
+    throw new Error(`Unknown suite state: ${state}`)
   }
 
   /**
    * Errors that happened outside of the test run during collection, like syntax errors.
    */
-  public errors(): TestError[] {
-    return (this.task.result?.errors as TestError[] | undefined) || []
+  public errors(): SerializedError[] {
+    return (this.task.result?.errors as SerializedError[] | undefined) || []
   }
 }
 
@@ -402,8 +455,8 @@ export class TestModule extends SuiteImplementation {
 
   /**
    * This is usually an absolute UNIX file path.
-   * It can be a virtual id if the file is not on the disk.
-   * This value corresponds to Vite's `ModuleGraph` id.
+   * It can be a virtual ID if the file is not on the disk.
+   * This value corresponds to the ID in the Vite's module graph.
    */
   public readonly moduleId: string
 
@@ -420,9 +473,9 @@ export class TestModule extends SuiteImplementation {
   declare public ok: () => boolean
 
   /**
-   * Checks if the module was skipped and didn't run.
+   * Checks the running state of the test file.
    */
-  declare public skipped: () => boolean
+  declare public state: () => TestSuiteState
 
   /**
    * Useful information about the module like duration, memory usage, etc.
@@ -446,6 +499,7 @@ export class TestModule extends SuiteImplementation {
 
 export interface TaskOptions {
   each: boolean | undefined
+  fails: boolean | undefined
   concurrent: boolean | undefined
   shuffle: boolean | undefined
   retry: number | undefined
@@ -454,10 +508,11 @@ export interface TaskOptions {
 }
 
 function buildOptions(
-  task: RunnerTestCase | RunnerTestFile | RunnerTestSuite,
+  task: RunnerTestCase | RunnerTestSuite,
 ): TaskOptions {
   return {
     each: task.each,
+    fails: task.type === 'test' && task.fails,
     concurrent: task.concurrent,
     shuffle: task.shuffle,
     retry: task.retry,
@@ -466,7 +521,24 @@ function buildOptions(
   }
 }
 
-export type TestResult = TestResultPassed | TestResultFailed | TestResultSkipped
+export type TestSuiteState = 'skipped' | 'pending' | 'queued' | 'failed' | 'passed'
+
+export type TestResult =
+  | TestResultPassed
+  | TestResultFailed
+  | TestResultSkipped
+  | TestResultPending
+
+export interface TestResultPending {
+  /**
+   * The test was collected, but didn't finish running yet.
+   */
+  state: 'pending'
+  /**
+   * Pending tests have no errors.
+   */
+  errors: undefined
+}
 
 export interface TestResultPassed {
   /**

packages/vitest/src/node/reporters/task-parser.ts

@@ -1,3 +1,5 @@
+// TODO: Remove once Reporter API implements these life cycles
+
 import type { File, Task, TaskResultPack, Test } from '@vitest/runner'
 import type { Vitest } from '../core'
 import { getTests } from '@vitest/runner/utils'

packages/vitest/src/node/state.ts

@@ -184,7 +184,11 @@ export class StateManager {
     }
   }
 
-  getReportedEntity(task: Task) {
+  getReportedEntity(task?: Task) {
+    if (!task) {
+      return undefined
+    }
+
     return this.reportedTasksMap.get(task)
   }
 

packages/vitest/src/node/test-run.ts

@@ -0,0 +1,128 @@
+import type { TaskResultPack } from '@vitest/runner'
+import type { Vitest } from './core'
+import type { TestCase, TestModule } from './reporters/reported-tasks'
+import type { TestSpecification } from './spec'
+
+export class TestRun {
+  private tests = emptyCounters()
+  private suites = emptyCounters()
+
+  // Internal state to prevent reporting duplicates and guaranteeing correct order
+  private runningTestModules = new Set<TestModule['id']>()
+  private finishedTestModules = new Set<TestModule['id']>()
+
+  private runningTestCases = new Set<TestCase['id']>()
+  private finishedTestCases = new Set<TestCase['id']>()
+
+  constructor(private vitest: Vitest) {}
+
+  async start(specifications: TestSpecification[]) {
+    this.tests = emptyCounters()
+    this.suites = emptyCounters()
+    this.suites.total = specifications.length
+
+    this.runningTestModules.clear()
+    this.finishedTestModules.clear()
+    this.runningTestCases.clear()
+    this.finishedTestCases.clear()
+
+    await this.vitest.report('onTestRunStart', specifications)
+  }
+
+  enqueued(_module: TestModule) {
+    // TODO
+  }
+
+  collected(_modules: TestModule[]) {
+    // TODO
+  }
+
+  async updated(update: TaskResultPack[]) {
+    const runningTestModules: TestModule[] = []
+    const finishedTestModules: TestModule[] = []
+
+    const runningTestCases: TestCase[] = []
+    const finishedTestCases: TestCase[] = []
+
+    for (const [id] of update) {
+      const entity = this.vitest.state.getReportedEntity(this.vitest.state.idMap.get(id))
+
+      if (!entity) {
+        continue
+      }
+
+      if (entity.type === 'module') {
+        const state = entity.state()
+
+        if (state === 'pending' && !this.runningTestModules.has(entity.id)) {
+          this.runningTestModules.add(entity.id)
+          runningTestModules.push(entity)
+        }
+
+        if (state !== 'pending' && state !== 'queued' && !this.finishedTestModules.has(entity.id)) {
+          this.finishedTestModules.add(entity.id)
+          finishedTestModules.push(entity)
+
+          // If module run was fast, it's possible that it was never reported to be running
+          if (!this.runningTestModules.has(entity.id)) {
+            this.runningTestModules.add(entity.id)
+            runningTestModules.push(entity)
+          }
+
+          // Skipped tests need to be reported manually once test module has finished
+          for (const test of entity.children.tests()) {
+            if (!this.finishedTestCases.has(test.id)) {
+              this.finishedTestCases.add(test.id)
+              finishedTestCases.push(test)
+            }
+          }
+        }
+      }
+
+      if (entity.type === 'test') {
+        const state = entity.result().state
+
+        if (state === 'pending' && !this.runningTestCases.has(entity.id)) {
+          this.runningTestCases.add(entity.id)
+          runningTestCases.push(entity)
+        }
+
+        if (state !== 'pending' && !this.finishedTestCases.has(entity.id)) {
+          this.finishedTestCases.add(entity.id)
+          finishedTestCases.push(entity)
+
+          // If test finished quickly, it's possible that it was never reported as running
+          if (!this.runningTestCases.has(entity.id)) {
+            this.runningTestCases.add(entity.id)
+            runningTestCases.push(entity)
+          }
+        }
+      }
+    }
+
+    // Order of reporting is important here
+    await Promise.all(finishedTestCases.map(testCase => this.vitest.report('onTestCaseFinished', testCase)))
+    await Promise.all(finishedTestModules.map(module => this.vitest.report('onTestModuleFinished', module)))
+
+    await Promise.all(runningTestModules.map(module => this.vitest.report('onTestModulePrepare', module)))
+    await Promise.all(runningTestCases.map(testCase => this.vitest.report('onTestCasePrepare', testCase)))
+  }
+
+  async end() {
+    // TODO
+    await this.vitest.report('onTestRunEnd', [], [], 'passed')
+  }
+}
+
+interface Counter {
+  total: number
+  completed: number
+  passed: number
+  failed: number
+  skipped: number
+  todo: number
+}
+
+function emptyCounters(): Counter {
+  return { completed: 0, passed: 0, failed: 0, skipped: 0, todo: 0, total: 0 }
+}