feat: annotation API (#7953)

Author: sheremet-va

.gitignore

@@ -32,3 +32,4 @@ test/cli/fixtures/browser-multiple/basic-*
 # exclude static html reporter folder
 test/browser/html/
 test/core/html/
+.vitest-attachments

docs/.vitepress/config.ts

@@ -516,6 +516,10 @@ function guide(): DefaultTheme.SidebarItem[] {
       text: 'Test Context',
       link: '/guide/test-context',
     },
+    {
+      text: 'Test Annotations',
+      link: '/guide/test-annotations',
+    },
     {
       text: 'Environment',
       link: '/guide/environment',

docs/advanced/api/reporters.md

@@ -15,6 +15,7 @@ Vitest has its own test run lifecycle. These are represented by reporter's metho
       - [`onHookStart(beforeAll)`](#onhookstart)
       - [`onHookEnd(beforeAll)`](#onhookend)
         - [`onTestCaseReady`](#ontestcaseready)
+          - [`onTestAnnotate`](#ontestannotate) <Version>3.2.0</Version>
           - [`onHookStart(beforeEach)`](#onhookstart)
           - [`onHookEnd(beforeEach)`](#onhookend)
           - [`onHookStart(afterEach)`](#onhookstart)
@@ -317,3 +318,16 @@ function onTestCaseResult(testCase: TestCase): Awaitable<void>
 This method is called when the test has finished running or was just skipped. Note that this will be called after the `afterEach` hook is finished, if there are any.
 
 At this point, [`testCase.result()`](/advanced/api/test-case#result) will have non-pending state.
+
+## onTestAnnotate <Version>3.2.0</Version> {#ontestannotate}
+
+```ts
+function onTestAnnotate(
+  testCase: TestCase,
+  annotation: TestAnnotation,
+): Awaitable<void>
+```
+
+The `onTestAnnotate` hook is associated with the [`context.annotate`](/guide/test-context#annotate) method. When `annotate` is invoked, Vitest serialises it and sends the same attachment to the main thread where reporter can interact with it.
+
+If the path is specified, Vitest stores it in a separate directory (configured by [`attachmentsDir`](/config/#attachmentsdir)) and modifies the `path` property to reference it.

docs/config/index.md

@@ -2645,3 +2645,10 @@ Polling timeout in milliseconds
 - **Default:** `false`
 
 Always print console traces when calling any `console` method. This is useful for debugging.
+
+### attachmentsDir <Version>3.2.0</Version>
+
+- **Type:** `string`
+- **Default:** `'.vitest-attachments'`
+
+Directory path for storing attachments created by [`context.annotate`](/guide/test-context#annotate) relative to the project root.

docs/guide/test-annotations.md

@@ -0,0 +1,98 @@
+---
+title: Test Annotations | Guide
+outline: deep
+---
+
+# Test Annotations
+
+Vitest supports annotating your tests with custom messages and files via the [`context.annotate`](/guide/test-context#annotate) API. These annotations will be attached to the test case and passed down to reporters in the [`onTestAnnotate`](/advanced/api/reporters#ontestannotate) hook.
+
+```ts
+test('hello world', async ({ annotate }) => {
+  await annotate('this is my test')
+
+  if (condition) {
+    await annotate('this should\'ve errored', 'error')
+  }
+
+  const file = createTestSpecificFile()
+  await annotate('creates a file', { body: file })
+})
+```
+
+Depending on your reporter, you will see these annotations differently.
+
+## Built-in Reporters
+### default
+
+The `default` reporter prints annotations only if the test has failed:
+
+```
+  ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯
+
+  FAIL  example.test.js > an example of a test with annotation
+Error: thrown error
+  ❯ example.test.js:11:21
+      9 |    await annotate('annotation 1')
+      10|    await annotate('annotation 2', 'warning')
+      11|    throw new Error('thrown error')
+        |          ^
+      12|  })
+
+  ❯ example.test.js:9:15 notice
+    ↳ annotation 1
+  ❯ example.test.js:10:15 warning
+    ↳ annotation 2
+
+  ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯
+```
+
+### verbose
+
+In a TTY terminal, the `verbose` reporter works similarly to the `default` reporter. However, in a non-TTY environment, the `verbose` reporter will also print annotations after every test.
+
+```
+✓ example.test.js > an example of a test with annotation
+
+  ❯ example.test.js:9:15 notice
+    ↳ annotation 1
+  ❯ example.test.js:10:15 warning
+    ↳ annotation 2
+
+```
+
+### html
+
+The HTML reporter shows annotations the same way the UI does. You can see the annotation on the line where it was called. At the moment, if the annotation wasn't called in a test file, you cannot see it in the UI. We are planning to support a separate test summary view where it will be visible.
+
+<img alt="Vitest UI" img-light src="/annotations-html-light.png">
+<img alt="Vitest UI" img-dark src="/annotations-html-dark.png">
+
+### junit
+
+The `junit` reporter lists annotations inside the testcase's `properties` tag. The JUnit reporter will ignore all attachments and will print only the type and the message.
+
+```xml
+<testcase classname="basic/example.test.js" name="an example of a test with annotation" time="0.14315">
+    <properties>
+        <property name="notice" value="the message of the annotation">
+        </property>
+    </properties>
+</testcase>
+```
+
+### github-actions
+
+The `github-actions` reporter will print the annotation as a [notice message](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-a-notice-message) by default. You can configure the `type` by passing down the second argument as `notice`, `warning` or `error`. If type is none of these, Vitest will show the message as a notice.
+
+<img alt="GitHub Actions" img-light src="/annotations-gha-light.png">
+<img alt="GitHub Actions" img-dark src="/annotations-gha-dark.png">
+
+### tap
+
+The `tap` and `tap-flat` reporters print annotations as diagnostic messages on a new line starting with a `#` symbol. They will ignore all attachments and will print only the type and message:
+
+```
+ok 1 - an example of a test with annotation # time=143.15ms
+    # notice: the message of the annotation
+```

docs/guide/test-context.md

@@ -79,7 +79,30 @@ it('math is hard', ({ skip, mind }) => {
 })
 ```
 
-#### `context.signal` <Version>3.2.0</Version> {#context-signal}
+#### `annotate` <Version>3.2.0</Version> {#annotate}
+
+```ts
+function annotate(
+  message: string,
+  attachment?: TestAttachment,
+): Promise<TestAnnotation>
+
+function annotate(
+  message: string,
+  type?: string,
+  attachment?: TestAttachment,
+): Promise<TestAnnotation>
+```
+
+Add a [test annotation](/guide/test-annotations) that will be displayed by your [reporter](/config/#reporter).
+
+```ts
+test('annotations API', async ({ annotate }) => {
+  await annotate('https://github.com/vitest-dev/vitest/pull/7953', 'issues')
+})
+```
+
+#### `signal` <Version>3.2.0</Version> {#signal}
 
 An [`AbortSignal`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal) that can be aborted by Vitest. The signal is aborted in these situations:
 

docs/public/annotations-gha-dark.png

@@ -1,5 +1,5 @@
-import type { CancelReason, File, Suite, Task, TaskEventPack, TaskResultPack, VitestRunner } from '@vitest/runner'
-import type { SerializedConfig, TestExecutionMethod, WorkerGlobalState } from 'vitest'
+import type { CancelReason, File, Suite, Task, TaskEventPack, TaskResultPack, TestAnnotation, VitestRunner } from '@vitest/runner'
+import type { RunnerTestCase, SerializedConfig, TestExecutionMethod, WorkerGlobalState } from 'vitest'
 import type { VitestExecutor } from 'vitest/execute'
 import type { VitestBrowserClientMocker } from './mocker'
 import { globalChannel, onCancel } from '@vitest/browser/client'
@@ -146,6 +146,40 @@ export function createBrowserRunner(
       return rpc().onCollected(this.method, files)
     }
 
+    onTestAnnotate = (test: RunnerTestCase, annotation: TestAnnotation): Promise<TestAnnotation> => {
+      if (annotation.location) {
+        // the file should be the test file
+        // tests from other files are not supported
+        const map = this.sourceMapCache.get(annotation.location.file)
+        if (!map) {
+          return rpc().onTaskAnnotate(test.id, annotation)
+        }
+
+        const traceMap = new TraceMap(map as any)
+        const { line, column, source } = originalPositionFor(traceMap, annotation.location)
+        if (line != null && column != null && source != null) {
+          let file: string = annotation.location.file
+          if (source) {
+            const fileUrl = annotation.location.file.startsWith('file://')
+              ? annotation.location.file
+              : `file://${annotation.location.file}`
+            const sourceRootUrl = map.sourceRoot
+              ? new URL(map.sourceRoot, fileUrl)
+              : fileUrl
+            file = new URL(source, sourceRootUrl).pathname
+          }
+
+          annotation.location = {
+            line,
+            column: column + 1,
+            // if the file path is on windows, we need to remove the starting slash
+            file: file.match(/\/\w:\//) ? file.slice(1) : file,
+          }
+        }
+      }
+      return rpc().onTaskAnnotate(test.id, annotation)
+    }
+
     onTaskUpdate = (task: TaskResultPack[], events: TaskEventPack[]): Promise<void> => {
       return rpc().onTaskUpdate(this.method, task, events)
     }
@@ -220,14 +254,24 @@ export async function initiateRunner(
   return runner
 }
 
+async function getTraceMap(file: string, sourceMaps: Map<string, any>) {
+  const result = sourceMaps.get(file) || await rpc().getBrowserFileSourceMap(file).then((map) => {
+    sourceMaps.set(file, map)
+    return map
+  })
+  if (!result) {
+    return null
+  }
+  return new TraceMap(result as any)
+}
+
 async function updateTestFilesLocations(files: File[], sourceMaps: Map<string, any>) {
   const promises = files.map(async (file) => {
-    const result = sourceMaps.get(file.filepath) || await rpc().getBrowserFileSourceMap(file.filepath)
-    if (!result) {
+    const traceMap = await getTraceMap(file.filepath, sourceMaps)
+    if (!traceMap) {
       return null
     }
-    const traceMap = new TraceMap(result as any)
-    function updateLocation(task: Task) {
+    const updateLocation = (task: Task) => {
       if (task.location) {
         const { line, column } = originalPositionFor(traceMap, task.location)
         if (line != null && column != null) {

docs/public/annotations-gha-light.png

@@ -149,6 +149,9 @@ export function setupBrowserRpc(globalServer: ParentBrowserProject, defaultMocke
             await vitest._testRun.collected(project, files)
           }
         },
+        async onTaskAnnotate(id, annotation) {
+          return vitest._testRun.annotate(id, annotation)
+        },
         async onTaskUpdate(method, packs, events) {
           if (method === 'collect') {
             vitest.state.updateTasks(packs)

docs/public/annotations-html-dark.png

@@ -1,6 +1,6 @@
 import type { MockedModuleSerialized } from '@vitest/mocker'
 import type { ServerIdResolution, ServerMockResolution } from '@vitest/mocker/node'
-import type { TaskEventPack, TaskResultPack } from '@vitest/runner'
+import type { TaskEventPack, TaskResultPack, TestAnnotation } from '@vitest/runner'
 import type { BirpcReturn } from 'birpc'
 import type {
   AfterSuiteRunMeta,
@@ -19,6 +19,7 @@ export interface WebSocketBrowserHandlers {
   onUnhandledError: (error: unknown, type: string) => Promise<void>
   onQueued: (method: TestExecutionMethod, file: RunnerTestFile) => void
   onCollected: (method: TestExecutionMethod, files: RunnerTestFile[]) => Promise<void>
+  onTaskAnnotate: (testId: string, annotation: TestAnnotation) => Promise<TestAnnotation>
   onTaskUpdate: (method: TestExecutionMethod, packs: TaskResultPack[], events: TaskEventPack[]) => void
   onAfterSuiteRun: (meta: AfterSuiteRunMeta) => void
   cancelCurrentRun: (reason: CancelReason) => void