feat: add soft assertions feature (#1836)

* feat: add soft assertions feature

* chore: implement suggestion

* Update docs/API.md

* Update src/index.ts

* chore: fix tests

* chore: adjust vitest threshold

* Update docs/API.md

---------

Co-authored-by: Christian Bromann <git@bromann.dev>

Author: JustasMonkev

docs/API.md

@@ -2,6 +2,115 @@
 
 When you're writing tests, you often need to check that values meet certain conditions. `expect` gives you access to a number of "matchers" that let you validate different things on the `browser`, an `element` or `mock` object.
 
+## Soft Assertions
+
+Soft assertions allow you to continue test execution even when an assertion fails. This is useful when you want to check multiple conditions in a test and collect all failures rather than stopping at the first failure. Failures are collected and reported at the end of the test.
+
+### Usage
+
+```js
+// Mocha example
+it('product page smoke', async () => {
+  // These won't throw immediately if they fail
+  await expect.soft(await $('h1').getText()).toEqual('Basketball Shoes');
+  await expect.soft(await $('#price').getText()).toMatch(/€\d+/);
+
+  // Regular assertions still throw immediately
+  await expect(await $('.add-to-cart').isClickable()).toBe(true);
+});
+
+// At the end of the test, all soft assertion failures
+// will be reported together with their details
+```
+
+### Soft Assertion API
+
+#### expect.soft()
+
+Creates a soft assertion that collects failures instead of immediately throwing errors.
+
+```js
+await expect.soft(actual).toBeDisplayed();
+await expect.soft(actual).not.toHaveText('Wrong text');
+```
+
+#### expect.getSoftFailures()
+
+Get all collected soft assertion failures for the current test.
+
+```js
+const failures = expect.getSoftFailures();
+console.log(`There are ${failures.length} soft assertion failures`);
+```
+
+#### expect.assertSoftFailures()
+
+Manually assert all collected soft failures. This will throw an aggregated error if any soft assertions have failed.
+
+```js
+// Manually throw if any soft assertions have failed
+expect.assertSoftFailures();
+```
+
+#### expect.clearSoftFailures()
+
+Clear all collected soft assertion failures for the current test.
+
+```js
+// Clear all collected failures
+expect.clearSoftFailures();
+```
+
+### Integration with Test Frameworks
+
+The soft assertions feature integrates with WebdriverIO's test runner automatically. By default, it will report all soft assertion failures at the end of each test (Mocha/Jasmine) or step (Cucumber).
+
+To use with WebdriverIO, add the SoftAssertionService to your services list:
+
+```js
+// wdio.conf.js
+import { SoftAssertionService } from 'expect-webdriverio'
+
+export const config = {
+  // ...
+  services: [
+    // ...other services
+    [SoftAssertionService]
+  ],
+  // ...
+}
+```
+
+#### Configuration Options
+
+The SoftAssertionService can be configured with options to control its behavior:
+
+```js
+// wdio.conf.js
+import { SoftAssertionService } from 'expect-webdriverio'
+
+export const config = {
+  // ...
+  services: [
+    // ...other services
+    [SoftAssertionService, {
+      // Disable automatic assertion at the end of tests (default: true)
+      autoAssertOnTestEnd: false
+    }]
+  ],
+  // ...
+}
+```
+
+##### autoAssertOnTestEnd
+
+- **Type**: `boolean`
+- **Default**: `true`
+
+When set to `true` (default), the service will automatically assert all soft assertions at the end of each test and throw an aggregated error if any failures are found. When set to `false`, you must manually call `expect.assertSoftFailures()` to verify soft assertions.
+
+This is useful if you want full control over when soft assertions are verified or if you want to handle soft assertion failures in a custom way.
+
 ## Default Options
 
 These default options below are connected to the [`waitforTimeout`](https://webdriver.io/docs/options#waitfortimeout) and [`waitforInterval`](https://webdriver.io/docs/options#waitforinterval) options set in the config.
@@ -46,7 +155,7 @@ Every matcher can take several options that allows you to modify the assertion:
 
 ##### String Options
 
-This option can be applied in addition to the command options when strings are being asserted. 
+This option can be applied in addition to the command options when strings are being asserted.
 
 | Name | Type | Details |
 | ---- | ---- | ------- |

src/index.ts

@@ -4,6 +4,8 @@ import type { RawMatcherFn } from './types.js'
 
 import wdioMatchers from './matchers.js'
 import { DEFAULT_OPTIONS } from './constants.js'
+import createSoftExpect from './softExpect.js'
+import { SoftAssertService } from './softAssert.js'
 
 export const matchers = new Map<string, RawMatcherFn>()
 
@@ -20,7 +22,28 @@ expectLib.extend = (m) => {
 type MatchersObject = Parameters<typeof expectLib.extend>[0]
 
 expectLib.extend(wdioMatchers as MatchersObject)
-export const expect = expectLib as unknown as ExpectWebdriverIO.Expect
+
+// Extend the expect object with soft assertions
+const expectWithSoft = expectLib as unknown as ExpectWebdriverIO.Expect
+Object.defineProperty(expectWithSoft, 'soft', {
+    value: <T = unknown>(actual: T) => createSoftExpect(actual)
+})
+
+// Add soft assertions utility methods
+Object.defineProperty(expectWithSoft, 'getSoftFailures', {
+    value: (testId?: string) => SoftAssertService.getInstance().getFailures(testId)
+})
+
+Object.defineProperty(expectWithSoft, 'assertSoftFailures', {
+    value: (testId?: string) => SoftAssertService.getInstance().assertNoFailures(testId)
+})
+
+Object.defineProperty(expectWithSoft, 'clearSoftFailures', {
+    value: (testId?: string) => SoftAssertService.getInstance().clearFailures(testId)
+})
+
+export const expect = expectWithSoft
+
 export const getConfig = (): ExpectWebdriverIO.DefaultOptions => DEFAULT_OPTIONS
 export const setDefaultOptions = (options = {}): void => {
     Object.entries(options).forEach(([key, value]) => {
@@ -37,6 +60,12 @@ export const setOptions = setDefaultOptions
  */
 export { SnapshotService } from './snapshot.js'
 
+/**
+ * export soft assertion utilities
+ */
+export { SoftAssertService } from './softAssert.js'
+export { SoftAssertionService, type SoftAssertionServiceOptions } from './softAssertService.js'
+
 /**
  * export utils
  */

src/matchers/snapshot.ts

@@ -1,7 +1,7 @@
 import path from 'node:path'
 import type { AssertionError } from 'node:assert'
 
-import { expect } from '../index.js'
+import { expect } from 'expect'
 import { SnapshotService } from '../snapshot.js'
 
 interface InlineSnapshotOptions {

src/softAssert.ts

@@ -0,0 +1,139 @@
+import type { AssertionError } from 'node:assert'
+
+interface SoftFailure {
+    error: AssertionError | Error;
+    matcherName: string;
+    location?: string;
+}
+
+interface TestIdentifier {
+    id: string;
+    name?: string;
+    file?: string;
+}
+
+/**
+ * Soft assertion service to collect failures without stopping test execution
+ */
+export class SoftAssertService {
+    private static instance: SoftAssertService
+    private failureMap: Map<string, SoftFailure[]> = new Map()
+    private currentTest: TestIdentifier | null = null
+
+    private constructor() { }
+
+    /**
+     * Get singleton instance
+     */
+    public static getInstance(): SoftAssertService {
+        if (!SoftAssertService.instance) {
+            SoftAssertService.instance = new SoftAssertService()
+        }
+        return SoftAssertService.instance
+    }
+
+    /**
+     * Set the current test context
+     */
+    public setCurrentTest(testId: string, testName?: string, testFile?: string): void {
+        this.currentTest = { id: testId, name: testName, file: testFile }
+        if (!this.failureMap.has(testId)) {
+            this.failureMap.set(testId, [])
+        }
+    }
+
+    /**
+     * Clear the current test context
+     */
+    public clearCurrentTest(): void {
+        this.currentTest = null
+    }
+
+    /**
+     * Get current test ID
+     */
+    public getCurrentTestId(): string | null {
+        return this.currentTest?.id || null
+    }
+
+    /**
+     * Add a soft failure for the current test
+     */
+    public addFailure(error: Error, matcherName: string): void {
+        const testId = this.getCurrentTestId()
+        if (!testId) {
+            throw error // If no test context, throw the error immediately
+        }
+
+        // Extract stack information to get file and line number
+        const stackLines = error.stack?.split('\n') || []
+        let location = ''
+
+        // Find the first non-expect-webdriverio line in the stack
+        for (const line of stackLines) {
+            if (line && !line.includes('expect-webdriverio') && !line.includes('node_modules')) {
+                location = line.trim()
+                break
+            }
+        }
+
+        const failures = this.failureMap.get(testId) || []
+        failures.push({ error, matcherName, location })
+        this.failureMap.set(testId, failures)
+    }
+
+    /**
+     * Get all failures for a specific test
+     */
+    public getFailures(testId?: string): SoftFailure[] {
+        const id = testId || this.getCurrentTestId()
+        if (!id) {
+            return []
+        }
+        return this.failureMap.get(id) || []
+    }
+
+    /**
+     * Clear failures for a specific test
+     */
+    public clearFailures(testId?: string): void {
+        const id = testId || this.getCurrentTestId()
+        if (id) {
+            this.failureMap.delete(id)
+        }
+    }
+
+    /**
+     * Throw an aggregated error if there are failures for the current test
+     */
+    public assertNoFailures(testId?: string): void {
+        const id = testId || this.getCurrentTestId()
+        if (!id) {
+            return
+        }
+
+        const failures = this.getFailures(id)
+        if (failures.length === 0) {
+            return
+        }
+
+        // Create a formatted error message with all failures
+        let message = `${failures.length} soft assertion failure${failures.length > 1 ? 's' : ''}:\n\n`
+
+        failures.forEach((failure, index) => {
+            message += `${index + 1}) ${failure.matcherName}: ${failure.error.message}\n`
+            if (failure.location) {
+                message += `   at ${failure.location}\n`
+            }
+            message += '\n'
+        })
+
+        // Clear failures for this test to prevent duplicate reporting
+        this.clearFailures(id)
+
+        // Throw an aggregated error
+        const error = new Error(message)
+        error.name = 'SoftAssertionsError'
+        throw error
+    }
+}