|
| 1 | +/** |
| 2 | + * A **flaky** test is defined as a test which passed after auto-retrying. |
| 3 | + * - By default, all tests run once if they pass. |
| 4 | + * - If a test fails, it will automatically re-run at most 2 times. |
| 5 | + * - If it pass after retrying (below 2 times), then it's marked as **flaky** |
| 6 | + * but displayed as **passed** in the original test suite. |
| 7 | + * - If it fail all 3 times, then it's a **failed** test. |
| 8 | + */ |
| 9 | +/** |
| 10 | + * External dependencies |
| 11 | + */ |
| 12 | +import fs from 'fs'; |
| 13 | +import type { Reporter, TestCase, TestResult } from '@playwright/test/reporter'; |
| 14 | +import filenamify from 'filenamify'; |
| 15 | + |
| 16 | +type FormattedTestResult = Omit< TestResult, 'steps' >; |
| 17 | + |
| 18 | +// Remove "steps" to prevent stringify circular structure. |
| 19 | +function formatTestResult( testResult: TestResult ): FormattedTestResult { |
| 20 | + const result = { ...testResult, steps: undefined }; |
| 21 | + delete result.steps; |
| 22 | + return result; |
| 23 | +} |
| 24 | + |
| 25 | +class FlakyTestsReporter implements Reporter { |
| 26 | + failingTestCaseResults = new Map< string, FormattedTestResult[] >(); |
| 27 | + |
| 28 | + onBegin() { |
| 29 | + try { |
| 30 | + fs.mkdirSync( 'flaky-tests' ); |
| 31 | + } catch ( err ) { |
| 32 | + if ( |
| 33 | + err instanceof Error && |
| 34 | + ( err as NodeJS.ErrnoException ).code === 'EEXIST' |
| 35 | + ) { |
| 36 | + // Ignore the error if the directory already exists. |
| 37 | + } else { |
| 38 | + throw err; |
| 39 | + } |
| 40 | + } |
| 41 | + } |
| 42 | + |
| 43 | + onTestEnd( test: TestCase, testCaseResult: TestResult ) { |
| 44 | + const testPath = test.location.file; |
| 45 | + const testTitle = test.title; |
| 46 | + |
| 47 | + switch ( test.outcome() ) { |
| 48 | + case 'unexpected': { |
| 49 | + if ( ! this.failingTestCaseResults.has( testTitle ) ) { |
| 50 | + this.failingTestCaseResults.set( testTitle, [] ); |
| 51 | + } |
| 52 | + this.failingTestCaseResults |
| 53 | + .get( testTitle )! |
| 54 | + .push( formatTestResult( testCaseResult ) ); |
| 55 | + break; |
| 56 | + } |
| 57 | + case 'flaky': { |
| 58 | + fs.writeFileSync( |
| 59 | + `flaky-tests/${ filenamify( testTitle ) }.json`, |
| 60 | + JSON.stringify( { |
| 61 | + version: 1, |
| 62 | + runner: '@playwright/test', |
| 63 | + title: testTitle, |
| 64 | + path: testPath, |
| 65 | + results: this.failingTestCaseResults.get( testTitle ), |
| 66 | + } ), |
| 67 | + 'utf-8' |
| 68 | + ); |
| 69 | + break; |
| 70 | + } |
| 71 | + default: |
| 72 | + break; |
| 73 | + } |
| 74 | + } |
| 75 | + |
| 76 | + onEnd() { |
| 77 | + this.failingTestCaseResults.clear(); |
| 78 | + } |
| 79 | + |
| 80 | + printsToStdio() { |
| 81 | + return false; |
| 82 | + } |
| 83 | +} |
| 84 | + |
| 85 | +module.exports = FlakyTestsReporter; |
0 commit comments