diff --git a/src/reportBuilder.ts b/src/reportBuilder.ts index 89ea9b5..be05095 100644 --- a/src/reportBuilder.ts +++ b/src/reportBuilder.ts @@ -156,14 +156,29 @@ async function toFKSuites(context: ProcessingContext, pwSuite: Suite, parentFKSu return [suite]; } +// Tests might get results even if they did not run to completion. +// Playwright ignores such results when it computes a test outcome, so we do not report them either. +function isExecutionResult(pwTest: TestCase, result: TestResult) { + // A test that was interrupted in-flight gets the "interrupted" status. + if (result.status === 'interrupted') + return false; + // A test that was skipped without calling `test.skip()` did not run: e.g. it is a part + // of a `.serial` suite where an earlier test failed. + if (result.status === 'skipped' && pwTest.expectedStatus !== 'skipped') + return false; + return true; +} + async function toFKTest(context: ProcessingContext, pwTest: TestCase, parentFKSuites: FK.Suite[]): Promise { + const results = Array.from(context.results.get(pwTest) ?? new Set()) + .filter(result => isExecutionResult(pwTest, result)); const test: FK.Test = { title: pwTest.title, // Playwright Test tags must start with '@' so we cut it off. tags: pwTest.tags.map(tag => tag.startsWith('@') ? tag.substring(1) : tag), location: createLocation(context, pwTest.location), // de-duplication of tests will happen later, so here we will have all attempts. - attempts: await Promise.all(Array.from(context.results.get(pwTest) ?? new Set()).map(result => toFKRunAttempt(context, pwTest, result))), + attempts: await Promise.all(results.map(result => toFKRunAttempt(context, pwTest, result))), }; const fkTestId = computeFKTestId(test, parentFKSuites); diff --git a/tests/retries.spec.ts b/tests/retries.spec.ts index 2f9c7e6..2aa7b67 100644 --- a/tests/retries.spec.ts +++ b/tests/retries.spec.ts @@ -28,3 +28,76 @@ test('should handle retries', async ({}, testInfo) => { assertCount(testNoRetry.attempts, 1); }); + +test('should drop attempts that did not run', async ({}, testInfo) => { + const { report } = await generateFlakinessReport(testInfo, { + 'smoke.spec.ts': ` + import { test, expect } from '@playwright/test'; + + test.describe.serial('serial', () => { + test('first', async ({}, testInfo) => { + expect(testInfo.retry).toBeGreaterThan(0); + }); + + test('second', async () => { + expect(1).toBe(2); + }); + + test('third', async () => {}); + }); + `, + }, {}, { + retries: 2, + }); + + const [fileSuite] = assertCount(report.suites, 1); + const [serialSuite] = assertCount(fileSuite.suites, 1); + const [first, second, third] = assertCount(serialSuite.tests, 3); + expect(first.attempts.map(a => a.status ?? 'passed')).toEqual(['failed', 'passed', 'passed']); + // Playwright: failed. The first attempt did not run, so it is not reported. + expect(second.attempts.map(a => a.status ?? 'passed')).toEqual(['failed', 'failed']); + // Playwright: did not run. Same as tests that never started because of --max-failures. + expect(third.attempts).toEqual([]); +}); + +test('should keep explicit skips on retry', async ({}, testInfo) => { + const { report } = await generateFlakinessReport(testInfo, { + 'smoke.spec.ts': ` + import { test, expect } from '@playwright/test'; + + test('skips on retry', async ({}, testInfo) => { + test.skip(testInfo.retry > 0, 'skip on retry'); + expect(1).toBe(2); + }); + `, + }, {}, { + retries: 1, + }); + + const [fileSuite] = assertCount(report.suites, 1); + const [skipsOnRetry] = assertCount(fileSuite.tests, 1); + expect(skipsOnRetry.attempts.map(a => a.status ?? 'passed')).toEqual(['failed', 'skipped']); + expect(skipsOnRetry.attempts.map(a => a.expectedStatus ?? 'passed')).toEqual(['skipped', 'skipped']); +}); + +test('should drop interrupted attempts', async ({}, testInfo) => { + test.skip(process.platform === 'win32', 'process.kill() cannot deliver SIGINT on Windows'); + const { report } = await generateFlakinessReport(testInfo, { + 'smoke.spec.ts': ` + import { test, expect } from '@playwright/test'; + + test('interrupted on retry', async ({}, testInfo) => { + expect(testInfo.retry).toBeGreaterThan(0); + // Interrupt the test runner, as if a user pressed Ctrl-C. + process.kill(process.ppid, 'SIGINT'); + await new Promise(() => {}); + }); + `, + }, {}, { + retries: 1, + }); + + const [fileSuite] = assertCount(report.suites, 1); + const [interruptedOnRetry] = assertCount(fileSuite.tests, 1); + expect(interruptedOnRetry.attempts.map(a => a.status ?? 'passed')).toEqual(['failed']); +});