fix: better support for serial suites

src/reportBuilder.ts

@@ -156,14 +156,29 @@ async function toFKSuites(context: ProcessingContext, pwSuite: Suite, parentFKSu
   return [suite];
 }
 
+// Tests might get results even if they did not run to completion.
+// Playwright ignores such results when it computes a test outcome, so we do not report them either.
+function isExecutionResult(pwTest: TestCase, result: TestResult) {
+  // A test that was interrupted in-flight gets the "interrupted" status.
+  if (result.status === 'interrupted')
+    return false;
+  // A test that was skipped without calling `test.skip()` did not run: e.g. it is a part
+  // of a `.serial` suite where an earlier test failed.
+  if (result.status === 'skipped' && pwTest.expectedStatus !== 'skipped')
+    return false;
+  return true;
+}
+
 async function toFKTest(context: ProcessingContext, pwTest: TestCase, parentFKSuites: FK.Suite[]): Promise<FK.Test> {
+  const results = Array.from(context.results.get(pwTest) ?? new Set<TestResult>())
+      .filter(result => isExecutionResult(pwTest, result));
   const test: FK.Test = {
     title: pwTest.title,
     // Playwright Test tags must start with '@' so we cut it off.
     tags: pwTest.tags.map(tag => tag.startsWith('@') ? tag.substring(1) : tag),
     location: createLocation(context, pwTest.location),
     // de-duplication of tests will happen later, so here we will have all attempts.
-    attempts: await Promise.all(Array.from(context.results.get(pwTest) ?? new Set<TestResult>()).map(result => toFKRunAttempt(context, pwTest, result))),
+    attempts: await Promise.all(results.map(result => toFKRunAttempt(context, pwTest, result))),
   };
 
   const fkTestId = computeFKTestId(test, parentFKSuites);

tests/retries.spec.ts

@@ -28,3 +28,76 @@ test('should handle retries', async ({}, testInfo) => {
 
   assertCount(testNoRetry.attempts, 1);
 });
+
+test('should drop attempts that did not run', async ({}, testInfo) => {
+  const { report } = await generateFlakinessReport(testInfo, {
+    'smoke.spec.ts': `
+      import { test, expect } from '@playwright/test';
+
+      test.describe.serial('serial', () => {
+        test('first', async ({}, testInfo) => {
+          expect(testInfo.retry).toBeGreaterThan(0);
+        });
+
+        test('second', async () => {
+          expect(1).toBe(2);
+        });
+
+        test('third', async () => {});
+      });
+    `,
+  }, {}, {
+    retries: 2,
+  });
+
+  const [fileSuite] = assertCount(report.suites, 1);
+  const [serialSuite] = assertCount(fileSuite.suites, 1);
+  const [first, second, third] = assertCount(serialSuite.tests, 3);
+  expect(first.attempts.map(a => a.status ?? 'passed')).toEqual(['failed', 'passed', 'passed']);
+  // Playwright: failed. The first attempt did not run, so it is not reported.
+  expect(second.attempts.map(a => a.status ?? 'passed')).toEqual(['failed', 'failed']);
+  // Playwright: did not run. Same as tests that never started because of --max-failures.
+  expect(third.attempts).toEqual([]);
+});
+
+test('should keep explicit skips on retry', async ({}, testInfo) => {
+  const { report } = await generateFlakinessReport(testInfo, {
+    'smoke.spec.ts': `
+      import { test, expect } from '@playwright/test';
+
+      test('skips on retry', async ({}, testInfo) => {
+        test.skip(testInfo.retry > 0, 'skip on retry');
+        expect(1).toBe(2);
+      });
+    `,
+  }, {}, {
+    retries: 1,
+  });
+
+  const [fileSuite] = assertCount(report.suites, 1);
+  const [skipsOnRetry] = assertCount(fileSuite.tests, 1);
+  expect(skipsOnRetry.attempts.map(a => a.status ?? 'passed')).toEqual(['failed', 'skipped']);
+  expect(skipsOnRetry.attempts.map(a => a.expectedStatus ?? 'passed')).toEqual(['skipped', 'skipped']);
+});
+
+test('should drop interrupted attempts', async ({}, testInfo) => {
+  test.skip(process.platform === 'win32', 'process.kill() cannot deliver SIGINT on Windows');
+  const { report } = await generateFlakinessReport(testInfo, {
+    'smoke.spec.ts': `
+      import { test, expect } from '@playwright/test';
+
+      test('interrupted on retry', async ({}, testInfo) => {
+        expect(testInfo.retry).toBeGreaterThan(0);
+        // Interrupt the test runner, as if a user pressed Ctrl-C.
+        process.kill(process.ppid, 'SIGINT');
+        await new Promise(() => {});
+      });
+    `,
+  }, {}, {
+    retries: 1,
+  });
+
+  const [fileSuite] = assertCount(report.suites, 1);
+  const [interruptedOnRetry] = assertCount(fileSuite.tests, 1);
+  expect(interruptedOnRetry.attempts.map(a => a.status ?? 'passed')).toEqual(['failed']);
+});