feat(testing): publish automated visual diff test report artifacts in CI (#1363)

.github/workflows/visual-regression.yml

@@ -43,10 +43,15 @@ jobs:
         run: npx playwright install --with-deps chromium
 
       - name: Run visual regression tests
+        id: visual
         working-directory: frontend
         run: npm run test:visual
         continue-on-error: true
 
+      - name: Publish visual diff summary
+        if: always()
+        run: node scripts/visual-diff-summary.mjs frontend/test-results
+
       - name: Upload test report
         if: always()
         uses: actions/upload-artifact@v4
@@ -56,12 +61,17 @@ jobs:
           retention-days: 30
 
       - name: Upload diff images
-        if: failure()
+        if: always() && steps.visual.outcome == 'failure'
         uses: actions/upload-artifact@v4
         with:
           name: visual-regression-diffs
           path: |
-            frontend/tests/visual/__snapshots__/**/*-diff.png
-            frontend/tests/visual/__snapshots__/**/*-actual.png
+            frontend/test-results/**/*-diff.png
+            frontend/test-results/**/*-actual.png
+            frontend/test-results/**/*-expected.png
           retention-days: 30
           if-no-files-found: ignore
+
+      - name: Fail if visual tests failed
+        if: steps.visual.outcome == 'failure'
+        run: exit 1

scripts/visual-diff-summary.mjs

@@ -0,0 +1,82 @@
+#!/usr/bin/env node
+// Builds a GitHub Actions run summary for Playwright visual diffs (#1363).
+// Usage: node scripts/visual-diff-summary.mjs [resultsDir] [--artifact=name] [--out=file]
+// Writes markdown to --out, or $GITHUB_STEP_SUMMARY, or stdout.
+import { readdirSync, statSync, existsSync, appendFileSync } from 'node:fs';
+import { join, relative, sep } from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+const KINDS = ['expected', 'actual', 'diff'];
+const IMAGE_RE = /^(.*)-(expected|actual|diff)\.png$/;
+
+export function walk(dir) {
+  if (!existsSync(dir)) return [];
+  const out = [];
+  for (const name of readdirSync(dir)) {
+    const p = join(dir, name);
+    let st;
+    try {
+      st = statSync(p);
+    } catch {
+      continue;
+    }
+    if (st.isDirectory()) out.push(...walk(p));
+    else out.push(p);
+  }
+  return out;
+}
+
+/** Group *-expected/actual/diff.png files by snapshot; only groups with a diff are failures. */
+export function collectDiffs(dir) {
+  const groups = new Map();
+  for (const file of walk(dir)) {
+    const m = IMAGE_RE.exec(file.split(sep).pop());
+    if (!m) continue;
+    const key = join(file.slice(0, file.length - file.split(sep).pop().length), m[1]);
+    const g = groups.get(key) ?? { name: relative(dir, key).split(sep).join('/') };
+    g[m[2]] = relative(dir, file).split(sep).join('/');
+    groups.set(key, g);
+  }
+  return [...groups.values()]
+    .filter((g) => g.diff)
+    .sort((a, b) => a.name.localeCompare(b.name));
+}
+
+export function renderSummary(diffs, { artifact = 'visual-regression-diffs', runUrl = '' } = {}) {
+  const lines = ['## Visual regression report', ''];
+  if (diffs.length === 0) {
+    lines.push('No visual differences detected.');
+    return lines.join('\n') + '\n';
+  }
+  lines.push(`**${diffs.length}** screenshot(s) differ from the baseline.`, '');
+  const where = runUrl ? `[${artifact}](${runUrl})` : `\`${artifact}\``;
+  lines.push(`Expected, actual and diff images are in the ${where} artifact of this run.`, '');
+  lines.push('| Snapshot | Expected | Actual | Diff |', '| --- | --- | --- | --- |');
+  for (const d of diffs) {
+    const cell = (k) => (d[k] ? `\`${d[k].split('/').pop()}\`` : '—');
+    lines.push(`| ${d.name.replace(/\|/g, '\\|')} | ${cell('expected')} | ${cell('actual')} | ${cell('diff')} |`);
+  }
+  lines.push('', 'If the change is intentional, run `npm run test:visual:update` in `frontend/` and commit the new baselines.');
+  return lines.join('\n') + '\n';
+}
+
+function main(argv) {
+  const opts = Object.fromEntries(
+    argv.filter((a) => a.startsWith('--')).map((a) => {
+      const [k, ...v] = a.slice(2).split('=');
+      return [k, v.join('=')];
+    }),
+  );
+  const dir = argv.find((a) => !a.startsWith('--')) ?? 'frontend/test-results';
+  const { GITHUB_SERVER_URL, GITHUB_REPOSITORY, GITHUB_RUN_ID } = process.env;
+  const runUrl =
+    GITHUB_SERVER_URL && GITHUB_REPOSITORY && GITHUB_RUN_ID
+      ? `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}#artifacts`
+      : '';
+  const md = renderSummary(collectDiffs(dir), { artifact: opts.artifact || undefined, runUrl });
+  const target = opts.out || process.env.GITHUB_STEP_SUMMARY;
+  if (target) appendFileSync(target, md);
+  else process.stdout.write(md);
+}
+
+if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) main(process.argv.slice(2));

scripts/visual-diff-summary.test.mjs

@@ -0,0 +1,49 @@
+// Run: node --test scripts/visual-diff-summary.test.mjs
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtempSync, mkdirSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { collectDiffs, renderSummary } from './visual-diff-summary.mjs';
+
+function fixture(files) {
+  const dir = mkdtempSync(join(tmpdir(), 'vds-'));
+  for (const f of files) {
+    const p = join(dir, f);
+    mkdirSync(join(p, '..'), { recursive: true });
+    writeFileSync(p, 'x');
+  }
+  return dir;
+}
+
+test('missing directory yields no diffs', () => {
+  assert.deepEqual(collectDiffs('/nonexistent-dir-1363'), []);
+});
+
+test('groups images per snapshot and ignores non-diff groups', () => {
+  const dir = fixture([
+    'a/home-expected.png',
+    'a/home-actual.png',
+    'a/home-diff.png',
+    'b/other-actual.png',
+    'b/trace.zip',
+  ]);
+  const diffs = collectDiffs(dir);
+  assert.equal(diffs.length, 1);
+  assert.equal(diffs[0].name, 'a/home');
+  assert.equal(diffs[0].diff, 'a/home-diff.png');
+});
+
+test('renders clean summary when there are no diffs', () => {
+  assert.match(renderSummary([]), /No visual differences/);
+});
+
+test('renders table with artifact link and escapes pipes', () => {
+  const md = renderSummary([{ name: 'a|b', diff: 'x/a-diff.png', actual: 'x/a-actual.png' }], {
+    runUrl: 'https://example.com/run',
+  });
+  assert.match(md, /\*\*1\*\* screenshot/);
+  assert.match(md, /\[visual-regression-diffs\]\(https:\/\/example.com\/run\)/);
+  assert.match(md, /a\\\|b/);
+  assert.match(md, /\| — \|/);
+});