diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index f529aa3e..03e27213 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -43,10 +43,15 @@ jobs: run: npx playwright install --with-deps chromium - name: Run visual regression tests + id: visual working-directory: frontend run: npm run test:visual continue-on-error: true + - name: Publish visual diff summary + if: always() + run: node scripts/visual-diff-summary.mjs frontend/test-results + - name: Upload test report if: always() uses: actions/upload-artifact@v4 @@ -56,12 +61,17 @@ jobs: retention-days: 30 - name: Upload diff images - if: failure() + if: always() && steps.visual.outcome == 'failure' uses: actions/upload-artifact@v4 with: name: visual-regression-diffs path: | - frontend/tests/visual/__snapshots__/**/*-diff.png - frontend/tests/visual/__snapshots__/**/*-actual.png + frontend/test-results/**/*-diff.png + frontend/test-results/**/*-actual.png + frontend/test-results/**/*-expected.png retention-days: 30 if-no-files-found: ignore + + - name: Fail if visual tests failed + if: steps.visual.outcome == 'failure' + run: exit 1 diff --git a/scripts/visual-diff-summary.mjs b/scripts/visual-diff-summary.mjs new file mode 100644 index 00000000..60234e6d --- /dev/null +++ b/scripts/visual-diff-summary.mjs @@ -0,0 +1,82 @@ +#!/usr/bin/env node +// Builds a GitHub Actions run summary for Playwright visual diffs (#1363). +// Usage: node scripts/visual-diff-summary.mjs [resultsDir] [--artifact=name] [--out=file] +// Writes markdown to --out, or $GITHUB_STEP_SUMMARY, or stdout. +import { readdirSync, statSync, existsSync, appendFileSync } from 'node:fs'; +import { join, relative, sep } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +const KINDS = ['expected', 'actual', 'diff']; +const IMAGE_RE = /^(.*)-(expected|actual|diff)\.png$/; + +export function walk(dir) { + if (!existsSync(dir)) return []; + const out = []; + for (const name of readdirSync(dir)) { + const p = join(dir, name); + let st; + try { + st = statSync(p); + } catch { + continue; + } + if (st.isDirectory()) out.push(...walk(p)); + else out.push(p); + } + return out; +} + +/** Group *-expected/actual/diff.png files by snapshot; only groups with a diff are failures. */ +export function collectDiffs(dir) { + const groups = new Map(); + for (const file of walk(dir)) { + const m = IMAGE_RE.exec(file.split(sep).pop()); + if (!m) continue; + const key = join(file.slice(0, file.length - file.split(sep).pop().length), m[1]); + const g = groups.get(key) ?? { name: relative(dir, key).split(sep).join('/') }; + g[m[2]] = relative(dir, file).split(sep).join('/'); + groups.set(key, g); + } + return [...groups.values()] + .filter((g) => g.diff) + .sort((a, b) => a.name.localeCompare(b.name)); +} + +export function renderSummary(diffs, { artifact = 'visual-regression-diffs', runUrl = '' } = {}) { + const lines = ['## Visual regression report', '']; + if (diffs.length === 0) { + lines.push('No visual differences detected.'); + return lines.join('\n') + '\n'; + } + lines.push(`**${diffs.length}** screenshot(s) differ from the baseline.`, ''); + const where = runUrl ? `[${artifact}](${runUrl})` : `\`${artifact}\``; + lines.push(`Expected, actual and diff images are in the ${where} artifact of this run.`, ''); + lines.push('| Snapshot | Expected | Actual | Diff |', '| --- | --- | --- | --- |'); + for (const d of diffs) { + const cell = (k) => (d[k] ? `\`${d[k].split('/').pop()}\`` : '—'); + lines.push(`| ${d.name.replace(/\|/g, '\\|')} | ${cell('expected')} | ${cell('actual')} | ${cell('diff')} |`); + } + lines.push('', 'If the change is intentional, run `npm run test:visual:update` in `frontend/` and commit the new baselines.'); + return lines.join('\n') + '\n'; +} + +function main(argv) { + const opts = Object.fromEntries( + argv.filter((a) => a.startsWith('--')).map((a) => { + const [k, ...v] = a.slice(2).split('='); + return [k, v.join('=')]; + }), + ); + const dir = argv.find((a) => !a.startsWith('--')) ?? 'frontend/test-results'; + const { GITHUB_SERVER_URL, GITHUB_REPOSITORY, GITHUB_RUN_ID } = process.env; + const runUrl = + GITHUB_SERVER_URL && GITHUB_REPOSITORY && GITHUB_RUN_ID + ? `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}#artifacts` + : ''; + const md = renderSummary(collectDiffs(dir), { artifact: opts.artifact || undefined, runUrl }); + const target = opts.out || process.env.GITHUB_STEP_SUMMARY; + if (target) appendFileSync(target, md); + else process.stdout.write(md); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) main(process.argv.slice(2)); diff --git a/scripts/visual-diff-summary.test.mjs b/scripts/visual-diff-summary.test.mjs new file mode 100644 index 00000000..e44f8ea1 --- /dev/null +++ b/scripts/visual-diff-summary.test.mjs @@ -0,0 +1,49 @@ +// Run: node --test scripts/visual-diff-summary.test.mjs +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, mkdirSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { collectDiffs, renderSummary } from './visual-diff-summary.mjs'; + +function fixture(files) { + const dir = mkdtempSync(join(tmpdir(), 'vds-')); + for (const f of files) { + const p = join(dir, f); + mkdirSync(join(p, '..'), { recursive: true }); + writeFileSync(p, 'x'); + } + return dir; +} + +test('missing directory yields no diffs', () => { + assert.deepEqual(collectDiffs('/nonexistent-dir-1363'), []); +}); + +test('groups images per snapshot and ignores non-diff groups', () => { + const dir = fixture([ + 'a/home-expected.png', + 'a/home-actual.png', + 'a/home-diff.png', + 'b/other-actual.png', + 'b/trace.zip', + ]); + const diffs = collectDiffs(dir); + assert.equal(diffs.length, 1); + assert.equal(diffs[0].name, 'a/home'); + assert.equal(diffs[0].diff, 'a/home-diff.png'); +}); + +test('renders clean summary when there are no diffs', () => { + assert.match(renderSummary([]), /No visual differences/); +}); + +test('renders table with artifact link and escapes pipes', () => { + const md = renderSummary([{ name: 'a|b', diff: 'x/a-diff.png', actual: 'x/a-actual.png' }], { + runUrl: 'https://example.com/run', + }); + assert.match(md, /\*\*1\*\* screenshot/); + assert.match(md, /\[visual-regression-diffs\]\(https:\/\/example.com\/run\)/); + assert.match(md, /a\\\|b/); + assert.match(md, /\| — \|/); +});