From dc6ae1ef4b4588e04d06ecd5ced3aef90b9282b5 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Fri, 23 Oct 2026 20:05:01 -0700 Subject: [PATCH 01/93] test(interaction): real touch input for browser interaction proofs Adds internal/interaction-harness (@astryxdesign/interaction-harness) and playwright.interaction.config.ts: phone-sized touch projects in Chromium and WebKit for *.interaction.browser.spec.ts specs. Touch scripts are data (tap, drag with hold and rest, flick with an exact release velocity and velocity profiles, hold-then-drag, pinch, rotate, any path, a second finger mid-gesture), played as real engine input: CDP Input.dispatchTouchEvent with scripted timestamps in Chromium, WebKit's own input command in WebKit. The harness's self-test proves the timestamps, a native fling and multi-touch delivery; a pr-interaction CI job runs the proofs against each PR's Storybook build, and a container script runs the WebKit project where host libraries are missing. --- .github/scripts/ci-test-routing.test.mjs | 49 +++ .github/workflows/ci.yml | 88 ++++ .github/workflows/deploy.yml | 3 + internal/README.md | 13 +- internal/interaction-harness/README.md | 67 +++ internal/interaction-harness/package.json | 21 + .../scripts/webkit-container.mjs | 72 ++++ .../src/harness.interaction.browser.spec.ts | 168 ++++++++ internal/interaction-harness/src/index.ts | 17 + .../interaction-harness/src/scripts.test.ts | 142 +++++++ internal/interaction-harness/src/scripts.ts | 395 ++++++++++++++++++ internal/interaction-harness/src/story.ts | 103 +++++ internal/interaction-harness/src/touch.ts | 295 +++++++++++++ internal/interaction-harness/tsconfig.json | 4 + package.json | 1 + playwright.interaction.config.ts | 77 ++++ pnpm-lock.yaml | 15 + 17 files changed, 1524 insertions(+), 6 deletions(-) create mode 100644 internal/interaction-harness/README.md create mode 100644 internal/interaction-harness/package.json create mode 100644 internal/interaction-harness/scripts/webkit-container.mjs create mode 100644 internal/interaction-harness/src/harness.interaction.browser.spec.ts create mode 100644 internal/interaction-harness/src/index.ts create mode 100644 internal/interaction-harness/src/scripts.test.ts create mode 100644 internal/interaction-harness/src/scripts.ts create mode 100644 internal/interaction-harness/src/story.ts create mode 100644 internal/interaction-harness/src/touch.ts create mode 100644 internal/interaction-harness/tsconfig.json create mode 100644 playwright.interaction.config.ts diff --git a/.github/scripts/ci-test-routing.test.mjs b/.github/scripts/ci-test-routing.test.mjs index 3f98eef8df4b8..1128817685567 100644 --- a/.github/scripts/ci-test-routing.test.mjs +++ b/.github/scripts/ci-test-routing.test.mjs @@ -588,3 +588,52 @@ describe('deploy.yml push gating', () => { } }); }); + +describe('ci.yml interaction proofs', () => { + const workflow = load('ci.yml'); + const job = workflow.jobs['pr-interaction']; + const step = name => job.steps.find(candidate => candidate.name === name); + + it('runs the interaction config in Chromium against the exact PR head', () => { + expect(KNOWN_RUNNERS.has(job['runs-on'])).toBe(true); + expect(job.needs).toEqual( + expect.arrayContaining(['check-scope', 'build-storybook']), + ); + const checkout = job.steps.find(candidate => + candidate.uses?.startsWith('actions/checkout@'), + ); + expect(checkout?.with?.ref).toBe( + '${{ github.event.pull_request.head.sha }}', + ); + const run = step('Run interaction proofs').run; + expect(run).toContain('--config playwright.interaction.config.ts'); + expect(run).toContain('--project chromium'); + const packageJson = JSON.parse(read('package.json')); + expect(packageJson.scripts['test:interaction']).toBe( + 'playwright test --config playwright.interaction.config.ts', + ); + }); + + it('runs whenever the harness, its config, a component or a story changes', () => { + const scope = step('Check interaction scope').run; + expect(scope).toContain('.github/scripts/pr-scope-touches.sh'); + for (const path of [ + 'internal/interaction-harness/', + 'playwright\\.interaction\\.config\\.ts', + 'packages/[^/]+/src/', + 'apps/storybook/', + ]) { + expect(scope).toContain(path); + } + // Unknown scope (exit 2) runs the proofs; only "out of scope" skips them. + expect(scope).toContain('if [ "$scope" = 1 ]; then'); + }); + + it('typechecks the harness in both workflows', () => { + for (const file of ['ci.yml', 'deploy.yml']) { + expect(runLines(load(file).jobs.typecheck), file).toContain( + '@astryxdesign/interaction-harness typecheck', + ); + } + }); +}); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c4430a1336b8..8cc8e7d9a951e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1252,6 +1252,12 @@ jobs: if: needs.check-scope.outputs.docsite_only != 'true' && needs.check-scope.outputs.spec_only != 'true' && needs.check-scope.outputs.tooling_only != 'true' run: pnpm -F @astryxdesign/native typecheck:types + # The interaction harness is consumed as TypeScript source by Vitest and + # Playwright, which do not typecheck; this is the only gate over it. + - name: Typecheck interaction harness + if: needs.check-scope.outputs.docsite_only != 'true' && needs.check-scope.outputs.spec_only != 'true' && needs.check-scope.outputs.tooling_only != 'true' + run: pnpm -F @astryxdesign/interaction-harness typecheck + # Sandbox preview build — the single slowest piece of PR CI (~4min), run as # its own job so it overlaps build-storybook instead of serializing after it. # PR-only: merge_group runs never deploy previews, and docsite-only PRs skip @@ -3764,6 +3770,88 @@ jobs: retention-days: 30 delete-merged: true + # Touch interaction and animation proofs: the harness in + # internal/interaction-harness and every *.interaction.browser.spec.ts. + # Real touch input, composited frames, rendering traces and presented + # animation values only exist in a real engine, and the specs read stories + # from this PR's own Storybook build. The harness's self-test runs with + # them, so a change to an instrument is proven by its negative controls + # before a component spec relies on it. Chromium only on these runners; + # see the harness README for running WebKit. + pr-interaction: + needs: [check-scope, build-storybook] + if: github.event_name == 'pull_request' && needs.check-scope.outputs.docsite_only != 'true' && needs.check-scope.outputs.spec_only != 'true' && needs.check-scope.outputs.tooling_only != 'true' + # Frame evidence measures timing: four cores, one Playwright worker. + runs-on: 4-core-ubuntu + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Verify evidence checkout + env: + EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: test "$(git rev-parse HEAD)" = "$EXPECTED_HEAD_SHA" + + # The helper comes from the base, since an older PR head lacks it. + # Exit 1 means out of scope; 0 in scope; 2 unknown, so run. + - name: Check interaction scope + id: scope + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + scope=2 + helper="$(mktemp)" + if git fetch --quiet --no-tags --depth=1 origin "$BASE_SHA" \ + && git show "$BASE_SHA:.github/scripts/pr-scope-touches.sh" > "$helper"; then + scope=0 + bash "$helper" "$BASE_SHA" '^(internal/interaction-harness/|playwright\.interaction\.config\.ts$|packages/[^/]+/src/|apps/storybook/)' || scope=$? + fi + if [ "$scope" = 1 ]; then + echo "Neither the interaction harness nor a component changed; skipping the interaction proofs." + echo 'run=false' >> "$GITHUB_OUTPUT" + else + echo 'run=true' >> "$GITHUB_OUTPUT" + fi + + - name: Setup Node and pnpm + if: steps.scope.outputs.run == 'true' + uses: ./.github/actions/setup + with: + install: 'false' + + - name: Download Storybook artifact + if: steps.scope.outputs.run == 'true' + uses: actions/download-artifact@v8 + with: + name: storybook-${{ needs.build-storybook.outputs.short_hash }} + path: apps/storybook/dist + + - name: Install Playwright + if: steps.scope.outputs.run == 'true' + run: pnpm install --frozen-lockfile && npx playwright install chromium + + - name: Run interaction proofs + if: steps.scope.outputs.run == 'true' + run: | + pnpm exec playwright test \ + --config playwright.interaction.config.ts \ + --project chromium \ + --output interaction-proof-output + + # Storyboards (PNG contact sheets) and curve traces, for review. + - name: Upload interaction evidence + if: always() && steps.scope.outputs.run == 'true' + uses: actions/upload-artifact@v7 + with: + name: interaction-evidence + path: test-results/interaction-evidence + if-no-files-found: ignore + retention-days: 7 + # Build package test owner. The plugin's production CSS partition is a Build # integration contract: source inspection cannot prove which declaration # paints after Vite emits the final bundle. The `test-build` name makes that diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 3eaee6914b595..f3b572c1dd629 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -132,6 +132,9 @@ jobs: - name: Typecheck accessibility spec-test contracts run: pnpm -F @astryxdesign/a11y-spec typecheck + - name: Typecheck interaction harness + run: pnpm -F @astryxdesign/interaction-harness typecheck + # Stable join for the three post-merge validation lanes. test: needs: [test-ui, test-node, typecheck] diff --git a/internal/README.md b/internal/README.md index 101fd8b07c28f..650fbd245d1a6 100644 --- a/internal/README.md +++ b/internal/README.md @@ -4,9 +4,10 @@ Internal tooling packages not published to npm. -| Directory | Role | Purpose | -| ------------- | ------- | ----------------------------------------------------------------------- | -| `a11y-spec/` | Testing | Reusable accessibility spec-test contracts and their evidence harnesses | -| `scripts/` | Tooling | One-off automation scripts (Crowdin screenshot upload, etc.) | -| `test-utils/` | Testing | Shared testing utilities and setup for Vitest | -| `vibe-tests/` | Testing | LLM vibeability test harness for component system | +| Directory | Role | Purpose | +| ---------------------- | ------- | --------------------------------------------------------------------------------- | +| `a11y-spec/` | Testing | Reusable accessibility spec-test contracts and their evidence harnesses | +| `interaction-harness/` | Testing | Playwright instruments for touch interaction and animation proofs and storyboards | +| `scripts/` | Tooling | One-off automation scripts (Crowdin screenshot upload, etc.) | +| `test-utils/` | Testing | Shared testing utilities and setup for Vitest | +| `vibe-tests/` | Testing | LLM vibeability test harness for component system | diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md new file mode 100644 index 0000000000000..a0b56d854c6b0 --- /dev/null +++ b/internal/interaction-harness/README.md @@ -0,0 +1,67 @@ +# @astryxdesign/interaction-harness + +Playwright instruments for touch interaction proofs: real touch input. +**Internal and unpublished.** Specs live next to their component as +`*.interaction.browser.spec.ts` and run under +[`playwright.interaction.config.ts`](../../playwright.interaction.config.ts) on +a phone-sized touch screen (402 × 714 CSS px page, 402 × 874 screen, 3 device +pixels per CSS px, touch, mobile viewport rules), in Chromium and WebKit. + +The harness proves its own instruments in +[`src/harness.interaction.browser.spec.ts`](src/harness.interaction.browser.spec.ts), +on fixture pages written inline. + +## Running + +```sh +pnpm storybook:build # component specs read stories from the built Storybook +pnpm exec playwright install chromium webkit +pnpm test:interaction --project chromium # everything, Chromium +pnpm test:interaction # one spec, both engines +``` + +On Linux distributions whose system libraries are older than Playwright's +WebKit build needs, run the WebKit project inside the official Playwright +image (podman or docker; the repository is mounted at the same path): + +```sh +node internal/interaction-harness/scripts/webkit-container.mjs [playwright test args…] +``` + +## Instruments + +| Module | What it gives a spec | +| ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact release velocity, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. | +| `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | +| `story.ts` | `phone`, `withStorybook`, `openStory`, `frames`. | + +## Real touch + +Untrusted DOM events are not input: they start no native scroll, claim no +gesture, and read `isTrusted: false`. Chromium input goes through CDP +`Input.dispatchTouchEvent` with an explicit timestamp per event: the page's +`event.timeStamp` is the scripted time to the microsecond, whatever the +dispatch latency, and the events go through the compositor's gesture detection +(native scrolling, flings with velocity from those timestamps, `touch-action`, +`pointercancel`). WebKit input goes through the engine's own input command; see +the limits below. Read positions a frame after a move: Chromium delivers +`pointermove` aligned to animation frames. + +## Engine limits (measured with Playwright 1.61) + +| | Chromium 149 | WebKit 26.5 (Linux build) | +| ------------------------------------------------------ | ------------ | ----------------------------------------------------------------------------------- | +| Trusted touch + pointer events, multi-touch | yes | yes (a `touchstart` lists only the fingers that landed in it; pointer events exact) | +| Page sees the scripted timestamps | yes, exactly | no: real dispatch time; real-time pacing keeps velocities within ~1 ms | +| Sub-pixel touch positions | yes | no: integer CSS px | +| Native scroll, fling, pinch-zoom from injected touches | yes | no | +| Click synthesised from a touch | yes | only from `driver.tap` (WebKit's tap gesture), not from a raw touch sequence | +| Input log | yes | yes | + +What WebKit cannot show here — native scroll hand-off, flings, and anything +iOS-only — needs Mobile Safari itself, on a device or the iOS simulator. + +The WebKit input command is reached through Playwright's in-process server +object, so it needs a locally launched browser (the default), not `connect()`. +Firefox has no touch input path and no project. diff --git a/internal/interaction-harness/package.json b/internal/interaction-harness/package.json new file mode 100644 index 0000000000000..08b1d02f61634 --- /dev/null +++ b/internal/interaction-harness/package.json @@ -0,0 +1,21 @@ +{ + "name": "@astryxdesign/interaction-harness", + "version": "0.0.0", + "private": true, + "type": "module", + "description": "Internal: Playwright helpers for touch interaction proofs (real touch input). Never published.", + "main": "./src/index.ts", + "exports": { + ".": "./src/index.ts", + "./scripts": "./src/scripts.ts" + }, + "scripts": { + "typecheck": "tsc --project tsconfig.json --noEmit" + }, + "devDependencies": { + "@astryxdesign/a11y-spec": "workspace:*", + "@playwright/test": "^1.61.1", + "typescript": "^6.0.3", + "vitest": "^4.1.11" + } +} diff --git a/internal/interaction-harness/scripts/webkit-container.mjs b/internal/interaction-harness/scripts/webkit-container.mjs new file mode 100644 index 0000000000000..32cc0452396df --- /dev/null +++ b/internal/interaction-harness/scripts/webkit-container.mjs @@ -0,0 +1,72 @@ +#!/usr/bin/env node +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * Runs the interaction proofs' WebKit project inside the official Playwright + * container image, for hosts where WebKit cannot launch (Playwright's Linux + * WebKit build needs newer system libraries than some distributions ship). + * + * node internal/interaction-harness/scripts/webkit-container.mjs [playwright test args…] + * + * The repository is mounted at the same path, so node_modules, the built + * Storybook and the evidence directory are the host's own. The image tag + * matches the installed playwright, so its browsers match the client. + * Uses podman when present, else docker. `--network=host` lets the specs' + * local Storybook server and the browser reach each other. + */ +import {spawnSync} from 'node:child_process'; +import fs from 'node:fs'; +import {createRequire} from 'node:module'; +import path from 'node:path'; +import {fileURLToPath} from 'node:url'; + +const root = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '../../..', +); +const require = createRequire(path.join(root, 'package.json')); +const {version} = JSON.parse( + fs.readFileSync(require.resolve('playwright/package.json'), 'utf8'), +); +const image = + process.env.PLAYWRIGHT_IMAGE ?? + `mcr.microsoft.com/playwright:v${version}-noble`; +const engine = ['podman', 'docker'].find( + cmd => spawnSync(cmd, ['--version'], {stdio: 'ignore'}).status === 0, +); +if (engine == null) { + console.error('webkit-container: neither podman nor docker is installed'); + process.exit(1); +} +const args = process.argv.slice(2); +const env = [ + 'ASTRYX_STORYBOOK_DIR', + 'ASTRYX_INTERACTION_EVIDENCE', + 'INTERACTION_WORKERS', + 'CI', +] + .filter(name => process.env[name] != null) + .flatMap(name => ['-e', `${name}=${process.env[name]}`]); +const command = [ + 'run', + '--rm', + '--network=host', + '--ipc=host', + '-v', + `${root}:${root}`, + '-w', + root, + ...env, + image, + 'node', + path.join(root, 'node_modules/@playwright/test/cli.js'), + 'test', + '--config', + 'playwright.interaction.config.ts', + '--project', + 'webkit', + ...args, +]; +console.log(`${engine} ${command.join(' ')}`); +const result = spawnSync(engine, command, {stdio: 'inherit'}); +process.exit(result.status ?? 1); diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts new file mode 100644 index 0000000000000..10d41e668c9bf --- /dev/null +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -0,0 +1,168 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file harness.interaction.browser.spec.ts + * @input Fixture pages written inline (no Storybook), real Chromium/WebKit + * @output Proof that each touch instrument delivers real engine input: + * scripted timestamps reach the page; a flick flings natively (Chromium); + * a second finger and a pinch arrive as distinct pointers. + * @position The harness's own test. A component spec trusts these + * instruments; this file is why it may. + */ + +import {expect, test, type Page} from '@playwright/test'; +import { + drag, + flick, + installInputLog, + merge, + pinch, + readInputLog, + tap, + touchDriver, + withFinger, +} from './index'; + +const FIXTURE = ` + +
+
+
${Array.from({length: 200}, (_, i) => `
row ${i}
`).join('')}
`; + +// A real navigation (not setContent, whose document.open() erases the window +// listeners the input log installs). +async function fixture(page: Page) { + await installInputLog(page); + await page.route('http://harness.test/', route => + route.fulfill({contentType: 'text/html', body: FIXTURE}), + ); + await page.goto('http://harness.test/'); +} + +test.describe('touch input', () => { + test('the page receives trusted touches and pointers at the scripted positions and times', async ({ + page, + browserName, + }) => { + await fixture(page); + const driver = await touchDriver(page); + const script = drag( + {x: 100, y: 100}, + {x: 100, y: 220}, + {durationMs: 200, hz: 60, restMs: 120}, + ); + await driver.play(script); + const log = await readInputLog(page); + const moves = log.filter(r => r.type === 'pointermove'); + const down = log.find(r => r.type === 'pointerdown'); + const up = log.find(r => r.type === 'pointerup'); + expect(down?.isTrusted).toBe(true); + expect(down?.pointerType).toBe('touch'); + expect(up, JSON.stringify(log.map(r => r.type))).toBeDefined(); + // (Not `cancelable`: with only passive listeners an engine may dispatch + // touchmove as non-blocking, which reads cancelable false.) + expect( + log.filter(r => r.type === 'touchmove').every(r => r.isTrusted), + ).toBe(true); + expect(moves.at(-1)?.y).toBe(220); + if (browserName === 'chromium') { + // The scripted 16.67 ms spacing arrives exactly, and the 120 ms rest + // before the lift is in the timestamps too. + const deltas = moves + .slice(1) + .map((m, i) => m.timeStamp - moves[i].timeStamp); + for (const delta of deltas) { + expect(delta).toBeCloseTo(1000 / 60, 0); + } + expect( + (up?.timeStamp ?? NaN) - (moves.at(-1)?.timeStamp ?? NaN), + ).toBeCloseTo(120, 0); + } else { + // WebKit stamps the real dispatch time: real-time pacing keeps it close. + const deltas = moves + .slice(1) + .map((m, i) => m.timeStamp - moves[i].timeStamp); + const mean = deltas.reduce((a, b) => a + b, 0) / deltas.length; + expect(mean).toBeGreaterThan(14); + expect(mean).toBeLessThan(20); + } + }); + + test('a flick flings a native scroller, farther for a faster finger', async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'injected WebKit touches drive no native scrolling (README: engine limits)', + ); + await fixture(page); + const driver = await touchDriver(page); + const scroller = page.locator('#scroller'); + const distanceFor = async (velocity: number) => { + await scroller.evaluate(el => (el.scrollTop = 0)); + await driver.play( + flick( + {x: 200, y: 680}, + {velocity: {x: 0, y: -velocity}, durationMs: 60, hz: 120}, + ), + ); + await expect + .poll( + async () => { + const a = await scroller.evaluate(el => el.scrollTop); + await page.waitForTimeout(150); + return a === (await scroller.evaluate(el => el.scrollTop)); + }, + {timeout: 8_000}, + ) + .toBe(true); + return scroller.evaluate(el => el.scrollTop); + }; + const slow = await distanceFor(800); + const fast = await distanceFor(2400); + expect(slow).toBeGreaterThan(48); // it coasted past the finger's own travel + expect(fast).toBeGreaterThan(slow * 1.8); + }); + + test('a second finger and a pinch arrive as distinct touch pointers', async ({ + page, + }) => { + await fixture(page); + const driver = await touchDriver(page); + const first = drag( + {x: 80, y: 80}, + {x: 80, y: 200}, + {durationMs: 300, hz: 60}, + ); + const second = withFinger(tap({x: 300, y: 120}, {holdMs: 100}), 1, 120); + await driver.play(merge(first, second)); + const downs = (await readInputLog(page, {clear: true})).filter( + r => r.type === 'pointerdown', + ); + expect(downs).toHaveLength(2); + expect(new Set(downs.map(d => d.pointerId)).size).toBe(2); + + await driver.play( + pinch({ + center: {x: 200, y: 150}, + fromDistance: 60, + toDistance: 200, + durationMs: 200, + }), + ); + const log = await readInputLog(page); + expect(log.filter(r => r.type === 'pointerdown')).toHaveLength(2); + expect( + Math.max( + ...log.filter(r => r.type === 'touchmove').map(r => r.touches ?? 0), + ), + ).toBe(2); + }); +}); diff --git a/internal/interaction-harness/src/index.ts b/internal/interaction-harness/src/index.ts new file mode 100644 index 0000000000000..3216b3c084322 --- /dev/null +++ b/internal/interaction-harness/src/index.ts @@ -0,0 +1,17 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file index.ts + * @input Re-exports the harness modules + * @output The `@astryxdesign/interaction-harness` entry point for Playwright + * specs. Pure helpers are also reachable without Playwright through + * `@astryxdesign/interaction-harness/math` and `/scripts` (unit tests). + * @position Imported only from `*.interaction.browser.spec.ts` files (run by + * playwright.interaction.config.ts) and from this package's own tests. + * + * SYNC: When a module is added, export it here and describe it in README.md. + */ + +export * from './scripts'; +export * from './story'; +export * from './touch'; diff --git a/internal/interaction-harness/src/scripts.test.ts b/internal/interaction-harness/src/scripts.test.ts new file mode 100644 index 0000000000000..b85d4b526de55 --- /dev/null +++ b/internal/interaction-harness/src/scripts.test.ts @@ -0,0 +1,142 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +import {describe, expect, it} from 'vitest'; +import { + drag, + flick, + holdThenDrag, + merge, + path, + pinch, + releaseVelocity, + rotate, + tap, + toTouchEvents, + withFinger, +} from './scripts'; + +describe('touch scripts', () => { + it('tap presses and lifts in place', () => { + expect( + toTouchEvents(tap({x: 5, y: 6}, {holdMs: 80})).map(e => [e.t, e.type]), + ).toEqual([ + [0, 'touchStart'], + [80, 'touchEnd'], + ]); + }); + + it('drag moves at 60 Hz, holds, rests and lifts', () => { + const script = drag( + {x: 0, y: 0}, + {x: 0, y: 100}, + {durationMs: 100, hz: 60, holdMs: 500, restMs: 150}, + ); + const events = toTouchEvents(script); + expect(events[0]).toMatchObject({t: 0, type: 'touchStart'}); + const moves = events.filter(e => e.type === 'touchMove'); + expect(moves).toHaveLength(6); + expect(moves[0].t).toBeCloseTo(516.667, 3); + expect(moves.at(-1)).toMatchObject({ + t: 600, + fingers: [{id: 0, x: 0, y: 100}], + }); + expect(events.at(-1)).toMatchObject({ + t: 750, + type: 'touchEnd', + fingers: [], + }); + // A rest of 100 ms or more releases nothing in motion. + expect(releaseVelocity(script)).toEqual({x: 0, y: 0}); + }); + + it('flick releases at exactly the scripted velocity', () => { + const script = flick( + {x: 10, y: 10}, + {velocity: {x: 0, y: 1500}, durationMs: 80, hz: 120}, + ); + expect(releaseVelocity(script).y).toBeCloseTo(1500, 6); + const last = script.at(-2)!; + expect(last.fingers[0].y).toBeCloseTo(10 + 1500 * 0.08, 6); + // The lift lands a quarter frame after the last move, still moving. + expect(script.at(-1)!.t - last.t).toBeCloseTo(1000 / 120 / 4, 6); + }); + + it('flick follows a velocity profile', () => { + const script = flick( + {x: 0, y: 0}, + {velocity: {x: 1000, y: 0}, durationMs: 100, hz: 100, profile: p => p}, + ); + // Linear ramp: distance = v * T / 2. + expect(script.at(-2)!.fingers[0].x).toBeCloseTo(50, 6); + }); + + it('pinch and rotate keep two fingers about the centre', () => { + const spread = pinch({ + center: {x: 100, y: 100}, + fromDistance: 40, + toDistance: 200, + durationMs: 100, + }); + const end = spread.at(-2)!.fingers; + expect(Math.hypot(end[1].x - end[0].x, end[1].y - end[0].y)).toBeCloseTo( + 200, + 6, + ); + const turn = rotate({ + center: {x: 0, y: 0}, + distance: 100, + fromAngle: 0, + toAngle: 90, + durationMs: 100, + }); + expect(turn.at(-2)!.fingers[1].x).toBeCloseTo(0, 6); + expect(turn.at(-2)!.fingers[1].y).toBeCloseTo(50, 6); + expect(toTouchEvents(spread)[0]).toMatchObject({ + type: 'touchStart', + changed: [{id: 0}, {id: 1}], + }); + }); + + it('merge lands a second finger mid-drag and lifts it first', () => { + const first = drag({x: 0, y: 0}, {x: 0, y: 90}, {durationMs: 300, hz: 10}); + const second = withFinger(tap({x: 50, y: 50}, {holdMs: 100}), 1, 120); + const events = toTouchEvents(merge(first, second)); + const starts = events.filter(e => e.type === 'touchStart'); + expect( + starts.map(e => [e.t, e.changed.map(f => f.id), e.fingers.length]), + ).toEqual([ + [0, [0], 1], + [120, [1], 2], + ]); + const ends = events.filter(e => e.type === 'touchEnd'); + expect(ends[0]).toMatchObject({t: 220, changed: [{id: 1}]}); + expect(ends[0].fingers.map(f => f.id)).toEqual([0]); + expect(ends.at(-1)!.fingers).toEqual([]); + expect(() => merge(first, first)).toThrow(/two scripts/); + }); + + it('path follows any curve; holdThenDrag holds before the first move', () => { + const arc = path( + p => ({x: 100 * Math.cos(p * Math.PI), y: 100 * Math.sin(p * Math.PI)}), + {durationMs: 100, hz: 40}, + ); + const last = arc.at(-2)!.fingers[0]; + expect(last.x).toBeCloseTo(-100, 6); + expect(last.y).toBeCloseTo(0, 6); + expect(arc.at(-1)!.fingers).toEqual([]); + const held = holdThenDrag( + {x: 0, y: 0}, + {x: 30, y: 0}, + {durationMs: 50, hz: 20, holdMs: 500}, + ); + const moves = toTouchEvents(held).filter(e => e.type === 'touchMove'); + expect(moves[0].t).toBeCloseTo(550, 6); + expect(moves.at(-1)!.fingers[0].x).toBeCloseTo(30, 6); + }); + + it('refuses a script that leaves a finger down', () => { + expect(() => + toTouchEvents([{t: 0, fingers: [{id: 0, x: 0, y: 0}]}]), + ).toThrow(/lifted/); + }); +}); diff --git a/internal/interaction-harness/src/scripts.ts b/internal/interaction-harness/src/scripts.ts new file mode 100644 index 0000000000000..111d9e01234aa --- /dev/null +++ b/internal/interaction-harness/src/scripts.ts @@ -0,0 +1,395 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file scripts.ts + * @input Nothing: pure functions + * @output Touch scripts — timed lists of the fingers that are down — and the + * builders that make them: `tap`, `drag`, `flick`, `holdThenDrag`, `pinch`, + * `rotate`, `path`, `withFinger`, `merge`, plus `toTouchEvents`, which turns + * a script into the per-event sequence a touch driver dispatches. + * @position The input half of every interaction proof. A script is data, so a + * test states exactly what the finger does — positions and timestamps — and + * a unit test can check a builder without a browser. + * + * A script is a list of frames ordered by time. Each frame lists every finger + * that is down at that moment; a finger missing from a frame has lifted. The + * last frame is always empty (every finger up). Times are milliseconds from + * the gesture's start; coordinates are CSS px in the viewport. + * + * SYNC: Keep ./scripts.test.ts covering every builder. + */ + +export interface Point { + readonly x: number; + readonly y: number; +} + +export interface Finger extends Point { + /** Stable per finger for the whole gesture. */ + readonly id: number; +} + +export interface TouchFrame { + /** Milliseconds from the start of the gesture. */ + readonly t: number; + /** Every finger that is down at `t`. Empty = all lifted. */ + readonly fingers: readonly Finger[]; +} + +export type TouchScript = readonly TouchFrame[]; + +/** One dispatchable event: what changed at `t`, and the fingers down after it. */ +export interface TouchEventStep { + readonly t: number; + readonly type: 'touchStart' | 'touchMove' | 'touchEnd' | 'touchCancel'; + /** Fingers down after this event (CDP semantics: the full active set). */ + readonly fingers: readonly Finger[]; + /** Fingers this event pressed, moved or released. */ + readonly changed: readonly Finger[]; +} + +const DEFAULT_HZ = 60; + +function frameTimes(durationMs: number, hz: number): number[] { + const step = 1000 / hz; + const count = Math.max(1, Math.round(durationMs / step)); + return Array.from( + {length: count}, + (_, i) => +(((i + 1) * durationMs) / count).toFixed(3), + ); +} + +function lerp(a: Point, b: Point, p: number): Point { + return {x: a.x + (b.x - a.x) * p, y: a.y + (b.y - a.y) * p}; +} + +/** A press and lift in place. */ +export function tap( + at: Point, + {holdMs = 60, id = 0}: {holdMs?: number; id?: number} = {}, +): TouchScript { + return [ + {t: 0, fingers: [{id, ...at}]}, + {t: holdMs, fingers: []}, + ]; +} + +export interface DragOptions { + /** Time from the first move to the last, ms. */ + readonly durationMs: number; + /** Move samples per second (iOS reports touches at 60 Hz, 120 Hz on ProMotion). */ + readonly hz?: number; + /** Rest before the first move (press-and-hold), ms. */ + readonly holdMs?: number; + /** Rest after the last move before the lift, ms: >= 100 makes the release velocity 0. */ + readonly restMs?: number; + /** Progress easing over the move, default linear (constant velocity). */ + readonly easing?: (progress: number) => number; + readonly id?: number; +} + +/** Press at `from`, move to `to`, lift. Constant velocity unless `easing` says otherwise. */ +export function drag( + from: Point, + to: Point, + options: DragOptions, +): TouchScript { + const { + durationMs, + hz = DEFAULT_HZ, + holdMs = 0, + restMs = 0, + easing = (p: number) => p, + id = 0, + } = options; + const frames: TouchFrame[] = [{t: 0, fingers: [{id, ...from}]}]; + for (const t of frameTimes(durationMs, hz)) { + frames.push({ + t: holdMs + t, + fingers: [{id, ...lerp(from, to, easing(t / durationMs))}], + }); + } + // A real finger reports no move while it rests: the lift simply comes later. + const last = frames[frames.length - 1].t; + frames.push({t: last + Math.max(restMs, 1000 / hz / 4), fingers: []}); + return frames; +} + +/** `drag` after a press-and-hold (a long press that turns into a drag). */ +export function holdThenDrag( + from: Point, + to: Point, + options: DragOptions & {readonly holdMs: number}, +): TouchScript { + return drag(from, to, options); +} + +export interface FlickOptions { + /** Velocity at the lift, px/s. */ + readonly velocity: Point; + /** How long the finger moves before the lift, ms. */ + readonly durationMs?: number; + readonly hz?: number; + /** + * Velocity over the move as a fraction of the final velocity (0..1 → 0..∞), + * default constant. `p => p` is a finger accelerating linearly into the lift. + */ + readonly profile?: (progress: number) => number; + readonly id?: number; +} + +/** + * A throw: the finger moves with the given velocity profile and lifts while + * still moving (the lift lands a quarter frame after the last move, as on a + * device), so the release velocity is exactly `velocity`. + */ +export function flick(from: Point, options: FlickOptions): TouchScript { + const { + velocity, + durationMs = 80, + hz = 120, + profile = () => 1, + id = 0, + } = options; + const step = 1000 / hz; + const frames: TouchFrame[] = [{t: 0, fingers: [{id, ...from}]}]; + let x = from.x; + let y = from.y; + let previous = 0; + for (const t of frameTimes(durationMs, hz)) { + // Integrate the profile over (previous, t] with the midpoint rule. + const scale = profile((previous + t) / 2 / durationMs); + x += (velocity.x * scale * (t - previous)) / 1000; + y += (velocity.y * scale * (t - previous)) / 1000; + previous = t; + frames.push({t, fingers: [{id, x, y}]}); + } + frames.push({t: previous + step / 4, fingers: []}); + return frames; +} + +/** Any single-finger path: `at(progress)` for progress 0..1 over `durationMs`. */ +export function path( + at: (progress: number) => Point, + { + durationMs, + hz = DEFAULT_HZ, + holdMs = 0, + restMs = 0, + id = 0, + }: Omit, +): TouchScript { + const frames: TouchFrame[] = [{t: 0, fingers: [{id, ...at(0)}]}]; + for (const t of frameTimes(durationMs, hz)) { + frames.push({t: holdMs + t, fingers: [{id, ...at(t / durationMs)}]}); + } + frames.push({ + t: frames[frames.length - 1].t + Math.max(restMs, 1000 / hz / 4), + fingers: [], + }); + return frames; +} + +export interface PinchOptions { + readonly center: Point; + /** Finger separation at the start and at the end, px. */ + readonly fromDistance: number; + readonly toDistance: number; + /** Angle of the line between the fingers, degrees (0 = horizontal). */ + readonly fromAngle?: number; + readonly toAngle?: number; + readonly durationMs: number; + readonly hz?: number; + readonly restMs?: number; +} + +/** Two fingers (ids 0 and 1) pressed together, spread or pinched and turned, lifted together. */ +export function pinch(options: PinchOptions): TouchScript { + const { + center, + fromDistance, + toDistance, + fromAngle = 0, + toAngle = fromAngle, + durationMs, + hz = DEFAULT_HZ, + restMs = 0, + } = options; + const at = (p: number): Finger[] => { + const distance = fromDistance + (toDistance - fromDistance) * p; + const angle = ((fromAngle + (toAngle - fromAngle) * p) * Math.PI) / 180; + const dx = (Math.cos(angle) * distance) / 2; + const dy = (Math.sin(angle) * distance) / 2; + return [ + {id: 0, x: center.x - dx, y: center.y - dy}, + {id: 1, x: center.x + dx, y: center.y + dy}, + ]; + }; + const frames: TouchFrame[] = [{t: 0, fingers: at(0)}]; + for (const t of frameTimes(durationMs, hz)) { + frames.push({t, fingers: at(t / durationMs)}); + } + frames.push({t: durationMs + Math.max(restMs, 1000 / hz / 4), fingers: []}); + return frames; +} + +/** Two fingers turning around `center` at a fixed separation. */ +export function rotate( + options: Omit & { + readonly distance: number; + }, +): TouchScript { + return pinch({ + ...options, + fromDistance: options.distance, + toDistance: options.distance, + }); +} + +/** Re-labels a single-finger script's finger, and shifts it in time. */ +export function withFinger( + script: TouchScript, + id: number, + delayMs = 0, +): TouchScript { + return script.map(frame => ({ + t: frame.t + delayMs, + fingers: frame.fingers.map(f => ({...f, id})), + })); +} + +/** + * Plays several scripts at once (each with its own finger ids): at every time + * any of them changes, each finger is where its own script last put it. + * Used for a second finger that lands mid-gesture. + */ +export function merge(...scripts: TouchScript[]): TouchScript { + const times = [...new Set(scripts.flatMap(s => s.map(f => f.t)))].sort( + (a, b) => a - b, + ); + const stateAt = (script: TouchScript, t: number): readonly Finger[] => { + let current: readonly Finger[] = []; + for (const frame of script) { + if (frame.t > t) { + break; + } + current = frame.fingers; + } + return current; + }; + const owner = new Map(); + scripts.forEach((script, index) => { + for (const frame of script) { + for (const f of frame.fingers) { + if ((owner.get(f.id) ?? index) !== index) { + throw new Error(`merge: finger ${f.id} is used by two scripts`); + } + owner.set(f.id, index); + } + } + }); + return times.map(t => ({t, fingers: scripts.flatMap(s => stateAt(s, t))})); +} + +/** + * Turns frames into events: a finger that appears is a `touchStart`, one that + * disappears a `touchEnd`, one that moved a `touchMove`. A frame that both + * moves some fingers and lifts others yields the move first, then the lift. + * Unchanged frames yield nothing (a resting finger reports nothing). + */ +export function toTouchEvents(script: TouchScript): TouchEventStep[] { + const events: TouchEventStep[] = []; + let down = new Map(); + for (const frame of script) { + const next = new Map(frame.fingers.map(f => [f.id, f])); + const pressed = frame.fingers.filter(f => !down.has(f.id)); + const released = [...down.values()].filter(f => !next.has(f.id)); + const moved = frame.fingers.filter(f => { + const before = down.get(f.id); + return before != null && (before.x !== f.x || before.y !== f.y); + }); + if (moved.length > 0) { + const after = [...down.values()].map(f => next.get(f.id) ?? f); + events.push({ + t: frame.t, + type: 'touchMove', + fingers: after, + changed: moved, + }); + down = new Map(after.map(f => [f.id, f])); + } + if (released.length > 0) { + const remaining = [...down.values()].filter(f => next.has(f.id)); + events.push({ + t: frame.t, + type: 'touchEnd', + fingers: remaining, + changed: released.map(f => down.get(f.id) ?? f), + }); + down = new Map(remaining.map(f => [f.id, f])); + } + if (pressed.length > 0) { + const all = [...down.values(), ...pressed]; + events.push({ + t: frame.t, + type: 'touchStart', + fingers: all, + changed: pressed, + }); + down = new Map(all.map(f => [f.id, f])); + } + } + if (down.size > 0) { + throw new Error( + 'toTouchEvents: a script must end with every finger lifted', + ); + } + return events; +} + +/** + * The velocity (px/s) of finger `id` at its lift: the slope over the last + * `windowMs` of moves, or zero when the finger rested `windowMs` or longer + * before lifting (a finger that rested released nothing in motion). + */ +export function releaseVelocity( + script: TouchScript, + id = 0, + windowMs = 100, +): Point { + const samples = script.flatMap(frame => + frame.fingers + .filter(f => f.id === id) + .map(f => ({t: frame.t, x: f.x, y: f.y})), + ); + if (samples.length < 2) { + return {x: 0, y: 0}; + } + const lift = script.find( + frame => frame.t > samples[0].t && !frame.fingers.some(f => f.id === id), + ); + // The last sample that actually moved: a resting finger reports nothing. + let lastMove = samples.length - 1; + while ( + lastMove > 0 && + samples[lastMove].x === samples[lastMove - 1].x && + samples[lastMove].y === samples[lastMove - 1].y + ) { + lastMove -= 1; + } + const last = samples[lastMove]; + if (lift != null && lift.t - last.t >= windowMs) { + return {x: 0, y: 0}; + } + const recent = samples + .slice(0, lastMove + 1) + .filter(s => s.t >= last.t - windowMs); + const first = recent[0]; + const dt = last.t - first.t; + return dt <= 0 + ? {x: 0, y: 0} + : { + x: ((last.x - first.x) / dt) * 1000, + y: ((last.y - first.y) / dt) * 1000, + }; +} diff --git a/internal/interaction-harness/src/story.ts b/internal/interaction-harness/src/story.ts new file mode 100644 index 0000000000000..9e4323436ff27 --- /dev/null +++ b/internal/interaction-harness/src/story.ts @@ -0,0 +1,103 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file story.ts + * @input Uses a Playwright `Page` and test object, and the a11y-spec package's + * static server over a built Storybook + * @output `phone` (the touch-phone browser context every interaction proof + * runs in), `withStorybook` (serve the built Storybook for a spec file), and + * `openStory` (load one story and wait until it has rendered). + * @position Stories are the reproduction path, as for every other browser + * check in this repository: a scenario used for verification is checked in, + * not built in a one-off page. + * + * The phone is an iPhone 17 Pro in Safari: 402 × 874 pt screen, 402 × 714 pt + * visible page with the toolbars shown, 3 device pixels per point, touch, + * mobile viewport rules. Measured on the iOS 26.2 simulator + * (`innerWidth`/`innerHeight`, `screen`, `devicePixelRatio`). + * + * SYNC: Keep `phone` equal to the projects in playwright.interaction.config.ts. + */ + +import type {Page, TestType} from '@playwright/test'; +import { + DEFAULT_STORYBOOK_DIR, + serveStorybook, + type StaticServer, +} from '@astryxdesign/a11y-spec/storybook'; + +export const phone = { + viewport: {width: 402, height: 714}, + screen: {width: 402, height: 874}, + deviceScaleFactor: 3, + isMobile: true, + hasTouch: true, +} as const; + +/** + * Serves the built Storybook for the spec file's tests (`ASTRYX_STORYBOOK_DIR` + * overrides the default `apps/storybook/dist`). Returns its origin getter. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any -- any fixture set +export function withStorybook(test: TestType): () => string { + let server: StaticServer | undefined; + test.beforeAll(async () => { + server = await serveStorybook( + process.env.ASTRYX_STORYBOOK_DIR ?? DEFAULT_STORYBOOK_DIR, + ); + }); + test.afterAll(async () => { + await server?.close(); + }); + return () => { + if (server == null) { + throw new Error( + 'withStorybook: the Storybook server is not running (call inside a test)', + ); + } + return server.origin; + }; +} + +/** Loads a story in the page and waits for its first render, its fonts and two frames. */ +export async function openStory( + page: Page, + origin: string, + storyId: string, + args?: Readonly>, +): Promise { + const params = new URLSearchParams({id: storyId, viewMode: 'story'}); + if (args != null) { + params.set( + 'args', + Object.entries(args) + .map(([k, v]) => `${k}:${v}`) + .join(';'), + ); + } + await page.goto(`${origin}/iframe.html?${params.toString()}`); + await page + .locator('#storybook-root > *') + .first() + .waitFor({state: 'attached'}); + await page.evaluate(async () => { + await document.fonts.ready; + await new Promise(resolve => + requestAnimationFrame(() => requestAnimationFrame(() => resolve())), + ); + }); +} + +/** Resolves after `count` animation frames have been presented in the page. */ +export async function frames(page: Page, count = 2): Promise { + await page.evaluate( + n => + new Promise(resolve => { + let left = n; + const tick = () => + --left <= 0 ? resolve() : requestAnimationFrame(tick); + requestAnimationFrame(tick); + }), + count, + ); +} diff --git a/internal/interaction-harness/src/touch.ts b/internal/interaction-harness/src/touch.ts new file mode 100644 index 0000000000000..56cdf8aa50289 --- /dev/null +++ b/internal/interaction-harness/src/touch.ts @@ -0,0 +1,295 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file touch.ts + * @input Uses a Playwright `Page` (Chromium or WebKit) and a touch script from + * ./scripts + * @output `touchDriver(page)` — plays touch scripts as REAL browser touch input + * — and `installInputLog` / `readInputLog`, the page-side record of what the + * page actually received. + * @position The input half of every interaction proof. Untrusted DOM events + * are not input: they never start native scrolling, never claim a gesture, + * and a component can tell them apart (`isTrusted`). Both drivers here go + * through the engine's own input pipeline instead. + * + * Chromium: CDP `Input.dispatchTouchEvent` with an explicit `timestamp` per + * event. The page's `event.timeStamp` IS that timestamp, whatever the + * dispatch latency, so a velocity profile reaches the page exactly. The + * events go through the compositor's gesture detection: native scrolling, + * flings (velocity from those timestamps), `touch-action`, pointer events + * and `pointercancel` all behave as for a finger. + * + * WebKit (Playwright's Linux build): the engine's own inspector command + * `Input.dispatchTouchEvent` on the page proxy. Touches and the pointer + * events derived from them are trusted and cancelable, multi-touch works. + * Limits, measured with WebKit 26.5: no timestamp parameter (the page sees + * the real dispatch time, so velocity comes from real-time pacing, ±1 ms); + * integer coordinates only; no native scrolling, fling or pinch-zoom from + * these touches; a `touchstart` lists only the fingers that landed in it + * (pointer events are exact); no click synthesised from a raw touch sequence (use + * `tap`, which dispatches WebKit's tap gesture and does produce the + * compatibility mouse events and click). The command is reached through + * Playwright's in-process server object, so it needs a locally launched + * browser (the default), not a remote `connect()`. + * + * SYNC: Keep the engine limits above equal to the README's table and to the + * harness self-test (../src/harness.interaction.browser.spec.ts). + */ + +import type {CDPSession, Page} from '@playwright/test'; +import { + toTouchEvents, + type Finger, + type Point, + type TouchEventStep, + type TouchScript, +} from './scripts'; + +export type Engine = 'chromium' | 'webkit'; + +export interface PlayOptions { + /** + * Pace dispatch on the wall clock so each event leaves at its scripted time + * (default true). Required in WebKit for a velocity to mean anything; in + * Chromium the page sees the scripted timestamps either way, but native + * behaviors that run on the compositor clock (a fling) want real pacing. + */ + readonly realtime?: boolean; + /** Called after each event is dispatched, before the next one (storyboards). */ + readonly onEvent?: ( + step: TouchEventStep, + index: number, + ) => Promise | void; +} + +export interface PlayedEvent extends TouchEventStep { + /** Wall-clock milliseconds (epoch) at which the event was dispatched. */ + readonly dispatchedAt: number; + /** The timestamp handed to the engine, epoch ms (Chromium only). */ + readonly timestamp: number | null; +} + +export interface TouchDriver { + readonly engine: Engine; + /** True when the page's `event.timeStamp` follows the script exactly. */ + readonly scriptedTimestamps: boolean; + /** True when these touches can scroll, fling and pinch-zoom natively. */ + readonly drivesNativeGestures: boolean; + /** Plays a script; resolves when its last event has been dispatched. */ + play(script: TouchScript, options?: PlayOptions): Promise; + /** A tap through the engine's tap path (WebKit synthesises click only from this). */ + tap(at: Point): Promise; +} + +const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); + +/** Waits until `performance.now()` reaches `deadline` with ~0.1 ms precision. */ +async function until(deadline: number): Promise { + const remaining = deadline - performance.now(); + if (remaining > 3) { + await sleep(remaining - 2); + } + while (performance.now() < deadline) { + // spin: setTimeout alone is ±1-4 ms late + } +} + +async function playSteps( + script: TouchScript, + options: PlayOptions, + dispatch: (step: TouchEventStep, timestamp: number) => Promise, + stampFor: (t: number) => number, +): Promise { + const {realtime = true, onEvent} = options; + const steps = toTouchEvents(script); + const played: PlayedEvent[] = []; + const origin = performance.now(); + for (const [index, step] of steps.entries()) { + if (realtime) { + await until(origin + step.t); + } + const timestamp = stampFor(step.t); + const dispatchedAt = Date.now(); + await dispatch(step, timestamp); + played.push({...step, dispatchedAt, timestamp}); + await onEvent?.(step, index); + } + return played; +} + +async function chromiumDriver(page: Page): Promise { + const cdp: CDPSession = await page.context().newCDPSession(page); + const touchPoints = (fingers: readonly Finger[]) => + fingers.map(f => ({ + x: f.x, + y: f.y, + id: f.id, + radiusX: 1, + radiusY: 1, + force: 1, + })); + return { + engine: 'chromium', + scriptedTimestamps: true, + drivesNativeGestures: true, + async play(script, options = {}) { + // Stamps follow the wall clock the page's own clock is on: with real-time + // pacing an event's stamp and its dispatch agree to ~0.1 ms. + const base = Date.now(); + return playSteps( + script, + options, + async (step, timestamp) => { + // CDP diffs each event's active points against the previous event's: + // a missing point is released, a new one pressed. touchEnd must carry + // no points, so lifting one finger of several is a touchMove that + // lists the fingers still down. + const partialLift = + step.type === 'touchEnd' && step.fingers.length > 0; + await cdp.send('Input.dispatchTouchEvent', { + type: partialLift ? 'touchMove' : step.type, + touchPoints: + step.type === 'touchEnd' && !partialLift + ? [] + : touchPoints(step.fingers), + timestamp: timestamp / 1000, + }); + }, + t => base + t, + ); + }, + async tap(at) { + await page.touchscreen.tap(at.x, at.y); + }, + }; +} + +interface WebKitPageProxySession { + send(method: string, params?: Record): Promise; + on(event: string, listener: (payload: unknown) => void): unknown; + off?(event: string, listener: (payload: unknown) => void): unknown; +} + +/** + * Playwright's server-side WebKit page session, reached through the in-process + * connection. Exported for helpers that use WebKit's own inspector protocol. + */ +export function webkitPageSession(page: Page): WebKitPageProxySession { + const connection = ( + page as unknown as {_connection?: {toImpl?: (o: unknown) => unknown}} + )._connection; + const impl = connection?.toImpl?.(page) as + {delegate?: {_pageProxySession?: WebKitPageProxySession}} | undefined; + const session = impl?.delegate?._pageProxySession; + if (session == null) { + throw new Error( + "WebKit touch input needs Playwright's in-process server (a locally launched browser); this page has none.", + ); + } + return session; +} + +async function webkitDriver(page: Page): Promise { + const session = webkitPageSession(page); + // WebKit's command takes integer CSS px and applies the event's phase to + // every point it lists: a start lists only the fingers that land (listing a + // finger already down presses it again), an end only the fingers that lift + // (an empty list releases nothing), a move every finger down. + const points = (fingers: readonly Finger[]) => + fingers.map(f => ({x: Math.round(f.x), y: Math.round(f.y), id: f.id})); + return { + engine: 'webkit', + scriptedTimestamps: false, + drivesNativeGestures: false, + async play(script, options = {}) { + return playSteps( + script, + options, + async step => { + await session.send('Input.dispatchTouchEvent', { + type: step.type, + touchPoints: points( + step.type === 'touchMove' ? step.fingers : step.changed, + ), + }); + }, + () => Date.now(), + ).then(played => played.map(p => ({...p, timestamp: null}))); + }, + async tap(at) { + await page.touchscreen.tap(Math.round(at.x), Math.round(at.y)); + }, + }; +} + +/** A touch driver for the page's engine. Firefox is not supported (no touch input path). */ +export async function touchDriver(page: Page): Promise { + const name = page.context().browser()?.browserType().name(); + if (name === 'chromium') { + return chromiumDriver(page); + } + if (name === 'webkit') { + return webkitDriver(page); + } + throw new Error( + `touchDriver: no real touch input path for ${name ?? 'this browser'}`, + ); +} + +export interface InputRecord { + readonly type: string; + /** The event's own `timeStamp`, ms since the page's time origin. */ + readonly timeStamp: number; + readonly isTrusted: boolean; + readonly cancelable: boolean; + readonly defaultPrevented: boolean; + readonly pointerId?: number; + readonly pointerType?: string; + readonly x?: number; + readonly y?: number; + readonly touches?: number; + readonly target: string; +} + +const INPUT_LOG_SCRIPT = `(() => { + if (window.__harnessInput) return; + const log = []; + window.__harnessInput = log; + const describe = node => node instanceof Element + ? node.tagName.toLowerCase() + (node.id ? '#' + node.id : '') + (node.getAttribute('aria-label') ? '[' + node.getAttribute('aria-label') + ']' : '') + : String(node && node.nodeName); + const types = ['pointerdown', 'pointermove', 'pointerup', 'pointercancel', 'touchstart', 'touchmove', + 'touchend', 'touchcancel', 'click', 'contextmenu', 'gotpointercapture', 'lostpointercapture']; + for (const type of types) { + // Bubble phase on window, so defaultPrevented reflects the page's own handlers. + window.addEventListener(type, event => { + const record = {type, timeStamp: event.timeStamp, isTrusted: event.isTrusted, cancelable: event.cancelable, + defaultPrevented: event.defaultPrevented, target: describe(event.target)}; + if ('pointerId' in event) Object.assign(record, {pointerId: event.pointerId, pointerType: event.pointerType, x: event.clientX, y: event.clientY}); + if (event.touches) Object.assign(record, {touches: event.touches.length, x: event.changedTouches[0] && event.changedTouches[0].clientX, y: event.changedTouches[0] && event.changedTouches[0].clientY}); + log.push(record); + }, {passive: true}); + } +})()`; + +/** Records every pointer/touch/click the page receives (install before navigation). */ +export async function installInputLog(page: Page): Promise { + await page.addInitScript(INPUT_LOG_SCRIPT); +} + +/** What the page received since navigation (or since the last `clear`). */ +export async function readInputLog( + page: Page, + {clear = false}: {clear?: boolean} = {}, +): Promise { + return page.evaluate(shouldClear => { + const log = + (window as unknown as {__harnessInput?: InputRecord[]}).__harnessInput ?? + []; + const copy = log.slice(); + if (shouldClear) { + log.length = 0; + } + return copy; + }, clear); +} diff --git a/internal/interaction-harness/tsconfig.json b/internal/interaction-harness/tsconfig.json new file mode 100644 index 0000000000000..596e2cf729a9f --- /dev/null +++ b/internal/interaction-harness/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.json", + "include": ["src"] +} diff --git a/package.json b/package.json index 9052c34ad02d0..a4ca9f2ae830e 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "test:coverage": "vitest --coverage", "test:a11y-contract": "playwright test --config playwright.config.ts --project chromium", "test:docsite-browser": "playwright test --config apps/docsite/playwright.config.ts", + "test:interaction": "playwright test --config playwright.interaction.config.ts", "check:package-boundaries": "node scripts/check-package-boundaries.js", "check:repo": "pnpm check:visual-owner && pnpm check:sync && pnpm check:neutral-palette-refs && pnpm check:stone-palette-refs && pnpm check:gothic-palette-refs && pnpm check:matcha-palette-refs && pnpm check:butter-palette-refs && pnpm check:y2k-palette-refs && pnpm check:data-tokens && pnpm check:token-docs && pnpm check:knowledge && pnpm check:package-boundaries && pnpm check:changesets && pnpm check:demo-media && pnpm check:executable-bits && pnpm check:cli-structure && pnpm check:use-client && pnpm check:portable-scripts && pnpm check:i18n-runtime && pnpm check:i18n-catalog && pnpm check:cldr-weekdays && pnpm check:fixtures && pnpm check:setup-contract", "check:visual-owner": "node scripts/check-visual-workflow-owner.mjs", diff --git a/playwright.interaction.config.ts b/playwright.interaction.config.ts new file mode 100644 index 0000000000000..357a5dc3b7642 --- /dev/null +++ b/playwright.interaction.config.ts @@ -0,0 +1,77 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file playwright.interaction.config.ts + * @input Uses @playwright/test + * @output Touch interaction proofs on a phone-sized touch screen, in Chromium + * and WebKit. + * @position Run with `pnpm test:interaction`. Separate from `pnpm test` (no + * browser needed there) and from playwright.config.ts (accessibility + * contracts on desktop profiles). + * + * Specs are named `*.interaction.browser.spec.ts` and read stories from a + * built Storybook, like the accessibility specs: + * + * pnpm storybook:build + * pnpm exec playwright install chromium webkit + * pnpm test:interaction --project chromium + * pnpm test:interaction harness.interaction.browser.spec.ts + * + * A spec that needs Chromium-only evidence (CDP tracing, scripted touch + * timestamps) skips itself in WebKit with the reason; see the harness README. + * Where WebKit cannot launch on the host (missing system libraries on some + * Linux distributions), internal/interaction-harness/scripts/webkit-container.mjs + * runs the WebKit project inside the official Playwright image. + * + * SYNC: Keep the phone profile here equal to `phone` in + * internal/interaction-harness/src/story.ts. + */ + +import {defineConfig} from '@playwright/test'; + +// The phone: an iPhone 17 Pro in Safari (402 × 874 pt screen, 402 × 714 pt +// page with the toolbars shown, 3 device px per point), measured on the iOS +// 26.2 simulator. Inline rather than imported: the config loads as CommonJS and +// the harness package is ESM. +const phone = { + viewport: {width: 402, height: 714}, + screen: {width: 402, height: 874}, + deviceScaleFactor: 3, + isMobile: true, + hasTouch: true, +} as const; + +const isCI = (process.env.CI ?? '') !== ''; + +function workers(): number { + const raw = process.env.INTERACTION_WORKERS; + const count = raw ? Number(raw) : 1; + if (!Number.isInteger(count) || count < 1) { + throw new Error( + `INTERACTION_WORKERS must be a positive integer, got "${raw}"`, + ); + } + return count; +} + +export default defineConfig({ + testDir: '.', + testMatch: [ + 'internal/interaction-harness/src/**/*.interaction.browser.spec.ts', + 'packages/*/src/**/*.interaction.browser.spec.ts', + ], + // Playwright wipes its output directory per run; evidence (storyboards, + // curve traces) goes to test-results/interaction-evidence instead. + outputDir: 'test-results/interaction-output', + // Frame evidence measures timing: one worker by default, so a proof never + // shares the CPU with another browser. + workers: workers(), + fullyParallel: false, + timeout: 60_000, + forbidOnly: isCI, + reporter: isCI ? [['list'], ['github']] : [['list']], + projects: [ + {name: 'chromium', use: {browserName: 'chromium', ...phone}}, + {name: 'webkit', use: {browserName: 'webkit', ...phone}}, + ], +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f2e4d844ccbfb..c93bc6ef3838a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -581,6 +581,21 @@ importers: specifier: ^8.62.1 version: 8.62.1(eslint@10.6.0(jiti@2.7.0))(typescript@6.0.3) + internal/interaction-harness: + devDependencies: + '@astryxdesign/a11y-spec': + specifier: workspace:* + version: link:../a11y-spec + '@playwright/test': + specifier: ^1.61.1 + version: 1.61.1 + typescript: + specifier: ^6.0.3 + version: 6.0.3 + vitest: + specifier: ^4.1.11 + version: 4.1.11(@types/node@25.9.4)(@vitest/coverage-v8@4.1.11)(happy-dom@20.11.1)(jsdom@27.4.0)(vite@8.1.3(@types/node@25.9.4)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.0)(yaml@2.9.0)) + internal/test-utils: dependencies: '@testing-library/jest-dom': From 75fc6cb1869b4100a9a1732f815bf49fb124932b Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Fri, 23 Oct 2026 23:31:44 -0700 Subject: [PATCH 02/93] fix(interaction): release only the lifted finger on a partial lift in Chromium The Chromium driver sent a lift of one finger of several as a touchMove listing the fingers still down, assuming CDP releases a point a move omits. It does not: the lifted finger stayed down. A touchEnd that lists the lifting fingers releases exactly those while the others keep moving, so the driver sends that, and a self-test lifts one of two fingers and checks the release, the other finger's moves and the touchend's remaining touches. Playwright's WebKit sends that touchend with no remaining touches, which iOS never does. The harness does not work around it: drivers expose partialLifts, and PARTIAL_LIFT_SKIP is the reason a proof cites when it skips; the README lists it with the other engine limits. --- internal/interaction-harness/README.md | 23 ++++++----- .../src/harness.interaction.browser.spec.ts | 40 +++++++++++++++++++ internal/interaction-harness/src/touch.ts | 37 ++++++++++++----- 3 files changed, 80 insertions(+), 20 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index a0b56d854c6b0..7b2ab8ec65067 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -50,17 +50,20 @@ the limits below. Read positions a frame after a move: Chromium delivers ## Engine limits (measured with Playwright 1.61) -| | Chromium 149 | WebKit 26.5 (Linux build) | -| ------------------------------------------------------ | ------------ | ----------------------------------------------------------------------------------- | -| Trusted touch + pointer events, multi-touch | yes | yes (a `touchstart` lists only the fingers that landed in it; pointer events exact) | -| Page sees the scripted timestamps | yes, exactly | no: real dispatch time; real-time pacing keeps velocities within ~1 ms | -| Sub-pixel touch positions | yes | no: integer CSS px | -| Native scroll, fling, pinch-zoom from injected touches | yes | no | -| Click synthesised from a touch | yes | only from `driver.tap` (WebKit's tap gesture), not from a raw touch sequence | -| Input log | yes | yes | +| | Chromium 149 | WebKit 26.5 (Linux build) | +| ------------------------------------------------------ | ------------ | ----------------------------------------------------------------------------------------------------- | +| Trusted touch + pointer events, multi-touch | yes | yes (a `touchstart` lists only the fingers that landed in it; pointer events exact) | +| Page sees the scripted timestamps | yes, exactly | no: real dispatch time; real-time pacing keeps velocities within ~1 ms | +| Sub-pixel touch positions | yes | no: integer CSS px | +| Native scroll, fling, pinch-zoom from injected touches | yes | no | +| Lifting one finger while others stay down | yes | pointer events right, but that `touchend` lists no remaining `touches`: skip with `PARTIAL_LIFT_SKIP` | +| Click synthesised from a touch | yes | only from `driver.tap` (WebKit's tap gesture), not from a raw touch sequence | +| Input log | yes | yes | -What WebKit cannot show here — native scroll hand-off, flings, and anything -iOS-only — needs Mobile Safari itself, on a device or the iOS simulator. +What WebKit cannot show here — native scroll hand-off, flings, a finger +lifting while another stays down, and anything iOS-only — needs Mobile Safari +itself, on a device or the iOS simulator. Skip such a proof on the driver's +flag, citing the reason: `test.skip(!driver.partialLifts, PARTIAL_LIFT_SKIP)`. The WebKit input command is reached through Playwright's in-process server object, so it needs a locally launched browser (the default), not `connect()`. diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 10d41e668c9bf..38aa9c4876b1c 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -16,6 +16,7 @@ import { flick, installInputLog, merge, + PARTIAL_LIFT_SKIP, pinch, readInputLog, tap, @@ -131,6 +132,45 @@ test.describe('touch input', () => { expect(fast).toBeGreaterThan(slow * 1.8); }); + test('lifting one of two fingers releases only that finger', async ({ + page, + }) => { + await fixture(page); + const driver = await touchDriver(page); + test.skip(!driver.partialLifts, PARTIAL_LIFT_SKIP); + const first = drag( + {x: 80, y: 80}, + {x: 80, y: 240}, + {durationMs: 400, hz: 60}, + ); + const second = withFinger(tap({x: 300, y: 120}, {holdMs: 100}), 1, 100); + await driver.play(merge(first, second)); + const log = await readInputLog(page); + const [stays, lifts] = log.filter(r => r.type === 'pointerdown'); + const upOf = (down?: {pointerId?: number}) => + log.find(r => r.type === 'pointerup' && r.pointerId === down?.pointerId); + const lifted = upOf(lifts); + const last = upOf(stays); + if (stays == null || lifted == null || last == null) { + throw new Error('expected two fingers down, each released'); + } + // Released when it lifted (~200 ms in), while the first finger was down. + expect(lifted.timeStamp).toBeLessThan(last.timeStamp - 150); + expect( + log.filter( + r => + r.type === 'pointermove' && + r.pointerId === stays.pointerId && + r.timeStamp > lifted.timeStamp, + ).length, + 'the other finger keeps moving', + ).toBeGreaterThan(5); + // As iOS sends it: the lift's touchend still lists the finger down. + expect(log.filter(r => r.type === 'touchend').map(r => r.touches)).toEqual([ + 1, 0, + ]); + }); + test('a second finger and a pinch arrive as distinct touch pointers', async ({ page, }) => { diff --git a/internal/interaction-harness/src/touch.ts b/internal/interaction-harness/src/touch.ts index 56cdf8aa50289..1c16c4b1f82e0 100644 --- a/internal/interaction-harness/src/touch.ts +++ b/internal/interaction-harness/src/touch.ts @@ -75,12 +75,27 @@ export interface TouchDriver { readonly scriptedTimestamps: boolean; /** True when these touches can scroll, fling and pinch-zoom natively. */ readonly drivesNativeGestures: boolean; + /** + * True when lifting one finger while others stay down reaches the page as + * iOS sends it: a `touchend` whose `touches` still lists the fingers down. + * Skip such proofs with `PARTIAL_LIFT_SKIP` where it is false. + */ + readonly partialLifts: boolean; /** Plays a script; resolves when its last event has been dispatched. */ play(script: TouchScript, options?: PlayOptions): Promise; /** A tap through the engine's tap path (WebKit synthesises click only from this). */ tap(at: Point): Promise; } +/** + * The skip reason for a proof that lifts one finger while another stays down, + * on a driver whose `partialLifts` is false: Playwright's WebKit (Linux build) + * sends that lift's `touchend` with no remaining `touches`, an emulation quirk + * iOS does not have. Those cases need a simulator capture. + */ +export const PARTIAL_LIFT_SKIP = + 'injected WebKit touches report no remaining touches when one finger of several lifts (README: engine limits); capture it on the iOS simulator'; + const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); /** Waits until `performance.now()` reaches `deadline` with ~0.1 ms precision. */ @@ -132,6 +147,7 @@ async function chromiumDriver(page: Page): Promise { engine: 'chromium', scriptedTimestamps: true, drivesNativeGestures: true, + partialLifts: true, async play(script, options = {}) { // Stamps follow the wall clock the page's own clock is on: with real-time // pacing an event's stamp and its dispatch agree to ~0.1 ms. @@ -140,18 +156,18 @@ async function chromiumDriver(page: Page): Promise { script, options, async (step, timestamp) => { - // CDP diffs each event's active points against the previous event's: - // a missing point is released, a new one pressed. touchEnd must carry - // no points, so lifting one finger of several is a touchMove that - // lists the fingers still down. - const partialLift = - step.type === 'touchEnd' && step.fingers.length > 0; + // A start or move lists every finger down (CDP presses the new ones). + // An end lists the fingers that lift: that releases exactly those + // while the rest stay down and keep moving; the last lift lists none. + // (A move that omits a finger does not release it.) await cdp.send('Input.dispatchTouchEvent', { - type: partialLift ? 'touchMove' : step.type, + type: step.type, touchPoints: - step.type === 'touchEnd' && !partialLift - ? [] - : touchPoints(step.fingers), + step.type !== 'touchEnd' + ? touchPoints(step.fingers) + : step.fingers.length > 0 + ? touchPoints(step.changed) + : [], timestamp: timestamp / 1000, }); }, @@ -201,6 +217,7 @@ async function webkitDriver(page: Page): Promise { engine: 'webkit', scriptedTimestamps: false, drivesNativeGestures: false, + partialLifts: false, async play(script, options = {}) { return playSteps( script, From 0a07aa13e07b1e3b3c9017fcf0cf989c3f3a09d5 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sat, 24 Oct 2026 14:46:20 -0700 Subject: [PATCH 03/93] fix(interaction): join the proofs to test, keep their setup in scope, lift profiled flicks at their velocity - pr-interaction now always reports and the `test` join requires its success (AST-030 FR7, FR11), shaped like test-native-swift: a Linux interaction-scope job decides, with the base's scope helper and failing closed, whether the harness, a component, Storybook, internal/a11y-spec, the workflow, package.json or pnpm-lock.yaml changed. Only then do the proofs take a 4-core runner; otherwise, and in merge groups (as with pr-a11y), the job passes without work. - A profiled flick's last move runs at velocity times profile(1), so the finger lifts at the documented velocity, and releaseVelocity reads the last move instead of a 100 ms window mean. - Docs: drop the /math subpath this package does not export yet; scripted timestamps arrive to 0.1 ms; the job comment describes the self-test it runs. --- .github/scripts/ci-interaction.test.mjs | 126 ++++++++++++++++++ .github/scripts/ci-test-join.mjs | 1 + .github/scripts/ci-test-join.test.mjs | 12 +- .github/scripts/ci-test-routing.test.mjs | 17 ++- .github/scripts/ci-tooling-routing.test.mjs | 2 + .github/workflows/ci.yml | 126 +++++++++++++----- internal/interaction-harness/README.md | 2 +- internal/interaction-harness/src/index.ts | 4 +- .../interaction-harness/src/scripts.test.ts | 35 ++++- internal/interaction-harness/src/scripts.ts | 46 ++++--- 10 files changed, 312 insertions(+), 59 deletions(-) create mode 100644 .github/scripts/ci-interaction.test.mjs diff --git a/.github/scripts/ci-interaction.test.mjs b/.github/scripts/ci-interaction.test.mjs new file mode 100644 index 0000000000000..e8b393ffbf486 --- /dev/null +++ b/.github/scripts/ci-interaction.test.mjs @@ -0,0 +1,126 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file Routing contract for the touch interaction proofs in the broad CI lane. + * + * The proofs are coverage inside the shared surface (AST-030 FR11), so the + * `test` join requires pr-interaction, and a failing proof fails `test` (FR7). + * A Linux job decides whether the change touches the harness, a component, + * Storybook, or the proofs' own setup; only then does pr-interaction take a + * 4-core runner, and otherwise it passes on Linux without work. + */ + +import fs from 'node:fs'; +import path from 'node:path'; + +import {describe, expect, it} from 'vitest'; +import yaml from 'yaml'; + +const root = path.resolve(import.meta.dirname, '../..'); +const ci = yaml.parse( + fs.readFileSync(path.join(root, '.github/workflows/ci.yml'), 'utf8'), +); +const scope = ci.jobs['interaction-scope']; +const proofs = ci.jobs['pr-interaction']; +const step = (job, name) => + job.steps.find(candidate => candidate.name === name); +const decide = () => step(scope, 'Decide whether the interaction proofs apply'); +const applies = "needs.interaction-scope.outputs.applies == 'true'"; + +describe('interaction proof routing', () => { + it('is joined by test', () => { + expect(ci.jobs.test.needs).toContain('pr-interaction'); + expect( + step(ci.jobs.test, 'Require expected test-owner outcomes').env, + ).toMatchObject({ + PR_INTERACTION_RESULT: '${{ needs.pr-interaction.result }}', + }); + }); + + it('always reports, so the join never reads a missing result', () => { + expect(proofs.if).toBe(ci.jobs['test-native-swift'].if); + expect(scope.if).toBe(ci.jobs['native-swift-scope'].if); + }); + + it('decides scope on Linux and takes four cores only when the proofs apply', () => { + expect(scope['runs-on']).toBe('ubuntu-slim'); + expect(proofs.needs).toEqual([ + 'check-scope', + 'interaction-scope', + 'build-storybook', + ]); + expect(proofs['runs-on']).toBe( + "${{ needs.interaction-scope.outputs.applies == 'true' && '4-core-ubuntu' || 'ubuntu-slim' }}", + ); + const guards = new Set([ + 'Require a decided interaction scope', + "Require this pull request's Storybook build", + 'Skip the proofs outside their scope', + ]); + for (const candidate of proofs.steps) { + if (guards.has(candidate.name)) continue; + expect(candidate.if, candidate.name ?? candidate.uses).toMatch( + new RegExp( + `^(always\\(\\) && )?${applies.replace(/[.()|]/g, '\\$&')}$`, + ), + ); + } + }); + + it('fails closed when the scope cannot be decided or Storybook did not build', () => { + expect(step(scope, 'Require successful scope classification').if).toBe( + "needs.check-scope.result != 'success'", + ); + expect(step(proofs, 'Require a decided interaction scope').if).toBe( + "needs.interaction-scope.result != 'success'", + ); + expect(step(proofs, "Require this pull request's Storybook build").if).toBe( + `${applies} && needs.build-storybook.result != 'success'`, + ); + expect(decide().run).toContain( + 'git show "$BASE_SHA:.github/scripts/pr-scope-touches.sh"', + ); + // Only "out of scope" (exit 1) skips; unknown (2) runs the proofs. + expect(decide().run).toContain('if [ "$scope" = 1 ]; then'); + }); + + it('keeps the harness, components, Storybook and its own setup in scope', () => { + const pattern = new RegExp( + decide().run.match(/bash "\$helper" "\$BASE_SHA" '([^']+)'/)[1], + ); + for (const touched of [ + 'internal/interaction-harness/src/scripts.ts', + 'internal/a11y-spec/src/storybook.ts', + 'playwright.interaction.config.ts', + 'packages/core/src/BottomSheet/BottomSheet.tsx', + 'apps/storybook/.storybook/preview.tsx', + '.github/workflows/ci.yml', + 'package.json', + 'pnpm-lock.yaml', + ]) { + expect(pattern.test(touched), touched).toBe(true); + } + for (const untouched of [ + 'docs/specs/AST-030/spec.md', + 'apps/docsite/src/app/page.tsx', + 'packages/core/package.json', + '.github/scripts/native-swift.sh', + ]) { + expect(pattern.test(untouched), untouched).toBe(false); + } + }); + + it('passes without work on docsite-only, tooling-only and merge-group runs', () => { + expect(decide().env).toMatchObject({ + EVENT_NAME: '${{ github.event_name }}', + DOCSITE_ONLY: '${{ needs.check-scope.outputs.docsite_only }}', + TOOLING_ONLY: '${{ needs.check-scope.outputs.tooling_only }}', + }); + expect(decide().run).toContain( + 'if [ "$DOCSITE_ONLY" = true ] || [ "$TOOLING_ONLY" = true ]; then', + ); + expect(decide().run).toContain( + 'if [ "$EVENT_NAME" != pull_request ]; then', + ); + }); +}); diff --git a/.github/scripts/ci-test-join.mjs b/.github/scripts/ci-test-join.mjs index c3cdf35fc16d0..61e9701c163e6 100644 --- a/.github/scripts/ci-test-join.mjs +++ b/.github/scripts/ci-test-join.mjs @@ -15,6 +15,7 @@ const requiredSuccess = [ ['test-node', process.env.TEST_NODE_RESULT], ['registry-contract', process.env.REGISTRY_CONTRACT_RESULT], ['test-native-swift', process.env.TEST_NATIVE_SWIFT_RESULT], + ['pr-interaction', process.env.PR_INTERACTION_RESULT], ]; function fail(message) { diff --git a/.github/scripts/ci-test-join.test.mjs b/.github/scripts/ci-test-join.test.mjs index 3344ba3d100a5..0786a00d86cde 100644 --- a/.github/scripts/ci-test-join.test.mjs +++ b/.github/scripts/ci-test-join.test.mjs @@ -18,13 +18,19 @@ const broad = { TEST_BUILD_RESULT: 'success', REGISTRY_CONTRACT_RESULT: 'success', TEST_NATIVE_SWIFT_RESULT: 'success', + PR_INTERACTION_RESULT: 'success', }; +// An override of `null` removes the variable, as a missing `needs` result would. function run(overrides = {}) { + const env = {...process.env, ...broad, ...overrides}; + for (const [name, value] of Object.entries(env)) { + if (value === null) delete env[name]; + } try { const stdout = execFileSync(process.execPath, [script], { encoding: 'utf8', - env: {...process.env, ...broad, ...overrides}, + env, stdio: ['ignore', 'pipe', 'pipe'], }); return {status: 0, output: stdout}; @@ -46,6 +52,10 @@ describe('pull-request test join', () => { ['REGISTRY_CONTRACT_RESULT', 'failure'], ['TEST_NATIVE_SWIFT_RESULT', 'failure'], ['TEST_NATIVE_SWIFT_RESULT', 'skipped'], + ['PR_INTERACTION_RESULT', 'failure'], + ['PR_INTERACTION_RESULT', 'skipped'], + ['PR_INTERACTION_RESULT', 'cancelled'], + ['PR_INTERACTION_RESULT', null], ]) { expect(run({[name]: value}), name).toMatchObject({status: 1}); } diff --git a/.github/scripts/ci-test-routing.test.mjs b/.github/scripts/ci-test-routing.test.mjs index 1128817685567..6908a2d8ec4d7 100644 --- a/.github/scripts/ci-test-routing.test.mjs +++ b/.github/scripts/ci-test-routing.test.mjs @@ -595,7 +595,13 @@ describe('ci.yml interaction proofs', () => { const step = name => job.steps.find(candidate => candidate.name === name); it('runs the interaction config in Chromium against the exact PR head', () => { - expect(KNOWN_RUNNERS.has(job['runs-on'])).toBe(true); + // Four cores when the proofs apply, a slim runner when it passes without work. + const labels = [...job['runs-on'].matchAll(/'([^']+)'/g)] + .map(match => match[1]) + .filter(label => label !== 'true'); + expect(labels).toEqual(['4-core-ubuntu', 'ubuntu-slim']); + for (const label of labels) + expect(KNOWN_RUNNERS.has(label), label).toBe(true); expect(job.needs).toEqual( expect.arrayContaining(['check-scope', 'build-storybook']), ); @@ -615,13 +621,20 @@ describe('ci.yml interaction proofs', () => { }); it('runs whenever the harness, its config, a component or a story changes', () => { - const scope = step('Check interaction scope').run; + const scope = workflow.jobs['interaction-scope'].steps.find( + candidate => + candidate.name === 'Decide whether the interaction proofs apply', + ).run; expect(scope).toContain('.github/scripts/pr-scope-touches.sh'); for (const path of [ 'internal/interaction-harness/', + 'internal/a11y-spec/', 'playwright\\.interaction\\.config\\.ts', 'packages/[^/]+/src/', 'apps/storybook/', + '\\.github/workflows/ci\\.yml$', + 'package\\.json$', + 'pnpm-lock\\.yaml$', ]) { expect(scope).toContain(path); } diff --git a/.github/scripts/ci-tooling-routing.test.mjs b/.github/scripts/ci-tooling-routing.test.mjs index 5561eb4f5add6..eb64d74f8cadf 100644 --- a/.github/scripts/ci-tooling-routing.test.mjs +++ b/.github/scripts/ci-tooling-routing.test.mjs @@ -179,6 +179,7 @@ describe('Node-tooling CI routing', () => { 'test-build', 'registry-contract', 'test-native-swift', + 'pr-interaction', ]); expect(ci.jobs.build.needs).toEqual([ 'build-storybook', @@ -198,6 +199,7 @@ describe('Node-tooling CI routing', () => { TEST_BUILD_RESULT: '${{ needs.test-build.result }}', REGISTRY_CONTRACT_RESULT: '${{ needs.registry-contract.result }}', TEST_NATIVE_SWIFT_RESULT: '${{ needs.test-native-swift.result }}', + PR_INTERACTION_RESULT: '${{ needs.pr-interaction.result }}', }); const buildJoin = step( ci.jobs.build, diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8cc8e7d9a951e..f5ca0f3930b8f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -881,9 +881,10 @@ jobs: ASTRYX_SHADCN_USE_PUBLISHED_PACKAGES: '1' run: node internal/shadcn-registry/verify-full-catalog.mjs - # Historical required check name. The Vitest lanes, Build test owner, and - # public registry contract always report or explicitly skip, so this join - # projects their combined result without creating another surface lane. + # Historical required check name. The Vitest lanes, Build test owner, + # public registry contract, native Swift and interaction proofs always report + # or explicitly skip, so this join projects their combined result without + # creating another surface lane. test: needs: [ @@ -893,6 +894,7 @@ jobs: test-build, registry-contract, test-native-swift, + pr-interaction, ] if: ${{ github.event_name != 'workflow_dispatch' && always() && !cancelled() && needs.check-scope.outputs.spec_only != 'true' }} runs-on: ubuntu-slim @@ -912,6 +914,7 @@ jobs: TEST_BUILD_RESULT: ${{ needs.test-build.result }} REGISTRY_CONTRACT_RESULT: ${{ needs.registry-contract.result }} TEST_NATIVE_SWIFT_RESULT: ${{ needs.test-native-swift.result }} + PR_INTERACTION_RESULT: ${{ needs.pr-interaction.result }} run: node .github/scripts/ci-test-join.mjs # Build storybook + typecheck + analyze components (parallel with test and @@ -3774,68 +3777,125 @@ jobs: # internal/interaction-harness and every *.interaction.browser.spec.ts. # Real touch input, composited frames, rendering traces and presented # animation values only exist in a real engine, and the specs read stories - # from this PR's own Storybook build. The harness's self-test runs with - # them, so a change to an instrument is proven by its negative controls - # before a component spec relies on it. Chromium only on these runners; - # see the harness README for running WebKit. - pr-interaction: - needs: [check-scope, build-storybook] - if: github.event_name == 'pull_request' && needs.check-scope.outputs.docsite_only != 'true' && needs.check-scope.outputs.spec_only != 'true' && needs.check-scope.outputs.tooling_only != 'true' - # Frame evidence measures timing: four cores, one Playwright worker. - runs-on: 4-core-ubuntu + # from this PR's own Storybook build. The harness's self-test runs with them, + # so a change to an instrument is proven by its own checks before a component + # spec relies on it. Chromium only on these runners; see the harness README + # for running WebKit. + # + # The proofs are coverage inside the shared surface (AST-030 FR11), so `test` + # joins pr-interaction. A Linux job first decides whether the change touches + # the harness, a component, Storybook, or the proofs' own setup; only then + # does pr-interaction take a 4-core runner, and otherwise it passes on Linux + # without work. Like pr-a11y, the proofs run on pull requests; a merge group + # passes without them. + interaction-scope: + needs: [check-scope] + if: ${{ github.event_name != 'workflow_dispatch' && always() && !cancelled() && needs.check-scope.outputs.spec_only != 'true' }} + runs-on: ubuntu-slim permissions: contents: read + outputs: + applies: ${{ steps.scope.outputs.applies }} steps: - - name: Checkout - uses: actions/checkout@v7 - with: - ref: ${{ github.event.pull_request.head.sha }} + - name: Require successful scope classification + if: needs.check-scope.result != 'success' + run: | + echo "Change scope could not be classified; refusing to skip required work." + exit 1 - - name: Verify evidence checkout - env: - EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - run: test "$(git rev-parse HEAD)" = "$EXPECTED_HEAD_SHA" + - uses: actions/checkout@v7 + if: github.event_name == 'pull_request' && needs.check-scope.outputs.docsite_only != 'true' && needs.check-scope.outputs.tooling_only != 'true' - # The helper comes from the base, since an older PR head lacks it. - # Exit 1 means out of scope; 0 in scope; 2 unknown, so run. - - name: Check interaction scope + # The helper comes from the base, since an older head lacks it. Exit 1 + # means out of scope; 0 in scope; 2 unknown, so the proofs run. + - name: Decide whether the interaction proofs apply id: scope env: + EVENT_NAME: ${{ github.event_name }} + DOCSITE_ONLY: ${{ needs.check-scope.outputs.docsite_only }} + TOOLING_ONLY: ${{ needs.check-scope.outputs.tooling_only }} BASE_SHA: ${{ github.event.pull_request.base.sha }} run: | + if [ "$DOCSITE_ONLY" = true ] || [ "$TOOLING_ONLY" = true ]; then + echo 'Specialized lane; no interaction proofs.' + echo 'applies=false' >> "$GITHUB_OUTPUT" + exit 0 + fi + if [ "$EVENT_NAME" != pull_request ]; then + echo 'Not a pull request; the proofs ran on the pull request itself.' + echo 'applies=false' >> "$GITHUB_OUTPUT" + exit 0 + fi scope=2 helper="$(mktemp)" - if git fetch --quiet --no-tags --depth=1 origin "$BASE_SHA" \ + if [ -n "$BASE_SHA" ] \ + && git fetch --quiet --no-tags --depth=1 origin "$BASE_SHA" \ && git show "$BASE_SHA:.github/scripts/pr-scope-touches.sh" > "$helper"; then scope=0 - bash "$helper" "$BASE_SHA" '^(internal/interaction-harness/|playwright\.interaction\.config\.ts$|packages/[^/]+/src/|apps/storybook/)' || scope=$? + bash "$helper" "$BASE_SHA" '^(internal/interaction-harness/|internal/a11y-spec/|playwright\.interaction\.config\.ts$|packages/[^/]+/src/|apps/storybook/|\.github/workflows/ci\.yml$|package\.json$|pnpm-lock\.yaml$)' || scope=$? fi if [ "$scope" = 1 ]; then - echo "Neither the interaction harness nor a component changed; skipping the interaction proofs." - echo 'run=false' >> "$GITHUB_OUTPUT" + echo 'No harness, component, Storybook or proof-setup change; skipping the interaction proofs.' + echo 'applies=false' >> "$GITHUB_OUTPUT" else - echo 'run=true' >> "$GITHUB_OUTPUT" + echo 'applies=true' >> "$GITHUB_OUTPUT" fi + pr-interaction: + needs: [check-scope, interaction-scope, build-storybook] + if: ${{ github.event_name != 'workflow_dispatch' && always() && !cancelled() && needs.check-scope.outputs.spec_only != 'true' }} + # Frame evidence measures timing: four cores, one Playwright worker. + runs-on: ${{ needs.interaction-scope.outputs.applies == 'true' && '4-core-ubuntu' || 'ubuntu-slim' }} + permissions: + contents: read + steps: + - name: Require a decided interaction scope + if: needs.interaction-scope.result != 'success' + run: | + echo "Interaction scope could not be decided; refusing to skip required work." + exit 1 + + - name: Require this pull request's Storybook build + if: needs.interaction-scope.outputs.applies == 'true' && needs.build-storybook.result != 'success' + run: | + echo "The proofs read this pull request's Storybook build, which did not succeed." + exit 1 + + - name: Skip the proofs outside their scope + if: needs.interaction-scope.outputs.applies != 'true' + run: echo "No interaction proofs for this change." + + - name: Checkout + if: needs.interaction-scope.outputs.applies == 'true' + uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Verify evidence checkout + if: needs.interaction-scope.outputs.applies == 'true' + env: + EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: test "$(git rev-parse HEAD)" = "$EXPECTED_HEAD_SHA" + - name: Setup Node and pnpm - if: steps.scope.outputs.run == 'true' + if: needs.interaction-scope.outputs.applies == 'true' uses: ./.github/actions/setup with: install: 'false' - name: Download Storybook artifact - if: steps.scope.outputs.run == 'true' + if: needs.interaction-scope.outputs.applies == 'true' uses: actions/download-artifact@v8 with: name: storybook-${{ needs.build-storybook.outputs.short_hash }} path: apps/storybook/dist - name: Install Playwright - if: steps.scope.outputs.run == 'true' + if: needs.interaction-scope.outputs.applies == 'true' run: pnpm install --frozen-lockfile && npx playwright install chromium - name: Run interaction proofs - if: steps.scope.outputs.run == 'true' + if: needs.interaction-scope.outputs.applies == 'true' run: | pnpm exec playwright test \ --config playwright.interaction.config.ts \ @@ -3844,7 +3904,7 @@ jobs: # Storyboards (PNG contact sheets) and curve traces, for review. - name: Upload interaction evidence - if: always() && steps.scope.outputs.run == 'true' + if: always() && needs.interaction-scope.outputs.applies == 'true' uses: actions/upload-artifact@v7 with: name: interaction-evidence diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 7b2ab8ec65067..8139a9fa33348 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -41,7 +41,7 @@ node internal/interaction-harness/scripts/webkit-container.mjs [playwright test Untrusted DOM events are not input: they start no native scroll, claim no gesture, and read `isTrusted: false`. Chromium input goes through CDP `Input.dispatchTouchEvent` with an explicit timestamp per event: the page's -`event.timeStamp` is the scripted time to the microsecond, whatever the +`event.timeStamp` is the scripted time to 0.1 ms, whatever the dispatch latency, and the events go through the compositor's gesture detection (native scrolling, flings with velocity from those timestamps, `touch-action`, `pointercancel`). WebKit input goes through the engine's own input command; see diff --git a/internal/interaction-harness/src/index.ts b/internal/interaction-harness/src/index.ts index 3216b3c084322..587b9f82d233a 100644 --- a/internal/interaction-harness/src/index.ts +++ b/internal/interaction-harness/src/index.ts @@ -4,8 +4,8 @@ * @file index.ts * @input Re-exports the harness modules * @output The `@astryxdesign/interaction-harness` entry point for Playwright - * specs. Pure helpers are also reachable without Playwright through - * `@astryxdesign/interaction-harness/math` and `/scripts` (unit tests). + * specs. The pure touch scripts are also reachable without Playwright + * through `@astryxdesign/interaction-harness/scripts` (unit tests). * @position Imported only from `*.interaction.browser.spec.ts` files (run by * playwright.interaction.config.ts) and from this package's own tests. * diff --git a/internal/interaction-harness/src/scripts.test.ts b/internal/interaction-harness/src/scripts.test.ts index b85d4b526de55..072adeddc0cd4 100644 --- a/internal/interaction-harness/src/scripts.test.ts +++ b/internal/interaction-harness/src/scripts.test.ts @@ -66,8 +66,39 @@ describe('touch scripts', () => { {x: 0, y: 0}, {velocity: {x: 1000, y: 0}, durationMs: 100, hz: 100, profile: p => p}, ); - // Linear ramp: distance = v * T / 2. - expect(script.at(-2)!.fingers[0].x).toBeCloseTo(50, 6); + // Linear ramp: the first nine 10 ms moves integrate it by the midpoint rule + // (40.5 px) and the last runs at the full 1,000 px/s (10 px). + expect(script.at(-2)!.fingers[0].x).toBeCloseTo(50.5, 6); + }); + + it.each([ + ['linear', (p: number) => p], + ['quadratic', (p: number) => p * p], + ])( + 'a %s-profile flick still lifts at the scripted velocity', + (_, profile) => { + const script = flick( + {x: 0, y: 0}, + {velocity: {x: 0, y: 1500}, durationMs: 80, hz: 120, profile}, + ); + const [before, last] = script.slice(-3, -1); + const lastMove = + ((last.fingers[0].y - before.fingers[0].y) / (last.t - before.t)) * + 1000; + expect(lastMove).toBeCloseTo(1500, 6); + expect(releaseVelocity(script).y).toBeCloseTo(1500, 6); + }, + ); + + it('release velocity is the last move, not a window mean', () => { + // 500 px/s for 100 ms, then 2,000 px/s for one 10 ms move; path() lifts a + // quarter frame later. + const script = path( + p => ({x: 0, y: p <= 10 / 11 ? 55 * p : 50 + 20 * (p * 11 - 10)}), + {durationMs: 110, hz: 100}, + ); + expect(script.at(-1)!.fingers).toEqual([]); + expect(releaseVelocity(script).y).toBeCloseTo(2000, 6); }); it('pinch and rotate keep two fingers about the centre', () => { diff --git a/internal/interaction-harness/src/scripts.ts b/internal/interaction-harness/src/scripts.ts index 111d9e01234aa..2fa861f665e7d 100644 --- a/internal/interaction-harness/src/scripts.ts +++ b/internal/interaction-harness/src/scripts.ts @@ -131,8 +131,10 @@ export interface FlickOptions { readonly durationMs?: number; readonly hz?: number; /** - * Velocity over the move as a fraction of the final velocity (0..1 → 0..∞), - * default constant. `p => p` is a finger accelerating linearly into the lift. + * Velocity over the move as a fraction of `velocity` (0..1 → 0..∞), default + * constant. `p => p` is a finger accelerating linearly into the lift. The + * moves before the last follow it (midpoint rule); the last move runs at + * `profile(1)`. */ readonly profile?: (progress: number) => number; readonly id?: number; @@ -141,7 +143,10 @@ export interface FlickOptions { /** * A throw: the finger moves with the given velocity profile and lifts while * still moving (the lift lands a quarter frame after the last move, as on a - * device), so the release velocity is exactly `velocity`. + * device). Its last move runs at `velocity` times `profile(1)`, so with the + * default constant profile, or any profile that ends at 1, the release + * velocity (what `releaseVelocity` and a velocity tracker read from the last + * move) is exactly `velocity`. */ export function flick(from: Point, options: FlickOptions): TouchScript { const { @@ -156,9 +161,14 @@ export function flick(from: Point, options: FlickOptions): TouchScript { let x = from.x; let y = from.y; let previous = 0; - for (const t of frameTimes(durationMs, hz)) { - // Integrate the profile over (previous, t] with the midpoint rule. - const scale = profile((previous + t) / 2 / durationMs); + const times = frameTimes(durationMs, hz); + for (const [index, t] of times.entries()) { + // Integrate the profile over (previous, t] with the midpoint rule; the last + // move runs at the release velocity itself, so the finger lifts at it. + const scale = + index === times.length - 1 + ? profile(1) + : profile((previous + t) / 2 / durationMs); x += (velocity.x * scale * (t - previous)) / 1000; y += (velocity.y * scale * (t - previous)) / 1000; previous = t; @@ -348,14 +358,14 @@ export function toTouchEvents(script: TouchScript): TouchEventStep[] { } /** - * The velocity (px/s) of finger `id` at its lift: the slope over the last - * `windowMs` of moves, or zero when the finger rested `windowMs` or longer - * before lifting (a finger that rested released nothing in motion). + * The velocity (px/s) of finger `id` as it lifts: the slope of its last move, + * or zero when it never moved or rested `restMs` or longer before lifting (a + * finger that rested released nothing in motion). */ export function releaseVelocity( script: TouchScript, id = 0, - windowMs = 100, + restMs = 100, ): Point { const samples = script.flatMap(frame => frame.fingers @@ -377,19 +387,19 @@ export function releaseVelocity( ) { lastMove -= 1; } + if (lastMove === 0) { + return {x: 0, y: 0}; + } const last = samples[lastMove]; - if (lift != null && lift.t - last.t >= windowMs) { + if (lift != null && lift.t - last.t >= restMs) { return {x: 0, y: 0}; } - const recent = samples - .slice(0, lastMove + 1) - .filter(s => s.t >= last.t - windowMs); - const first = recent[0]; - const dt = last.t - first.t; + const before = samples[lastMove - 1]; + const dt = last.t - before.t; return dt <= 0 ? {x: 0, y: 0} : { - x: ((last.x - first.x) / dt) * 1000, - y: ((last.y - first.y) / dt) * 1000, + x: ((last.x - before.x) / dt) * 1000, + y: ((last.y - before.y) / dt) * 1000, }; } From 5f2ed5076b8edc1886a68a23de9aaf8e72e737d6 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sat, 24 Oct 2026 15:26:39 -0700 Subject: [PATCH 04/93] fix(interaction): releaseVelocity measures from when the finger reached its previous position A sample that repeats a position is not a move: frames that merge() inserts for another finger, where this finger did not move, made the last move look shorter, so a 10 Hz drag merged with a tap read 375 px/s instead of 300 and a 60 Hz drag whose other finger lifted 5 ms before its last move read about 3.6x too fast. The previous sample is now the first one at the previous position, as the page sees it, with a merged case in scripts.test.ts. --- .../interaction-harness/src/scripts.test.ts | 23 +++++++++++++++++++ internal/interaction-harness/src/scripts.ts | 16 ++++++++++--- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/internal/interaction-harness/src/scripts.test.ts b/internal/interaction-harness/src/scripts.test.ts index 072adeddc0cd4..321b76d1a49c5 100644 --- a/internal/interaction-harness/src/scripts.test.ts +++ b/internal/interaction-harness/src/scripts.test.ts @@ -90,6 +90,29 @@ describe('touch scripts', () => { }, ); + it('release velocity ignores frames merged in for another finger', () => { + // A 10 Hz drag with a tap from 120 to 220 ms: merge inserts frames where + // the dragging finger did not move, and the page sees no move there. + const slow = drag({x: 0, y: 0}, {x: 0, y: 90}, {durationMs: 300, hz: 10}); + const tapping = withFinger(tap({x: 50, y: 50}, {holdMs: 100}), 1, 120); + expect(releaseVelocity(merge(slow, tapping)).y).toBeCloseTo( + releaseVelocity(slow).y, + 6, + ); + // A 60 Hz drag whose second finger lifts 5 ms before the drag's last move. + const fast = drag({x: 0, y: 0}, {x: 0, y: 300}, {durationMs: 300, hz: 60}); + const lastMove = fast.at(-2)!.t; + const lifting = withFinger( + tap({x: 50, y: 50}, {holdMs: lastMove - 5 - 100}), + 1, + 100, + ); + expect(releaseVelocity(merge(fast, lifting)).y).toBeCloseTo( + releaseVelocity(fast).y, + 6, + ); + }); + it('release velocity is the last move, not a window mean', () => { // 500 px/s for 100 ms, then 2,000 px/s for one 10 ms move; path() lifts a // quarter frame later. diff --git a/internal/interaction-harness/src/scripts.ts b/internal/interaction-harness/src/scripts.ts index 2fa861f665e7d..79e8744060f78 100644 --- a/internal/interaction-harness/src/scripts.ts +++ b/internal/interaction-harness/src/scripts.ts @@ -359,8 +359,10 @@ export function toTouchEvents(script: TouchScript): TouchEventStep[] { /** * The velocity (px/s) of finger `id` as it lifts: the slope of its last move, - * or zero when it never moved or rested `restMs` or longer before lifting (a - * finger that rested released nothing in motion). + * measured from when it reached its previous position (samples that repeat a + * position, such as frames `merge` inserts for another finger, are not moves: + * the page sees none there), or zero when it never moved or rested `restMs` or + * longer before lifting (a finger that rested released nothing in motion). */ export function releaseVelocity( script: TouchScript, @@ -394,7 +396,15 @@ export function releaseVelocity( if (lift != null && lift.t - last.t >= restMs) { return {x: 0, y: 0}; } - const before = samples[lastMove - 1]; + let previous = lastMove - 1; + while ( + previous > 0 && + samples[previous - 1].x === samples[previous].x && + samples[previous - 1].y === samples[previous].y + ) { + previous -= 1; + } + const before = samples[previous]; const dt = last.t - before.t; return dt <= 0 ? {x: 0, y: 0} From d3c13cc11f0bdf238ea5046ca4f70addb07365d7 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Fri, 23 Oct 2026 20:06:07 -0700 Subject: [PATCH 05/93] test(interaction): curve sampling and deterministic storyboards Pause every Web Animation, seek it to a time and read the presented transform and opacity, then compare the curve with oracles that share no code with the motion under test: cubic-bezier, the SwiftUI spring (response, dampingFraction, initial velocity), UIScrollView deceleration and UIKit's rubber band. Storyboards photograph seeked animations at t = 0, 16.7, 33.3 ... ms or each step of a scripted touch, and write a PNG contact sheet plus the JSON trace a spec asserts, under the gitignored test-results/interaction-evidence. A timer gate holds a component's own timers while a filmstrip is taken. pnpm storyboards runs the tagged tests. --- internal/interaction-harness/README.md | 54 ++- internal/interaction-harness/package.json | 3 +- internal/interaction-harness/src/curves.ts | 313 ++++++++++++++++++ .../src/harness.interaction.browser.spec.ts | 224 ++++++++++++- internal/interaction-harness/src/index.ts | 4 + internal/interaction-harness/src/math.test.ts | 154 +++++++++ internal/interaction-harness/src/math.ts | 291 ++++++++++++++++ internal/interaction-harness/src/output.ts | 66 ++++ internal/interaction-harness/src/story.ts | 60 ++++ .../interaction-harness/src/storyboard.ts | 302 +++++++++++++++++ package.json | 1 + playwright.interaction.config.ts | 10 +- 12 files changed, 1460 insertions(+), 22 deletions(-) create mode 100644 internal/interaction-harness/src/curves.ts create mode 100644 internal/interaction-harness/src/math.test.ts create mode 100644 internal/interaction-harness/src/math.ts create mode 100644 internal/interaction-harness/src/output.ts create mode 100644 internal/interaction-harness/src/storyboard.ts diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 8139a9fa33348..ec30c5bfaf2af 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -1,8 +1,8 @@ # @astryxdesign/interaction-harness -Playwright instruments for touch interaction proofs: real touch input. -**Internal and unpublished.** Specs live next to their component as -`*.interaction.browser.spec.ts` and run under +Playwright instruments for touch interaction and animation proofs: real touch +input, curve sampling and storyboards. **Internal and unpublished.** Specs live +next to their component as `*.interaction.browser.spec.ts` and run under [`playwright.interaction.config.ts`](../../playwright.interaction.config.ts) on a phone-sized touch screen (402 × 714 CSS px page, 402 × 874 screen, 3 device pixels per CSS px, touch, mobile viewport rules), in Chromium and WebKit. @@ -18,6 +18,7 @@ pnpm storybook:build # component specs read stories from th pnpm exec playwright install chromium webkit pnpm test:interaction --project chromium # everything, Chromium pnpm test:interaction # one spec, both engines +pnpm storyboards # only the tests tagged @storyboard ``` On Linux distributions whose system libraries are older than Playwright's @@ -28,26 +29,51 @@ image (podman or docker; the repository is mounted at the same path): node internal/interaction-harness/scripts/webkit-container.mjs [playwright test args…] ``` +Evidence (storyboards, curve traces) is written to +`test-results/interaction-evidence/` (override with +`ASTRYX_INTERACTION_EVIDENCE`) and attached to the test report. It is never +committed: a pull request links or attaches it. + ## Instruments -| Module | What it gives a spec | -| ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact release velocity, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. | -| `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | -| `story.ts` | `phone`, `withStorybook`, `openStory`, `frames`. | +| Module | What it gives a spec | +| --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact release velocity, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. | +| `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | +| `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t), `waitForAnimation`, `waitForRest`, `releaseAnimations`. | +| `math.ts` | Oracles independent of the code under test: `cubicBezier`, `timingFunction`, `springStep` (SwiftUI response / dampingFraction, initial velocity), `projectDecay`, `rubberBand`, `parseTransform`, `compareCurve`, `timeline`. | +| `storyboard.ts` | `animationStoryboard` (seek to t = 0, 16.7, 33.3 … ms), `touchStoryboard` (a frame per scripted event): PNG contact sheet + JSON trace. | +| `output.ts` | `evidenceDir`, `writeEvidence`: where evidence goes and how it is attached to the report. | +| `story.ts` | `phone`, `withStorybook`, `openStory`, `frames`, and the timer gate (`installTimerGate`, `freezeTimers`, `thawTimers`). | -## Real touch +## The proofs -Untrusted DOM events are not input: they start no native scroll, claim no -gesture, and read `isTrusted: false`. Chromium input goes through CDP -`Input.dispatchTouchEvent` with an explicit timestamp per event: the page's -`event.timeStamp` is the scripted time to 0.1 ms, whatever the +**Real touch.** Untrusted DOM events are not input: they start no native +scroll, claim no gesture, and read `isTrusted: false`. Chromium input goes +through CDP `Input.dispatchTouchEvent` with an explicit timestamp per event: the +page's `event.timeStamp` is the scripted time to 0.1 ms, whatever the dispatch latency, and the events go through the compositor's gesture detection (native scrolling, flings with velocity from those timestamps, `touch-action`, `pointercancel`). WebKit input goes through the engine's own input command; see the limits below. Read positions a frame after a move: Chromium delivers `pointermove` aligned to animation frames. +**Curve sampling.** Pause every animation (CSS transitions and animations are +Web Animations too), seek to a time, read `getComputedStyle`: the presented +value at that exact time, independent of frame timing. Compare it with an +oracle from `math.ts` through `compareCurve`; a tolerance states the error +budget (for a spring baked into `linear()` stops of `h` ms, chords err by up to +`h² / 8 · |x''|`). A seek past an animation's end lands half a millisecond +inside it, the last frame it presents. + +**Storyboards.** Deterministic: frames come from seeking held animations or +from stepping scripted input, never from wall-clock capture. The JSON trace +carries each frame's measurements and the spec asserts on it; the sheet is +evidence. A component that finishes an animation from a timer (a transitionend +backstop, a cleanup render) would do so behind a held storyboard's back: +`installTimerGate` before the animation starts and `freezeTimers` while +photographing. + ## Engine limits (measured with Playwright 1.61) | | Chromium 149 | WebKit 26.5 (Linux build) | @@ -58,7 +84,7 @@ the limits below. Read positions a frame after a move: Chromium delivers | Native scroll, fling, pinch-zoom from injected touches | yes | no | | Lifting one finger while others stay down | yes | pointer events right, but that `touchend` lists no remaining `touches`: skip with `PARTIAL_LIFT_SKIP` | | Click synthesised from a touch | yes | only from `driver.tap` (WebKit's tap gesture), not from a raw touch sequence | -| Input log | yes | yes | +| Curve sampling, storyboards, input log | yes | yes | What WebKit cannot show here — native scroll hand-off, flings, a finger lifting while another stays down, and anything iOS-only — needs Mobile Safari diff --git a/internal/interaction-harness/package.json b/internal/interaction-harness/package.json index 08b1d02f61634..40cccb26ca206 100644 --- a/internal/interaction-harness/package.json +++ b/internal/interaction-harness/package.json @@ -3,10 +3,11 @@ "version": "0.0.0", "private": true, "type": "module", - "description": "Internal: Playwright helpers for touch interaction proofs (real touch input). Never published.", + "description": "Internal: Playwright helpers for touch interaction and animation proofs (real touch input, curve sampling, storyboards). Never published.", "main": "./src/index.ts", "exports": { ".": "./src/index.ts", + "./math": "./src/math.ts", "./scripts": "./src/scripts.ts" }, "scripts": { diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts new file mode 100644 index 0000000000000..c003d97b0dab6 --- /dev/null +++ b/internal/interaction-harness/src/curves.ts @@ -0,0 +1,313 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file curves.ts + * @input Uses a Playwright `Page`, a `Locator` for the animated element, and + * the Web Animations API in the page (CSS transitions and CSS animations are + * Web Animations too) + * @output `holdAnimations` (pause every running animation and describe it), + * `seekAnimations` (move them all to t ms), `sampleCurve` (the presented + * transform/opacity at each t), `releaseAnimations`, and `curveOf`, which + * picks one channel out of the samples for ./math's `compareCurve`. + * @position Curve sampling: the exact presented value of an animation at a + * fixed time, independent of frame timing, so a curve can be compared with + * the UIKit reference to a fraction of a pixel. A paused animation's effect + * is applied at its current time and `getComputedStyle` reads it. + * + * SYNC: Keep ./storyboard.ts using these for its animation filmstrips. + */ + +import type {Locator, Page} from '@playwright/test'; +import {parseTransform, type TimedValue, type Transform2D} from './math'; + +/** How far inside its end a seek past an animation's end lands, ms. */ +export const END_INSET_MS = 0.5; + +export interface HeldAnimation { + readonly index: number; + /** `CSSTransition`, `CSSAnimation` or `Animation` (script-created). */ + readonly kind: string; + /** Transitioned property or animation name, when CSS made it. */ + readonly name: string | null; + readonly target: string | null; + readonly durationMs: number; + readonly delayMs: number; + readonly easing: string; + readonly keyframes: readonly Record[]; + /** Where it was when held, ms. */ + readonly heldAtMs: number | null; +} + +/** + * Pauses every animation in the document (optionally only those whose target + * is inside `scope`) and returns what was held. Later `seekAnimations` calls + * move exactly this set. + */ +export async function holdAnimations( + page: Page, + scope?: Locator, +): Promise { + const scopeHandle = scope ? await scope.elementHandle() : null; + return page.evaluate(root => { + const describe = (node: Element | null) => + node == null + ? null + : node.tagName.toLowerCase() + + (node.id ? `#${node.id}` : '') + + (node.classList.length + ? `.${[...node.classList].slice(0, 2).join('.')}` + : ''); + const all = document.getAnimations().filter(animation => { + const target = + (animation.effect as KeyframeEffect | null)?.target ?? null; + return ( + root == null || + (target != null && (root === target || root.contains(target))) + ); + }); + const held = all.map(animation => { + animation.pause(); + return animation; + }); + (window as unknown as {__harnessHeld: Animation[]}).__harnessHeld = held; + return held.map((animation, index) => { + const effect = animation.effect as KeyframeEffect | null; + const timing = effect?.getComputedTiming(); + const cssName = + 'transitionProperty' in animation + ? (animation as unknown as {transitionProperty: string}) + .transitionProperty + : 'animationName' in animation + ? (animation as unknown as {animationName: string}).animationName + : null; + return { + index, + kind: animation.constructor.name, + name: cssName, + target: describe((effect?.target as Element | null) ?? null), + durationMs: Number(timing?.duration ?? 0), + delayMs: Number(timing?.delay ?? 0), + easing: String(effect?.getTiming().easing ?? 'linear'), + keyframes: (effect?.getKeyframes() ?? []).map( + (k): Record => ({...k}), + ), + heldAtMs: + animation.currentTime == null ? null : Number(animation.currentTime), + }; + }); + }, scopeHandle); +} + +/** + * Moves every held animation to `timeMs` (from its own start, delay included). + * A time at or past an animation's end is clamped to END_INSET_MS before it: + * an animation without `fill` stops applying its effect exactly at the end, + * and the frame wanted there is the last one it presents. (Half a millisecond, + * not less: engines round `currentTime` writes to their timer resolution.) + */ +export async function seekAnimations( + page: Page, + timeMs: number, +): Promise { + await page.evaluate( + ({t, inset}) => { + for (const animation of ( + window as unknown as {__harnessHeld?: Animation[]} + ).__harnessHeld ?? []) { + const end = Number( + animation.effect?.getComputedTiming().endTime ?? Infinity, + ); + animation.currentTime = Math.min(t, end - inset); + } + }, + {t: timeMs, inset: END_INSET_MS}, + ); +} + +/** Resumes (default) or finishes every held animation. */ +export async function releaseAnimations( + page: Page, + {finish = false}: {finish?: boolean} = {}, +): Promise { + await page.evaluate(shouldFinish => { + const held = + (window as unknown as {__harnessHeld?: Animation[]}).__harnessHeld ?? []; + for (const animation of held) { + if (shouldFinish) { + animation.finish(); + } else { + animation.play(); + } + } + (window as unknown as {__harnessHeld?: Animation[]}).__harnessHeld = []; + }, finish); +} + +export interface CurveSample { + readonly t: number; + readonly transform: Transform2D; + readonly opacity: number; + /** Raw computed values, for the JSON trace. */ + readonly raw: {readonly transform: string; readonly opacity: string}; +} + +/** + * The presented transform and opacity of `locator` at each time (ms from the + * held animations' start). Hold the animations first. + */ +export async function sampleCurve( + locator: Locator, + times: readonly number[], +): Promise { + const raw = await locator.evaluate( + (element, {ts, inset}) => { + const held = + (window as unknown as {__harnessHeld?: Animation[]}).__harnessHeld ?? + []; + if (held.length === 0) { + throw new Error( + 'sampleCurve: no held animations (call holdAnimations first)', + ); + } + return ts.map(t => { + for (const animation of held) { + // Just inside the end: see seekAnimations. + const end = Number( + animation.effect?.getComputedTiming().endTime ?? Infinity, + ); + animation.currentTime = Math.min(t, end - inset); + } + const style = getComputedStyle(element); + return {t, transform: style.transform, opacity: style.opacity}; + }); + }, + {ts: [...times], inset: END_INSET_MS}, + ); + return raw.map(r => ({ + t: r.t, + transform: parseTransform(r.transform), + opacity: Number(r.opacity), + raw: {transform: r.transform, opacity: r.opacity}, + })); +} + +export type Channel = + 'translateX' | 'translateY' | 'scaleX' | 'scaleY' | 'rotate' | 'opacity'; + +/** One channel of a sampled curve, ready for `compareCurve`. */ +export function curveOf( + samples: readonly CurveSample[], + channel: Channel, +): TimedValue[] { + return samples.map(s => ({ + t: s.t, + value: channel === 'opacity' ? s.opacity : s.transform[channel], + })); +} + +export interface ActiveAnimation { + readonly kind: string; + readonly name: string | null; + /** Epoch ms when the active interval starts and ends (page clock). */ + readonly from: number; + readonly to: number; + readonly durationMs: number; + readonly easing: string; + readonly keyframes: readonly Record[]; +} + +/** + * Waits until an animation is running on `locator`'s element (or inside it) + * and has presented at least one frame, and returns its active interval on + * the page clock. Use it before blocking the main thread: an animation that + * has not started yet cannot start while the thread is blocked. + */ +export async function waitForAnimation( + locator: Locator, + { + timeout = 5_000, + minProgressMs = 1, + }: {timeout?: number; minProgressMs?: number} = {}, +): Promise { + const deadline = Date.now() + timeout; + for (;;) { + const found = await locator.evaluate((element, minProgress) => { + return element + .getAnimations({subtree: true}) + .filter( + a => + a.playState === 'running' && + a.startTime != null && + Number(a.currentTime ?? 0) >= minProgress, + ) + .map(a => { + const effect = a.effect as KeyframeEffect | null; + const timing = effect?.getComputedTiming(); + const start = + performance.timeOrigin + + Number(a.startTime) + + Number(timing?.delay ?? 0); + const name = + 'transitionProperty' in a + ? (a as unknown as {transitionProperty: string}) + .transitionProperty + : 'animationName' in a + ? (a as unknown as {animationName: string}).animationName + : null; + return { + kind: a.constructor.name, + name, + from: start, + to: start + Number(timing?.activeDuration ?? 0), + durationMs: Number(timing?.duration ?? 0), + easing: String(effect?.getTiming().easing ?? 'linear'), + keyframes: (effect?.getKeyframes() ?? []).map( + (k): Record => ({...k}), + ), + }; + }); + }, minProgressMs); + if (found.length > 0) { + return found; + } + if (Date.now() > deadline) { + throw new Error('waitForAnimation: no animation started on the element'); + } + await new Promise(resolve => setTimeout(resolve, 4)); + } +} + +/** + * Resolves once nothing animates on `locator`'s element (or inside it) and its + * transform has held still for two frames: the element is at rest. + */ +export async function waitForRest( + locator: Locator, + {timeout = 5_000}: {timeout?: number} = {}, +): Promise { + await locator.evaluate( + (element, ms) => + new Promise((resolve, reject) => { + const deadline = performance.now() + ms; + let last = ''; + let still = 0; + const tick = () => { + const transform = getComputedStyle(element).transform; + const animating = element + .getAnimations({subtree: true}) + .some(a => a.playState === 'running'); + still = !animating && transform === last ? still + 1 : 0; + last = transform; + if (still >= 2) { + resolve(); + } else if (performance.now() > deadline) { + reject(new Error('waitForRest: still moving')); + } else { + requestAnimationFrame(tick); + } + }; + requestAnimationFrame(tick); + }), + timeout, + ); +} diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 38aa9c4876b1c..0698e603fe4ac 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -3,24 +3,42 @@ /** * @file harness.interaction.browser.spec.ts * @input Fixture pages written inline (no Storybook), real Chromium/WebKit - * @output Proof that each touch instrument delivers real engine input: + * @output Proof that each harness instrument measures what it claims: * scripted timestamps reach the page; a flick flings natively (Chromium); - * a second finger and a pinch arrive as distinct pointers. + * a second finger and a pinch arrive as distinct pointers; sampled curves + * match the oracles; the animation-state waits report a running transition + * and its end; storyboards write a sheet and a trace that follows the curve, + * and the timer gate holds a component timer until the filmstrip is done. * @position The harness's own test. A component spec trusts these * instruments; this file is why it may. */ import {expect, test, type Page} from '@playwright/test'; import { + animationStoryboard, + compareCurve, + cubicBezier, + curveOf, drag, flick, + freezeTimers, + holdAnimations, installInputLog, + installTimerGate, merge, + parseTransform, PARTIAL_LIFT_SKIP, pinch, readInputLog, + sampleCurve, + springStep, tap, + thawTimers, + timeline, touchDriver, + touchStoryboard, + waitForAnimation, + waitForRest, withFinger, } from './index'; @@ -206,3 +224,205 @@ test.describe('touch input', () => { ).toBe(2); }); }); + +test.describe('curve sampling', () => { + test('a CSS transition samples onto its cubic-bezier to 0.05 px', async ({ + page, + }) => { + await fixture(page); + const box = page.locator('#box'); + await box.evaluate(el => { + el.style.transition = 'transform 400ms cubic-bezier(0.24, 1, 0.4, 1)'; + void (el as HTMLElement).offsetWidth; + el.style.transform = 'translateY(200px)'; + }); + const held = await holdAnimations(page, box); + expect(held).toHaveLength(1); + expect(held[0].kind).toBe('CSSTransition'); + const samples = curveOf( + await sampleCurve(box, timeline(400)), + 'translateY', + ); + const ease = cubicBezier(0.24, 1, 0.4, 1); + const result = compareCurve(samples, t => 200 * ease(t / 400), 0.05); + expect(result.maxError, JSON.stringify(result.worst)).toBeLessThanOrEqual( + 0.05, + ); + }); + + test('a spring baked into linear() matches the SwiftUI spring oracle', async ({ + page, + }) => { + await fixture(page); + const box = page.locator('#box'); + const spring = {response: 0.4, dampingFraction: 0.75}; + const duration = 1200; + await box.evaluate( + (el, {stops, ms}) => { + el.animate( + [{transform: 'translateX(0px)'}, {transform: 'translateX(200px)'}], + {duration: ms, easing: `linear(${stops})`}, + ); + }, + { + stops: timeline(duration, 4) + .map(t => springStep(spring, t).toFixed(6)) + .join(', '), + ms: duration, + }, + ); + await holdAnimations(page, box); + const samples = curveOf( + await sampleCurve(box, timeline(duration, 1000 / 60)), + 'translateX', + ); + // linear() draws chords between the 4 ms stops: at most h²/8 · |x''| = + // 0.004² / 8 · 200 · (2π / 0.4)² ≈ 0.1 px off the exact oracle. + const result = compareCurve( + samples, + t => 200 * springStep(spring, t), + 0.15, + ); + expect(result.maxError, JSON.stringify(result.worst)).toBeLessThanOrEqual( + 0.15, + ); + expect(Math.max(...samples.map(s => s.value))).toBeGreaterThan(200); // the overshoot is presented + }); +}); + +test.describe('animation state', () => { + test('waitForAnimation reports a running transition and waitForRest resolves after it', async ({ + page, + }) => { + await fixture(page); + const box = page.locator('#box'); + await box.evaluate(el => { + el.style.transition = 'transform 300ms linear'; + void (el as HTMLElement).offsetWidth; + el.style.transform = 'translateY(100px)'; + }); + const [running] = await waitForAnimation(box); + expect(running.kind).toBe('CSSTransition'); + expect(running.name).toBe('transform'); + expect(running.durationMs).toBe(300); + expect(running.to - running.from).toBeCloseTo(300, 0); + await waitForRest(box); + const settledAt = await page.evaluate( + () => performance.timeOrigin + performance.now(), + ); + expect(settledAt).toBeGreaterThanOrEqual(running.to); + const transform = await box.evaluate(el => getComputedStyle(el).transform); + expect(parseTransform(transform).translateY).toBeCloseTo(100, 1); + }); +}); + +test.describe('storyboards', () => { + test('an animation filmstrip records every frame and its trace follows the curve', async ({ + page, + }, testInfo) => { + await fixture(page); + const box = page.locator('#box'); + await box.evaluate(el => + el.animate( + [{transform: 'translateX(0)'}, {transform: 'translateX(240px)'}], + {duration: 200, easing: 'ease-out'}, + ), + ); + const {storyboard} = await animationStoryboard(page, { + name: 'harness-self-test-animation', + clip: {x: 0, y: 340, width: 402, height: 120}, + measure: async () => ({ + x: await box.evaluate(el => el.getBoundingClientRect().x), + }), + columns: 4, + }); + const files = await storyboard.write(testInfo); + const xs = storyboard.frames.map(f => f.data.x as number); + expect(xs).toHaveLength(13); // t = 0, 16.7 … 200 + expect(xs[0]).toBeCloseTo(20, 1); + expect(xs.at(-1)).toBeCloseTo(260, 1); + expect(xs.every((x, i) => i === 0 || x >= xs[i - 1])).toBe(true); + expect(files['sheet.png']).toMatch(/sheet\.png$/); + }); + + test('a touch filmstrip photographs every scripted event', async ({ + page, + }, testInfo) => { + await fixture(page); + await page.evaluate(() => { + const pad = document.getElementById('pad'); + if (pad == null) { + throw new Error('no pad'); + } + const dot = document.createElement('div'); + dot.id = 'dot'; + dot.style.cssText = + 'position:absolute;width:24px;height:24px;border-radius:12px;background:#c33;left:-30px;top:0'; + pad.append(dot); + pad.addEventListener('pointermove', e => { + dot.style.transform = `translate(${e.clientX - 12 + 30}px, ${e.clientY - 12}px)`; + }); + }); + const driver = await touchDriver(page); + const storyboard = await touchStoryboard( + page, + driver, + drag({x: 60, y: 60}, {x: 300, y: 240}, {durationMs: 100, hz: 60}), + { + name: 'harness-self-test-touch', + clip: {x: 0, y: 0, width: 402, height: 300}, + measure: async () => ({ + x: await page + .locator('#dot') + .evaluate(el => el.getBoundingClientRect().x + 12), + }), + columns: 4, + }, + ); + await storyboard.write(testInfo); + // before + touchStart + 6 moves + touchEnd + expect(storyboard.frames).toHaveLength(9); + expect(storyboard.frames.at(-2)?.data.x).toBeCloseTo(300, 0); + }); + + test('the timer gate holds a component timer until the filmstrip is done', async ({ + page, + }, testInfo) => { + await fixture(page); + await installTimerGate(page); + const box = page.locator('#box'); + // A component that ends its own animation from a timer (a transitionend + // backstop): without the gate it would finish behind the held storyboard. + await box.evaluate(el => { + const animation = el.animate( + [{transform: 'translateX(0px)'}, {transform: 'translateX(200px)'}], + {duration: 300, easing: 'linear'}, + ); + setTimeout(() => { + animation.finish(); + (window as unknown as {timerRan?: boolean}).timerRan = true; + }, 40); + }); + await freezeTimers(page); + await page.waitForTimeout(120); // the timer is due now, and held + const {storyboard} = await animationStoryboard(page, { + name: 'harness-self-test-timer-gate', + clip: {x: 0, y: 340, width: 402, height: 120}, + measure: async () => ({ + x: await box.evaluate(el => el.getBoundingClientRect().x), + }), + columns: 6, + }); + const timerRan = () => + page.evaluate( + () => (window as unknown as {timerRan?: boolean}).timerRan === true, + ); + expect(await timerRan()).toBe(false); + await thawTimers(page); + expect(await timerRan()).toBe(true); + await storyboard.write(testInfo); + const xs = storyboard.frames.map(f => f.data.x as number); + expect(xs).toHaveLength(19); // t = 0 … 300 ms every 16.7 ms + expect(xs.every((x, i) => i === 0 || x > xs[i - 1])).toBe(true); + }); +}); diff --git a/internal/interaction-harness/src/index.ts b/internal/interaction-harness/src/index.ts index 587b9f82d233a..28a43db50d47e 100644 --- a/internal/interaction-harness/src/index.ts +++ b/internal/interaction-harness/src/index.ts @@ -12,6 +12,10 @@ * SYNC: When a module is added, export it here and describe it in README.md. */ +export * from './curves'; +export * from './math'; +export * from './output'; export * from './scripts'; export * from './story'; +export * from './storyboard'; export * from './touch'; diff --git a/internal/interaction-harness/src/math.test.ts b/internal/interaction-harness/src/math.test.ts new file mode 100644 index 0000000000000..645e42fa6b221 --- /dev/null +++ b/internal/interaction-harness/src/math.test.ts @@ -0,0 +1,154 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +import {describe, expect, it} from 'vitest'; +import { + compareCurve, + cubicBezier, + parseTransform, + projectDecay, + rubberBand, + springStep, + timeline, + timingFunction, + velocityAt, +} from './math'; + +describe('cubicBezier', () => { + it('matches the CSS keyword curves at known points', () => { + const linear = cubicBezier(0, 0, 1, 1); + expect(linear(0.3)).toBeCloseTo(0.3, 6); + // ease (0.25, 0.1, 0.25, 1) at x = 0.5 is 0.8024 (CSS reference value). + expect(cubicBezier(0.25, 0.1, 0.25, 1)(0.5)).toBeCloseTo(0.8024, 3); + expect(cubicBezier(0.24, 1, 0.4, 1)(0)).toBe(0); + expect(cubicBezier(0.24, 1, 0.4, 1)(1)).toBe(1); + }); + it('parses timing functions', () => { + expect(timingFunction('cubic-bezier(0, 0, 1, 1)')(0.25)).toBeCloseTo( + 0.25, + 6, + ); + expect(timingFunction('ease-out')(0.5)).toBeCloseTo( + cubicBezier(0, 0, 0.58, 1)(0.5), + 9, + ); + expect(() => timingFunction('steps(4)')).toThrow(); + }); +}); + +describe('springStep (SwiftUI response / dampingFraction)', () => { + const critical = {response: 0.5, dampingFraction: 1}; + it('starts at 0 with the given velocity and ends at 1', () => { + expect(springStep(critical, 0)).toBe(0); + expect(springStep(critical, 3000)).toBeCloseTo(1, 6); + const v = 4; // progress units per second + const slope = springStep({...critical, initialVelocity: v}, 0.01) / 0.00001; + expect(slope).toBeCloseTo(v, 1); + }); + it('matches the closed forms in each regime', () => { + const w0 = (2 * Math.PI) / 0.5; + const t = 0.12; + expect(springStep(critical, t * 1000)).toBeCloseTo( + 1 - Math.exp(-w0 * t) * (1 + w0 * t), + 9, + ); + const z = 0.5; + const wd = w0 * Math.sqrt(1 - z * z); + const under = + 1 - + Math.exp(-z * w0 * t) * + (Math.cos(wd * t) + ((z * w0) / wd) * Math.sin(wd * t)); + expect( + springStep({response: 0.5, dampingFraction: z}, t * 1000), + ).toBeCloseTo(under, 9); + // Over-damped and critical agree as damping approaches 1 from above. + expect( + springStep({response: 0.5, dampingFraction: 1.000001}, 120), + ).toBeCloseTo(springStep(critical, 120), 4); + }); + it('overshoots only when under-damped', () => { + const peak = (dampingFraction: number) => + Math.max( + ...timeline(2000, 4).map(ms => + springStep({response: 0.4, dampingFraction}, ms), + ), + ); + expect(peak(0.6)).toBeGreaterThan(1.05); + expect(peak(1)).toBeLessThanOrEqual(1); + expect(peak(1.5)).toBeLessThanOrEqual(1); + }); +}); + +describe('UIKit scroll physics', () => { + it('projects a release with UIScrollView deceleration', () => { + // 1000 pt/s at 0.998/ms coasts 499 pt; at 0.99/ms (fast) 99 pt. + expect(projectDecay(1000)).toBeCloseTo(499, 6); + expect(projectDecay(1000, 0.99)).toBeCloseTo(99, 6); + expect(projectDecay(-500)).toBeCloseTo(-249.5, 6); + }); + it('rubber-bands with c = 0.55', () => { + expect(rubberBand(0, 800)).toBeCloseTo(0, 9); + expect(rubberBand(100, 800)).toBeCloseTo( + (1 - 1 / ((100 * 0.55) / 800 + 1)) * 800, + 9, + ); + expect(rubberBand(-100, 800)).toBeCloseTo(-rubberBand(100, 800), 9); + expect(rubberBand(1e9, 800)).toBeLessThan(800); + }); +}); + +describe('parseTransform', () => { + it('decomposes computed transforms', () => { + expect(parseTransform('none')).toEqual({ + translateX: 0, + translateY: 0, + scaleX: 1, + scaleY: 1, + rotate: 0, + }); + expect(parseTransform('matrix(1, 0, 0, 1, 12.5, -40)')).toMatchObject({ + translateX: 12.5, + translateY: -40, + scaleX: 1, + }); + const rotated = parseTransform( + `matrix(${Math.cos(0.5)}, ${Math.sin(0.5)}, ${-Math.sin(0.5)}, ${Math.cos(0.5)}, 0, 0)`, + ); + expect(rotated.rotate).toBeCloseTo((0.5 * 180) / Math.PI, 6); + expect( + parseTransform( + 'matrix3d(2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 5, 6, 0, 1)', + ), + ).toMatchObject({translateX: 5, translateY: 6, scaleX: 2, scaleY: 2}); + expect(() => parseTransform('translateX(4px)')).toThrow(); + }); +}); + +describe('velocityAt', () => { + it('fits the last window and reports zero for a rest', () => { + const samples = timeline(200, 10).map(t => ({t, value: t * 0.5})); + expect(velocityAt(samples, 200)).toBeCloseTo(500, 6); + expect(velocityAt(samples, 400)).toBe(0); + }); +}); + +describe('compareCurve', () => { + it('reports the worst point', () => { + const result = compareCurve( + [ + {t: 0, value: 0}, + {t: 10, value: 1.2}, + ], + t => t / 10, + 0.1, + ); + expect(result.pass).toBe(false); + expect(result.worst.t).toBe(10); + expect(result.maxError).toBeCloseTo(0.2, 9); + }); +}); + +describe('timeline', () => { + it('includes both ends', () => { + expect(timeline(50, 1000 / 60)).toEqual([0, 16.667, 33.333, 50]); + }); +}); diff --git a/internal/interaction-harness/src/math.ts b/internal/interaction-harness/src/math.ts new file mode 100644 index 0000000000000..0006424980b3b --- /dev/null +++ b/internal/interaction-harness/src/math.ts @@ -0,0 +1,291 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file math.ts + * @input Nothing: pure functions + * @output Test oracles for motion: `cubicBezier`, `springStep` (SwiftUI + * response / dampingFraction semantics, with initial velocity), + * `projectDecay` (UIScrollView deceleration), `rubberBand` (UIKit's + * formula), `parseTransform`, `velocityAt`, and curve comparison helpers. + * @position Imported by the browser helpers and by the specs that compare a + * sampled curve against an expected one. Deliberately independent of the + * motion code under test in packages/core: an oracle that shares code with + * the implementation cannot catch the implementation's mistakes. + * + * Units: time in milliseconds, distance in CSS px (= points), velocity in + * px/s unless a name says otherwise. Spring progress is unitless (0 -> 1). + * + * SYNC: Keep ./math.test.ts covering every export. + */ + +/** A CSS `cubic-bezier(x1, y1, x2, y2)` timing function: progress in, eased progress out. */ +export function cubicBezier( + x1: number, + y1: number, + x2: number, + y2: number, +): (progress: number) => number { + const cx = 3 * x1; + const bx = 3 * (x2 - x1) - cx; + const ax = 1 - cx - bx; + const cy = 3 * y1; + const by = 3 * (y2 - y1) - cy; + const ay = 1 - cy - by; + const sampleX = (s: number) => ((ax * s + bx) * s + cx) * s; + const sampleY = (s: number) => ((ay * s + by) * s + cy) * s; + const slopeX = (s: number) => (3 * ax * s + 2 * bx) * s + cx; + return progress => { + if (progress <= 0) { + return 0; + } + if (progress >= 1) { + return 1; + } + // Newton first (fast, converges for well-behaved curves), bisection as the fallback. + let s = progress; + for (let i = 0; i < 8; i += 1) { + const error = sampleX(s) - progress; + if (Math.abs(error) < 1e-7) { + return sampleY(s); + } + const slope = slopeX(s); + if (Math.abs(slope) < 1e-6) { + break; + } + s -= error / slope; + } + let lo = 0; + let hi = 1; + s = progress; + for (let i = 0; i < 64; i += 1) { + const x = sampleX(s); + if (Math.abs(x - progress) < 1e-7) { + break; + } + if (x < progress) { + lo = s; + } else { + hi = s; + } + s = (lo + hi) / 2; + } + return sampleY(s); + }; +} + +/** Parses `cubic-bezier(a, b, c, d)` or one of the CSS keywords into a timing function. */ +export function timingFunction(css: string): (progress: number) => number { + const named: Record = { + ease: [0.25, 0.1, 0.25, 1], + 'ease-in': [0.42, 0, 1, 1], + 'ease-out': [0, 0, 0.58, 1], + 'ease-in-out': [0.42, 0, 0.58, 1], + }; + const value = css.trim(); + if (value === 'linear') { + return progress => Math.min(1, Math.max(0, progress)); + } + const match = /^cubic-bezier\(([^)]+)\)$/.exec(value); + const parts = match + ? match[1].split(',').map(Number) + : (named[value] as number[] | undefined); + if (parts == null || parts.length !== 4 || parts.some(Number.isNaN)) { + throw new Error(`unsupported timing function: ${css}`); + } + return cubicBezier(parts[0], parts[1], parts[2], parts[3]); +} + +export interface SpringParameters { + /** SwiftUI `response`: the period of the undamped spring, in seconds. */ + readonly response: number; + /** SwiftUI `dampingFraction` (UIKit `dampingRatio`): 1 = critical, < 1 bounces. */ + readonly dampingFraction: number; + /** + * Initial velocity in progress units per second, UIKit's `initialVelocity` + * convention: release velocity (px/s) divided by the distance to travel (px). + */ + readonly initialVelocity?: number; +} + +/** + * Progress (0 -> 1, may overshoot) of a spring released from rest-or-moving at + * progress 0 towards 1, after `timeMs`. Mass 1, stiffness (2π/response)², + * damping 4π·dampingFraction/response: SwiftUI's mapping. + */ +export function springStep( + {response, dampingFraction, initialVelocity = 0}: SpringParameters, + timeMs: number, +): number { + if (timeMs <= 0) { + return 0; + } + const t = timeMs / 1000; + const w0 = (2 * Math.PI) / response; + const z = dampingFraction; + // y = x - 1: displacement from the target, y(0) = -1, y'(0) = v0. + const v0 = initialVelocity; + let y: number; + if (Math.abs(z - 1) < 1e-9) { + y = Math.exp(-w0 * t) * (-1 + (v0 - w0) * t); + } else if (z < 1) { + const wd = w0 * Math.sqrt(1 - z * z); + const b = (v0 - z * w0) / wd; + y = Math.exp(-z * w0 * t) * (-Math.cos(wd * t) + b * Math.sin(wd * t)); + } else { + const root = Math.sqrt(z * z - 1); + const r1 = -w0 * (z - root); + const r2 = -w0 * (z + root); + const c1 = (v0 + r2) / (r1 - r2); + const c2 = -1 - c1; + y = c1 * Math.exp(r1 * t) + c2 * Math.exp(r2 * t); + } + return 1 + y; +} + +/** + * Where a surface released at `velocity` px/s comes to rest, relative to the + * release point, under UIScrollView deceleration (`rate` per millisecond: + * 0.998 normal, 0.99 fast). UIKit's own projection formula. + */ +export function projectDecay(velocity: number, rate = 0.998): number { + return ((velocity / 1000) * rate) / (1 - rate); +} + +/** UIKit's rubber band: `(1 - 1 / (x * c / d + 1)) * d`, c = 0.55. */ +export function rubberBand( + offset: number, + dimension: number, + constant = 0.55, +): number { + const sign = Math.sign(offset); + const x = Math.abs(offset); + return sign * (1 - 1 / ((x * constant) / dimension + 1)) * dimension; +} + +export interface Transform2D { + readonly translateX: number; + readonly translateY: number; + readonly scaleX: number; + readonly scaleY: number; + /** Rotation in degrees. */ + readonly rotate: number; +} + +const IDENTITY: Transform2D = { + translateX: 0, + translateY: 0, + scaleX: 1, + scaleY: 1, + rotate: 0, +}; + +/** Decomposes a computed `transform` (`none`, `matrix(...)`, `matrix3d(...)`) into 2D parts. */ +export function parseTransform(css: string): Transform2D { + const value = css.trim(); + if (value === '' || value === 'none') { + return IDENTITY; + } + const match = /^matrix(3d)?\(([^)]+)\)$/.exec(value); + if (match == null) { + throw new Error(`not a computed transform: ${css}`); + } + const n = match[2].split(',').map(Number); + const [a, b, c, d, e, f] = match[1] + ? [n[0], n[1], n[4], n[5], n[12], n[13]] + : n; + const scaleX = Math.hypot(a, b); + const rotate = (Math.atan2(b, a) * 180) / Math.PI; + const scaleY = (a * d - b * c) / (scaleX || 1); + return {translateX: e, translateY: f, scaleX, scaleY, rotate}; +} + +export interface TimedValue { + /** Milliseconds. */ + readonly t: number; + readonly value: number; +} + +/** + * Velocity (units/s) at the end of `samples`, by least squares over the last + * `windowMs`, the way a release velocity is judged: one slow sample before the + * lift must not cancel a throw, and a finger that rested has no velocity. + */ +export function velocityAt( + samples: readonly TimedValue[], + endMs: number, + windowMs = 100, +): number { + const recent = samples.filter(s => s.t <= endMs && s.t >= endMs - windowMs); + if (recent.length < 2) { + return 0; + } + const meanT = recent.reduce((sum, s) => sum + s.t, 0) / recent.length; + const meanV = recent.reduce((sum, s) => sum + s.value, 0) / recent.length; + let num = 0; + let den = 0; + for (const s of recent) { + num += (s.t - meanT) * (s.value - meanV); + den += (s.t - meanT) ** 2; + } + return den === 0 ? 0 : (num / den) * 1000; +} + +export interface CurveComparison { + readonly pass: boolean; + readonly tolerance: number; + readonly maxError: number; + readonly rmsError: number; + readonly worst: { + readonly t: number; + readonly actual: number; + readonly expected: number; + }; + readonly points: readonly { + readonly t: number; + readonly actual: number; + readonly expected: number; + }[]; +} + +/** Compares sampled values with an expected curve, point by point. */ +export function compareCurve( + samples: readonly TimedValue[], + expected: (timeMs: number) => number, + tolerance: number, +): CurveComparison { + if (samples.length === 0) { + throw new Error('compareCurve: no samples'); + } + const points = samples.map(s => ({ + t: s.t, + actual: s.value, + expected: expected(s.t), + })); + let worst = points[0]; + let sumSq = 0; + for (const p of points) { + const error = Math.abs(p.actual - p.expected); + sumSq += error * error; + if (error > Math.abs(worst.actual - worst.expected)) { + worst = p; + } + } + const maxError = Math.abs(worst.actual - worst.expected); + return { + pass: maxError <= tolerance, + tolerance, + maxError, + rmsError: Math.sqrt(sumSq / points.length), + worst, + points, + }; +} + +/** `from`, `from + step`, … up to and including `to` (within float error). */ +export function timeline(to: number, step = 1000 / 60, from = 0): number[] { + const out: number[] = []; + for (let i = 0; from + i * step <= to + 1e-6; i += 1) { + out.push(+(from + i * step).toFixed(3)); + } + return out; +} diff --git a/internal/interaction-harness/src/output.ts b/internal/interaction-harness/src/output.ts new file mode 100644 index 0000000000000..59926b82c9aef --- /dev/null +++ b/internal/interaction-harness/src/output.ts @@ -0,0 +1,66 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file output.ts + * @input Uses node:fs, node:path and the Playwright `TestInfo` + * @output `evidenceDir` / `writeEvidence` — where storyboards and curve traces + * go, and the one way they are written and attached to the test report. + * @position Every artifact the harness produces lands under one gitignored + * directory, `test-results/interaction-evidence/` by default + * (`ASTRYX_INTERACTION_EVIDENCE` overrides it). Playwright wipes its own + * output directory per run; this one is separate, so a storyboard run never + * deletes another's evidence. Generated evidence is never committed: a pull + * request links or attaches it. + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import type {TestInfo} from '@playwright/test'; + +export function evidenceDir(): string { + return path.resolve( + process.env.ASTRYX_INTERACTION_EVIDENCE ?? + 'test-results/interaction-evidence', + ); +} + +/** A filesystem-safe name: letters, digits, dot, dash and underscore. */ +export function safeName(name: string): string { + return ( + name + .replace(/[^A-Za-z0-9._-]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 120) || 'evidence' + ); +} + +/** + * Writes `files` (name → contents) under `//` and attaches + * them to the test's report when a `TestInfo` is given. Returns the paths. + */ +export async function writeEvidence( + group: string, + files: Readonly>, + testInfo?: TestInfo, +): Promise> { + // `group` may nest ("storyboards/"): each segment is made safe on its own. + const dir = path.join(evidenceDir(), ...group.split('/').map(safeName)); + fs.mkdirSync(dir, {recursive: true}); + const written: Record = {}; + for (const [name, contents] of Object.entries(files)) { + const file = path.join(dir, safeName(name)); + fs.writeFileSync(file, contents); + written[name] = file; + if (testInfo != null) { + await testInfo.attach(`${group}/${safeName(name)}`, { + path: file, + contentType: name.endsWith('.png') + ? 'image/png' + : name.endsWith('.json') + ? 'application/json' + : 'text/plain', + }); + } + } + return written; +} diff --git a/internal/interaction-harness/src/story.ts b/internal/interaction-harness/src/story.ts index 9e4323436ff27..0058fe5b579e2 100644 --- a/internal/interaction-harness/src/story.ts +++ b/internal/interaction-harness/src/story.ts @@ -101,3 +101,63 @@ export async function frames(page: Page, count = 2): Promise { count, ); } + +const TIMER_GATE = `(() => { + if (window.__harnessTimerGate) return; + const gate = {frozen: false, queue: []}; + window.__harnessTimerGate = gate; + const realTimeout = window.setTimeout.bind(window); + const realInterval = window.setInterval.bind(window); + const realFrame = window.requestAnimationFrame.bind(window); + const hold = run => (gate.frozen ? gate.queue.push(run) : run()); + window.setTimeout = (fn, ms, ...args) => realTimeout(() => hold(() => (typeof fn === 'function' ? fn(...args) : 0)), ms); + window.setInterval = (fn, ms, ...args) => realInterval(() => { if (!gate.frozen && typeof fn === 'function') fn(...args); }, ms); + window.requestAnimationFrame = fn => realFrame(time => hold(() => fn(gate.frozen ? performance.now() : time))); +})()`; + +/** + * Installs a gate on the page's setTimeout / setInterval / + * requestAnimationFrame. Install it BEFORE the animation under test starts: + * only callbacks scheduled through the gate can be held. Event timestamps, + * Date, performance.now and the Web Animations timeline are untouched. + */ +export async function installTimerGate(page: Page): Promise { + await page.evaluate(TIMER_GATE); +} + +/** + * Holds every gated timer and frame callback until `thawTimers`, so a + * component cannot finish an animation behind a held storyboard's back (a + * transitionend backstop timer, a cleanup render). Callbacks that come due + * meanwhile run, in order, on thaw; intervals skip their frozen ticks. + */ +export async function freezeTimers(page: Page): Promise { + await page.evaluate(() => { + const gate = (window as unknown as {__harnessTimerGate?: {frozen: boolean}}) + .__harnessTimerGate; + if (gate == null) { + throw new Error( + 'freezeTimers: call installTimerGate before the animation starts', + ); + } + gate.frozen = true; + }); +} + +/** Releases the gate: held callbacks run now, in the order they came due. */ +export async function thawTimers(page: Page): Promise { + await page.evaluate(() => { + const gate = ( + window as unknown as { + __harnessTimerGate?: {frozen: boolean; queue: (() => void)[]}; + } + ).__harnessTimerGate; + if (gate == null) { + return; + } + gate.frozen = false; + for (const run of gate.queue.splice(0)) { + run(); + } + }); +} diff --git a/internal/interaction-harness/src/storyboard.ts b/internal/interaction-harness/src/storyboard.ts new file mode 100644 index 0000000000000..e2a7d264b43d5 --- /dev/null +++ b/internal/interaction-harness/src/storyboard.ts @@ -0,0 +1,302 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file storyboard.ts + * @input Uses a Playwright `Page`, a clip (`Locator` or box), ./curves to seek + * animations, ./touch to play scripted touches, and ./output to write + * @output `Storyboard` (collect frames, write a PNG contact sheet + JSON trace), + * `animationStoryboard` (seek every held animation to t = 0, 16.7, 33.3 … ms + * and photograph each), `touchStoryboard` (photograph the page after every + * scripted touch event). + * @position Deterministic filmstrips: frames come from seeking paused + * animations or from stepping scripted input, never from wall-clock capture, + * so the same code produces the same sheet. The JSON trace carries the + * measured value of every frame and is what the spec asserts; the sheet is + * the human evidence attached to the report (and linked from a pull + * request, never committed). + * + * The contact sheet is composed in a scratch page of the same browser (a + * canvas grid with a label per cell), so the harness needs no image library. + * + * SYNC: Keep the README's storyboard section and `pnpm storyboards` in step. + */ + +import type {Locator, Page, TestInfo} from '@playwright/test'; +import {holdAnimations, seekAnimations, type HeldAnimation} from './curves'; +import {timeline} from './math'; +import {writeEvidence} from './output'; +import type {TouchScript} from './scripts'; +import type {TouchDriver} from './touch'; + +export interface ClipBox { + readonly x: number; + readonly y: number; + readonly width: number; + readonly height: number; +} + +export interface StoryboardFrame { + readonly label: string; + /** Milliseconds on the storyboard's own timeline. */ + readonly t: number; + /** Measurements taken at this frame; the spec asserts these. */ + readonly data: Readonly>; + readonly png: Buffer; +} + +export interface StoryboardOptions { + readonly name: string; + /** What to photograph: an element, a box, or the whole viewport (default). */ + readonly clip?: Locator | ClipBox; + /** Grid columns in the sheet (default: up to 8). */ + readonly columns?: number; + /** Cell scale relative to CSS px (default 0.5). */ + readonly cellScale?: number; +} + +export class Storyboard { + readonly frames: StoryboardFrame[] = []; + private clipBox: ClipBox | undefined; + + constructor( + private readonly page: Page, + readonly options: StoryboardOptions, + ) {} + + /** Photographs the clip now and records `data` with it. */ + async capture( + label: string, + t: number, + data: Readonly> = {}, + ): Promise { + const clip = await this.resolveClip(); + const png = await this.page.screenshot({ + clip, + scale: 'css', + animations: 'allow', + caret: 'hide', + }); + this.frames.push({label, t, data, png}); + } + + private async resolveClip(): Promise { + if (this.clipBox != null) { + return this.clipBox; + } + const {clip} = this.options; + if (clip == null) { + return undefined; + } + if ('boundingBox' in clip) { + // A locator is measured once, at the first frame, so every cell shows the same region. + const box = await clip.boundingBox(); + if (box == null) { + throw new Error( + `storyboard ${this.options.name}: clip element is not visible`, + ); + } + const viewport = this.page.viewportSize(); + const x = Math.max(0, Math.floor(box.x)); + const y = Math.max(0, Math.floor(box.y)); + const right = Math.min( + viewport?.width ?? Infinity, + Math.ceil(box.x + box.width), + ); + const bottom = Math.min( + viewport?.height ?? Infinity, + Math.ceil(box.y + box.height), + ); + this.clipBox = {x, y, width: right - x, height: bottom - y}; + } else { + this.clipBox = clip; + } + return this.clipBox; + } + + /** The JSON trace: every frame's label, time and data. */ + trace( + extra: Readonly> = {}, + ): Record { + return { + name: this.options.name, + browser: this.page.context().browser()?.browserType().name() ?? null, + viewport: this.page.viewportSize(), + clip: this.clipBox ?? null, + ...extra, + frames: this.frames.map(({label, t, data}) => ({label, t, data})), + }; + } + + /** Composes the sheet, writes `sheet.png` + `trace.json`, attaches both. */ + async write( + testInfo?: TestInfo, + extra: Readonly> = {}, + ): Promise> { + if (this.frames.length === 0) { + throw new Error(`storyboard ${this.options.name}: no frames`); + } + const sheet = await composeSheet(this.page, this.frames, this.options); + const browser = + this.page.context().browser()?.browserType().name() ?? 'browser'; + return writeEvidence( + `storyboards/${this.options.name}.${browser}`, + { + 'sheet.png': sheet, + 'trace.json': JSON.stringify(this.trace(extra), null, 2), + }, + testInfo, + ); + } +} + +async function composeSheet( + page: Page, + frames: readonly StoryboardFrame[], + options: StoryboardOptions, +): Promise { + const scratch = await page.context().newPage(); + try { + const dataUrl = await scratch.evaluate( + async ({cells, columns, scale, title}) => { + const images = await Promise.all( + cells.map(async cell => + createImageBitmap( + await (await fetch(`data:image/png;base64,${cell.png}`)).blob(), + ), + ), + ); + const cellW = Math.ceil(images[0].width * scale); + const cellH = Math.ceil(images[0].height * scale); + const labelH = 30; + const pad = 8; + const cols = Math.min(columns, cells.length); + const rows = Math.ceil(cells.length / cols); + const canvas = document.createElement('canvas'); + canvas.width = pad + cols * (cellW + pad); + canvas.height = 28 + pad + rows * (cellH + labelH + pad); + const context = canvas.getContext('2d'); + if (context == null) { + throw new Error('no 2d context'); + } + context.fillStyle = '#ffffff'; + context.fillRect(0, 0, canvas.width, canvas.height); + context.fillStyle = '#111111'; + context.font = '600 14px sans-serif'; + context.fillText(title, pad, 19); + cells.forEach((cell, i) => { + const x = pad + (i % cols) * (cellW + pad); + const y = 28 + pad + Math.floor(i / cols) * (cellH + labelH + pad); + context.drawImage(images[i], x, y, cellW, cellH); + context.strokeStyle = '#999999'; + context.lineWidth = 1; + context.strokeRect(x - 0.5, y - 0.5, cellW + 1, cellH + 1); + context.fillStyle = '#111111'; + context.font = '600 11px sans-serif'; + context.fillText(cell.label, x, y + cellH + 13, cellW); + context.fillStyle = '#555555'; + context.font = '10px monospace'; + context.fillText(cell.caption, x, y + cellH + 26, cellW); + }); + return canvas.toDataURL('image/png'); + }, + { + cells: frames.map(f => ({ + png: f.png.toString('base64'), + label: f.label, + caption: Object.entries(f.data) + .filter( + ([, v]) => + typeof v === 'number' || + typeof v === 'string' || + typeof v === 'boolean', + ) + .map( + ([k, v]) => + `${k}=${typeof v === 'number' ? +v.toFixed(1) : String(v)}`, + ) + .join(' '), + })), + columns: options.columns ?? 8, + scale: options.cellScale ?? 0.5, + title: options.name, + }, + ); + return Buffer.from(dataUrl.slice(dataUrl.indexOf(',') + 1), 'base64'); + } finally { + await scratch.close(); + } +} + +export interface AnimationStoryboardOptions extends StoryboardOptions { + /** Frame times, ms (default every 16.7 ms through the longest held animation). */ + readonly times?: readonly number[]; + /** Restrict to animations inside this element. */ + readonly scope?: Locator; + /** Measurements per frame (runs after the seek). */ + readonly measure?: () => Promise>>; +} + +/** + * Holds every running animation, then seeks them together to each time and + * photographs the clip. Call it right after starting the animation under test. + */ +export async function animationStoryboard( + page: Page, + options: AnimationStoryboardOptions, +): Promise<{storyboard: Storyboard; held: HeldAnimation[]}> { + const held = await holdAnimations(page, options.scope); + if (held.length === 0) { + throw new Error(`storyboard ${options.name}: no running animation to hold`); + } + const longest = Math.max(...held.map(a => a.delayMs + a.durationMs)); + const times = options.times ?? timeline(longest); + const storyboard = new Storyboard(page, options); + for (const t of times) { + await seekAnimations(page, t); + const data = (await options.measure?.()) ?? {}; + await storyboard.capture(`t=${+t.toFixed(1)} ms`, t, data); + } + return {storyboard, held}; +} + +export interface TouchStoryboardOptions extends StoryboardOptions { + readonly measure?: () => Promise>>; +} + +/** + * Plays a touch script without wall-clock pacing and photographs the page + * after every event. In Chromium the page still sees the scripted timestamps, + * so velocity-dependent behavior is unchanged; in WebKit the page sees the + * real (slower) dispatch times. + */ +export async function touchStoryboard( + page: Page, + driver: TouchDriver, + script: TouchScript, + options: TouchStoryboardOptions, +): Promise { + const storyboard = new Storyboard(page, options); + await storyboard.capture('before', 0, (await options.measure?.()) ?? {}); + await driver.play(script, { + realtime: false, + onEvent: async step => { + // Let the page run its handlers and present the frame before the photo. + await page.evaluate( + () => + new Promise(resolve => + requestAnimationFrame(() => requestAnimationFrame(() => resolve())), + ), + ); + const data = (await options.measure?.()) ?? {}; + const where = step.changed + .map(f => `${Math.round(f.x)},${Math.round(f.y)}`) + .join(' '); + await storyboard.capture( + `${step.type} t=${+step.t.toFixed(1)} @${where}`, + step.t, + data, + ); + }, + }); + return storyboard; +} diff --git a/package.json b/package.json index a4ca9f2ae830e..e49d57a474542 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "test:a11y-contract": "playwright test --config playwright.config.ts --project chromium", "test:docsite-browser": "playwright test --config apps/docsite/playwright.config.ts", "test:interaction": "playwright test --config playwright.interaction.config.ts", + "storyboards": "playwright test --config playwright.interaction.config.ts --grep @storyboard", "check:package-boundaries": "node scripts/check-package-boundaries.js", "check:repo": "pnpm check:visual-owner && pnpm check:sync && pnpm check:neutral-palette-refs && pnpm check:stone-palette-refs && pnpm check:gothic-palette-refs && pnpm check:matcha-palette-refs && pnpm check:butter-palette-refs && pnpm check:y2k-palette-refs && pnpm check:data-tokens && pnpm check:token-docs && pnpm check:knowledge && pnpm check:package-boundaries && pnpm check:changesets && pnpm check:demo-media && pnpm check:executable-bits && pnpm check:cli-structure && pnpm check:use-client && pnpm check:portable-scripts && pnpm check:i18n-runtime && pnpm check:i18n-catalog && pnpm check:cldr-weekdays && pnpm check:fixtures && pnpm check:setup-contract", "check:visual-owner": "node scripts/check-visual-workflow-owner.mjs", diff --git a/playwright.interaction.config.ts b/playwright.interaction.config.ts index 357a5dc3b7642..f0c61e2e6e563 100644 --- a/playwright.interaction.config.ts +++ b/playwright.interaction.config.ts @@ -3,11 +3,11 @@ /** * @file playwright.interaction.config.ts * @input Uses @playwright/test - * @output Touch interaction proofs on a phone-sized touch screen, in Chromium - * and WebKit. - * @position Run with `pnpm test:interaction`. Separate from `pnpm test` (no - * browser needed there) and from playwright.config.ts (accessibility - * contracts on desktop profiles). + * @output Touch interaction and animation proofs on a phone-sized touch + * screen, in Chromium and WebKit. + * @position Run with `pnpm test:interaction`; filmstrips only with + * `pnpm storyboards`. Separate from `pnpm test` (no browser needed there) and + * from playwright.config.ts (accessibility contracts on desktop profiles). * * Specs are named `*.interaction.browser.spec.ts` and read stories from a * built Storybook, like the accessibility specs: From 65c017ed0cd12d248a6932dae66f97ea8e845714 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Fri, 23 Oct 2026 20:42:13 -0700 Subject: [PATCH 06/93] fix(interaction): curves end before a held end value; rest judges presented values An animation that keeps its end value in a run of identical final keyframes (held so a later interruption can read it) reported its whole duration as the curve: storyboards filmed the hold, waitForAnimation's interval reached past the landing, and waitForRest, which also required every animation to leave the running state and read only transform, timed out. The curve now ends where that run starts (HeldAnimation.curveEndMs, waitForAnimation's to, the default storyboard times), and rest is no animation pending or before its curve's end plus the presented transform, translate, rotate, scale and opacity (and ::backdrop opacity) still for 3 frames, whatever the playState. sampleCurve composes translate, rotate and scale with transform, so curves of the individual properties are sampled. --- internal/interaction-harness/README.md | 7 +- internal/interaction-harness/src/curves.ts | 208 +++++++++++++++--- .../src/harness.interaction.browser.spec.ts | 47 +++- .../interaction-harness/src/storyboard.ts | 7 +- 4 files changed, 237 insertions(+), 32 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index ec30c5bfaf2af..2977189fa1500 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -64,7 +64,12 @@ value at that exact time, independent of frame timing. Compare it with an oracle from `math.ts` through `compareCurve`; a tolerance states the error budget (for a spring baked into `linear()` stops of `h` ms, chords err by up to `h² / 8 · |x''|`). A seek past an animation's end lands half a millisecond -inside it, the last frame it presents. +inside it, the last frame it presents. An animation whose keyframes end in a run of +identical values (an end value held so a later interruption can read it) +ends where that run starts: `curveEndMs`, `waitForAnimation`'s interval and +the default storyboard times stop there, and `waitForRest`, which judges the +presented `transform`, `translate`, `rotate`, `scale` and `opacity` rather +than `playState`, does not wait the hold out. **Storyboards.** Deterministic: frames come from seeking held animations or from stepping scripted input, never from wall-clock capture. The JSON trace diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index c003d97b0dab6..5e9a04511b453 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -7,8 +7,10 @@ * Web Animations too) * @output `holdAnimations` (pause every running animation and describe it), * `seekAnimations` (move them all to t ms), `sampleCurve` (the presented - * transform/opacity at each t), `releaseAnimations`, and `curveOf`, which - * picks one channel out of the samples for ./math's `compareCurve`. + * transform/opacity at each t, with the individual `translate`, `rotate` and + * `scale` properties composed in), `releaseAnimations`, `curveOf`, which + * picks one channel out of the samples for ./math's `compareCurve`, and the + * animation-state waits `waitForAnimation` and `waitForRest`. * @position Curve sampling: the exact presented value of an animation at a * fixed time, independent of frame timing, so a curve can be compared with * the UIKit reference to a fraction of a pixel. A paused animation's effect @@ -23,6 +25,56 @@ import {parseTransform, type TimedValue, type Transform2D} from './math'; /** How far inside its end a seek past an animation's end lands, ms. */ export const END_INSET_MS = 0.5; +/** + * Installs `window.__harnessCurveEnd(animation)`: the local time (ms, delay + * included) where the animation's effect stops changing. An animation whose + * keyframes end in a run of identical values (an end value held so that a + * later interruption can read it) ends where that run starts; any other ends + * at its end time. + */ +async function installCurveEnd(page: Page): Promise { + await page.evaluate(() => { + const w = window as unknown as { + __harnessCurveEnd?: (animation: Animation) => number; + }; + if (w.__harnessCurveEnd) { + return; + } + const timingKeys = new Set([ + 'offset', + 'computedOffset', + 'easing', + 'composite', + ]); + const values = (k: ComputedKeyframe) => + JSON.stringify( + Object.keys(k) + .filter(p => !timingKeys.has(p)) + .sort() + .map(p => [p, String(k[p])]), + ); + w.__harnessCurveEnd = animation => { + const effect = animation.effect as KeyframeEffect | null; + const timing = effect?.getComputedTiming(); + const delay = Number(timing?.delay ?? 0); + const active = Number(timing?.activeDuration ?? 0); + if ( + effect == null || + !Number.isFinite(active) || + Number(timing?.iterations ?? 1) !== 1 + ) { + return delay + active; + } + const frames = effect.getKeyframes(); + let first = frames.length - 1; + while (first > 0 && values(frames[first - 1]) === values(frames[first])) { + first -= 1; + } + return delay + Number(frames[first]?.computedOffset ?? 1) * active; + }; + }); +} + export interface HeldAnimation { readonly index: number; /** `CSSTransition`, `CSSAnimation` or `Animation` (script-created). */ @@ -32,6 +84,11 @@ export interface HeldAnimation { readonly target: string | null; readonly durationMs: number; readonly delayMs: number; + /** + * Where its curve ends, ms from its start (delay included): before a held + * end value, if its keyframes end in one; otherwise its end time. + */ + readonly curveEndMs: number; readonly easing: string; readonly keyframes: readonly Record[]; /** Where it was when held, ms. */ @@ -48,7 +105,11 @@ export async function holdAnimations( scope?: Locator, ): Promise { const scopeHandle = scope ? await scope.elementHandle() : null; + await installCurveEnd(page); return page.evaluate(root => { + const curveEnd = ( + window as unknown as {__harnessCurveEnd: (a: Animation) => number} + ).__harnessCurveEnd; const describe = (node: Element | null) => node == null ? null @@ -87,6 +148,7 @@ export async function holdAnimations( target: describe((effect?.target as Element | null) ?? null), durationMs: Number(timing?.duration ?? 0), delayMs: Number(timing?.delay ?? 0), + curveEndMs: curveEnd(animation), easing: String(effect?.getTiming().easing ?? 'linear'), keyframes: (effect?.getKeyframes() ?? []).map( (k): Record => ({...k}), @@ -145,15 +207,27 @@ export async function releaseAnimations( export interface CurveSample { readonly t: number; + /** + * The presented 2D transform: `translate` · `rotate` · `scale` · + * `transform`, composed in the order CSS applies them. + */ readonly transform: Transform2D; readonly opacity: number; /** Raw computed values, for the JSON trace. */ - readonly raw: {readonly transform: string; readonly opacity: string}; + readonly raw: { + readonly transform: string; + readonly translate: string; + readonly rotate: string; + readonly scale: string; + readonly opacity: string; + }; } /** * The presented transform and opacity of `locator` at each time (ms from the - * held animations' start). Hold the animations first. + * held animations' start). Hold the animations first. An animation of the + * individual `translate`, `rotate` or `scale` properties is included: they + * are composed with `transform` into one matrix. */ export async function sampleCurve( locator: Locator, @@ -178,16 +252,65 @@ export async function sampleCurve( animation.currentTime = Math.min(t, end - inset); } const style = getComputedStyle(element); - return {t, transform: style.transform, opacity: style.opacity}; + // translate · rotate · scale · transform, as CSS applies them (all + // about the same transform origin, which `transform` excludes too). + const box = element as HTMLElement; + const length = (value: string, size: number) => + value.endsWith('%') + ? (parseFloat(value) / 100) * size + : parseFloat(value) || 0; + const matrix = new DOMMatrix(); + if (style.translate !== 'none') { + const [x = '0px', y = '0px'] = style.translate.split(' '); + matrix.translateSelf( + length(x, box.offsetWidth ?? 0), + length(y, box.offsetHeight ?? 0), + ); + } + if (style.rotate !== 'none') { + // "45deg", "z 45deg" or "0 0 1 45deg": the angle is the last token. + const angle = style.rotate.split(' ').at(-1) ?? '0deg'; + const turns: Record = { + deg: 1, + rad: 180 / Math.PI, + grad: 0.9, + turn: 360, + }; + const unit = /[a-z]+$/.exec(angle)?.[0] ?? 'deg'; + matrix.rotateSelf(parseFloat(angle) * (turns[unit] ?? 1)); + } + if (style.scale !== 'none') { + const [sx = 1, sy = sx] = style.scale.split(' ').map(Number); + matrix.scaleSelf(sx, sy); + } + if (style.transform !== 'none') { + matrix.multiplySelf(new DOMMatrix(style.transform)); + } + const {a, b, c, d, e, f} = matrix; + return { + t, + matrix: `matrix(${a}, ${b}, ${c}, ${d}, ${e}, ${f})`, + transform: style.transform, + translate: style.translate, + rotate: style.rotate, + scale: style.scale, + opacity: style.opacity, + }; }); }, {ts: [...times], inset: END_INSET_MS}, ); return raw.map(r => ({ t: r.t, - transform: parseTransform(r.transform), + transform: parseTransform(r.matrix), opacity: Number(r.opacity), - raw: {transform: r.transform, opacity: r.opacity}, + raw: { + transform: r.transform, + translate: r.translate, + rotate: r.rotate, + scale: r.scale, + opacity: r.opacity, + }, })); } @@ -208,7 +331,10 @@ export function curveOf( export interface ActiveAnimation { readonly kind: string; readonly name: string | null; - /** Epoch ms when the active interval starts and ends (page clock). */ + /** + * Epoch ms (page clock) when the animation's curve starts and ends. A held + * end value (see `HeldAnimation.curveEndMs`) is not part of the curve. + */ readonly from: number; readonly to: number; readonly durationMs: number; @@ -230,8 +356,12 @@ export async function waitForAnimation( }: {timeout?: number; minProgressMs?: number} = {}, ): Promise { const deadline = Date.now() + timeout; + await installCurveEnd(locator.page()); for (;;) { const found = await locator.evaluate((element, minProgress) => { + const curveEnd = ( + window as unknown as {__harnessCurveEnd: (a: Animation) => number} + ).__harnessCurveEnd; return element .getAnimations({subtree: true}) .filter( @@ -243,10 +373,7 @@ export async function waitForAnimation( .map(a => { const effect = a.effect as KeyframeEffect | null; const timing = effect?.getComputedTiming(); - const start = - performance.timeOrigin + - Number(a.startTime) + - Number(timing?.delay ?? 0); + const startedAt = performance.timeOrigin + Number(a.startTime); const name = 'transitionProperty' in a ? (a as unknown as {transitionProperty: string}) @@ -257,8 +384,8 @@ export async function waitForAnimation( return { kind: a.constructor.name, name, - from: start, - to: start + Number(timing?.activeDuration ?? 0), + from: startedAt + Number(timing?.delay ?? 0), + to: startedAt + curveEnd(a), durationMs: Number(timing?.duration ?? 0), easing: String(effect?.getTiming().easing ?? 'linear'), keyframes: (effect?.getKeyframes() ?? []).map( @@ -278,36 +405,63 @@ export async function waitForAnimation( } /** - * Resolves once nothing animates on `locator`'s element (or inside it) and its - * transform has held still for two frames: the element is at rest. + * Resolves once `locator`'s element is at rest: no animation on it (or inside + * it) is pending or still before the end of its curve, and its presented + * `transform`, `translate`, `rotate`, `scale` and `opacity` (and `::backdrop` + * opacity) have held still for `frames` consecutive frames. + * + * It judges curves and presented values, not `playState`: an animation that + * holds its end value (a run of identical final keyframes kept so a later + * interruption can read it) still reports `running` while nothing moves. */ export async function waitForRest( locator: Locator, - {timeout = 5_000}: {timeout?: number} = {}, + {timeout = 5_000, frames = 3}: {timeout?: number; frames?: number} = {}, ): Promise { + await installCurveEnd(locator.page()); await locator.evaluate( - (element, ms) => + (element, {ms, need}) => new Promise((resolve, reject) => { const deadline = performance.now() + ms; - let last = ''; + const curveEnd = ( + window as unknown as {__harnessCurveEnd: (a: Animation) => number} + ).__harnessCurveEnd; + const read = () => { + const style = getComputedStyle(element); + const backdrop = getComputedStyle(element, '::backdrop').opacity; + return [ + style.transform, + style.translate, + style.rotate, + style.scale, + style.opacity, + backdrop, + ].join(' | '); + }; + let last = read(); let still = 0; const tick = () => { - const transform = getComputedStyle(element).transform; - const animating = element + const now = read(); + const moving = element .getAnimations({subtree: true}) - .some(a => a.playState === 'running'); - still = !animating && transform === last ? still + 1 : 0; - last = transform; - if (still >= 2) { + .some( + a => + a.pending || + (a.playState === 'running' && + Number(a.currentTime ?? 0) < curveEnd(a)), + ); + still = !moving && now === last ? still + 1 : 0; + last = now; + if (still >= need) { resolve(); } else if (performance.now() > deadline) { - reject(new Error('waitForRest: still moving')); + reject(new Error(`waitForRest: still moving (${now})`)); } else { requestAnimationFrame(tick); } }; requestAnimationFrame(tick); }), - timeout, + {ms: timeout, need: frames}, ); } diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 0698e603fe4ac..d5177019faf16 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -7,8 +7,10 @@ * scripted timestamps reach the page; a flick flings natively (Chromium); * a second finger and a pinch arrive as distinct pointers; sampled curves * match the oracles; the animation-state waits report a running transition - * and its end; storyboards write a sheet and a trace that follows the curve, - * and the timer gate holds a component timer until the filmstrip is done. + * and its end, and judge rest by presented values even while an animation + * holds its end; individual transform properties are sampled; storyboards + * write a sheet and a trace that follows the curve, and the timer gate + * holds a component timer until the filmstrip is done. * @position The harness's own test. A component spec trusts these * instruments; this file is why it may. */ @@ -314,6 +316,47 @@ test.describe('animation state', () => { const transform = await box.evaluate(el => getComputedStyle(el).transform); expect(parseTransform(transform).translateY).toBeCloseTo(100, 1); }); + + test('an end value held by the keyframes is not part of the curve, and individual transform properties are sampled', async ({ + page, + }) => { + await fixture(page); + const box = page.locator('#box'); + // 300 ms of motion on the individual `translate` and `scale` properties, + // then 5 s holding the end values: playState stays 'running' throughout. + await box.evaluate(el => { + el.animate( + [ + {translate: '0px 0px', scale: '1', offset: 0}, + {translate: '0px 120px', scale: '1.5', offset: 300 / 5300}, + {translate: '0px 120px', scale: '1.5', offset: 1}, + ], + {duration: 5300, fill: 'forwards'}, + ); + }); + const startedAt = await page.evaluate(() => performance.now()); + const [active] = await waitForAnimation(box); + expect(active.to - active.from).toBeCloseTo(300, 0); // the curve, not the hold + await waitForRest(box, {timeout: 3_000}); + const after = await box.evaluate(el => ({ + at: performance.now(), + running: el.getAnimations().some(a => a.playState === 'running'), + translate: getComputedStyle(el).translate, + })); + expect(after.running).toBe(true); // still holding its end value + expect(after.at - startedAt).toBeGreaterThanOrEqual(300); + expect(after.at - startedAt).toBeLessThan(1_500); + expect(after.translate).toBe('0px 120px'); + + const held = await holdAnimations(page, box); + expect(held[0].curveEndMs).toBeCloseTo(300, 0); + const [mid, end] = await sampleCurve(box, [150, 300]); + expect(mid.raw.translate).toBe('0px 60px'); + expect(mid.transform.translateY).toBeCloseTo(60, 1); + expect(mid.transform.scaleX).toBeCloseTo(1.25, 3); + expect(end.transform.translateY).toBeCloseTo(120, 1); + expect(end.transform.scaleY).toBeCloseTo(1.5, 3); + }); }); test.describe('storyboards', () => { diff --git a/internal/interaction-harness/src/storyboard.ts b/internal/interaction-harness/src/storyboard.ts index e2a7d264b43d5..50b34adffc73d 100644 --- a/internal/interaction-harness/src/storyboard.ts +++ b/internal/interaction-harness/src/storyboard.ts @@ -228,7 +228,10 @@ async function composeSheet( } export interface AnimationStoryboardOptions extends StoryboardOptions { - /** Frame times, ms (default every 16.7 ms through the longest held animation). */ + /** + * Frame times, ms (default every 16.7 ms to the end of the longest held + * animation's curve; a held end value is not filmed). + */ readonly times?: readonly number[]; /** Restrict to animations inside this element. */ readonly scope?: Locator; @@ -248,7 +251,7 @@ export async function animationStoryboard( if (held.length === 0) { throw new Error(`storyboard ${options.name}: no running animation to hold`); } - const longest = Math.max(...held.map(a => a.delayMs + a.durationMs)); + const longest = Math.max(...held.map(a => a.curveEndMs)); const times = options.times ?? timeline(longest); const storyboard = new Storyboard(page, options); for (const t of times) { From d8af73a7a05dc58f16673576cb546b2c67951cc6 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sat, 24 Oct 2026 14:56:33 -0700 Subject: [PATCH 07/93] fix(interaction): hold only running animations at their own offsets, film no endless curve, let the page cancel gated callbacks - holdAnimations holds only running document-timeline animations: a finished fill animation keeps its end value, an animation the page paused stays paused (release resumes only what was held), and a scroll-driven one is not seeked. Each held animation keeps its offset from the earliest start, so staggered starts stay staggered. - A sample or storyboard frame at or past an end reports the time it was read at (presentedAt, half a millisecond inside the end), and curveOf reads oracles there. - An animation that repeats forever no longer sets a storyboard's default length (it ran the worker out of memory); with nothing else to film, animationStoryboard asks for times. - The timer gate queues held callbacks by id, so the page's clearTimeout and cancelAnimationFrame still cancel them; a held frame callback gets the time it runs; frames(), waitForRest and touchStoryboard use the real requestAnimationFrame, so they resolve while frozen. - The storyboard self-tests carry the @storyboard tag that `pnpm storyboards` selects. --- internal/interaction-harness/README.md | 17 +- internal/interaction-harness/src/curves.ts | 122 ++++++++--- .../src/harness.interaction.browser.spec.ts | 205 +++++++++++++++++- internal/interaction-harness/src/index.ts | 4 +- internal/interaction-harness/src/story.ts | 54 ++++- .../interaction-harness/src/storyboard.ts | 32 ++- 6 files changed, 370 insertions(+), 64 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 2977189fa1500..21d594e28b3de 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -58,13 +58,17 @@ dispatch latency, and the events go through the compositor's gesture detection the limits below. Read positions a frame after a move: Chromium delivers `pointermove` aligned to animation frames. -**Curve sampling.** Pause every animation (CSS transitions and animations are -Web Animations too), seek to a time, read `getComputedStyle`: the presented -value at that exact time, independent of frame timing. Compare it with an +**Curve sampling.** Pause every running animation (CSS transitions and +animations are Web Animations too), seek to a time, read `getComputedStyle`: +the presented value at that exact time, independent of frame timing. Finished +animations, ones the page paused and ones on a scroll or view timeline are left +alone; held animations keep their offsets from the earliest start, and release +resumes only them. Compare it with an oracle from `math.ts` through `compareCurve`; a tolerance states the error budget (for a spring baked into `linear()` stops of `h` ms, chords err by up to `h² / 8 · |x''|`). A seek past an animation's end lands half a millisecond -inside it, the last frame it presents. An animation whose keyframes end in a run of +inside it, the last frame it presents, and the sample says so (`presentedAt`, +where oracles are read). An animation whose keyframes end in a run of identical values (an end value held so a later interruption can read it) ends where that run starts: `curveEndMs`, `waitForAnimation`'s interval and the default storyboard times stop there, and `waitForRest`, which judges the @@ -77,7 +81,10 @@ carries each frame's measurements and the spec asserts on it; the sheet is evidence. A component that finishes an animation from a timer (a transitionend backstop, a cleanup render) would do so behind a held storyboard's back: `installTimerGate` before the animation starts and `freezeTimers` while -photographing. +photographing. The page can still cancel a held callback, a held frame callback +gets the time it actually runs, and the harness's own frame waits are not held. +An animation that repeats forever does not set a storyboard's default length; +with nothing else to film, pass `times`. ## Engine limits (measured with Playwright 1.61) diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index 5e9a04511b453..87d79aaa0449d 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -6,7 +6,8 @@ * the Web Animations API in the page (CSS transitions and CSS animations are * Web Animations too) * @output `holdAnimations` (pause every running animation and describe it), - * `seekAnimations` (move them all to t ms), `sampleCurve` (the presented + * `seekAnimations` (move them to t ms, each keeping its offset from the + * earliest), `sampleCurve` (the presented * transform/opacity at each t, with the individual `translate`, `rotate` and * `scale` properties composed in), `releaseAnimations`, `curveOf`, which * picks one channel out of the samples for ./math's `compareCurve`, and the @@ -93,12 +94,20 @@ export interface HeldAnimation { readonly keyframes: readonly Record[]; /** Where it was when held, ms. */ readonly heldAtMs: number | null; + /** + * How much later than the earliest held animation it started, ms: a seek to + * t moves it to its own t - offsetMs, so staggered starts stay staggered. + */ + readonly offsetMs: number; } /** - * Pauses every animation in the document (optionally only those whose target - * is inside `scope`) and returns what was held. Later `seekAnimations` calls - * move exactly this set. + * Pauses every running animation in the document (optionally only those whose + * target is inside `scope`) and returns what was held. Finished animations, + * animations the page paused, and animations on a scroll or view timeline are + * left as they are. Later `seekAnimations` calls move exactly the held set, + * each by its offset from the earliest start, and `releaseAnimations` resumes + * only it. */ export async function holdAnimations( page: Page, @@ -118,19 +127,36 @@ export async function holdAnimations( (node.classList.length ? `.${[...node.classList].slice(0, 2).join('.')}` : ''); + const now = Number(document.timeline.currentTime ?? performance.now()); const all = document.getAnimations().filter(animation => { const target = (animation.effect as KeyframeEffect | null)?.target ?? null; return ( - root == null || - (target != null && (root === target || root.contains(target))) + animation.playState === 'running' && + animation.timeline instanceof DocumentTimeline && + (root == null || + (target != null && (root === target || root.contains(target)))) ); }); + // Each animation's start on the document timeline, read before the pause + // clears it (a pending play has no start time yet: it starts now). + const starts = all.map(animation => + animation.startTime != null + ? Number(animation.startTime) + : now - Number(animation.currentTime ?? 0), + ); + const earliest = Math.min(...starts); + const offsets = starts.map(start => start - earliest); const held = all.map(animation => { animation.pause(); return animation; }); - (window as unknown as {__harnessHeld: Animation[]}).__harnessHeld = held; + const w = window as unknown as { + __harnessHeld: Animation[]; + __harnessHeldOffsets: number[]; + }; + w.__harnessHeld = held; + w.__harnessHeldOffsets = offsets; return held.map((animation, index) => { const effect = animation.effect as KeyframeEffect | null; const timing = effect?.getComputedTiming(); @@ -155,38 +181,52 @@ export async function holdAnimations( ), heldAtMs: animation.currentTime == null ? null : Number(animation.currentTime), + offsetMs: offsets[index], }; }); }, scopeHandle); } /** - * Moves every held animation to `timeMs` (from its own start, delay included). - * A time at or past an animation's end is clamped to END_INSET_MS before it: - * an animation without `fill` stops applying its effect exactly at the end, - * and the frame wanted there is the last one it presents. (Half a millisecond, - * not less: engines round `currentTime` writes to their timer resolution.) + * Moves every held animation to `timeMs` from the earliest held start (delay + * included): each to its own `timeMs - offsetMs`. A time at or past an + * animation's end is clamped to END_INSET_MS before it: an animation without + * `fill` stops applying its effect exactly at the end, and the frame wanted + * there is the last one it presents. (Half a millisecond, not less: engines + * round `currentTime` writes to their timer resolution.) Returns the time the + * page then presents: `timeMs`, or the latest clamped end when every held + * animation was clamped. */ export async function seekAnimations( page: Page, timeMs: number, -): Promise { - await page.evaluate( +): Promise { + return page.evaluate( ({t, inset}) => { - for (const animation of ( - window as unknown as {__harnessHeld?: Animation[]} - ).__harnessHeld ?? []) { + const w = window as unknown as { + __harnessHeld?: Animation[]; + __harnessHeldOffsets?: number[]; + }; + let presented = -Infinity; + (w.__harnessHeld ?? []).forEach((animation, index) => { + const offset = w.__harnessHeldOffsets?.[index] ?? 0; const end = Number( animation.effect?.getComputedTiming().endTime ?? Infinity, ); - animation.currentTime = Math.min(t, end - inset); - } + const local = Math.min(t - offset, end - inset); + animation.currentTime = local; + presented = Math.max(presented, local + offset); + }); + return Number.isFinite(presented) ? presented : t; }, {t: timeMs, inset: END_INSET_MS}, ); } -/** Resumes (default) or finishes every held animation. */ +/** + * Resumes (default) or finishes every held animation: only those + * `holdAnimations` paused, so an animation the page paused stays paused. + */ export async function releaseAnimations( page: Page, {finish = false}: {finish?: boolean} = {}, @@ -206,7 +246,13 @@ export async function releaseAnimations( } export interface CurveSample { + /** The requested time. */ readonly t: number; + /** + * The time the page presents: `t`, or just inside the end when the sample + * was taken at or past it (see `seekAnimations`). Oracles are read here. + */ + readonly presentedAt: number; /** * The presented 2D transform: `translate` · `rotate` · `scale` · * `transform`, composed in the order CSS applies them. @@ -235,22 +281,29 @@ export async function sampleCurve( ): Promise { const raw = await locator.evaluate( (element, {ts, inset}) => { - const held = - (window as unknown as {__harnessHeld?: Animation[]}).__harnessHeld ?? - []; + const w = window as unknown as { + __harnessHeld?: Animation[]; + __harnessHeldOffsets?: number[]; + }; + const held = w.__harnessHeld ?? []; if (held.length === 0) { throw new Error( 'sampleCurve: no held animations (call holdAnimations first)', ); } return ts.map(t => { - for (const animation of held) { - // Just inside the end: see seekAnimations. + let presented = -Infinity; + held.forEach((animation, index) => { + // Each at its own offset, just inside its end: see seekAnimations. + const offset = w.__harnessHeldOffsets?.[index] ?? 0; const end = Number( animation.effect?.getComputedTiming().endTime ?? Infinity, ); - animation.currentTime = Math.min(t, end - inset); - } + const local = Math.min(t - offset, end - inset); + animation.currentTime = local; + presented = Math.max(presented, local + offset); + }); + const presentedAt = Number.isFinite(presented) ? presented : t; const style = getComputedStyle(element); // translate · rotate · scale · transform, as CSS applies them (all // about the same transform origin, which `transform` excludes too). @@ -289,6 +342,7 @@ export async function sampleCurve( const {a, b, c, d, e, f} = matrix; return { t, + presentedAt, matrix: `matrix(${a}, ${b}, ${c}, ${d}, ${e}, ${f})`, transform: style.transform, translate: style.translate, @@ -302,6 +356,7 @@ export async function sampleCurve( ); return raw.map(r => ({ t: r.t, + presentedAt: r.presentedAt, transform: parseTransform(r.matrix), opacity: Number(r.opacity), raw: { @@ -323,7 +378,7 @@ export function curveOf( channel: Channel, ): TimedValue[] { return samples.map(s => ({ - t: s.t, + t: s.presentedAt, value: channel === 'opacity' ? s.opacity : s.transform[channel], })); } @@ -438,6 +493,13 @@ export async function waitForRest( backdrop, ].join(' | '); }; + // The page's real frames, even while a timer gate holds its callbacks. + const frame = + ( + window as unknown as { + __harnessTimerGate?: {frame: typeof requestAnimationFrame}; + } + ).__harnessTimerGate?.frame ?? requestAnimationFrame; let last = read(); let still = 0; const tick = () => { @@ -457,10 +519,10 @@ export async function waitForRest( } else if (performance.now() > deadline) { reject(new Error(`waitForRest: still moving (${now})`)); } else { - requestAnimationFrame(tick); + frame(tick); } }; - requestAnimationFrame(tick); + frame(tick); }), {ms: timeout, need: frames}, ); diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index d5177019faf16..f4e69adb46190 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -8,9 +8,14 @@ * a second finger and a pinch arrive as distinct pointers; sampled curves * match the oracles; the animation-state waits report a running transition * and its end, and judge rest by presented values even while an animation - * holds its end; individual transform properties are sampled; storyboards - * write a sheet and a trace that follows the curve, and the timer gate - * holds a component timer until the filmstrip is done. + * holds its end; individual transform properties are sampled; a sample at + * an end reports the time it was read at; holding takes only running + * document-timeline animations, each at its own offset, and releases only + * them; storyboards write a sheet and a trace that follows the curve, and an + * endless animation does not set their length; the timer gate holds a + * component timer until the filmstrip is done, drops callbacks the page + * cancels, stamps held frames with the time they run, and leaves the + * harness its own frames. * @position The harness's own test. A component spec trusts these * instruments; this file is why it may. */ @@ -23,6 +28,7 @@ import { curveOf, drag, flick, + frames, freezeTimers, holdAnimations, installInputLog, @@ -32,7 +38,9 @@ import { PARTIAL_LIFT_SKIP, pinch, readInputLog, + releaseAnimations, sampleCurve, + seekAnimations, springStep, tap, thawTimers, @@ -292,6 +300,109 @@ test.describe('curve sampling', () => { }); }); +test.describe('holding', () => { + test('a sample at an end reports the time it was read at', async ({page}) => { + await fixture(page); + const box = page.locator('#box'); + await box.evaluate(el => + el.animate( + [{transform: 'translateY(0px)'}, {transform: 'translateY(200px)'}], + {duration: 300, easing: 'linear'}, + ), + ); + await holdAnimations(page, box); + const samples = await sampleCurve(box, [0, 150, 300]); + // Read just inside the end, and labelled so. + expect(samples.at(-1)).toMatchObject({t: 300, presentedAt: 299.5}); + const result = compareCurve( + curveOf(samples, 'translateY'), + t => (200 * t) / 300, + 0.05, + ); + expect(result.maxError, JSON.stringify(result.worst)).toBeLessThanOrEqual( + 0.05, + ); + await releaseAnimations(page); + }); + + test('only running document-timeline animations are held, each at its own offset', async ({ + page, + }) => { + await fixture(page); + // Besides #box: a finished fade, an animation the page paused, one on a + // scroll timeline, and a second runner started 100 ms after #box's. + await page.evaluate(async () => { + const make = (id: string) => { + const el = document.createElement('div'); + el.id = id; + el.style.cssText = + 'position:absolute;left:0;top:300px;width:40px;height:40px;background:#c63'; + document.body.append(el); + return el; + }; + make('finished').animate([{opacity: 0}, {opacity: 1}], { + duration: 50, + fill: 'both', + }); + const paused = make('paused').animate( + [{transform: 'translateX(0px)'}, {transform: 'translateX(100px)'}], + {duration: 1000}, + ); + paused.pause(); + paused.currentTime = 500; + const timelines = window as unknown as { + ScrollTimeline?: new (options: object) => AnimationTimeline; + }; + if (timelines.ScrollTimeline != null) { + new Animation( + new KeyframeEffect(make('scrolled'), [{opacity: 0}, {opacity: 1}], { + duration: 1000, + }), + new timelines.ScrollTimeline({source: document.scrollingElement}), + ).play(); + } + await new Promise(resolve => setTimeout(resolve, 120)); + const run = (el: Element | null) => + el?.animate( + [{transform: 'translateX(0px)'}, {transform: 'translateX(300px)'}], + {duration: 600, easing: 'linear'}, + ); + run(document.getElementById('box')); + await new Promise(resolve => setTimeout(resolve, 100)); + run(make('later')); + }); + const held = await holdAnimations(page); + expect(held.map(a => a.target)).toEqual(['div#box', 'div#later']); + const [first, later] = held; + expect(first.offsetMs).toBe(0); + expect(later.offsetMs).toBeGreaterThan(80); + const x = (id: string) => + page + .locator(`#${id}`) + .evaluate(el => new DOMMatrix(getComputedStyle(el).transform).m41); + expect(await seekAnimations(page, 300)).toBe(300); + expect(await x('box')).toBeCloseTo(150, 1); + expect(await x('later')).toBeCloseTo( + (300 * (300 - later.offsetMs)) / 600, + 1, + ); + // The finished fade kept its end value; the paused animation stays put. + expect( + await page + .locator('#finished') + .evaluate(el => getComputedStyle(el).opacity), + ).toBe('1'); + await releaseAnimations(page); + await page.waitForTimeout(100); + expect( + await page.evaluate(() => { + const animation = document.getElementById('paused')?.getAnimations()[0]; + return [animation?.playState, Number(animation?.currentTime)]; + }), + ).toEqual(['paused', 500]); + }); +}); + test.describe('animation state', () => { test('waitForAnimation reports a running transition and waitForRest resolves after it', async ({ page, @@ -359,7 +470,7 @@ test.describe('animation state', () => { }); }); -test.describe('storyboards', () => { +test.describe('storyboards', {tag: '@storyboard'}, () => { test('an animation filmstrip records every frame and its trace follows the curve', async ({ page, }, testInfo) => { @@ -468,4 +579,90 @@ test.describe('storyboards', () => { expect(xs).toHaveLength(19); // t = 0 … 300 ms every 16.7 ms expect(xs.every((x, i) => i === 0 || x > xs[i - 1])).toBe(true); }); + + test('an animation that repeats forever does not set the filmstrip length', async ({ + page, + }, testInfo) => { + await fixture(page); + await page.locator('#box').evaluate(el => { + el.animate([{rotate: '0deg'}, {rotate: '360deg'}], { + duration: 800, + iterations: Infinity, + }); + el.animate([{opacity: 1}, {opacity: 0.4}], {duration: 200}); + }); + const {storyboard, held} = await animationStoryboard(page, { + name: 'harness-self-test-endless', + clip: {x: 0, y: 340, width: 402, height: 120}, + columns: 4, + }); + expect(held.map(a => Number.isFinite(a.curveEndMs))).toEqual([false, true]); + expect(storyboard.frames).toHaveLength(13); // t = 0 … 200 ms + await releaseAnimations(page); + await storyboard.write(testInfo); + }); + + test('a filmstrip of endless animations alone asks for its times', async ({ + page, + }) => { + await fixture(page); + await page.locator('#box').evaluate(el => + el.animate([{rotate: '0deg'}, {rotate: '360deg'}], { + duration: 800, + iterations: Infinity, + }), + ); + await expect( + animationStoryboard(page, { + name: 'harness-self-test-endless-only', + clip: {x: 0, y: 340, width: 402, height: 120}, + }), + ).rejects.toThrow(/repeats forever/); + await releaseAnimations(page); + }); + + test('the timer gate drops callbacks the page cancels and stamps held frames with the time they run', async ({ + page, + }) => { + await fixture(page); + await installTimerGate(page); + await freezeTimers(page); + await page.evaluate(() => { + const w = window as unknown as Record; + w.frozenAt = performance.now(); + w.timer = setTimeout(() => { + w.cancelledTimerRan = true; + }, 20); + w.frame = requestAnimationFrame(() => { + w.cancelledFrameRan = true; + }); + requestAnimationFrame(time => { + w.heldFrameTime = time; + }); + }); + await page.waitForTimeout(150); // both are due now, and held + // The harness's own frame waits still resolve while the page's are held. + await Promise.race([ + frames(page, 2), + page.waitForTimeout(5000).then(() => { + throw new Error('frames() did not resolve while the gate was frozen'); + }), + ]); + await page.evaluate(() => { + const w = window as unknown as {timer: number; frame: number}; + clearTimeout(w.timer); + cancelAnimationFrame(w.frame); + }); + await thawTimers(page); + const state = await page.evaluate(() => { + const w = window as unknown as Record; + return { + timer: w.cancelledTimerRan === true, + frame: w.cancelledFrameRan === true, + heldFor: Number(w.heldFrameTime) - Number(w.frozenAt), + }; + }); + expect(state).toMatchObject({timer: false, frame: false}); + expect(state.heldFor).toBeGreaterThanOrEqual(140); + }); }); diff --git a/internal/interaction-harness/src/index.ts b/internal/interaction-harness/src/index.ts index 28a43db50d47e..537bfc4d4beca 100644 --- a/internal/interaction-harness/src/index.ts +++ b/internal/interaction-harness/src/index.ts @@ -4,8 +4,8 @@ * @file index.ts * @input Re-exports the harness modules * @output The `@astryxdesign/interaction-harness` entry point for Playwright - * specs. The pure touch scripts are also reachable without Playwright - * through `@astryxdesign/interaction-harness/scripts` (unit tests). + * specs. Pure helpers are also reachable without Playwright through + * `@astryxdesign/interaction-harness/math` and `/scripts` (unit tests). * @position Imported only from `*.interaction.browser.spec.ts` files (run by * playwright.interaction.config.ts) and from this package's own tests. * diff --git a/internal/interaction-harness/src/story.ts b/internal/interaction-harness/src/story.ts index 0058fe5b579e2..598f75d852cfe 100644 --- a/internal/interaction-harness/src/story.ts +++ b/internal/interaction-harness/src/story.ts @@ -93,26 +93,51 @@ export async function frames(page: Page, count = 2): Promise { await page.evaluate( n => new Promise(resolve => { + // The page's real frames, even while a timer gate holds its callbacks. + const frame = + ( + window as unknown as { + __harnessTimerGate?: {frame: typeof requestAnimationFrame}; + } + ).__harnessTimerGate?.frame ?? requestAnimationFrame; let left = n; - const tick = () => - --left <= 0 ? resolve() : requestAnimationFrame(tick); - requestAnimationFrame(tick); + const tick = () => (--left <= 0 ? resolve() : frame(tick)); + frame(tick); }), count, ); } +// A held callback is queued under its timer or frame id, so the page's own +// clearTimeout and cancelAnimationFrame still cancel it while frozen; a held +// frame callback gets the time it actually runs at. `frame` stays the real +// requestAnimationFrame, for the harness's own waits. const TIMER_GATE = `(() => { if (window.__harnessTimerGate) return; - const gate = {frozen: false, queue: []}; - window.__harnessTimerGate = gate; const realTimeout = window.setTimeout.bind(window); + const realClearTimeout = window.clearTimeout.bind(window); const realInterval = window.setInterval.bind(window); const realFrame = window.requestAnimationFrame.bind(window); - const hold = run => (gate.frozen ? gate.queue.push(run) : run()); - window.setTimeout = (fn, ms, ...args) => realTimeout(() => hold(() => (typeof fn === 'function' ? fn(...args) : 0)), ms); + const realCancelFrame = window.cancelAnimationFrame.bind(window); + const gate = {frozen: false, queue: [], frame: realFrame}; + window.__harnessTimerGate = gate; + const drop = key => { gate.queue = gate.queue.filter(entry => entry.key !== key); }; + window.setTimeout = (fn, ms, ...args) => { + const id = realTimeout(() => { + const run = () => (typeof fn === 'function' ? fn(...args) : 0); + if (gate.frozen) gate.queue.push({key: 't' + id, run}); else run(); + }, ms); + return id; + }; + window.clearTimeout = id => { realClearTimeout(id); drop('t' + id); }; window.setInterval = (fn, ms, ...args) => realInterval(() => { if (!gate.frozen && typeof fn === 'function') fn(...args); }, ms); - window.requestAnimationFrame = fn => realFrame(time => hold(() => fn(gate.frozen ? performance.now() : time))); + window.requestAnimationFrame = fn => { + const id = realFrame(time => { + if (gate.frozen) gate.queue.push({key: 'f' + id, run: () => fn(performance.now())}); else fn(time); + }); + return id; + }; + window.cancelAnimationFrame = id => { realCancelFrame(id); drop('f' + id); }; })()`; /** @@ -129,7 +154,8 @@ export async function installTimerGate(page: Page): Promise { * Holds every gated timer and frame callback until `thawTimers`, so a * component cannot finish an animation behind a held storyboard's back (a * transitionend backstop timer, a cleanup render). Callbacks that come due - * meanwhile run, in order, on thaw; intervals skip their frozen ticks. + * meanwhile run, in order, on thaw, unless the page cancels them first + * (clearTimeout, cancelAnimationFrame); intervals skip their frozen ticks. */ export async function freezeTimers(page: Page): Promise { await page.evaluate(() => { @@ -149,15 +175,19 @@ export async function thawTimers(page: Page): Promise { await page.evaluate(() => { const gate = ( window as unknown as { - __harnessTimerGate?: {frozen: boolean; queue: (() => void)[]}; + __harnessTimerGate?: { + frozen: boolean; + queue: {key: string; run: () => void}[]; + }; } ).__harnessTimerGate; if (gate == null) { return; } gate.frozen = false; - for (const run of gate.queue.splice(0)) { - run(); + // One at a time from the live queue: a callback may cancel a later one. + while (gate.queue.length > 0) { + gate.queue.shift()?.run(); } }); } diff --git a/internal/interaction-harness/src/storyboard.ts b/internal/interaction-harness/src/storyboard.ts index 50b34adffc73d..f9b79e27c7e63 100644 --- a/internal/interaction-harness/src/storyboard.ts +++ b/internal/interaction-harness/src/storyboard.ts @@ -26,6 +26,7 @@ import {holdAnimations, seekAnimations, type HeldAnimation} from './curves'; import {timeline} from './math'; import {writeEvidence} from './output'; import type {TouchScript} from './scripts'; +import {frames} from './story'; import type {TouchDriver} from './touch'; export interface ClipBox { @@ -230,7 +231,8 @@ async function composeSheet( export interface AnimationStoryboardOptions extends StoryboardOptions { /** * Frame times, ms (default every 16.7 ms to the end of the longest held - * animation's curve; a held end value is not filmed). + * curve that ends: a held end value is not filmed, and an animation that + * repeats forever does not set the length). */ readonly times?: readonly number[]; /** Restrict to animations inside this element. */ @@ -251,13 +253,26 @@ export async function animationStoryboard( if (held.length === 0) { throw new Error(`storyboard ${options.name}: no running animation to hold`); } - const longest = Math.max(...held.map(a => a.curveEndMs)); - const times = options.times ?? timeline(longest); + // An animation that repeats forever (a spinner, a pulsing skeleton) has no + // end to film to: the default length comes from the others. + const ends = held.map(a => a.curveEndMs).filter(Number.isFinite); + if (options.times == null && ends.length === 0) { + throw new Error( + `storyboard ${options.name}: every held animation repeats forever ` + + `(${held.map(a => a.name ?? a.kind).join(', ')}); pass \`times\``, + ); + } + const times = options.times ?? timeline(Math.max(...ends)); const storyboard = new Storyboard(page, options); for (const t of times) { - await seekAnimations(page, t); + // Labelled with the time the page presents (just inside an end). + const presentedAt = await seekAnimations(page, t); const data = (await options.measure?.()) ?? {}; - await storyboard.capture(`t=${+t.toFixed(1)} ms`, t, data); + await storyboard.capture( + `t=${+presentedAt.toFixed(1)} ms`, + presentedAt, + data, + ); } return {storyboard, held}; } @@ -284,12 +299,7 @@ export async function touchStoryboard( realtime: false, onEvent: async step => { // Let the page run its handlers and present the frame before the photo. - await page.evaluate( - () => - new Promise(resolve => - requestAnimationFrame(() => requestAnimationFrame(() => resolve())), - ), - ); + await frames(page, 2); const data = (await options.measure?.()) ?? {}; const where = step.changed .map(f => `${Math.round(f.x)},${Math.round(f.y)}`) From 2e170f54faea26c03d4691cf7880d6564c065d6c Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sat, 24 Oct 2026 19:12:05 -0700 Subject: [PATCH 08/93] fix(interaction): film from each animation's own offset, label samples from the element's own animations, resume endless ones - animationStoryboard's default times run from the earliest start to the latest curve end among the animations that end, each seek at origin + t: beside a spinner that started earlier the animation under test no longer shows its start in every frame, and a second animation that starts later is filmed to its end. - A sample's presentedAt (per channel: transform, opacity) comes from the sampled element's own animations, by keyframe property, not from any animation on the page. - releaseAnimations({finish: true}) resumes an animation that repeats forever instead of finishing it (which throws), and always clears the held set. - The interaction proofs' scope includes packages/themes/*/src/. --- .github/scripts/ci-interaction.test.mjs | 5 +- .github/workflows/ci.yml | 2 +- internal/interaction-harness/src/curves.ts | 83 ++++++++++-- .../src/harness.interaction.browser.spec.ts | 127 ++++++++++++++++++ .../interaction-harness/src/storyboard.ts | 32 +++-- 5 files changed, 225 insertions(+), 24 deletions(-) diff --git a/.github/scripts/ci-interaction.test.mjs b/.github/scripts/ci-interaction.test.mjs index e8b393ffbf486..1c5981150e3ed 100644 --- a/.github/scripts/ci-interaction.test.mjs +++ b/.github/scripts/ci-interaction.test.mjs @@ -84,7 +84,7 @@ describe('interaction proof routing', () => { expect(decide().run).toContain('if [ "$scope" = 1 ]; then'); }); - it('keeps the harness, components, Storybook and its own setup in scope', () => { + it('keeps the harness, components, themes, Storybook and its own setup in scope', () => { const pattern = new RegExp( decide().run.match(/bash "\$helper" "\$BASE_SHA" '([^']+)'/)[1], ); @@ -93,6 +93,8 @@ describe('interaction proof routing', () => { 'internal/a11y-spec/src/storybook.ts', 'playwright.interaction.config.ts', 'packages/core/src/BottomSheet/BottomSheet.tsx', + // Themes set the motion durations components animate with. + 'packages/themes/neutral/src/neutralTheme.ts', 'apps/storybook/.storybook/preview.tsx', '.github/workflows/ci.yml', 'package.json', @@ -104,6 +106,7 @@ describe('interaction proof routing', () => { 'docs/specs/AST-030/spec.md', 'apps/docsite/src/app/page.tsx', 'packages/core/package.json', + 'packages/themes/neutral/package.json', '.github/scripts/native-swift.sh', ]) { expect(pattern.test(untouched), untouched).toBe(false); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5ca0f3930b8f..740ac18b7bb3a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3832,7 +3832,7 @@ jobs: && git fetch --quiet --no-tags --depth=1 origin "$BASE_SHA" \ && git show "$BASE_SHA:.github/scripts/pr-scope-touches.sh" > "$helper"; then scope=0 - bash "$helper" "$BASE_SHA" '^(internal/interaction-harness/|internal/a11y-spec/|playwright\.interaction\.config\.ts$|packages/[^/]+/src/|apps/storybook/|\.github/workflows/ci\.yml$|package\.json$|pnpm-lock\.yaml$)' || scope=$? + bash "$helper" "$BASE_SHA" '^(internal/interaction-harness/|internal/a11y-spec/|playwright\.interaction\.config\.ts$|packages/[^/]+/src/|packages/themes/[^/]+/src/|apps/storybook/|\.github/workflows/ci\.yml$|package\.json$|pnpm-lock\.yaml$)' || scope=$? fi if [ "$scope" = 1 ]; then echo 'No harness, component, Storybook or proof-setup change; skipping the interaction proofs.' diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index 87d79aaa0449d..87578b845c566 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -225,23 +225,35 @@ export async function seekAnimations( /** * Resumes (default) or finishes every held animation: only those - * `holdAnimations` paused, so an animation the page paused stays paused. + * `holdAnimations` paused, so an animation the page paused stays paused. An + * animation that repeats forever has no end to finish at, so `finish` + * resumes it instead. The held set is cleared even if one throws. */ export async function releaseAnimations( page: Page, {finish = false}: {finish?: boolean} = {}, ): Promise { await page.evaluate(shouldFinish => { - const held = - (window as unknown as {__harnessHeld?: Animation[]}).__harnessHeld ?? []; - for (const animation of held) { - if (shouldFinish) { - animation.finish(); - } else { - animation.play(); + const w = window as unknown as { + __harnessHeld?: Animation[]; + __harnessHeldOffsets?: number[]; + }; + const held = w.__harnessHeld ?? []; + try { + for (const animation of held) { + const end = Number( + animation.effect?.getComputedTiming().endTime ?? Infinity, + ); + if (shouldFinish && Number.isFinite(end)) { + animation.finish(); + } else { + animation.play(); + } } + } finally { + w.__harnessHeld = []; + w.__harnessHeldOffsets = []; } - (window as unknown as {__harnessHeld?: Animation[]}).__harnessHeld = []; }, finish); } @@ -249,10 +261,22 @@ export interface CurveSample { /** The requested time. */ readonly t: number; /** - * The time the page presents: `t`, or just inside the end when the sample - * was taken at or past it (see `seekAnimations`). Oracles are read here. + * The time the sampled element presents: `t`, or just inside the end of + * its own animations when the sample was taken at or past it (see + * `seekAnimations`). Read from the element's own animations only (from + * every held one if none targets it). */ readonly presentedAt: number; + /** + * The same per channel group, from the element's animations of those + * properties: a 300 ms move beside a 400 ms fade on one element is read + * just inside 300 ms for the transform and at the requested time for the + * opacity. `curveOf` reads oracles here. + */ + readonly presentedAtFor: { + readonly transform: number; + readonly opacity: number; + }; /** * The presented 2D transform: `translate` · `rotate` · `scale` · * `transform`, composed in the order CSS applies them. @@ -291,8 +315,22 @@ export async function sampleCurve( 'sampleCurve: no held animations (call holdAnimations first)', ); } + // The properties each held animation of this element animates. + const own = held.map(animation => { + const effect = animation.effect as KeyframeEffect | null; + if (effect?.target !== element) {return null;} + const properties = new Set(); + for (const keyframe of effect.getKeyframes()) { + for (const key of Object.keys(keyframe)) {properties.add(key);} + } + return properties; + }); + const TRANSFORM = ['transform', 'translate', 'rotate', 'scale']; return ts.map(t => { let presented = -Infinity; + let presentedOwn = -Infinity; + let presentedTransform = -Infinity; + let presentedOpacity = -Infinity; held.forEach((animation, index) => { // Each at its own offset, just inside its end: see seekAnimations. const offset = w.__harnessHeldOffsets?.[index] ?? 0; @@ -302,8 +340,19 @@ export async function sampleCurve( const local = Math.min(t - offset, end - inset); animation.currentTime = local; presented = Math.max(presented, local + offset); + const properties = own[index]; + if (properties == null) {return;} + presentedOwn = Math.max(presentedOwn, local + offset); + if (TRANSFORM.some(p => properties.has(p))) { + presentedTransform = Math.max(presentedTransform, local + offset); + } + if (properties.has('opacity')) { + presentedOpacity = Math.max(presentedOpacity, local + offset); + } }); - const presentedAt = Number.isFinite(presented) ? presented : t; + const or = (value: number, fallback: number) => + Number.isFinite(value) ? value : fallback; + const presentedAt = or(presentedOwn, or(presented, t)); const style = getComputedStyle(element); // translate · rotate · scale · transform, as CSS applies them (all // about the same transform origin, which `transform` excludes too). @@ -343,6 +392,10 @@ export async function sampleCurve( return { t, presentedAt, + presentedAtFor: { + transform: or(presentedTransform, presentedAt), + opacity: or(presentedOpacity, presentedAt), + }, matrix: `matrix(${a}, ${b}, ${c}, ${d}, ${e}, ${f})`, transform: style.transform, translate: style.translate, @@ -357,6 +410,7 @@ export async function sampleCurve( return raw.map(r => ({ t: r.t, presentedAt: r.presentedAt, + presentedAtFor: r.presentedAtFor, transform: parseTransform(r.matrix), opacity: Number(r.opacity), raw: { @@ -378,7 +432,10 @@ export function curveOf( channel: Channel, ): TimedValue[] { return samples.map(s => ({ - t: s.presentedAt, + t: + channel === 'opacity' + ? s.presentedAtFor.opacity + : s.presentedAtFor.transform, value: channel === 'opacity' ? s.opacity : s.transform[channel], })); } diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index f4e69adb46190..e5ce66b29df84 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -401,6 +401,74 @@ test.describe('holding', () => { }), ).toEqual(['paused', 500]); }); + + test("a sample is labelled from its own element's animations, per channel", async ({ + page, + }) => { + await fixture(page); + // #box moves for 300 ms and fades for 400 ms; another element runs longer. + await page.evaluate(() => { + const box = document.getElementById('box'); + box?.animate( + [{transform: 'translateX(0px)'}, {transform: 'translateX(300px)'}], + {duration: 300, easing: 'linear'}, + ); + box?.animate([{opacity: 1}, {opacity: 0}], { + duration: 400, + easing: 'linear', + }); + const other = document.createElement('div'); + other.style.cssText = + 'position:absolute;top:300px;width:40px;height:40px'; + document.body.append(other); + other.animate([{opacity: 0}, {opacity: 1}], {duration: 600}); + }); + await holdAnimations(page); + const samples = await sampleCurve(page.locator('#box'), [0, 150, 300]); + // The move is read just inside its end, the fade where it is. + expect(samples.at(-1)?.presentedAtFor).toEqual({ + transform: 299.5, + opacity: 300, + }); + const move = compareCurve(curveOf(samples, 'translateX'), t => t, 0.05); + expect(move.maxError, JSON.stringify(move.worst)).toBeLessThanOrEqual(0.05); + const fade = compareCurve( + curveOf(samples, 'opacity'), + t => 1 - t / 400, + 0.002, + ); + expect(fade.maxError, JSON.stringify(fade.worst)).toBeLessThanOrEqual( + 0.002, + ); + await releaseAnimations(page); + }); + + test('finishing resumes an animation that repeats forever and still clears the held set', async ({ + page, + }) => { + await fixture(page); + await page.evaluate(() => { + const box = document.getElementById('box'); + box?.animate([{rotate: '0deg'}, {rotate: '360deg'}], { + duration: 800, + iterations: Infinity, + }); + const fade = document.createElement('div'); + fade.id = 'fade'; + document.body.append(fade); + fade.animate([{opacity: 0}, {opacity: 1}], {duration: 300}); + }); + expect(await holdAnimations(page)).toHaveLength(2); + await releaseAnimations(page, {finish: true}); + expect( + await page.evaluate(() => ({ + spinner: document.getElementById('box')?.getAnimations()[0]?.playState, + fade: document.getElementById('fade')?.getAnimations().length, + held: (window as unknown as {__harnessHeld: Animation[]}).__harnessHeld + .length, + })), + ).toEqual({spinner: 'running', fade: 0, held: 0}); + }); }); test.describe('animation state', () => { @@ -602,6 +670,65 @@ test.describe('storyboards', {tag: '@storyboard'}, () => { await storyboard.write(testInfo); }); + test('a filmstrip runs from the earliest start to the latest end among the animations that end', async ({ + page, + }, testInfo) => { + await fixture(page); + // A spinner that started a second earlier, a 300 ms move, and a 240 ms + // move that starts 83 ms after it. + await page.evaluate(async () => { + const make = (id: string, top: number) => { + const el = document.createElement('div'); + el.id = id; + el.style.cssText = `position:absolute;left:0;top:${top}px;width:40px;height:40px;background:#36c`; + document.body.append(el); + return el; + }; + document + .getElementById('box') + ?.animate([{rotate: '0deg'}, {rotate: '360deg'}], { + duration: 800, + iterations: Infinity, + }); + await new Promise(resolve => setTimeout(resolve, 1000)); + make('first', 360).animate( + [{transform: 'translateX(0px)'}, {transform: 'translateX(300px)'}], + {duration: 300, easing: 'linear'}, + ); + await new Promise(resolve => setTimeout(resolve, 83)); + make('second', 410).animate( + [{transform: 'translateX(0px)'}, {transform: 'translateX(240px)'}], + {duration: 240, easing: 'linear'}, + ); + }); + const x = (id: string) => + page + .locator(`#${id}`) + .evaluate(el => new DOMMatrix(getComputedStyle(el).transform).m41); + const {storyboard, held} = await animationStoryboard(page, { + name: 'harness-self-test-offsets', + clip: {x: 0, y: 340, width: 402, height: 120}, + columns: 6, + measure: async () => ({ + first: await x('first'), + second: await x('second'), + }), + }); + const second = held.find(a => a.target === 'div#second'); + const first = held.find(a => a.target === 'div#first'); + const lag = (second?.offsetMs ?? 0) - (first?.offsetMs ?? 0); + const firsts = storyboard.frames.map(f => Number(f.data.first)); + const seconds = storyboard.frames.map(f => Number(f.data.second)); + // The film starts with the first move at its start, the spinner aside... + expect(firsts[0]).toBeCloseTo(0, 1); + expect(firsts[6]).toBeCloseTo(100, 0); + // ...and ends with the second move at its end, not cut off. + expect(seconds.at(-1)).toBeGreaterThan(239); + expect(storyboard.frames.at(-1)?.t).toBeCloseTo(lag + 240, 0); + await releaseAnimations(page); + await storyboard.write(testInfo); + }); + test('a filmstrip of endless animations alone asks for its times', async ({ page, }) => { diff --git a/internal/interaction-harness/src/storyboard.ts b/internal/interaction-harness/src/storyboard.ts index f9b79e27c7e63..03d70d02222f0 100644 --- a/internal/interaction-harness/src/storyboard.ts +++ b/internal/interaction-harness/src/storyboard.ts @@ -230,9 +230,11 @@ async function composeSheet( export interface AnimationStoryboardOptions extends StoryboardOptions { /** - * Frame times, ms (default every 16.7 ms to the end of the longest held - * curve that ends: a held end value is not filmed, and an animation that - * repeats forever does not set the length). + * Frame times, ms from the film's start: the earliest start among the held + * animations that end (an animation that repeats forever, such as a + * spinner that started earlier, neither starts nor ends the film). Default: + * every 16.7 ms to the latest curve end among them, and that end itself, + * each animation at its own offset (a held end value is not filmed). */ readonly times?: readonly number[]; /** Restrict to animations inside this element. */ @@ -254,19 +256,31 @@ export async function animationStoryboard( throw new Error(`storyboard ${options.name}: no running animation to hold`); } // An animation that repeats forever (a spinner, a pulsing skeleton) has no - // end to film to: the default length comes from the others. - const ends = held.map(a => a.curveEndMs).filter(Number.isFinite); - if (options.times == null && ends.length === 0) { + // end to film to: the film runs over the animations that end, from the + // earliest of their starts to the latest of their curve ends. Seeks are on + // the held timeline (from the earliest held start), so both carry offsets. + const ending = held.filter(a => Number.isFinite(a.curveEndMs)); + if (options.times == null && ending.length === 0) { throw new Error( `storyboard ${options.name}: every held animation repeats forever ` + `(${held.map(a => a.name ?? a.kind).join(', ')}); pass \`times\``, ); } - const times = options.times ?? timeline(Math.max(...ends)); + const origin = ending.length ? Math.min(...ending.map(a => a.offsetMs)) : 0; + const length = + Math.max(...ending.map(a => a.offsetMs + a.curveEndMs)) - origin; + // The 16.7 ms grid, plus the end itself when the grid misses it: the last + // frame is the one the page presents at the end. + const grid = options.times == null ? timeline(length) : []; + if (options.times == null && length - (grid.at(-1) ?? 0) > 1e-3) { + grid.push(+length.toFixed(3)); + } + const times = options.times ?? grid; const storyboard = new Storyboard(page, options); for (const t of times) { - // Labelled with the time the page presents (just inside an end). - const presentedAt = await seekAnimations(page, t); + // Labelled with the time the page presents (just inside an end), from + // the film's start. + const presentedAt = (await seekAnimations(page, origin + t)) - origin; const data = (await options.measure?.()) ?? {}; await storyboard.capture( `t=${+presentedAt.toFixed(1)} ms`, From 04cbfc5854542e772a855606d04f278c6ba3df9e Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sat, 24 Oct 2026 19:23:51 -0700 Subject: [PATCH 09/93] fix(interaction): waitForAnimation sees a short animation that ended before it looked waitForAnimation polled getAnimations() after the action, so an entrance shorter than the first look (a cold page answers its first evaluate late) had already finished: "no animation started". The page now records every animation from the first call into curves.ts on it, and on every later document: a frame loop on its own requestAnimationFrame, the CSS animation and transition events, and Element.animate as it is called, pending animations included (WebKit can start and finish a short script animation between two frames of a fresh page). waitForAnimation also reports an animation that started after the last waitForRest or waitForAnimation and has finished, with finishedAt. watchAnimations(locator) arms the record before an action that may be the page's first contact with the harness. --- internal/interaction-harness/README.md | 12 +- internal/interaction-harness/src/curves.ts | 253 ++++++++++++++++-- .../src/harness.interaction.browser.spec.ts | 55 ++++ 3 files changed, 296 insertions(+), 24 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 21d594e28b3de..1066c22ffc374 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -40,7 +40,7 @@ committed: a pull request links or attaches it. | --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact release velocity, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. | | `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | -| `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t), `waitForAnimation`, `waitForRest`, `releaseAnimations`. | +| `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t), `waitForAnimation`, `watchAnimations`, `waitForRest`, `releaseAnimations`. | | `math.ts` | Oracles independent of the code under test: `cubicBezier`, `timingFunction`, `springStep` (SwiftUI response / dampingFraction, initial velocity), `projectDecay`, `rubberBand`, `parseTransform`, `compareCurve`, `timeline`. | | `storyboard.ts` | `animationStoryboard` (seek to t = 0, 16.7, 33.3 … ms), `touchStoryboard` (a frame per scripted event): PNG contact sheet + JSON trace. | | `output.ts` | `evidenceDir`, `writeEvidence`: where evidence goes and how it is attached to the report. | @@ -75,6 +75,16 @@ the default storyboard times stop there, and `waitForRest`, which judges the presented `transform`, `translate`, `rotate`, `scale` and `opacity` rather than `playState`, does not wait the hold out. +**Waiting for an animation.** The page records every animation from the first +call into `curves.ts` on it (and on every later document): a frame loop on the +page's own `requestAnimationFrame`, the CSS animation and transition events, +and `Element.animate` as it is called, pending animations included. So +`waitForAnimation` also reports one that started after the last +`waitForRest` or `waitForAnimation` and has already finished (`finishedAt` +set): a short entrance no longer races the first look on a cold page. When the +action may be the page's first contact with the harness, take +`watchAnimations(locator)` before it and read `watch.started()` after. + **Storyboards.** Deterministic: frames come from seeking held animations or from stepping scripted input, never from wall-clock capture. The JSON trace carries each frame's measurements and the spec asserts on it; the sheet is diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index 87578b845c566..3b0e572cf94b7 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -33,7 +33,90 @@ export const END_INSET_MS = 0.5; * later interruption can read it) ends where that run starts; any other ends * at its end time. */ +/** + * Page side: records every animation the document runs, from its first frame + * or its creation (a frame loop on the page's real requestAnimationFrame, and + * the CSS animation and transition events), so an animation that starts and + * ends before a test looks is still seen, with the time it finished. + */ +function animationRecorder(): void { + const w = window as unknown as {__harnessAnimations?: unknown}; + if (w.__harnessAnimations) { + return; + } + const frame = window.requestAnimationFrame.bind(window); + const seen = new WeakSet(); + const log: { + animation: Animation; + seenAt: number; + finishedAt: number | null; + }[] = []; + let armedAt = performance.now(); + const remember = (animation: Animation) => { + // A running animation is recorded while still pending: on a cold page the + // next frame can come only after a short animation has finished. + if (seen.has(animation) || animation.playState !== 'running') { + return; + } + seen.add(animation); + const entry = { + animation, + seenAt: performance.now(), + finishedAt: null as number | null, + }; + animation.addEventListener('finish', () => { + entry.finishedAt = performance.now(); + }); + log.push(entry); + if (log.length > 5000) { + log.splice(0, log.length - 5000); + } + }; + const record = () => { + for (const animation of document.getAnimations()) { + remember(animation); + } + }; + const tick = () => { + record(); + frame(tick); + }; + frame(tick); + for (const type of ['animationstart', 'transitionrun', 'transitionstart']) { + document.addEventListener(type, record, true); + } + // A script animation is recorded as it is made: WebKit can start and finish + // a short one between two frames of a fresh page. + const animate = Element.prototype.animate; + Element.prototype.animate = function ( + this: Element, + ...args: Parameters + ) { + const animation = animate.apply(this, args); + remember(animation); + return animation; + }; + w.__harnessAnimations = { + log, + record, + arm: () => (armedAt = performance.now()), + armedAt: () => armedAt, + }; +} + +const recordingPages = new WeakSet(); + +/** Starts the recorder now and on every later document of the page. */ +async function installRecorder(page: Page): Promise { + if (!recordingPages.has(page)) { + recordingPages.add(page); + await page.addInitScript(animationRecorder); + } + await page.evaluate(animationRecorder); +} + async function installCurveEnd(page: Page): Promise { + await installRecorder(page); await page.evaluate(() => { const w = window as unknown as { __harnessCurveEnd?: (animation: Animation) => number; @@ -318,10 +401,14 @@ export async function sampleCurve( // The properties each held animation of this element animates. const own = held.map(animation => { const effect = animation.effect as KeyframeEffect | null; - if (effect?.target !== element) {return null;} + if (effect?.target !== element) { + return null; + } const properties = new Set(); for (const keyframe of effect.getKeyframes()) { - for (const key of Object.keys(keyframe)) {properties.add(key);} + for (const key of Object.keys(keyframe)) { + properties.add(key); + } } return properties; }); @@ -341,7 +428,9 @@ export async function sampleCurve( animation.currentTime = local; presented = Math.max(presented, local + offset); const properties = own[index]; - if (properties == null) {return;} + if (properties == null) { + return; + } presentedOwn = Math.max(presentedOwn, local + offset); if (TRANSFORM.some(p => properties.has(p))) { presentedTransform = Math.max(presentedTransform, local + offset); @@ -452,37 +541,61 @@ export interface ActiveAnimation { readonly durationMs: number; readonly easing: string; readonly keyframes: readonly Record[]; + /** + * Epoch ms (page clock) when it finished, if it had already finished when + * it was reported; null while it runs. + */ + readonly finishedAt: number | null; } /** * Waits until an animation is running on `locator`'s element (or inside it) - * and has presented at least one frame, and returns its active interval on - * the page clock. Use it before blocking the main thread: an animation that - * has not started yet cannot start while the thread is blocked. + * and has presented `minProgressMs` of its curve, and returns its active + * interval on the page clock. Use it before blocking the main thread: an + * animation that has not started yet cannot start while the thread is blocked. + * + * It does not race a short animation: the page records every animation from + * its first frame (from the first call into this module on the page: this, + * `waitForRest`, `watchAnimations`, sampling or holding), so one that started + * after the last `waitForRest` or `waitForAnimation` and has already + * finished is reported too, with `finishedAt` set. For an action that may be + * the page's first contact with this module, take `watchAnimations(locator)` + * before the action instead. */ export async function waitForAnimation( locator: Locator, { timeout = 5_000, minProgressMs = 1, - }: {timeout?: number; minProgressMs?: number} = {}, + since, + }: { + timeout?: number; + minProgressMs?: number; + /** Page-clock ms (performance.now): only animations first seen after it. */ + since?: number; + } = {}, ): Promise { const deadline = Date.now() + timeout; await installCurveEnd(locator.page()); for (;;) { - const found = await locator.evaluate((element, minProgress) => { - const curveEnd = ( - window as unknown as {__harnessCurveEnd: (a: Animation) => number} - ).__harnessCurveEnd; - return element - .getAnimations({subtree: true}) - .filter( - a => - a.playState === 'running' && - a.startTime != null && - Number(a.currentTime ?? 0) >= minProgress, - ) - .map(a => { + const found = await locator.evaluate( + (element, {minProgress, since: watchedFrom}) => { + const w = window as unknown as { + __harnessCurveEnd: (a: Animation) => number; + __harnessAnimations: { + log: { + animation: Animation; + seenAt: number; + finishedAt: number | null; + }[]; + record: () => void; + armedAt: () => number; + }; + }; + const curveEnd = w.__harnessCurveEnd; + const recorder = w.__harnessAnimations; + recorder.record(); + const describe = (a: Animation, finishedAt: number | null) => { const effect = a.effect as KeyframeEffect | null; const timing = effect?.getComputedTiming(); const startedAt = performance.timeOrigin + Number(a.startTime); @@ -503,19 +616,109 @@ export async function waitForAnimation( keyframes: (effect?.getKeyframes() ?? []).map( (k): Record => ({...k}), ), + finishedAt: + finishedAt == null ? null : performance.timeOrigin + finishedAt, }; - }); - }, minProgressMs); + }; + const inside = (a: Animation) => { + const target = (a.effect as KeyframeEffect | null)?.target ?? null; + return ( + target != null && (target === element || element.contains(target)) + ); + }; + const after = watchedFrom ?? recorder.armedAt(); + const recorded = recorder.log.filter( + e => e.seenAt >= after && inside(e.animation), + ); + // Running: since the watch, or (without one) any on the element. + const running = ( + watchedFrom == null + ? element.getAnimations({subtree: true}) + : recorded.filter(e => e.finishedAt == null).map(e => e.animation) + ).filter(a => a.playState === 'running' && a.startTime != null); + const progressed = running.filter( + a => Number(a.currentTime ?? 0) >= minProgress, + ); + if (progressed.length > 0) { + return progressed.map(a => describe(a, null)); + } + if (running.length > 0) { + return []; + } + // Finished: by its finish event, or by its state when the event has not + // been delivered yet (then at its scheduled end). + const finishedAt = (e: (typeof recorded)[number]) => + e.finishedAt ?? + (e.animation.playState === 'finished' && e.animation.startTime != null + ? Number(e.animation.startTime) + + Number( + ( + e.animation.effect as KeyframeEffect | null + )?.getComputedTiming().endTime ?? 0, + ) + : null); + return recorded + .filter(e => finishedAt(e) != null) + .map(e => describe(e.animation, finishedAt(e))); + }, + {minProgress: minProgressMs, since: since ?? null}, + ); if (found.length > 0) { + await armRecorder(locator.page()); return found; } if (Date.now() > deadline) { - throw new Error('waitForAnimation: no animation started on the element'); + throw new Error( + 'waitForAnimation: no animation started on the element, none running ' + + 'and none recorded since the last wait. If the action is the ' + + "page's first contact with the harness, take " + + 'watchAnimations(locator) before it.', + ); } await new Promise(resolve => setTimeout(resolve, 4)); } } +/** Marks now as the point later waits count animations from. */ +async function armRecorder(page: Page): Promise { + return page.evaluate(() => + ( + window as unknown as {__harnessAnimations: {arm: () => number}} + ).__harnessAnimations.arm(), + ); +} + +export interface AnimationWatch { + /** + * Resolves with the animations on the element that started after the watch + * was taken: running ones once they have presented `minProgressMs`, or, + * when none runs, those that already finished. + */ + started(options?: { + timeout?: number; + minProgressMs?: number; + }): Promise; +} + +/** + * Starts watching `locator`'s element before an action, so the animation the + * action starts is caught even when it ends before the test looks (a short + * entrance on a cold page): + * + * const watch = await watchAnimations(sheet); + * await driver.play(tap(button)); + * const [entrance] = await watch.started(); + */ +export async function watchAnimations( + locator: Locator, +): Promise { + await installCurveEnd(locator.page()); + const since = await armRecorder(locator.page()); + return { + started: options => waitForAnimation(locator, {...options, since}), + }; +} + /** * Resolves once `locator`'s element is at rest: no animation on it (or inside * it) is pending or still before the end of its curve, and its presented @@ -572,6 +775,10 @@ export async function waitForRest( still = !moving && now === last ? still + 1 : 0; last = now; if (still >= need) { + // Later waits count animations from here. + ( + window as unknown as {__harnessAnimations?: {arm: () => number}} + ).__harnessAnimations?.arm(); resolve(); } else if (performance.now() > deadline) { reject(new Error(`waitForRest: still moving (${now})`)); diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index e5ce66b29df84..744c2c29cc512 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -49,6 +49,7 @@ import { touchStoryboard, waitForAnimation, waitForRest, + watchAnimations, withFinger, } from './index'; @@ -496,6 +497,60 @@ test.describe('animation state', () => { expect(parseTransform(transform).translateY).toBeCloseTo(100, 1); }); + test('waitForAnimation reports a short animation that ended before it looked', async ({ + page, + }) => { + await fixture(page); + const box = page.locator('#box'); + await waitForRest(box); + await box.evaluate(el => { + el.style.transition = 'transform 60ms linear'; + void (el as HTMLElement).offsetWidth; + el.style.transform = 'translateY(40px)'; + }); + // A cold page answers late: the transition is over before the first look. + await page.waitForTimeout(250); + const [ended] = await waitForAnimation(box); + expect(ended.kind).toBe('CSSTransition'); + expect(ended.durationMs).toBe(60); + expect(ended.finishedAt).not.toBeNull(); + expect(ended.finishedAt!).toBeGreaterThanOrEqual(ended.to - 1); + }); + + test("a watch taken before the action catches an entrance on the page's first contact", async ({ + page, + }) => { + await fixture(page); + const box = page.locator('#box'); + const watch = await watchAnimations(box); + await box.evaluate(el => + el.animate([{opacity: 0}, {opacity: 1}], {duration: 50}), + ); + await page.waitForTimeout(250); + const [entrance] = await watch.started(); + expect(entrance.kind).toBe('Animation'); + expect(entrance.durationMs).toBe(50); + expect(entrance.finishedAt).not.toBeNull(); + }); + + test('a watch reports only what started after it', async ({page}) => { + await fixture(page); + const box = page.locator('#box'); + await box.evaluate(el => + el.animate([{opacity: 0}, {opacity: 1}], {duration: 40}), + ); + await page.waitForTimeout(150); + const watch = await watchAnimations(box); + await expect(watch.started({timeout: 300})).rejects.toThrow( + /no animation started/, + ); + await box.evaluate(el => + el.animate([{opacity: 1}, {opacity: 0.5}], {duration: 70}), + ); + const started = await watch.started(); + expect(started.map(a => a.durationMs)).toEqual([70]); + }); + test('an end value held by the keyframes is not part of the curve, and individual transform properties are sampled', async ({ page, }) => { From d29981df9142b728d9091e8ec70067fff245c468 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sat, 24 Oct 2026 19:27:14 -0700 Subject: [PATCH 10/93] refactor(interaction): the storyboard layer moves to its own pull request This pull request keeps the curve-sampling instruments: holding and seeking animations, sampling presented values, the oracles, waiting for an animation and for rest. The storyboards (storyboard.ts, the evidence output, the timer gate and the `pnpm storyboards` script) and their tests move to a pull request stacked on this one. --- internal/interaction-harness/README.md | 41 +-- internal/interaction-harness/package.json | 2 +- internal/interaction-harness/src/curves.ts | 8 +- .../src/harness.interaction.browser.spec.ts | 264 +------------- internal/interaction-harness/src/index.ts | 2 - internal/interaction-harness/src/output.ts | 66 ---- internal/interaction-harness/src/story.ts | 96 +---- .../interaction-harness/src/storyboard.ts | 329 ------------------ package.json | 1 - playwright.interaction.config.ts | 6 +- 10 files changed, 20 insertions(+), 795 deletions(-) delete mode 100644 internal/interaction-harness/src/output.ts delete mode 100644 internal/interaction-harness/src/storyboard.ts diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 1066c22ffc374..4cf4778d5a5cd 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -1,7 +1,7 @@ # @astryxdesign/interaction-harness Playwright instruments for touch interaction and animation proofs: real touch -input, curve sampling and storyboards. **Internal and unpublished.** Specs live +input and curve sampling. **Internal and unpublished.** Specs live next to their component as `*.interaction.browser.spec.ts` and run under [`playwright.interaction.config.ts`](../../playwright.interaction.config.ts) on a phone-sized touch screen (402 × 714 CSS px page, 402 × 874 screen, 3 device @@ -18,7 +18,6 @@ pnpm storybook:build # component specs read stories from th pnpm exec playwright install chromium webkit pnpm test:interaction --project chromium # everything, Chromium pnpm test:interaction # one spec, both engines -pnpm storyboards # only the tests tagged @storyboard ``` On Linux distributions whose system libraries are older than Playwright's @@ -29,22 +28,15 @@ image (podman or docker; the repository is mounted at the same path): node internal/interaction-harness/scripts/webkit-container.mjs [playwright test args…] ``` -Evidence (storyboards, curve traces) is written to -`test-results/interaction-evidence/` (override with -`ASTRYX_INTERACTION_EVIDENCE`) and attached to the test report. It is never -committed: a pull request links or attaches it. - ## Instruments -| Module | What it gives a spec | -| --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact release velocity, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. | -| `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | -| `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t), `waitForAnimation`, `watchAnimations`, `waitForRest`, `releaseAnimations`. | -| `math.ts` | Oracles independent of the code under test: `cubicBezier`, `timingFunction`, `springStep` (SwiftUI response / dampingFraction, initial velocity), `projectDecay`, `rubberBand`, `parseTransform`, `compareCurve`, `timeline`. | -| `storyboard.ts` | `animationStoryboard` (seek to t = 0, 16.7, 33.3 … ms), `touchStoryboard` (a frame per scripted event): PNG contact sheet + JSON trace. | -| `output.ts` | `evidenceDir`, `writeEvidence`: where evidence goes and how it is attached to the report. | -| `story.ts` | `phone`, `withStorybook`, `openStory`, `frames`, and the timer gate (`installTimerGate`, `freezeTimers`, `thawTimers`). | +| Module | What it gives a spec | +| ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact release velocity, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. | +| `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | +| `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t), `waitForAnimation`, `watchAnimations`, `waitForRest`, `releaseAnimations`. | +| `math.ts` | Oracles independent of the code under test: `cubicBezier`, `timingFunction`, `springStep` (SwiftUI response / dampingFraction, initial velocity), `projectDecay`, `rubberBand`, `parseTransform`, `compareCurve`, `timeline`. | +| `story.ts` | `phone`, `withStorybook`, `openStory`, `frames`. | ## The proofs @@ -70,8 +62,8 @@ budget (for a spring baked into `linear()` stops of `h` ms, chords err by up to inside it, the last frame it presents, and the sample says so (`presentedAt`, where oracles are read). An animation whose keyframes end in a run of identical values (an end value held so a later interruption can read it) -ends where that run starts: `curveEndMs`, `waitForAnimation`'s interval and -the default storyboard times stop there, and `waitForRest`, which judges the +ends where that run starts: `curveEndMs` and `waitForAnimation`'s interval +stop there, and `waitForRest`, which judges the presented `transform`, `translate`, `rotate`, `scale` and `opacity` rather than `playState`, does not wait the hold out. @@ -85,17 +77,6 @@ set): a short entrance no longer races the first look on a cold page. When the action may be the page's first contact with the harness, take `watchAnimations(locator)` before it and read `watch.started()` after. -**Storyboards.** Deterministic: frames come from seeking held animations or -from stepping scripted input, never from wall-clock capture. The JSON trace -carries each frame's measurements and the spec asserts on it; the sheet is -evidence. A component that finishes an animation from a timer (a transitionend -backstop, a cleanup render) would do so behind a held storyboard's back: -`installTimerGate` before the animation starts and `freezeTimers` while -photographing. The page can still cancel a held callback, a held frame callback -gets the time it actually runs, and the harness's own frame waits are not held. -An animation that repeats forever does not set a storyboard's default length; -with nothing else to film, pass `times`. - ## Engine limits (measured with Playwright 1.61) | | Chromium 149 | WebKit 26.5 (Linux build) | @@ -106,7 +87,7 @@ with nothing else to film, pass `times`. | Native scroll, fling, pinch-zoom from injected touches | yes | no | | Lifting one finger while others stay down | yes | pointer events right, but that `touchend` lists no remaining `touches`: skip with `PARTIAL_LIFT_SKIP` | | Click synthesised from a touch | yes | only from `driver.tap` (WebKit's tap gesture), not from a raw touch sequence | -| Curve sampling, storyboards, input log | yes | yes | +| Curve sampling, input log | yes | yes | What WebKit cannot show here — native scroll hand-off, flings, a finger lifting while another stays down, and anything iOS-only — needs Mobile Safari diff --git a/internal/interaction-harness/package.json b/internal/interaction-harness/package.json index 40cccb26ca206..de2f8e87aa229 100644 --- a/internal/interaction-harness/package.json +++ b/internal/interaction-harness/package.json @@ -3,7 +3,7 @@ "version": "0.0.0", "private": true, "type": "module", - "description": "Internal: Playwright helpers for touch interaction and animation proofs (real touch input, curve sampling, storyboards). Never published.", + "description": "Internal: Playwright helpers for touch interaction and animation proofs (real touch input, curve sampling). Never published.", "main": "./src/index.ts", "exports": { ".": "./src/index.ts", diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index 3b0e572cf94b7..2dea7fe3597ae 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -753,13 +753,7 @@ export async function waitForRest( backdrop, ].join(' | '); }; - // The page's real frames, even while a timer gate holds its callbacks. - const frame = - ( - window as unknown as { - __harnessTimerGate?: {frame: typeof requestAnimationFrame}; - } - ).__harnessTimerGate?.frame ?? requestAnimationFrame; + const frame = requestAnimationFrame; let last = read(); let still = 0; const tick = () => { diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 744c2c29cc512..94007b3251463 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -22,17 +22,13 @@ import {expect, test, type Page} from '@playwright/test'; import { - animationStoryboard, compareCurve, cubicBezier, curveOf, drag, flick, - frames, - freezeTimers, holdAnimations, installInputLog, - installTimerGate, merge, parseTransform, PARTIAL_LIFT_SKIP, @@ -43,10 +39,8 @@ import { seekAnimations, springStep, tap, - thawTimers, timeline, touchDriver, - touchStoryboard, waitForAnimation, waitForRest, watchAnimations, @@ -514,7 +508,7 @@ test.describe('animation state', () => { expect(ended.kind).toBe('CSSTransition'); expect(ended.durationMs).toBe(60); expect(ended.finishedAt).not.toBeNull(); - expect(ended.finishedAt!).toBeGreaterThanOrEqual(ended.to - 1); + expect(ended.finishedAt ?? 0).toBeGreaterThanOrEqual(ended.to - 1); }); test("a watch taken before the action catches an entrance on the page's first contact", async ({ @@ -592,259 +586,3 @@ test.describe('animation state', () => { expect(end.transform.scaleY).toBeCloseTo(1.5, 3); }); }); - -test.describe('storyboards', {tag: '@storyboard'}, () => { - test('an animation filmstrip records every frame and its trace follows the curve', async ({ - page, - }, testInfo) => { - await fixture(page); - const box = page.locator('#box'); - await box.evaluate(el => - el.animate( - [{transform: 'translateX(0)'}, {transform: 'translateX(240px)'}], - {duration: 200, easing: 'ease-out'}, - ), - ); - const {storyboard} = await animationStoryboard(page, { - name: 'harness-self-test-animation', - clip: {x: 0, y: 340, width: 402, height: 120}, - measure: async () => ({ - x: await box.evaluate(el => el.getBoundingClientRect().x), - }), - columns: 4, - }); - const files = await storyboard.write(testInfo); - const xs = storyboard.frames.map(f => f.data.x as number); - expect(xs).toHaveLength(13); // t = 0, 16.7 … 200 - expect(xs[0]).toBeCloseTo(20, 1); - expect(xs.at(-1)).toBeCloseTo(260, 1); - expect(xs.every((x, i) => i === 0 || x >= xs[i - 1])).toBe(true); - expect(files['sheet.png']).toMatch(/sheet\.png$/); - }); - - test('a touch filmstrip photographs every scripted event', async ({ - page, - }, testInfo) => { - await fixture(page); - await page.evaluate(() => { - const pad = document.getElementById('pad'); - if (pad == null) { - throw new Error('no pad'); - } - const dot = document.createElement('div'); - dot.id = 'dot'; - dot.style.cssText = - 'position:absolute;width:24px;height:24px;border-radius:12px;background:#c33;left:-30px;top:0'; - pad.append(dot); - pad.addEventListener('pointermove', e => { - dot.style.transform = `translate(${e.clientX - 12 + 30}px, ${e.clientY - 12}px)`; - }); - }); - const driver = await touchDriver(page); - const storyboard = await touchStoryboard( - page, - driver, - drag({x: 60, y: 60}, {x: 300, y: 240}, {durationMs: 100, hz: 60}), - { - name: 'harness-self-test-touch', - clip: {x: 0, y: 0, width: 402, height: 300}, - measure: async () => ({ - x: await page - .locator('#dot') - .evaluate(el => el.getBoundingClientRect().x + 12), - }), - columns: 4, - }, - ); - await storyboard.write(testInfo); - // before + touchStart + 6 moves + touchEnd - expect(storyboard.frames).toHaveLength(9); - expect(storyboard.frames.at(-2)?.data.x).toBeCloseTo(300, 0); - }); - - test('the timer gate holds a component timer until the filmstrip is done', async ({ - page, - }, testInfo) => { - await fixture(page); - await installTimerGate(page); - const box = page.locator('#box'); - // A component that ends its own animation from a timer (a transitionend - // backstop): without the gate it would finish behind the held storyboard. - await box.evaluate(el => { - const animation = el.animate( - [{transform: 'translateX(0px)'}, {transform: 'translateX(200px)'}], - {duration: 300, easing: 'linear'}, - ); - setTimeout(() => { - animation.finish(); - (window as unknown as {timerRan?: boolean}).timerRan = true; - }, 40); - }); - await freezeTimers(page); - await page.waitForTimeout(120); // the timer is due now, and held - const {storyboard} = await animationStoryboard(page, { - name: 'harness-self-test-timer-gate', - clip: {x: 0, y: 340, width: 402, height: 120}, - measure: async () => ({ - x: await box.evaluate(el => el.getBoundingClientRect().x), - }), - columns: 6, - }); - const timerRan = () => - page.evaluate( - () => (window as unknown as {timerRan?: boolean}).timerRan === true, - ); - expect(await timerRan()).toBe(false); - await thawTimers(page); - expect(await timerRan()).toBe(true); - await storyboard.write(testInfo); - const xs = storyboard.frames.map(f => f.data.x as number); - expect(xs).toHaveLength(19); // t = 0 … 300 ms every 16.7 ms - expect(xs.every((x, i) => i === 0 || x > xs[i - 1])).toBe(true); - }); - - test('an animation that repeats forever does not set the filmstrip length', async ({ - page, - }, testInfo) => { - await fixture(page); - await page.locator('#box').evaluate(el => { - el.animate([{rotate: '0deg'}, {rotate: '360deg'}], { - duration: 800, - iterations: Infinity, - }); - el.animate([{opacity: 1}, {opacity: 0.4}], {duration: 200}); - }); - const {storyboard, held} = await animationStoryboard(page, { - name: 'harness-self-test-endless', - clip: {x: 0, y: 340, width: 402, height: 120}, - columns: 4, - }); - expect(held.map(a => Number.isFinite(a.curveEndMs))).toEqual([false, true]); - expect(storyboard.frames).toHaveLength(13); // t = 0 … 200 ms - await releaseAnimations(page); - await storyboard.write(testInfo); - }); - - test('a filmstrip runs from the earliest start to the latest end among the animations that end', async ({ - page, - }, testInfo) => { - await fixture(page); - // A spinner that started a second earlier, a 300 ms move, and a 240 ms - // move that starts 83 ms after it. - await page.evaluate(async () => { - const make = (id: string, top: number) => { - const el = document.createElement('div'); - el.id = id; - el.style.cssText = `position:absolute;left:0;top:${top}px;width:40px;height:40px;background:#36c`; - document.body.append(el); - return el; - }; - document - .getElementById('box') - ?.animate([{rotate: '0deg'}, {rotate: '360deg'}], { - duration: 800, - iterations: Infinity, - }); - await new Promise(resolve => setTimeout(resolve, 1000)); - make('first', 360).animate( - [{transform: 'translateX(0px)'}, {transform: 'translateX(300px)'}], - {duration: 300, easing: 'linear'}, - ); - await new Promise(resolve => setTimeout(resolve, 83)); - make('second', 410).animate( - [{transform: 'translateX(0px)'}, {transform: 'translateX(240px)'}], - {duration: 240, easing: 'linear'}, - ); - }); - const x = (id: string) => - page - .locator(`#${id}`) - .evaluate(el => new DOMMatrix(getComputedStyle(el).transform).m41); - const {storyboard, held} = await animationStoryboard(page, { - name: 'harness-self-test-offsets', - clip: {x: 0, y: 340, width: 402, height: 120}, - columns: 6, - measure: async () => ({ - first: await x('first'), - second: await x('second'), - }), - }); - const second = held.find(a => a.target === 'div#second'); - const first = held.find(a => a.target === 'div#first'); - const lag = (second?.offsetMs ?? 0) - (first?.offsetMs ?? 0); - const firsts = storyboard.frames.map(f => Number(f.data.first)); - const seconds = storyboard.frames.map(f => Number(f.data.second)); - // The film starts with the first move at its start, the spinner aside... - expect(firsts[0]).toBeCloseTo(0, 1); - expect(firsts[6]).toBeCloseTo(100, 0); - // ...and ends with the second move at its end, not cut off. - expect(seconds.at(-1)).toBeGreaterThan(239); - expect(storyboard.frames.at(-1)?.t).toBeCloseTo(lag + 240, 0); - await releaseAnimations(page); - await storyboard.write(testInfo); - }); - - test('a filmstrip of endless animations alone asks for its times', async ({ - page, - }) => { - await fixture(page); - await page.locator('#box').evaluate(el => - el.animate([{rotate: '0deg'}, {rotate: '360deg'}], { - duration: 800, - iterations: Infinity, - }), - ); - await expect( - animationStoryboard(page, { - name: 'harness-self-test-endless-only', - clip: {x: 0, y: 340, width: 402, height: 120}, - }), - ).rejects.toThrow(/repeats forever/); - await releaseAnimations(page); - }); - - test('the timer gate drops callbacks the page cancels and stamps held frames with the time they run', async ({ - page, - }) => { - await fixture(page); - await installTimerGate(page); - await freezeTimers(page); - await page.evaluate(() => { - const w = window as unknown as Record; - w.frozenAt = performance.now(); - w.timer = setTimeout(() => { - w.cancelledTimerRan = true; - }, 20); - w.frame = requestAnimationFrame(() => { - w.cancelledFrameRan = true; - }); - requestAnimationFrame(time => { - w.heldFrameTime = time; - }); - }); - await page.waitForTimeout(150); // both are due now, and held - // The harness's own frame waits still resolve while the page's are held. - await Promise.race([ - frames(page, 2), - page.waitForTimeout(5000).then(() => { - throw new Error('frames() did not resolve while the gate was frozen'); - }), - ]); - await page.evaluate(() => { - const w = window as unknown as {timer: number; frame: number}; - clearTimeout(w.timer); - cancelAnimationFrame(w.frame); - }); - await thawTimers(page); - const state = await page.evaluate(() => { - const w = window as unknown as Record; - return { - timer: w.cancelledTimerRan === true, - frame: w.cancelledFrameRan === true, - heldFor: Number(w.heldFrameTime) - Number(w.frozenAt), - }; - }); - expect(state).toMatchObject({timer: false, frame: false}); - expect(state.heldFor).toBeGreaterThanOrEqual(140); - }); -}); diff --git a/internal/interaction-harness/src/index.ts b/internal/interaction-harness/src/index.ts index 537bfc4d4beca..32c238e4a134a 100644 --- a/internal/interaction-harness/src/index.ts +++ b/internal/interaction-harness/src/index.ts @@ -14,8 +14,6 @@ export * from './curves'; export * from './math'; -export * from './output'; export * from './scripts'; export * from './story'; -export * from './storyboard'; export * from './touch'; diff --git a/internal/interaction-harness/src/output.ts b/internal/interaction-harness/src/output.ts deleted file mode 100644 index 59926b82c9aef..0000000000000 --- a/internal/interaction-harness/src/output.ts +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. - -/** - * @file output.ts - * @input Uses node:fs, node:path and the Playwright `TestInfo` - * @output `evidenceDir` / `writeEvidence` — where storyboards and curve traces - * go, and the one way they are written and attached to the test report. - * @position Every artifact the harness produces lands under one gitignored - * directory, `test-results/interaction-evidence/` by default - * (`ASTRYX_INTERACTION_EVIDENCE` overrides it). Playwright wipes its own - * output directory per run; this one is separate, so a storyboard run never - * deletes another's evidence. Generated evidence is never committed: a pull - * request links or attaches it. - */ - -import fs from 'node:fs'; -import path from 'node:path'; -import type {TestInfo} from '@playwright/test'; - -export function evidenceDir(): string { - return path.resolve( - process.env.ASTRYX_INTERACTION_EVIDENCE ?? - 'test-results/interaction-evidence', - ); -} - -/** A filesystem-safe name: letters, digits, dot, dash and underscore. */ -export function safeName(name: string): string { - return ( - name - .replace(/[^A-Za-z0-9._-]+/g, '-') - .replace(/^-+|-+$/g, '') - .slice(0, 120) || 'evidence' - ); -} - -/** - * Writes `files` (name → contents) under `//` and attaches - * them to the test's report when a `TestInfo` is given. Returns the paths. - */ -export async function writeEvidence( - group: string, - files: Readonly>, - testInfo?: TestInfo, -): Promise> { - // `group` may nest ("storyboards/"): each segment is made safe on its own. - const dir = path.join(evidenceDir(), ...group.split('/').map(safeName)); - fs.mkdirSync(dir, {recursive: true}); - const written: Record = {}; - for (const [name, contents] of Object.entries(files)) { - const file = path.join(dir, safeName(name)); - fs.writeFileSync(file, contents); - written[name] = file; - if (testInfo != null) { - await testInfo.attach(`${group}/${safeName(name)}`, { - path: file, - contentType: name.endsWith('.png') - ? 'image/png' - : name.endsWith('.json') - ? 'application/json' - : 'text/plain', - }); - } - } - return written; -} diff --git a/internal/interaction-harness/src/story.ts b/internal/interaction-harness/src/story.ts index 598f75d852cfe..9e4323436ff27 100644 --- a/internal/interaction-harness/src/story.ts +++ b/internal/interaction-harness/src/story.ts @@ -93,101 +93,11 @@ export async function frames(page: Page, count = 2): Promise { await page.evaluate( n => new Promise(resolve => { - // The page's real frames, even while a timer gate holds its callbacks. - const frame = - ( - window as unknown as { - __harnessTimerGate?: {frame: typeof requestAnimationFrame}; - } - ).__harnessTimerGate?.frame ?? requestAnimationFrame; let left = n; - const tick = () => (--left <= 0 ? resolve() : frame(tick)); - frame(tick); + const tick = () => + --left <= 0 ? resolve() : requestAnimationFrame(tick); + requestAnimationFrame(tick); }), count, ); } - -// A held callback is queued under its timer or frame id, so the page's own -// clearTimeout and cancelAnimationFrame still cancel it while frozen; a held -// frame callback gets the time it actually runs at. `frame` stays the real -// requestAnimationFrame, for the harness's own waits. -const TIMER_GATE = `(() => { - if (window.__harnessTimerGate) return; - const realTimeout = window.setTimeout.bind(window); - const realClearTimeout = window.clearTimeout.bind(window); - const realInterval = window.setInterval.bind(window); - const realFrame = window.requestAnimationFrame.bind(window); - const realCancelFrame = window.cancelAnimationFrame.bind(window); - const gate = {frozen: false, queue: [], frame: realFrame}; - window.__harnessTimerGate = gate; - const drop = key => { gate.queue = gate.queue.filter(entry => entry.key !== key); }; - window.setTimeout = (fn, ms, ...args) => { - const id = realTimeout(() => { - const run = () => (typeof fn === 'function' ? fn(...args) : 0); - if (gate.frozen) gate.queue.push({key: 't' + id, run}); else run(); - }, ms); - return id; - }; - window.clearTimeout = id => { realClearTimeout(id); drop('t' + id); }; - window.setInterval = (fn, ms, ...args) => realInterval(() => { if (!gate.frozen && typeof fn === 'function') fn(...args); }, ms); - window.requestAnimationFrame = fn => { - const id = realFrame(time => { - if (gate.frozen) gate.queue.push({key: 'f' + id, run: () => fn(performance.now())}); else fn(time); - }); - return id; - }; - window.cancelAnimationFrame = id => { realCancelFrame(id); drop('f' + id); }; -})()`; - -/** - * Installs a gate on the page's setTimeout / setInterval / - * requestAnimationFrame. Install it BEFORE the animation under test starts: - * only callbacks scheduled through the gate can be held. Event timestamps, - * Date, performance.now and the Web Animations timeline are untouched. - */ -export async function installTimerGate(page: Page): Promise { - await page.evaluate(TIMER_GATE); -} - -/** - * Holds every gated timer and frame callback until `thawTimers`, so a - * component cannot finish an animation behind a held storyboard's back (a - * transitionend backstop timer, a cleanup render). Callbacks that come due - * meanwhile run, in order, on thaw, unless the page cancels them first - * (clearTimeout, cancelAnimationFrame); intervals skip their frozen ticks. - */ -export async function freezeTimers(page: Page): Promise { - await page.evaluate(() => { - const gate = (window as unknown as {__harnessTimerGate?: {frozen: boolean}}) - .__harnessTimerGate; - if (gate == null) { - throw new Error( - 'freezeTimers: call installTimerGate before the animation starts', - ); - } - gate.frozen = true; - }); -} - -/** Releases the gate: held callbacks run now, in the order they came due. */ -export async function thawTimers(page: Page): Promise { - await page.evaluate(() => { - const gate = ( - window as unknown as { - __harnessTimerGate?: { - frozen: boolean; - queue: {key: string; run: () => void}[]; - }; - } - ).__harnessTimerGate; - if (gate == null) { - return; - } - gate.frozen = false; - // One at a time from the live queue: a callback may cancel a later one. - while (gate.queue.length > 0) { - gate.queue.shift()?.run(); - } - }); -} diff --git a/internal/interaction-harness/src/storyboard.ts b/internal/interaction-harness/src/storyboard.ts deleted file mode 100644 index 03d70d02222f0..0000000000000 --- a/internal/interaction-harness/src/storyboard.ts +++ /dev/null @@ -1,329 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. - -/** - * @file storyboard.ts - * @input Uses a Playwright `Page`, a clip (`Locator` or box), ./curves to seek - * animations, ./touch to play scripted touches, and ./output to write - * @output `Storyboard` (collect frames, write a PNG contact sheet + JSON trace), - * `animationStoryboard` (seek every held animation to t = 0, 16.7, 33.3 … ms - * and photograph each), `touchStoryboard` (photograph the page after every - * scripted touch event). - * @position Deterministic filmstrips: frames come from seeking paused - * animations or from stepping scripted input, never from wall-clock capture, - * so the same code produces the same sheet. The JSON trace carries the - * measured value of every frame and is what the spec asserts; the sheet is - * the human evidence attached to the report (and linked from a pull - * request, never committed). - * - * The contact sheet is composed in a scratch page of the same browser (a - * canvas grid with a label per cell), so the harness needs no image library. - * - * SYNC: Keep the README's storyboard section and `pnpm storyboards` in step. - */ - -import type {Locator, Page, TestInfo} from '@playwright/test'; -import {holdAnimations, seekAnimations, type HeldAnimation} from './curves'; -import {timeline} from './math'; -import {writeEvidence} from './output'; -import type {TouchScript} from './scripts'; -import {frames} from './story'; -import type {TouchDriver} from './touch'; - -export interface ClipBox { - readonly x: number; - readonly y: number; - readonly width: number; - readonly height: number; -} - -export interface StoryboardFrame { - readonly label: string; - /** Milliseconds on the storyboard's own timeline. */ - readonly t: number; - /** Measurements taken at this frame; the spec asserts these. */ - readonly data: Readonly>; - readonly png: Buffer; -} - -export interface StoryboardOptions { - readonly name: string; - /** What to photograph: an element, a box, or the whole viewport (default). */ - readonly clip?: Locator | ClipBox; - /** Grid columns in the sheet (default: up to 8). */ - readonly columns?: number; - /** Cell scale relative to CSS px (default 0.5). */ - readonly cellScale?: number; -} - -export class Storyboard { - readonly frames: StoryboardFrame[] = []; - private clipBox: ClipBox | undefined; - - constructor( - private readonly page: Page, - readonly options: StoryboardOptions, - ) {} - - /** Photographs the clip now and records `data` with it. */ - async capture( - label: string, - t: number, - data: Readonly> = {}, - ): Promise { - const clip = await this.resolveClip(); - const png = await this.page.screenshot({ - clip, - scale: 'css', - animations: 'allow', - caret: 'hide', - }); - this.frames.push({label, t, data, png}); - } - - private async resolveClip(): Promise { - if (this.clipBox != null) { - return this.clipBox; - } - const {clip} = this.options; - if (clip == null) { - return undefined; - } - if ('boundingBox' in clip) { - // A locator is measured once, at the first frame, so every cell shows the same region. - const box = await clip.boundingBox(); - if (box == null) { - throw new Error( - `storyboard ${this.options.name}: clip element is not visible`, - ); - } - const viewport = this.page.viewportSize(); - const x = Math.max(0, Math.floor(box.x)); - const y = Math.max(0, Math.floor(box.y)); - const right = Math.min( - viewport?.width ?? Infinity, - Math.ceil(box.x + box.width), - ); - const bottom = Math.min( - viewport?.height ?? Infinity, - Math.ceil(box.y + box.height), - ); - this.clipBox = {x, y, width: right - x, height: bottom - y}; - } else { - this.clipBox = clip; - } - return this.clipBox; - } - - /** The JSON trace: every frame's label, time and data. */ - trace( - extra: Readonly> = {}, - ): Record { - return { - name: this.options.name, - browser: this.page.context().browser()?.browserType().name() ?? null, - viewport: this.page.viewportSize(), - clip: this.clipBox ?? null, - ...extra, - frames: this.frames.map(({label, t, data}) => ({label, t, data})), - }; - } - - /** Composes the sheet, writes `sheet.png` + `trace.json`, attaches both. */ - async write( - testInfo?: TestInfo, - extra: Readonly> = {}, - ): Promise> { - if (this.frames.length === 0) { - throw new Error(`storyboard ${this.options.name}: no frames`); - } - const sheet = await composeSheet(this.page, this.frames, this.options); - const browser = - this.page.context().browser()?.browserType().name() ?? 'browser'; - return writeEvidence( - `storyboards/${this.options.name}.${browser}`, - { - 'sheet.png': sheet, - 'trace.json': JSON.stringify(this.trace(extra), null, 2), - }, - testInfo, - ); - } -} - -async function composeSheet( - page: Page, - frames: readonly StoryboardFrame[], - options: StoryboardOptions, -): Promise { - const scratch = await page.context().newPage(); - try { - const dataUrl = await scratch.evaluate( - async ({cells, columns, scale, title}) => { - const images = await Promise.all( - cells.map(async cell => - createImageBitmap( - await (await fetch(`data:image/png;base64,${cell.png}`)).blob(), - ), - ), - ); - const cellW = Math.ceil(images[0].width * scale); - const cellH = Math.ceil(images[0].height * scale); - const labelH = 30; - const pad = 8; - const cols = Math.min(columns, cells.length); - const rows = Math.ceil(cells.length / cols); - const canvas = document.createElement('canvas'); - canvas.width = pad + cols * (cellW + pad); - canvas.height = 28 + pad + rows * (cellH + labelH + pad); - const context = canvas.getContext('2d'); - if (context == null) { - throw new Error('no 2d context'); - } - context.fillStyle = '#ffffff'; - context.fillRect(0, 0, canvas.width, canvas.height); - context.fillStyle = '#111111'; - context.font = '600 14px sans-serif'; - context.fillText(title, pad, 19); - cells.forEach((cell, i) => { - const x = pad + (i % cols) * (cellW + pad); - const y = 28 + pad + Math.floor(i / cols) * (cellH + labelH + pad); - context.drawImage(images[i], x, y, cellW, cellH); - context.strokeStyle = '#999999'; - context.lineWidth = 1; - context.strokeRect(x - 0.5, y - 0.5, cellW + 1, cellH + 1); - context.fillStyle = '#111111'; - context.font = '600 11px sans-serif'; - context.fillText(cell.label, x, y + cellH + 13, cellW); - context.fillStyle = '#555555'; - context.font = '10px monospace'; - context.fillText(cell.caption, x, y + cellH + 26, cellW); - }); - return canvas.toDataURL('image/png'); - }, - { - cells: frames.map(f => ({ - png: f.png.toString('base64'), - label: f.label, - caption: Object.entries(f.data) - .filter( - ([, v]) => - typeof v === 'number' || - typeof v === 'string' || - typeof v === 'boolean', - ) - .map( - ([k, v]) => - `${k}=${typeof v === 'number' ? +v.toFixed(1) : String(v)}`, - ) - .join(' '), - })), - columns: options.columns ?? 8, - scale: options.cellScale ?? 0.5, - title: options.name, - }, - ); - return Buffer.from(dataUrl.slice(dataUrl.indexOf(',') + 1), 'base64'); - } finally { - await scratch.close(); - } -} - -export interface AnimationStoryboardOptions extends StoryboardOptions { - /** - * Frame times, ms from the film's start: the earliest start among the held - * animations that end (an animation that repeats forever, such as a - * spinner that started earlier, neither starts nor ends the film). Default: - * every 16.7 ms to the latest curve end among them, and that end itself, - * each animation at its own offset (a held end value is not filmed). - */ - readonly times?: readonly number[]; - /** Restrict to animations inside this element. */ - readonly scope?: Locator; - /** Measurements per frame (runs after the seek). */ - readonly measure?: () => Promise>>; -} - -/** - * Holds every running animation, then seeks them together to each time and - * photographs the clip. Call it right after starting the animation under test. - */ -export async function animationStoryboard( - page: Page, - options: AnimationStoryboardOptions, -): Promise<{storyboard: Storyboard; held: HeldAnimation[]}> { - const held = await holdAnimations(page, options.scope); - if (held.length === 0) { - throw new Error(`storyboard ${options.name}: no running animation to hold`); - } - // An animation that repeats forever (a spinner, a pulsing skeleton) has no - // end to film to: the film runs over the animations that end, from the - // earliest of their starts to the latest of their curve ends. Seeks are on - // the held timeline (from the earliest held start), so both carry offsets. - const ending = held.filter(a => Number.isFinite(a.curveEndMs)); - if (options.times == null && ending.length === 0) { - throw new Error( - `storyboard ${options.name}: every held animation repeats forever ` + - `(${held.map(a => a.name ?? a.kind).join(', ')}); pass \`times\``, - ); - } - const origin = ending.length ? Math.min(...ending.map(a => a.offsetMs)) : 0; - const length = - Math.max(...ending.map(a => a.offsetMs + a.curveEndMs)) - origin; - // The 16.7 ms grid, plus the end itself when the grid misses it: the last - // frame is the one the page presents at the end. - const grid = options.times == null ? timeline(length) : []; - if (options.times == null && length - (grid.at(-1) ?? 0) > 1e-3) { - grid.push(+length.toFixed(3)); - } - const times = options.times ?? grid; - const storyboard = new Storyboard(page, options); - for (const t of times) { - // Labelled with the time the page presents (just inside an end), from - // the film's start. - const presentedAt = (await seekAnimations(page, origin + t)) - origin; - const data = (await options.measure?.()) ?? {}; - await storyboard.capture( - `t=${+presentedAt.toFixed(1)} ms`, - presentedAt, - data, - ); - } - return {storyboard, held}; -} - -export interface TouchStoryboardOptions extends StoryboardOptions { - readonly measure?: () => Promise>>; -} - -/** - * Plays a touch script without wall-clock pacing and photographs the page - * after every event. In Chromium the page still sees the scripted timestamps, - * so velocity-dependent behavior is unchanged; in WebKit the page sees the - * real (slower) dispatch times. - */ -export async function touchStoryboard( - page: Page, - driver: TouchDriver, - script: TouchScript, - options: TouchStoryboardOptions, -): Promise { - const storyboard = new Storyboard(page, options); - await storyboard.capture('before', 0, (await options.measure?.()) ?? {}); - await driver.play(script, { - realtime: false, - onEvent: async step => { - // Let the page run its handlers and present the frame before the photo. - await frames(page, 2); - const data = (await options.measure?.()) ?? {}; - const where = step.changed - .map(f => `${Math.round(f.x)},${Math.round(f.y)}`) - .join(' '); - await storyboard.capture( - `${step.type} t=${+step.t.toFixed(1)} @${where}`, - step.t, - data, - ); - }, - }); - return storyboard; -} diff --git a/package.json b/package.json index e49d57a474542..a4ca9f2ae830e 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,6 @@ "test:a11y-contract": "playwright test --config playwright.config.ts --project chromium", "test:docsite-browser": "playwright test --config apps/docsite/playwright.config.ts", "test:interaction": "playwright test --config playwright.interaction.config.ts", - "storyboards": "playwright test --config playwright.interaction.config.ts --grep @storyboard", "check:package-boundaries": "node scripts/check-package-boundaries.js", "check:repo": "pnpm check:visual-owner && pnpm check:sync && pnpm check:neutral-palette-refs && pnpm check:stone-palette-refs && pnpm check:gothic-palette-refs && pnpm check:matcha-palette-refs && pnpm check:butter-palette-refs && pnpm check:y2k-palette-refs && pnpm check:data-tokens && pnpm check:token-docs && pnpm check:knowledge && pnpm check:package-boundaries && pnpm check:changesets && pnpm check:demo-media && pnpm check:executable-bits && pnpm check:cli-structure && pnpm check:use-client && pnpm check:portable-scripts && pnpm check:i18n-runtime && pnpm check:i18n-catalog && pnpm check:cldr-weekdays && pnpm check:fixtures && pnpm check:setup-contract", "check:visual-owner": "node scripts/check-visual-workflow-owner.mjs", diff --git a/playwright.interaction.config.ts b/playwright.interaction.config.ts index f0c61e2e6e563..9d8a4f00452f4 100644 --- a/playwright.interaction.config.ts +++ b/playwright.interaction.config.ts @@ -5,9 +5,9 @@ * @input Uses @playwright/test * @output Touch interaction and animation proofs on a phone-sized touch * screen, in Chromium and WebKit. - * @position Run with `pnpm test:interaction`; filmstrips only with - * `pnpm storyboards`. Separate from `pnpm test` (no browser needed there) and - * from playwright.config.ts (accessibility contracts on desktop profiles). + * @position Run with `pnpm test:interaction`. Separate from `pnpm test` (no + * browser needed there) and from playwright.config.ts (accessibility + * contracts on desktop profiles). * * Specs are named `*.interaction.browser.spec.ts` and read stories from a * built Storybook, like the accessibility specs: From beecc802b3c6fad6a230535818df604a9681b19a Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sat, 24 Oct 2026 21:21:22 -0700 Subject: [PATCH 11/93] test(interaction): a second hold adds to the held set `holdAnimations` replaced the held set on each call, so a second hold left the first set paused after `releaseAnimations` (it resumed only the newest set), and `waitForRest` then reported the frozen frame as rest. A later hold now adds to the set: everything held stays held until the release, and offsets count from the earliest start of all of it. The self-test holds two elements in turn, releases, and checks the first runs to its end; on the old code it stays frozen 50 px into a 300 px move. The self-test's header and the `curves.ts` SYNC line no longer describe the storyboard and timer-gate layer, which moved to its own pull request. --- internal/interaction-harness/src/curves.ts | 35 ++++++++------- .../src/harness.interaction.browser.spec.ts | 45 ++++++++++++++++--- 2 files changed, 59 insertions(+), 21 deletions(-) diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index 2dea7fe3597ae..ed4fe0df69a84 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -16,8 +16,6 @@ * fixed time, independent of frame timing, so a curve can be compared with * the UIKit reference to a fraction of a pixel. A paused animation's effect * is applied at its current time and `getComputedStyle` reads it. - * - * SYNC: Keep ./storyboard.ts using these for its animation filmstrips. */ import type {Locator, Page} from '@playwright/test'; @@ -190,7 +188,8 @@ export interface HeldAnimation { * animations the page paused, and animations on a scroll or view timeline are * left as they are. Later `seekAnimations` calls move exactly the held set, * each by its offset from the earliest start, and `releaseAnimations` resumes - * only it. + * only it. A second call adds to the held set: what the first held stays held + * until the release, and offsets count from the earliest start of all of it. */ export async function holdAnimations( page: Page, @@ -228,18 +227,20 @@ export async function holdAnimations( ? Number(animation.startTime) : now - Number(animation.currentTime ?? 0), ); - const earliest = Math.min(...starts); - const offsets = starts.map(start => start - earliest); const held = all.map(animation => { animation.pause(); return animation; }); + // A later hold adds to the held set: what an earlier one held stays held + // (it is paused, so it is not taken again above) until released. const w = window as unknown as { - __harnessHeld: Animation[]; - __harnessHeldOffsets: number[]; + __harnessHeld?: Animation[]; + __harnessHeldStarts?: number[]; }; - w.__harnessHeld = held; - w.__harnessHeldOffsets = offsets; + w.__harnessHeld = [...(w.__harnessHeld ?? []), ...held]; + w.__harnessHeldStarts = [...(w.__harnessHeldStarts ?? []), ...starts]; + const earliest = Math.min(...w.__harnessHeldStarts); + const offsets = starts.map(start => start - earliest); return held.map((animation, index) => { const effect = animation.effect as KeyframeEffect | null; const timing = effect?.getComputedTiming(); @@ -288,11 +289,13 @@ export async function seekAnimations( ({t, inset}) => { const w = window as unknown as { __harnessHeld?: Animation[]; - __harnessHeldOffsets?: number[]; + __harnessHeldStarts?: number[]; }; + const starts = w.__harnessHeldStarts ?? []; + const earliest = Math.min(...starts); let presented = -Infinity; (w.__harnessHeld ?? []).forEach((animation, index) => { - const offset = w.__harnessHeldOffsets?.[index] ?? 0; + const offset = (starts[index] ?? earliest) - earliest; const end = Number( animation.effect?.getComputedTiming().endTime ?? Infinity, ); @@ -319,7 +322,7 @@ export async function releaseAnimations( await page.evaluate(shouldFinish => { const w = window as unknown as { __harnessHeld?: Animation[]; - __harnessHeldOffsets?: number[]; + __harnessHeldStarts?: number[]; }; const held = w.__harnessHeld ?? []; try { @@ -335,7 +338,7 @@ export async function releaseAnimations( } } finally { w.__harnessHeld = []; - w.__harnessHeldOffsets = []; + w.__harnessHeldStarts = []; } }, finish); } @@ -390,9 +393,11 @@ export async function sampleCurve( (element, {ts, inset}) => { const w = window as unknown as { __harnessHeld?: Animation[]; - __harnessHeldOffsets?: number[]; + __harnessHeldStarts?: number[]; }; const held = w.__harnessHeld ?? []; + const starts = w.__harnessHeldStarts ?? []; + const earliest = Math.min(...starts); if (held.length === 0) { throw new Error( 'sampleCurve: no held animations (call holdAnimations first)', @@ -420,7 +425,7 @@ export async function sampleCurve( let presentedOpacity = -Infinity; held.forEach((animation, index) => { // Each at its own offset, just inside its end: see seekAnimations. - const offset = w.__harnessHeldOffsets?.[index] ?? 0; + const offset = (starts[index] ?? earliest) - earliest; const end = Number( animation.effect?.getComputedTiming().endTime ?? Infinity, ); diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 94007b3251463..0caeb7665653a 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -10,12 +10,8 @@ * and its end, and judge rest by presented values even while an animation * holds its end; individual transform properties are sampled; a sample at * an end reports the time it was read at; holding takes only running - * document-timeline animations, each at its own offset, and releases only - * them; storyboards write a sheet and a trace that follows the curve, and an - * endless animation does not set their length; the timer gate holds a - * component timer until the filmstrip is done, drops callbacks the page - * cancels, stamps held frames with the time they run, and leaves the - * harness its own frames. + * document-timeline animations, each at its own offset, adds a second hold + * to the held set, and releases only them. * @position The harness's own test. A component spec trusts these * instruments; this file is why it may. */ @@ -438,6 +434,43 @@ test.describe('holding', () => { await releaseAnimations(page); }); + test('a second hold adds to the held set, and releasing resumes both', async ({ + page, + }) => { + await fixture(page); + await page.evaluate(() => { + document + .getElementById('box') + ?.animate( + [{transform: 'translateX(0px)'}, {transform: 'translateX(300px)'}], + {duration: 300, easing: 'linear', fill: 'forwards'}, + ); + const other = document.createElement('div'); + other.id = 'other'; + document.body.append(other); + other.animate([{opacity: 0}, {opacity: 1}], { + duration: 300, + fill: 'forwards', + }); + }); + const box = page.locator('#box'); + expect(await holdAnimations(page, box)).toHaveLength(1); + expect(await holdAnimations(page, page.locator('#other'))).toHaveLength(1); + await releaseAnimations(page); + // The first hold was released too: the move runs to its end, and rest is + // its end, not the frame it was held on. + await waitForRest(box); + expect(await box.evaluate(el => getComputedStyle(el).transform)).toBe( + 'matrix(1, 0, 0, 1, 300, 0)', + ); + expect( + await page.evaluate( + () => + document.getAnimations().filter(a => a.playState === 'paused').length, + ), + ).toBe(0); + }); + test('finishing resumes an animation that repeats forever and still clears the held set', async ({ page, }) => { From 4a6dfbea97feab6c5a928af1730cc3aa45f2c033 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sat, 24 Oct 2026 19:28:58 -0700 Subject: [PATCH 12/93] test(interaction): deterministic storyboards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit animationStoryboard films held animations by seeking them (t = 0, 16.7, 33.3 … ms, from the earliest start to the latest curve end among the animations that end, each at its own offset); touchStoryboard takes a frame per scripted touch event. Each writes a PNG contact sheet and a JSON trace of its measurements to the evidence directory and attaches them to the report; the spec asserts on the trace. The timer gate (installTimerGate, freezeTimers, thawTimers) holds the page's timers and frame callbacks while a storyboard photographs, so a component that finishes an animation from a timer cannot do it behind the held animation's back; the page can still cancel a held callback, and the harness's own frame waits (and the animation recorder) keep the page's real requestAnimationFrame. `pnpm storyboards` runs only the tests tagged @storyboard. --- internal/interaction-harness/README.md | 41 ++- internal/interaction-harness/package.json | 2 +- internal/interaction-harness/src/curves.ts | 19 +- .../src/harness.interaction.browser.spec.ts | 268 +++++++++++++- internal/interaction-harness/src/index.ts | 2 + internal/interaction-harness/src/output.ts | 66 ++++ internal/interaction-harness/src/story.ts | 96 ++++- .../interaction-harness/src/storyboard.ts | 329 ++++++++++++++++++ package.json | 1 + playwright.interaction.config.ts | 6 +- 10 files changed, 809 insertions(+), 21 deletions(-) create mode 100644 internal/interaction-harness/src/output.ts create mode 100644 internal/interaction-harness/src/storyboard.ts diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 4cf4778d5a5cd..1066c22ffc374 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -1,7 +1,7 @@ # @astryxdesign/interaction-harness Playwright instruments for touch interaction and animation proofs: real touch -input and curve sampling. **Internal and unpublished.** Specs live +input, curve sampling and storyboards. **Internal and unpublished.** Specs live next to their component as `*.interaction.browser.spec.ts` and run under [`playwright.interaction.config.ts`](../../playwright.interaction.config.ts) on a phone-sized touch screen (402 × 714 CSS px page, 402 × 874 screen, 3 device @@ -18,6 +18,7 @@ pnpm storybook:build # component specs read stories from th pnpm exec playwright install chromium webkit pnpm test:interaction --project chromium # everything, Chromium pnpm test:interaction # one spec, both engines +pnpm storyboards # only the tests tagged @storyboard ``` On Linux distributions whose system libraries are older than Playwright's @@ -28,15 +29,22 @@ image (podman or docker; the repository is mounted at the same path): node internal/interaction-harness/scripts/webkit-container.mjs [playwright test args…] ``` +Evidence (storyboards, curve traces) is written to +`test-results/interaction-evidence/` (override with +`ASTRYX_INTERACTION_EVIDENCE`) and attached to the test report. It is never +committed: a pull request links or attaches it. + ## Instruments -| Module | What it gives a spec | -| ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact release velocity, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. | -| `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | -| `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t), `waitForAnimation`, `watchAnimations`, `waitForRest`, `releaseAnimations`. | -| `math.ts` | Oracles independent of the code under test: `cubicBezier`, `timingFunction`, `springStep` (SwiftUI response / dampingFraction, initial velocity), `projectDecay`, `rubberBand`, `parseTransform`, `compareCurve`, `timeline`. | -| `story.ts` | `phone`, `withStorybook`, `openStory`, `frames`. | +| Module | What it gives a spec | +| --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact release velocity, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. | +| `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | +| `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t), `waitForAnimation`, `watchAnimations`, `waitForRest`, `releaseAnimations`. | +| `math.ts` | Oracles independent of the code under test: `cubicBezier`, `timingFunction`, `springStep` (SwiftUI response / dampingFraction, initial velocity), `projectDecay`, `rubberBand`, `parseTransform`, `compareCurve`, `timeline`. | +| `storyboard.ts` | `animationStoryboard` (seek to t = 0, 16.7, 33.3 … ms), `touchStoryboard` (a frame per scripted event): PNG contact sheet + JSON trace. | +| `output.ts` | `evidenceDir`, `writeEvidence`: where evidence goes and how it is attached to the report. | +| `story.ts` | `phone`, `withStorybook`, `openStory`, `frames`, and the timer gate (`installTimerGate`, `freezeTimers`, `thawTimers`). | ## The proofs @@ -62,8 +70,8 @@ budget (for a spring baked into `linear()` stops of `h` ms, chords err by up to inside it, the last frame it presents, and the sample says so (`presentedAt`, where oracles are read). An animation whose keyframes end in a run of identical values (an end value held so a later interruption can read it) -ends where that run starts: `curveEndMs` and `waitForAnimation`'s interval -stop there, and `waitForRest`, which judges the +ends where that run starts: `curveEndMs`, `waitForAnimation`'s interval and +the default storyboard times stop there, and `waitForRest`, which judges the presented `transform`, `translate`, `rotate`, `scale` and `opacity` rather than `playState`, does not wait the hold out. @@ -77,6 +85,17 @@ set): a short entrance no longer races the first look on a cold page. When the action may be the page's first contact with the harness, take `watchAnimations(locator)` before it and read `watch.started()` after. +**Storyboards.** Deterministic: frames come from seeking held animations or +from stepping scripted input, never from wall-clock capture. The JSON trace +carries each frame's measurements and the spec asserts on it; the sheet is +evidence. A component that finishes an animation from a timer (a transitionend +backstop, a cleanup render) would do so behind a held storyboard's back: +`installTimerGate` before the animation starts and `freezeTimers` while +photographing. The page can still cancel a held callback, a held frame callback +gets the time it actually runs, and the harness's own frame waits are not held. +An animation that repeats forever does not set a storyboard's default length; +with nothing else to film, pass `times`. + ## Engine limits (measured with Playwright 1.61) | | Chromium 149 | WebKit 26.5 (Linux build) | @@ -87,7 +106,7 @@ action may be the page's first contact with the harness, take | Native scroll, fling, pinch-zoom from injected touches | yes | no | | Lifting one finger while others stay down | yes | pointer events right, but that `touchend` lists no remaining `touches`: skip with `PARTIAL_LIFT_SKIP` | | Click synthesised from a touch | yes | only from `driver.tap` (WebKit's tap gesture), not from a raw touch sequence | -| Curve sampling, input log | yes | yes | +| Curve sampling, storyboards, input log | yes | yes | What WebKit cannot show here — native scroll hand-off, flings, a finger lifting while another stays down, and anything iOS-only — needs Mobile Safari diff --git a/internal/interaction-harness/package.json b/internal/interaction-harness/package.json index de2f8e87aa229..40cccb26ca206 100644 --- a/internal/interaction-harness/package.json +++ b/internal/interaction-harness/package.json @@ -3,7 +3,7 @@ "version": "0.0.0", "private": true, "type": "module", - "description": "Internal: Playwright helpers for touch interaction and animation proofs (real touch input, curve sampling). Never published.", + "description": "Internal: Playwright helpers for touch interaction and animation proofs (real touch input, curve sampling, storyboards). Never published.", "main": "./src/index.ts", "exports": { ".": "./src/index.ts", diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index ed4fe0df69a84..47fdba03078cd 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -16,6 +16,8 @@ * fixed time, independent of frame timing, so a curve can be compared with * the UIKit reference to a fraction of a pixel. A paused animation's effect * is applied at its current time and `getComputedStyle` reads it. + * + * SYNC: Keep ./storyboard.ts using these for its animation filmstrips. */ import type {Locator, Page} from '@playwright/test'; @@ -42,7 +44,14 @@ function animationRecorder(): void { if (w.__harnessAnimations) { return; } - const frame = window.requestAnimationFrame.bind(window); + // The page's real frames, even when a timer gate already holds its callbacks. + const frame = ( + ( + window as unknown as { + __harnessTimerGate?: {frame: typeof requestAnimationFrame}; + } + ).__harnessTimerGate?.frame ?? window.requestAnimationFrame + ).bind(window); const seen = new WeakSet(); const log: { animation: Animation; @@ -758,7 +767,13 @@ export async function waitForRest( backdrop, ].join(' | '); }; - const frame = requestAnimationFrame; + // The page's real frames, even while a timer gate holds its callbacks. + const frame = + ( + window as unknown as { + __harnessTimerGate?: {frame: typeof requestAnimationFrame}; + } + ).__harnessTimerGate?.frame ?? requestAnimationFrame; let last = read(); let still = 0; const tick = () => { diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 0caeb7665653a..7a2e12ca7ae0f 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -11,20 +11,28 @@ * holds its end; individual transform properties are sampled; a sample at * an end reports the time it was read at; holding takes only running * document-timeline animations, each at its own offset, adds a second hold - * to the held set, and releases only them. + * to the held set, and releases only them; storyboards write a sheet and a + * trace that follows the curve, and an endless animation does not set their + * length; the timer gate holds a component timer until the filmstrip is + * done, drops callbacks the page cancels, stamps held frames with the time + * they run, and leaves the harness its own frames. * @position The harness's own test. A component spec trusts these * instruments; this file is why it may. */ import {expect, test, type Page} from '@playwright/test'; import { + animationStoryboard, compareCurve, cubicBezier, curveOf, drag, flick, + frames, + freezeTimers, holdAnimations, installInputLog, + installTimerGate, merge, parseTransform, PARTIAL_LIFT_SKIP, @@ -35,8 +43,10 @@ import { seekAnimations, springStep, tap, + thawTimers, timeline, touchDriver, + touchStoryboard, waitForAnimation, waitForRest, watchAnimations, @@ -619,3 +629,259 @@ test.describe('animation state', () => { expect(end.transform.scaleY).toBeCloseTo(1.5, 3); }); }); + +test.describe('storyboards', {tag: '@storyboard'}, () => { + test('an animation filmstrip records every frame and its trace follows the curve', async ({ + page, + }, testInfo) => { + await fixture(page); + const box = page.locator('#box'); + await box.evaluate(el => + el.animate( + [{transform: 'translateX(0)'}, {transform: 'translateX(240px)'}], + {duration: 200, easing: 'ease-out'}, + ), + ); + const {storyboard} = await animationStoryboard(page, { + name: 'harness-self-test-animation', + clip: {x: 0, y: 340, width: 402, height: 120}, + measure: async () => ({ + x: await box.evaluate(el => el.getBoundingClientRect().x), + }), + columns: 4, + }); + const files = await storyboard.write(testInfo); + const xs = storyboard.frames.map(f => f.data.x as number); + expect(xs).toHaveLength(13); // t = 0, 16.7 … 200 + expect(xs[0]).toBeCloseTo(20, 1); + expect(xs.at(-1)).toBeCloseTo(260, 1); + expect(xs.every((x, i) => i === 0 || x >= xs[i - 1])).toBe(true); + expect(files['sheet.png']).toMatch(/sheet\.png$/); + }); + + test('a touch filmstrip photographs every scripted event', async ({ + page, + }, testInfo) => { + await fixture(page); + await page.evaluate(() => { + const pad = document.getElementById('pad'); + if (pad == null) { + throw new Error('no pad'); + } + const dot = document.createElement('div'); + dot.id = 'dot'; + dot.style.cssText = + 'position:absolute;width:24px;height:24px;border-radius:12px;background:#c33;left:-30px;top:0'; + pad.append(dot); + pad.addEventListener('pointermove', e => { + dot.style.transform = `translate(${e.clientX - 12 + 30}px, ${e.clientY - 12}px)`; + }); + }); + const driver = await touchDriver(page); + const storyboard = await touchStoryboard( + page, + driver, + drag({x: 60, y: 60}, {x: 300, y: 240}, {durationMs: 100, hz: 60}), + { + name: 'harness-self-test-touch', + clip: {x: 0, y: 0, width: 402, height: 300}, + measure: async () => ({ + x: await page + .locator('#dot') + .evaluate(el => el.getBoundingClientRect().x + 12), + }), + columns: 4, + }, + ); + await storyboard.write(testInfo); + // before + touchStart + 6 moves + touchEnd + expect(storyboard.frames).toHaveLength(9); + expect(storyboard.frames.at(-2)?.data.x).toBeCloseTo(300, 0); + }); + + test('the timer gate holds a component timer until the filmstrip is done', async ({ + page, + }, testInfo) => { + await fixture(page); + await installTimerGate(page); + const box = page.locator('#box'); + // A component that ends its own animation from a timer (a transitionend + // backstop): without the gate it would finish behind the held storyboard. + await box.evaluate(el => { + const animation = el.animate( + [{transform: 'translateX(0px)'}, {transform: 'translateX(200px)'}], + {duration: 300, easing: 'linear'}, + ); + setTimeout(() => { + animation.finish(); + (window as unknown as {timerRan?: boolean}).timerRan = true; + }, 40); + }); + await freezeTimers(page); + await page.waitForTimeout(120); // the timer is due now, and held + const {storyboard} = await animationStoryboard(page, { + name: 'harness-self-test-timer-gate', + clip: {x: 0, y: 340, width: 402, height: 120}, + measure: async () => ({ + x: await box.evaluate(el => el.getBoundingClientRect().x), + }), + columns: 6, + }); + const timerRan = () => + page.evaluate( + () => (window as unknown as {timerRan?: boolean}).timerRan === true, + ); + expect(await timerRan()).toBe(false); + await thawTimers(page); + expect(await timerRan()).toBe(true); + await storyboard.write(testInfo); + const xs = storyboard.frames.map(f => f.data.x as number); + expect(xs).toHaveLength(19); // t = 0 … 300 ms every 16.7 ms + expect(xs.every((x, i) => i === 0 || x > xs[i - 1])).toBe(true); + }); + + test('an animation that repeats forever does not set the filmstrip length', async ({ + page, + }, testInfo) => { + await fixture(page); + await page.locator('#box').evaluate(el => { + el.animate([{rotate: '0deg'}, {rotate: '360deg'}], { + duration: 800, + iterations: Infinity, + }); + el.animate([{opacity: 1}, {opacity: 0.4}], {duration: 200}); + }); + const {storyboard, held} = await animationStoryboard(page, { + name: 'harness-self-test-endless', + clip: {x: 0, y: 340, width: 402, height: 120}, + columns: 4, + }); + expect(held.map(a => Number.isFinite(a.curveEndMs))).toEqual([false, true]); + expect(storyboard.frames).toHaveLength(13); // t = 0 … 200 ms + await releaseAnimations(page); + await storyboard.write(testInfo); + }); + + test('a filmstrip runs from the earliest start to the latest end among the animations that end', async ({ + page, + }, testInfo) => { + await fixture(page); + // A spinner that started a second earlier, a 300 ms move, and a 240 ms + // move that starts 83 ms after it. + await page.evaluate(async () => { + const make = (id: string, top: number) => { + const el = document.createElement('div'); + el.id = id; + el.style.cssText = `position:absolute;left:0;top:${top}px;width:40px;height:40px;background:#36c`; + document.body.append(el); + return el; + }; + document + .getElementById('box') + ?.animate([{rotate: '0deg'}, {rotate: '360deg'}], { + duration: 800, + iterations: Infinity, + }); + await new Promise(resolve => setTimeout(resolve, 1000)); + make('first', 360).animate( + [{transform: 'translateX(0px)'}, {transform: 'translateX(300px)'}], + {duration: 300, easing: 'linear'}, + ); + await new Promise(resolve => setTimeout(resolve, 83)); + make('second', 410).animate( + [{transform: 'translateX(0px)'}, {transform: 'translateX(240px)'}], + {duration: 240, easing: 'linear'}, + ); + }); + const x = (id: string) => + page + .locator(`#${id}`) + .evaluate(el => new DOMMatrix(getComputedStyle(el).transform).m41); + const {storyboard, held} = await animationStoryboard(page, { + name: 'harness-self-test-offsets', + clip: {x: 0, y: 340, width: 402, height: 120}, + columns: 6, + measure: async () => ({ + first: await x('first'), + second: await x('second'), + }), + }); + const second = held.find(a => a.target === 'div#second'); + const first = held.find(a => a.target === 'div#first'); + const lag = (second?.offsetMs ?? 0) - (first?.offsetMs ?? 0); + const firsts = storyboard.frames.map(f => Number(f.data.first)); + const seconds = storyboard.frames.map(f => Number(f.data.second)); + // The film starts with the first move at its start, the spinner aside... + expect(firsts[0]).toBeCloseTo(0, 1); + expect(firsts[6]).toBeCloseTo(100, 0); + // ...and ends with the second move at its end, not cut off. + expect(seconds.at(-1)).toBeGreaterThan(239); + expect(storyboard.frames.at(-1)?.t).toBeCloseTo(lag + 240, 0); + await releaseAnimations(page); + await storyboard.write(testInfo); + }); + + test('a filmstrip of endless animations alone asks for its times', async ({ + page, + }) => { + await fixture(page); + await page.locator('#box').evaluate(el => + el.animate([{rotate: '0deg'}, {rotate: '360deg'}], { + duration: 800, + iterations: Infinity, + }), + ); + await expect( + animationStoryboard(page, { + name: 'harness-self-test-endless-only', + clip: {x: 0, y: 340, width: 402, height: 120}, + }), + ).rejects.toThrow(/repeats forever/); + await releaseAnimations(page); + }); + + test('the timer gate drops callbacks the page cancels and stamps held frames with the time they run', async ({ + page, + }) => { + await fixture(page); + await installTimerGate(page); + await freezeTimers(page); + await page.evaluate(() => { + const w = window as unknown as Record; + w.frozenAt = performance.now(); + w.timer = setTimeout(() => { + w.cancelledTimerRan = true; + }, 20); + w.frame = requestAnimationFrame(() => { + w.cancelledFrameRan = true; + }); + requestAnimationFrame(time => { + w.heldFrameTime = time; + }); + }); + await page.waitForTimeout(150); // both are due now, and held + // The harness's own frame waits still resolve while the page's are held. + await Promise.race([ + frames(page, 2), + page.waitForTimeout(5000).then(() => { + throw new Error('frames() did not resolve while the gate was frozen'); + }), + ]); + await page.evaluate(() => { + const w = window as unknown as {timer: number; frame: number}; + clearTimeout(w.timer); + cancelAnimationFrame(w.frame); + }); + await thawTimers(page); + const state = await page.evaluate(() => { + const w = window as unknown as Record; + return { + timer: w.cancelledTimerRan === true, + frame: w.cancelledFrameRan === true, + heldFor: Number(w.heldFrameTime) - Number(w.frozenAt), + }; + }); + expect(state).toMatchObject({timer: false, frame: false}); + expect(state.heldFor).toBeGreaterThanOrEqual(140); + }); +}); diff --git a/internal/interaction-harness/src/index.ts b/internal/interaction-harness/src/index.ts index 32c238e4a134a..537bfc4d4beca 100644 --- a/internal/interaction-harness/src/index.ts +++ b/internal/interaction-harness/src/index.ts @@ -14,6 +14,8 @@ export * from './curves'; export * from './math'; +export * from './output'; export * from './scripts'; export * from './story'; +export * from './storyboard'; export * from './touch'; diff --git a/internal/interaction-harness/src/output.ts b/internal/interaction-harness/src/output.ts new file mode 100644 index 0000000000000..59926b82c9aef --- /dev/null +++ b/internal/interaction-harness/src/output.ts @@ -0,0 +1,66 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file output.ts + * @input Uses node:fs, node:path and the Playwright `TestInfo` + * @output `evidenceDir` / `writeEvidence` — where storyboards and curve traces + * go, and the one way they are written and attached to the test report. + * @position Every artifact the harness produces lands under one gitignored + * directory, `test-results/interaction-evidence/` by default + * (`ASTRYX_INTERACTION_EVIDENCE` overrides it). Playwright wipes its own + * output directory per run; this one is separate, so a storyboard run never + * deletes another's evidence. Generated evidence is never committed: a pull + * request links or attaches it. + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import type {TestInfo} from '@playwright/test'; + +export function evidenceDir(): string { + return path.resolve( + process.env.ASTRYX_INTERACTION_EVIDENCE ?? + 'test-results/interaction-evidence', + ); +} + +/** A filesystem-safe name: letters, digits, dot, dash and underscore. */ +export function safeName(name: string): string { + return ( + name + .replace(/[^A-Za-z0-9._-]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 120) || 'evidence' + ); +} + +/** + * Writes `files` (name → contents) under `//` and attaches + * them to the test's report when a `TestInfo` is given. Returns the paths. + */ +export async function writeEvidence( + group: string, + files: Readonly>, + testInfo?: TestInfo, +): Promise> { + // `group` may nest ("storyboards/"): each segment is made safe on its own. + const dir = path.join(evidenceDir(), ...group.split('/').map(safeName)); + fs.mkdirSync(dir, {recursive: true}); + const written: Record = {}; + for (const [name, contents] of Object.entries(files)) { + const file = path.join(dir, safeName(name)); + fs.writeFileSync(file, contents); + written[name] = file; + if (testInfo != null) { + await testInfo.attach(`${group}/${safeName(name)}`, { + path: file, + contentType: name.endsWith('.png') + ? 'image/png' + : name.endsWith('.json') + ? 'application/json' + : 'text/plain', + }); + } + } + return written; +} diff --git a/internal/interaction-harness/src/story.ts b/internal/interaction-harness/src/story.ts index 9e4323436ff27..598f75d852cfe 100644 --- a/internal/interaction-harness/src/story.ts +++ b/internal/interaction-harness/src/story.ts @@ -93,11 +93,101 @@ export async function frames(page: Page, count = 2): Promise { await page.evaluate( n => new Promise(resolve => { + // The page's real frames, even while a timer gate holds its callbacks. + const frame = + ( + window as unknown as { + __harnessTimerGate?: {frame: typeof requestAnimationFrame}; + } + ).__harnessTimerGate?.frame ?? requestAnimationFrame; let left = n; - const tick = () => - --left <= 0 ? resolve() : requestAnimationFrame(tick); - requestAnimationFrame(tick); + const tick = () => (--left <= 0 ? resolve() : frame(tick)); + frame(tick); }), count, ); } + +// A held callback is queued under its timer or frame id, so the page's own +// clearTimeout and cancelAnimationFrame still cancel it while frozen; a held +// frame callback gets the time it actually runs at. `frame` stays the real +// requestAnimationFrame, for the harness's own waits. +const TIMER_GATE = `(() => { + if (window.__harnessTimerGate) return; + const realTimeout = window.setTimeout.bind(window); + const realClearTimeout = window.clearTimeout.bind(window); + const realInterval = window.setInterval.bind(window); + const realFrame = window.requestAnimationFrame.bind(window); + const realCancelFrame = window.cancelAnimationFrame.bind(window); + const gate = {frozen: false, queue: [], frame: realFrame}; + window.__harnessTimerGate = gate; + const drop = key => { gate.queue = gate.queue.filter(entry => entry.key !== key); }; + window.setTimeout = (fn, ms, ...args) => { + const id = realTimeout(() => { + const run = () => (typeof fn === 'function' ? fn(...args) : 0); + if (gate.frozen) gate.queue.push({key: 't' + id, run}); else run(); + }, ms); + return id; + }; + window.clearTimeout = id => { realClearTimeout(id); drop('t' + id); }; + window.setInterval = (fn, ms, ...args) => realInterval(() => { if (!gate.frozen && typeof fn === 'function') fn(...args); }, ms); + window.requestAnimationFrame = fn => { + const id = realFrame(time => { + if (gate.frozen) gate.queue.push({key: 'f' + id, run: () => fn(performance.now())}); else fn(time); + }); + return id; + }; + window.cancelAnimationFrame = id => { realCancelFrame(id); drop('f' + id); }; +})()`; + +/** + * Installs a gate on the page's setTimeout / setInterval / + * requestAnimationFrame. Install it BEFORE the animation under test starts: + * only callbacks scheduled through the gate can be held. Event timestamps, + * Date, performance.now and the Web Animations timeline are untouched. + */ +export async function installTimerGate(page: Page): Promise { + await page.evaluate(TIMER_GATE); +} + +/** + * Holds every gated timer and frame callback until `thawTimers`, so a + * component cannot finish an animation behind a held storyboard's back (a + * transitionend backstop timer, a cleanup render). Callbacks that come due + * meanwhile run, in order, on thaw, unless the page cancels them first + * (clearTimeout, cancelAnimationFrame); intervals skip their frozen ticks. + */ +export async function freezeTimers(page: Page): Promise { + await page.evaluate(() => { + const gate = (window as unknown as {__harnessTimerGate?: {frozen: boolean}}) + .__harnessTimerGate; + if (gate == null) { + throw new Error( + 'freezeTimers: call installTimerGate before the animation starts', + ); + } + gate.frozen = true; + }); +} + +/** Releases the gate: held callbacks run now, in the order they came due. */ +export async function thawTimers(page: Page): Promise { + await page.evaluate(() => { + const gate = ( + window as unknown as { + __harnessTimerGate?: { + frozen: boolean; + queue: {key: string; run: () => void}[]; + }; + } + ).__harnessTimerGate; + if (gate == null) { + return; + } + gate.frozen = false; + // One at a time from the live queue: a callback may cancel a later one. + while (gate.queue.length > 0) { + gate.queue.shift()?.run(); + } + }); +} diff --git a/internal/interaction-harness/src/storyboard.ts b/internal/interaction-harness/src/storyboard.ts new file mode 100644 index 0000000000000..03d70d02222f0 --- /dev/null +++ b/internal/interaction-harness/src/storyboard.ts @@ -0,0 +1,329 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file storyboard.ts + * @input Uses a Playwright `Page`, a clip (`Locator` or box), ./curves to seek + * animations, ./touch to play scripted touches, and ./output to write + * @output `Storyboard` (collect frames, write a PNG contact sheet + JSON trace), + * `animationStoryboard` (seek every held animation to t = 0, 16.7, 33.3 … ms + * and photograph each), `touchStoryboard` (photograph the page after every + * scripted touch event). + * @position Deterministic filmstrips: frames come from seeking paused + * animations or from stepping scripted input, never from wall-clock capture, + * so the same code produces the same sheet. The JSON trace carries the + * measured value of every frame and is what the spec asserts; the sheet is + * the human evidence attached to the report (and linked from a pull + * request, never committed). + * + * The contact sheet is composed in a scratch page of the same browser (a + * canvas grid with a label per cell), so the harness needs no image library. + * + * SYNC: Keep the README's storyboard section and `pnpm storyboards` in step. + */ + +import type {Locator, Page, TestInfo} from '@playwright/test'; +import {holdAnimations, seekAnimations, type HeldAnimation} from './curves'; +import {timeline} from './math'; +import {writeEvidence} from './output'; +import type {TouchScript} from './scripts'; +import {frames} from './story'; +import type {TouchDriver} from './touch'; + +export interface ClipBox { + readonly x: number; + readonly y: number; + readonly width: number; + readonly height: number; +} + +export interface StoryboardFrame { + readonly label: string; + /** Milliseconds on the storyboard's own timeline. */ + readonly t: number; + /** Measurements taken at this frame; the spec asserts these. */ + readonly data: Readonly>; + readonly png: Buffer; +} + +export interface StoryboardOptions { + readonly name: string; + /** What to photograph: an element, a box, or the whole viewport (default). */ + readonly clip?: Locator | ClipBox; + /** Grid columns in the sheet (default: up to 8). */ + readonly columns?: number; + /** Cell scale relative to CSS px (default 0.5). */ + readonly cellScale?: number; +} + +export class Storyboard { + readonly frames: StoryboardFrame[] = []; + private clipBox: ClipBox | undefined; + + constructor( + private readonly page: Page, + readonly options: StoryboardOptions, + ) {} + + /** Photographs the clip now and records `data` with it. */ + async capture( + label: string, + t: number, + data: Readonly> = {}, + ): Promise { + const clip = await this.resolveClip(); + const png = await this.page.screenshot({ + clip, + scale: 'css', + animations: 'allow', + caret: 'hide', + }); + this.frames.push({label, t, data, png}); + } + + private async resolveClip(): Promise { + if (this.clipBox != null) { + return this.clipBox; + } + const {clip} = this.options; + if (clip == null) { + return undefined; + } + if ('boundingBox' in clip) { + // A locator is measured once, at the first frame, so every cell shows the same region. + const box = await clip.boundingBox(); + if (box == null) { + throw new Error( + `storyboard ${this.options.name}: clip element is not visible`, + ); + } + const viewport = this.page.viewportSize(); + const x = Math.max(0, Math.floor(box.x)); + const y = Math.max(0, Math.floor(box.y)); + const right = Math.min( + viewport?.width ?? Infinity, + Math.ceil(box.x + box.width), + ); + const bottom = Math.min( + viewport?.height ?? Infinity, + Math.ceil(box.y + box.height), + ); + this.clipBox = {x, y, width: right - x, height: bottom - y}; + } else { + this.clipBox = clip; + } + return this.clipBox; + } + + /** The JSON trace: every frame's label, time and data. */ + trace( + extra: Readonly> = {}, + ): Record { + return { + name: this.options.name, + browser: this.page.context().browser()?.browserType().name() ?? null, + viewport: this.page.viewportSize(), + clip: this.clipBox ?? null, + ...extra, + frames: this.frames.map(({label, t, data}) => ({label, t, data})), + }; + } + + /** Composes the sheet, writes `sheet.png` + `trace.json`, attaches both. */ + async write( + testInfo?: TestInfo, + extra: Readonly> = {}, + ): Promise> { + if (this.frames.length === 0) { + throw new Error(`storyboard ${this.options.name}: no frames`); + } + const sheet = await composeSheet(this.page, this.frames, this.options); + const browser = + this.page.context().browser()?.browserType().name() ?? 'browser'; + return writeEvidence( + `storyboards/${this.options.name}.${browser}`, + { + 'sheet.png': sheet, + 'trace.json': JSON.stringify(this.trace(extra), null, 2), + }, + testInfo, + ); + } +} + +async function composeSheet( + page: Page, + frames: readonly StoryboardFrame[], + options: StoryboardOptions, +): Promise { + const scratch = await page.context().newPage(); + try { + const dataUrl = await scratch.evaluate( + async ({cells, columns, scale, title}) => { + const images = await Promise.all( + cells.map(async cell => + createImageBitmap( + await (await fetch(`data:image/png;base64,${cell.png}`)).blob(), + ), + ), + ); + const cellW = Math.ceil(images[0].width * scale); + const cellH = Math.ceil(images[0].height * scale); + const labelH = 30; + const pad = 8; + const cols = Math.min(columns, cells.length); + const rows = Math.ceil(cells.length / cols); + const canvas = document.createElement('canvas'); + canvas.width = pad + cols * (cellW + pad); + canvas.height = 28 + pad + rows * (cellH + labelH + pad); + const context = canvas.getContext('2d'); + if (context == null) { + throw new Error('no 2d context'); + } + context.fillStyle = '#ffffff'; + context.fillRect(0, 0, canvas.width, canvas.height); + context.fillStyle = '#111111'; + context.font = '600 14px sans-serif'; + context.fillText(title, pad, 19); + cells.forEach((cell, i) => { + const x = pad + (i % cols) * (cellW + pad); + const y = 28 + pad + Math.floor(i / cols) * (cellH + labelH + pad); + context.drawImage(images[i], x, y, cellW, cellH); + context.strokeStyle = '#999999'; + context.lineWidth = 1; + context.strokeRect(x - 0.5, y - 0.5, cellW + 1, cellH + 1); + context.fillStyle = '#111111'; + context.font = '600 11px sans-serif'; + context.fillText(cell.label, x, y + cellH + 13, cellW); + context.fillStyle = '#555555'; + context.font = '10px monospace'; + context.fillText(cell.caption, x, y + cellH + 26, cellW); + }); + return canvas.toDataURL('image/png'); + }, + { + cells: frames.map(f => ({ + png: f.png.toString('base64'), + label: f.label, + caption: Object.entries(f.data) + .filter( + ([, v]) => + typeof v === 'number' || + typeof v === 'string' || + typeof v === 'boolean', + ) + .map( + ([k, v]) => + `${k}=${typeof v === 'number' ? +v.toFixed(1) : String(v)}`, + ) + .join(' '), + })), + columns: options.columns ?? 8, + scale: options.cellScale ?? 0.5, + title: options.name, + }, + ); + return Buffer.from(dataUrl.slice(dataUrl.indexOf(',') + 1), 'base64'); + } finally { + await scratch.close(); + } +} + +export interface AnimationStoryboardOptions extends StoryboardOptions { + /** + * Frame times, ms from the film's start: the earliest start among the held + * animations that end (an animation that repeats forever, such as a + * spinner that started earlier, neither starts nor ends the film). Default: + * every 16.7 ms to the latest curve end among them, and that end itself, + * each animation at its own offset (a held end value is not filmed). + */ + readonly times?: readonly number[]; + /** Restrict to animations inside this element. */ + readonly scope?: Locator; + /** Measurements per frame (runs after the seek). */ + readonly measure?: () => Promise>>; +} + +/** + * Holds every running animation, then seeks them together to each time and + * photographs the clip. Call it right after starting the animation under test. + */ +export async function animationStoryboard( + page: Page, + options: AnimationStoryboardOptions, +): Promise<{storyboard: Storyboard; held: HeldAnimation[]}> { + const held = await holdAnimations(page, options.scope); + if (held.length === 0) { + throw new Error(`storyboard ${options.name}: no running animation to hold`); + } + // An animation that repeats forever (a spinner, a pulsing skeleton) has no + // end to film to: the film runs over the animations that end, from the + // earliest of their starts to the latest of their curve ends. Seeks are on + // the held timeline (from the earliest held start), so both carry offsets. + const ending = held.filter(a => Number.isFinite(a.curveEndMs)); + if (options.times == null && ending.length === 0) { + throw new Error( + `storyboard ${options.name}: every held animation repeats forever ` + + `(${held.map(a => a.name ?? a.kind).join(', ')}); pass \`times\``, + ); + } + const origin = ending.length ? Math.min(...ending.map(a => a.offsetMs)) : 0; + const length = + Math.max(...ending.map(a => a.offsetMs + a.curveEndMs)) - origin; + // The 16.7 ms grid, plus the end itself when the grid misses it: the last + // frame is the one the page presents at the end. + const grid = options.times == null ? timeline(length) : []; + if (options.times == null && length - (grid.at(-1) ?? 0) > 1e-3) { + grid.push(+length.toFixed(3)); + } + const times = options.times ?? grid; + const storyboard = new Storyboard(page, options); + for (const t of times) { + // Labelled with the time the page presents (just inside an end), from + // the film's start. + const presentedAt = (await seekAnimations(page, origin + t)) - origin; + const data = (await options.measure?.()) ?? {}; + await storyboard.capture( + `t=${+presentedAt.toFixed(1)} ms`, + presentedAt, + data, + ); + } + return {storyboard, held}; +} + +export interface TouchStoryboardOptions extends StoryboardOptions { + readonly measure?: () => Promise>>; +} + +/** + * Plays a touch script without wall-clock pacing and photographs the page + * after every event. In Chromium the page still sees the scripted timestamps, + * so velocity-dependent behavior is unchanged; in WebKit the page sees the + * real (slower) dispatch times. + */ +export async function touchStoryboard( + page: Page, + driver: TouchDriver, + script: TouchScript, + options: TouchStoryboardOptions, +): Promise { + const storyboard = new Storyboard(page, options); + await storyboard.capture('before', 0, (await options.measure?.()) ?? {}); + await driver.play(script, { + realtime: false, + onEvent: async step => { + // Let the page run its handlers and present the frame before the photo. + await frames(page, 2); + const data = (await options.measure?.()) ?? {}; + const where = step.changed + .map(f => `${Math.round(f.x)},${Math.round(f.y)}`) + .join(' '); + await storyboard.capture( + `${step.type} t=${+step.t.toFixed(1)} @${where}`, + step.t, + data, + ); + }, + }); + return storyboard; +} diff --git a/package.json b/package.json index a4ca9f2ae830e..e49d57a474542 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "test:a11y-contract": "playwright test --config playwright.config.ts --project chromium", "test:docsite-browser": "playwright test --config apps/docsite/playwright.config.ts", "test:interaction": "playwright test --config playwright.interaction.config.ts", + "storyboards": "playwright test --config playwright.interaction.config.ts --grep @storyboard", "check:package-boundaries": "node scripts/check-package-boundaries.js", "check:repo": "pnpm check:visual-owner && pnpm check:sync && pnpm check:neutral-palette-refs && pnpm check:stone-palette-refs && pnpm check:gothic-palette-refs && pnpm check:matcha-palette-refs && pnpm check:butter-palette-refs && pnpm check:y2k-palette-refs && pnpm check:data-tokens && pnpm check:token-docs && pnpm check:knowledge && pnpm check:package-boundaries && pnpm check:changesets && pnpm check:demo-media && pnpm check:executable-bits && pnpm check:cli-structure && pnpm check:use-client && pnpm check:portable-scripts && pnpm check:i18n-runtime && pnpm check:i18n-catalog && pnpm check:cldr-weekdays && pnpm check:fixtures && pnpm check:setup-contract", "check:visual-owner": "node scripts/check-visual-workflow-owner.mjs", diff --git a/playwright.interaction.config.ts b/playwright.interaction.config.ts index 9d8a4f00452f4..f0c61e2e6e563 100644 --- a/playwright.interaction.config.ts +++ b/playwright.interaction.config.ts @@ -5,9 +5,9 @@ * @input Uses @playwright/test * @output Touch interaction and animation proofs on a phone-sized touch * screen, in Chromium and WebKit. - * @position Run with `pnpm test:interaction`. Separate from `pnpm test` (no - * browser needed there) and from playwright.config.ts (accessibility - * contracts on desktop profiles). + * @position Run with `pnpm test:interaction`; filmstrips only with + * `pnpm storyboards`. Separate from `pnpm test` (no browser needed there) and + * from playwright.config.ts (accessibility contracts on desktop profiles). * * Specs are named `*.interaction.browser.spec.ts` and read stories from a * built Storybook, like the accessibility specs: From eec2404fd85f9243fb94102c0724abb5e7ef7462 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sat, 24 Oct 2026 21:26:53 -0700 Subject: [PATCH 13/93] test(interaction): a touch filmstrip's handler clock follows its pacing `touchStoryboard` said velocity-dependent behavior is unchanged in Chromium. That holds for velocity read from event timestamps, which keep their scripted times, but a handler that reads `performance.now()` or `Date.now()` (Item's and Toast's swipes) sees one event per photograph, about 100 ms apart, so a flick that dismisses when played can settle back when filmed. The doc and the README now state that limit and point such gestures at a screencast, and a self-test shows both clocks: scripted 16.7 ms gaps in `timeStamp`, more than two frames in the handler's clock. --- internal/interaction-harness/README.md | 6 ++- .../src/harness.interaction.browser.spec.ts | 49 +++++++++++++++++-- .../interaction-harness/src/storyboard.ts | 13 +++-- 3 files changed, 61 insertions(+), 7 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 1066c22ffc374..47d373b4f72c7 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -94,7 +94,11 @@ backstop, a cleanup render) would do so behind a held storyboard's back: photographing. The page can still cancel a held callback, a held frame callback gets the time it actually runs, and the harness's own frame waits are not held. An animation that repeats forever does not set a storyboard's default length; -with nothing else to film, pass `times`. +with nothing else to film, pass `times`. A touch filmstrip keeps the scripted +event `timeStamp`s (Chromium), but a handler that reads `performance.now()` or +`Date.now()` sees one event per photograph, about 100 ms apart, so a +clock-timed release flings slower when filmed than when played: film such a +gesture from a screencast. ## Engine limits (measured with Playwright 1.61) diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 7a2e12ca7ae0f..e85e5fb8bfe6c 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -13,9 +13,10 @@ * document-timeline animations, each at its own offset, adds a second hold * to the held set, and releases only them; storyboards write a sheet and a * trace that follows the curve, and an endless animation does not set their - * length; the timer gate holds a component timer until the filmstrip is - * done, drops callbacks the page cancels, stamps held frames with the time - * they run, and leaves the harness its own frames. + * length, and a touch filmstrip keeps scripted event times while a handler + * clock sees its pacing; the timer gate holds a component timer until the + * filmstrip is done, drops callbacks the page cancels, stamps held frames + * with the time they run, and leaves the harness its own frames. * @position The harness's own test. A component spec trusts these * instruments; this file is why it may. */ @@ -699,6 +700,48 @@ test.describe('storyboards', {tag: '@storyboard'}, () => { expect(storyboard.frames.at(-2)?.data.x).toBeCloseTo(300, 0); }); + test('a touch filmstrip keeps the scripted event times, and a handler clock sees its pacing', async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'WebKit stamps events with their real dispatch time', + ); + await fixture(page); + await page.evaluate(() => { + const w = window as unknown as {__moves: [number, number][]}; + w.__moves = []; + document.getElementById('pad')?.addEventListener('pointermove', e => { + w.__moves.push([e.timeStamp, performance.now()]); + }); + }); + const driver = await touchDriver(page); + await touchStoryboard( + page, + driver, + drag({x: 60, y: 60}, {x: 300, y: 240}, {durationMs: 100, hz: 60}), + { + name: 'harness-self-test-touch-clock', + clip: {x: 0, y: 0, width: 402, height: 300}, + }, + ); + const moves = await page.evaluate( + () => (window as unknown as {__moves: [number, number][]}).__moves, + ); + const gaps = (column: 0 | 1) => + moves.slice(1).map((move, i) => move[column] - moves[i][column]); + expect(moves.length).toBeGreaterThanOrEqual(5); + // The events carry the script's 60 Hz times… + for (const gap of gaps(0)) { + expect(gap).toBeCloseTo(1000 / 60, 0); + } + // …but the clock a handler reads advances by a photograph per event. + for (const gap of gaps(1)) { + expect(gap).toBeGreaterThan(2 * (1000 / 60)); + } + }); + test('the timer gate holds a component timer until the filmstrip is done', async ({ page, }, testInfo) => { diff --git a/internal/interaction-harness/src/storyboard.ts b/internal/interaction-harness/src/storyboard.ts index 03d70d02222f0..8386edaf86323 100644 --- a/internal/interaction-harness/src/storyboard.ts +++ b/internal/interaction-harness/src/storyboard.ts @@ -297,9 +297,16 @@ export interface TouchStoryboardOptions extends StoryboardOptions { /** * Plays a touch script without wall-clock pacing and photographs the page - * after every event. In Chromium the page still sees the scripted timestamps, - * so velocity-dependent behavior is unchanged; in WebKit the page sees the - * real (slower) dispatch times. + * after every event. In Chromium each event's `timeStamp` is still its + * scripted time, so velocity read from event timestamps is unchanged; in + * WebKit it is the real (slower) dispatch time. + * + * A handler that reads the clock instead (`performance.now()`, `Date.now()`) + * sees the filmstrip's pacing in both engines: two frames, `measure` and a + * photograph per event, about 100 ms between moves instead of the script's + * 16.7. Its release velocity is that much lower, so a flick that dismisses + * when played can settle back when filmed: film such a gesture from a + * screencast (`startScreencast`, `castStoryboard`) instead. */ export async function touchStoryboard( page: Page, From 0954697e76428ad5b3358b2ff636882bd1d6c283 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Fri, 23 Oct 2026 20:09:04 -0700 Subject: [PATCH 14/93] test(interaction): frame evidence and the BottomSheet exemplar Timestamped composited frames (Chromium's screencast and WebKit's own), a marker the frames can find, and the busy-main-thread proof: block the page's main thread mid-flight and show the element still moving every frame, with a requestAnimationFrame animation that freezes as the negative control. Chromium trace counts of Layout, Paint and style work inside an animation window, and long tasks. The exemplar spec proves the BottomSheet drag to a detent with all of it: the drag follows the finger, a flick dismisses, the settle matches its timing function, keeps advancing through a blocked main thread in Chromium and WebKit, does no Layout or Paint, and gets a drag and settle storyboard. --- internal/interaction-harness/README.md | 52 +- internal/interaction-harness/package.json | 2 +- internal/interaction-harness/src/busy.ts | 139 +++++ internal/interaction-harness/src/frames.ts | 480 ++++++++++++++++++ .../src/harness.interaction.browser.spec.ts | 161 +++++- internal/interaction-harness/src/index.ts | 3 + .../interaction-harness/src/storyboard.ts | 61 +++ internal/interaction-harness/src/trace.ts | 203 ++++++++ packages/core/package.json | 1 + ...tomSheetDetent.interaction.browser.spec.ts | 353 +++++++++++++ playwright.interaction.config.ts | 2 +- pnpm-lock.yaml | 3 + 12 files changed, 1433 insertions(+), 27 deletions(-) create mode 100644 internal/interaction-harness/src/busy.ts create mode 100644 internal/interaction-harness/src/frames.ts create mode 100644 internal/interaction-harness/src/trace.ts create mode 100644 packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 47d373b4f72c7..6bb189611db4d 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -1,23 +1,27 @@ # @astryxdesign/interaction-harness Playwright instruments for touch interaction and animation proofs: real touch -input, curve sampling and storyboards. **Internal and unpublished.** Specs live -next to their component as `*.interaction.browser.spec.ts` and run under +input, frame evidence, curve sampling and storyboards. **Internal and +unpublished.** Specs live next to their component as +`*.interaction.browser.spec.ts` and run under [`playwright.interaction.config.ts`](../../playwright.interaction.config.ts) on a phone-sized touch screen (402 × 714 CSS px page, 402 × 874 screen, 3 device pixels per CSS px, touch, mobile viewport rules), in Chromium and WebKit. +The exemplar to copy is +[`BottomSheetDetent.interaction.browser.spec.ts`](../../packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts). The harness proves its own instruments in -[`src/harness.interaction.browser.spec.ts`](src/harness.interaction.browser.spec.ts), -on fixture pages written inline. +[`src/harness.interaction.browser.spec.ts`](src/harness.interaction.browser.spec.ts): +every positive check has a negative control beside it (a rAF animation that +freezes, a `top` animation that lays out every frame). ## Running ```sh -pnpm storybook:build # component specs read stories from the built Storybook +pnpm storybook:build # the specs read stories from the built Storybook pnpm exec playwright install chromium webkit pnpm test:interaction --project chromium # everything, Chromium -pnpm test:interaction # one spec, both engines +pnpm test:interaction BottomSheetDetent # one spec, both engines pnpm storyboards # only the tests tagged @storyboard ``` @@ -29,7 +33,7 @@ image (podman or docker; the repository is mounted at the same path): node internal/interaction-harness/scripts/webkit-container.mjs [playwright test args…] ``` -Evidence (storyboards, curve traces) is written to +Evidence (storyboards, curve traces, busy-main-thread frames) is written to `test-results/interaction-evidence/` (override with `ASTRYX_INTERACTION_EVIDENCE`) and attached to the test report. It is never committed: a pull request links or attaches it. @@ -40,9 +44,12 @@ committed: a pull request links or attaches it. | --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact release velocity, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. | | `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | +| `frames.ts` | `startScreencast` (timestamped composited frames), `markElement` (a marker chip or outline), `trackMarker` (marker box per frame), `sampleRects` (per-rAF rects), `frameStats`. | +| `busy.ts` | `blockMainThread(page, ms)` and `busyWindowProof(tracked, window)`: frames presented, frames that moved, longest stall. | +| `trace.ts` | `traceRendering(page, body)`: main-thread Layout / Paint / UpdateLayoutTree / PrePaint counts, `countBetween` for a sub-window, long tasks. `installLongTaskLog` / `readLongTasks`. | | `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t), `waitForAnimation`, `watchAnimations`, `waitForRest`, `releaseAnimations`. | | `math.ts` | Oracles independent of the code under test: `cubicBezier`, `timingFunction`, `springStep` (SwiftUI response / dampingFraction, initial velocity), `projectDecay`, `rubberBand`, `parseTransform`, `compareCurve`, `timeline`. | -| `storyboard.ts` | `animationStoryboard` (seek to t = 0, 16.7, 33.3 … ms), `touchStoryboard` (a frame per scripted event): PNG contact sheet + JSON trace. | +| `storyboard.ts` | `animationStoryboard` (seek to t = 0, 16.7, 33.3 … ms), `touchStoryboard` (a frame per scripted event), `castStoryboard` (real screencast frames): PNG contact sheet + JSON trace. | | `output.ts` | `evidenceDir`, `writeEvidence`: where evidence goes and how it is attached to the report. | | `story.ts` | `phone`, `withStorybook`, `openStory`, `frames`, and the timer gate (`installTimerGate`, `freezeTimers`, `thawTimers`). | @@ -85,6 +92,26 @@ set): a short entrance no longer races the first look on a cold page. When the action may be the page's first contact with the harness, take `watchAnimations(locator)` before it and read `watch.started()` after. +**Busy main thread.** Mark the element, start the screencast, start the +animation, wait until it runs (`waitForAnimation`: an animation cannot start +while the thread is held), block the main thread 500–1000 ms, stop. Always +start the gesture or animation first and block mid-flight, never before: +in headless Chromium a touch that starts while the main thread is busy does +not scroll or start a gesture even with only passive listeners (the input +waits for the main thread). Tracking a finger under a busy main thread is +therefore proven on the iOS simulator, not here. The +screencast is captured off the main thread, so a compositor animation shows a +new position every frame of the block, and a main-thread animation shows none +(nothing changed, so no frame is sent at all: `busyWindowProof` counts the +whole window as one stall). `settledWithin` stops judging at the easing tail +that moves less than a frame pixel per frame. `castStoryboard` turns the frames +of the block into a filmstrip. + +**No Layout / Paint.** `traceRendering` records a Chromium trace around the +interaction; `countBetween` counts the steady middle of an animation (its +first and last frames may commit). A transform/opacity animation does no +Layout or Paint there; a layout-property animation does one per frame. + **Storyboards.** Deterministic: frames come from seeking held animations or from stepping scripted input, never from wall-clock capture. The JSON trace carries each frame's measurements and the spec asserts on it; the sheet is @@ -111,11 +138,14 @@ gesture from a screencast. | Lifting one finger while others stay down | yes | pointer events right, but that `touchend` lists no remaining `touches`: skip with `PARTIAL_LIFT_SKIP` | | Click synthesised from a touch | yes | only from `driver.tap` (WebKit's tap gesture), not from a raw touch sequence | | Curve sampling, storyboards, input log | yes | yes | +| Screencast frame evidence (busy main thread) | yes | yes (WebKit's own screencast; frames keep coming while the main thread is blocked) | +| Rendering trace, long tasks | yes | no (Playwright exposes neither) | What WebKit cannot show here — native scroll hand-off, flings, a finger -lifting while another stays down, and anything iOS-only — needs Mobile Safari -itself, on a device or the iOS simulator. Skip such a proof on the driver's -flag, citing the reason: `test.skip(!driver.partialLifts, PARTIAL_LIFT_SKIP)`. +lifting while another stays down, finger tracking under a busy main thread, and +anything iOS-only — needs Mobile Safari itself, on a device or the iOS +simulator. Skip such a proof on the driver's flag, citing the reason: +`test.skip(!driver.partialLifts, PARTIAL_LIFT_SKIP)`. The WebKit input command is reached through Playwright's in-process server object, so it needs a locally launched browser (the default), not `connect()`. diff --git a/internal/interaction-harness/package.json b/internal/interaction-harness/package.json index 40cccb26ca206..942eccb9ed8d5 100644 --- a/internal/interaction-harness/package.json +++ b/internal/interaction-harness/package.json @@ -3,7 +3,7 @@ "version": "0.0.0", "private": true, "type": "module", - "description": "Internal: Playwright helpers for touch interaction and animation proofs (real touch input, curve sampling, storyboards). Never published.", + "description": "Internal: Playwright helpers for touch interaction and animation proofs (real touch input, frame evidence, curve sampling, storyboards). Never published.", "main": "./src/index.ts", "exports": { ".": "./src/index.ts", diff --git a/internal/interaction-harness/src/busy.ts b/internal/interaction-harness/src/busy.ts new file mode 100644 index 0000000000000..e8674d07044a5 --- /dev/null +++ b/internal/interaction-harness/src/busy.ts @@ -0,0 +1,139 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file busy.ts + * @input Uses a Playwright `Page` + * @output `blockMainThread` — a synchronous busy loop on the page's main thread + * that reports exactly when it ran — and `busyWindowProof`, which reads frame + * evidence for that window. + * @position The busy-main-thread proof: an animation that runs on the + * compositor keeps advancing at display rate while JavaScript holds the main + * thread; one driven from the main thread (rAF, style writes, layout + * properties) freezes for the whole block. Pair it with ./frames + * (`startScreencast` + `trackMarker`), which observes off the main thread. + * + * SYNC: Keep the README's busy-main-thread section equal to this contract. + */ + +import type {Page} from '@playwright/test'; +import {frameStats, type FrameStats, type TrackedFrame} from './frames'; + +export interface BusyWindow { + /** Epoch ms when the loop started and ended, on the page's own clock. */ + readonly from: number; + readonly to: number; +} + +/** + * Holds the page's main thread for `ms` (a synchronous loop: no task, frame, + * rAF, event or microtask runs meanwhile). Resolves with the window it held. + * + * Block MID-FLIGHT only: start the gesture or animation first. A touch that + * starts while the main thread is busy does not scroll or begin a gesture in + * headless Chromium, even with only passive listeners, and an animation that + * has not started cannot start (see `waitForAnimation`). Finger tracking + * under a busy main thread is a simulator proof, not a Playwright one. + */ +export async function blockMainThread( + page: Page, + ms: number, +): Promise { + return page.evaluate(duration => { + const from = performance.timeOrigin + performance.now(); + const end = performance.now() + duration; + let spins = 0; + while (performance.now() < end) { + spins += 1; + } + return {from, to: performance.timeOrigin + performance.now(), spins}; + }, ms); +} + +export interface BusyProof extends FrameStats { + readonly window: BusyWindow; + /** Frames the display should have shown in the window at the measured cadence. */ + readonly expectedFrames: number; + /** + * Longest time inside the window with no presented change, the window's own + * edges included. A frozen animation presents no new frame at all (nothing + * on screen changed, so the compositor sends none): its stall is the whole + * window. + */ + readonly longestStallMs: number; +} + +/** + * Frame evidence for a busy window: how many frames the screencast presented + * inside it, how many of them moved the tracked axis, and the longest stall. + * The window is trimmed by `marginMs` at both ends so the frame the block + * started on and the one it ended on are not counted. + */ +export function busyWindowProof( + tracked: readonly TrackedFrame[], + window: BusyWindow, + { + axis = 'y', + marginMs = 20, + settledWithin, + }: { + axis?: 'x' | 'y' | 'width' | 'height'; + marginMs?: number; + /** + * Stop judging once the value is within this many px of where it ends: + * an easing tail moves less than a frame pixel per frame, and the + * screencast resolves whole CSS pixels. + */ + settledWithin?: number; + } = {}, +): BusyProof { + let end = window.to - marginMs; + if (settledWithin != null) { + const valued = tracked.flatMap(f => + f.box == null ? [] : [{t: f.t, value: f.box[axis]}], + ); + const final = valued.length > 0 ? valued[valued.length - 1].value : null; + const arrived = valued.find( + f => + f.t >= window.from && + final != null && + Math.abs(f.value - final) <= settledWithin, + ); + if (arrived != null) { + end = Math.min(end, arrived.t); + } + } + const inner = {from: window.from + marginMs, to: end}; + const stats = frameStats( + tracked.map(f => ({t: f.t, value: f.box == null ? null : f.box[axis]})), + inner, + ); + const values = tracked + .filter(f => f.t >= inner.from && f.t <= inner.to) + .map(f => ({t: f.t, value: f.box == null ? null : f.box[axis]})); + const before = [...tracked].reverse().find(f => f.t < inner.from); + let previous: number | null = before?.box == null ? null : before.box[axis]; + const changes: number[] = []; + for (const v of values) { + if ( + v.value != null && + previous != null && + Math.abs(v.value - previous) > 0.25 + ) { + changes.push(v.t); + } + if (v.value != null) { + previous = v.value; + } + } + const edges = [inner.from, ...changes, inner.to]; + const longestStallMs = Math.max( + ...edges.slice(1).map((t, i) => t - edges[i]), + ); + const cadence = stats.medianFrameMs || 1000 / 60; + return { + ...stats, + window, + longestStallMs, + expectedFrames: Math.floor((inner.to - inner.from) / cadence), + }; +} diff --git a/internal/interaction-harness/src/frames.ts b/internal/interaction-harness/src/frames.ts new file mode 100644 index 0000000000000..927a4e12dd6a5 --- /dev/null +++ b/internal/interaction-harness/src/frames.ts @@ -0,0 +1,480 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file frames.ts + * @input Uses a Playwright `Page`, a `Locator` for the element under test, and + * the engine's screencast (Chromium CDP `Page.startScreencast`; WebKit's + * `Screencast` domain) + * @output `startScreencast` (timestamped composited frames), `markElement` + * (a marker outline the frames can find), `trackMarker` (per-frame marker + * box), `sampleRects` (per-rAF rect sampling), `frameStats`. + * @position Frame evidence for motion proofs. The screencast is captured off + * the page's main thread, from what the compositor presents, so it keeps + * delivering frames while a script blocks the main thread: that is what makes + * the busy-main-thread proof possible. Rect sampling is the cheap path when + * the main thread is free. + * + * Marker tracking needs no image library: frames are decoded and scanned in a + * scratch page of the same browser (createImageBitmap + canvas), so the + * harness adds no dependency. + * + * SYNC: Keep the README's frame-evidence section equal to what this measures. + */ + +import type {CDPSession, Locator, Page} from '@playwright/test'; +import {webkitPageSession} from './touch'; + +export interface CastFrame { + /** Presentation time, epoch ms (the compositor's frame timestamp). */ + readonly t: number; + /** Base64 PNG/JPEG. */ + readonly data: string; + readonly format: 'png' | 'jpeg'; + /** CSS px of the viewport the frame shows. */ + readonly viewportWidth: number; + readonly viewportHeight: number; +} + +export interface Screencast { + /** Stops and returns every frame captured, in order. */ + stop(): Promise; +} + +export interface ScreencastOptions { + /** Lossless frames by default: marker colours survive PNG, not JPEG. */ + readonly format?: 'png' | 'jpeg'; + readonly quality?: number; +} + +/** + * Starts capturing composited frames. Chromium delivers one per presented + * frame (16.7 ms at 60 Hz), timestamped by the compositor; WebKit's own + * screencast delivers one per composited update (about every 14 ms in the + * Linux build), stamped on its monotonic clock and rebased here onto the wall + * clock. Both keep delivering while the page's main thread is blocked, and + * neither sends a frame when nothing on screen changed. + */ +export async function startScreencast( + page: Page, + options: ScreencastOptions = {}, +): Promise { + const name = page.context().browser()?.browserType().name(); + const viewport = page.viewportSize() ?? {width: 0, height: 0}; + if (name === 'chromium') { + return chromiumScreencast(page, viewport, options); + } + if (name === 'webkit') { + return webkitScreencast(page, viewport, options); + } + throw new Error( + `startScreencast: unsupported browser ${name ?? '(unknown)'}`, + ); +} + +async function chromiumScreencast( + page: Page, + viewport: {width: number; height: number}, + {format = 'png', quality = 90}: ScreencastOptions, +): Promise { + const cdp: CDPSession = await page.context().newCDPSession(page); + const frames: CastFrame[] = []; + const onFrame = (frame: { + data: string; + sessionId: number; + metadata: {timestamp?: number; deviceWidth: number; deviceHeight: number}; + }) => { + frames.push({ + t: (frame.metadata.timestamp ?? Date.now() / 1000) * 1000, + data: frame.data, + format, + viewportWidth: frame.metadata.deviceWidth || viewport.width, + viewportHeight: frame.metadata.deviceHeight || viewport.height, + }); + cdp + .send('Page.screencastFrameAck', {sessionId: frame.sessionId}) + .catch(() => {}); + }; + cdp.on('Page.screencastFrame', onFrame); + // One CSS px per frame px keeps decoding cheap; everyNthFrame 1 = every presented frame. + await cdp.send('Page.startScreencast', { + format, + quality, + maxWidth: viewport.width, + maxHeight: viewport.height, + everyNthFrame: 1, + }); + return { + async stop() { + await cdp.send('Page.stopScreencast'); + cdp.off('Page.screencastFrame', onFrame); + await cdp.detach().catch(() => {}); + return frames; + }, + }; +} + +async function webkitScreencast( + page: Page, + viewport: {width: number; height: number}, + {quality = 90}: ScreencastOptions, +): Promise { + const session = webkitPageSession(page); + const frames: CastFrame[] = []; + let generation = 0; + const onFrame = (payload: unknown) => { + const frame = payload as {data: string; timestamp: number}; + frames.push({ + // WebKit stamps frames in seconds on the monotonic clock; rebased below. + t: frame.timestamp * 1000, + data: frame.data, + format: 'jpeg', + viewportWidth: viewport.width, + viewportHeight: viewport.height, + }); + session.send('Screencast.screencastFrameAck', {generation}).catch(() => {}); + }; + session.on('Screencast.screencastFrame', onFrame); + const started = (await session.send('Screencast.startScreencast', { + width: viewport.width, + height: viewport.height, + toolbarHeight: 0, + quality, + })) as {generation: number}; + generation = started.generation; + const wallAtStart = Date.now(); + return { + async stop() { + await session.send('Screencast.stopScreencast'); + session.off?.('Screencast.screencastFrame', onFrame); + if (frames.length === 0) { + return frames; + } + // Rebase the monotonic stamps onto the wall clock (first frame ≈ start). + const offset = wallAtStart - frames[0].t; + return frames.map(f => ({...f, t: f.t + offset})); + }, + }; +} + +/** The marker colour: magenta never occurs in Astryx themes. */ +export const MARKER_RGB: readonly [number, number, number] = [255, 0, 255]; + +export interface MarkerOptions { + readonly rgb?: readonly [number, number, number]; + /** + * `chip` (default): a small square child pinned inside the element's + * top-left corner, above every sibling (z-index max), so nothing the + * component paints can cover it. The element must be its containing block: + * a static element is made `position: relative` while marked. + * `outline`: an inset outline on the element itself (no DOM insertion), for + * elements that cannot take a child; children with their own layers can + * cover parts of it. + */ + readonly mode?: 'chip' | 'outline'; + /** Chip size or outline width, CSS px. */ + readonly size?: number; + /** Chip inset from the element's top-left corner, CSS px. */ + readonly inset?: number; +} + +/** + * Marks the element so screencast frames can find it: the mark moves, scales + * and fades with the element's own layer and never changes its layout. + * Returns an undo. + */ +export async function markElement( + locator: Locator, + options: MarkerOptions = {}, +): Promise<() => Promise> { + const { + rgb = MARKER_RGB, + mode = 'chip', + size = mode === 'chip' ? 16 : 6, + inset = 8, + } = options; + const id = `m${Math.random().toString(36).slice(2, 8)}`; + await locator.evaluate( + (element, {id: markerId, color, size: px, inset: offset, mode: kind}) => { + const el = element as HTMLElement; + el.setAttribute('data-harness-marker', markerId); + if (kind === 'outline') { + const style = document.createElement('style'); + style.setAttribute('data-harness-marker-style', markerId); + style.textContent = `[data-harness-marker="${markerId}"]{outline:${px}px solid rgb(${color.join(',')}) !important;outline-offset:-${px}px !important;}`; + document.head.append(style); + return; + } + if (getComputedStyle(el).position === 'static') { + el.setAttribute('data-harness-marker-position', el.style.position); + el.style.position = 'relative'; + } + const chip = document.createElement('div'); + chip.setAttribute('data-harness-marker-chip', markerId); + chip.setAttribute('aria-hidden', 'true'); + chip.style.cssText = `position:absolute;top:${offset}px;left:${offset}px;width:${px}px;height:${px}px;background:rgb(${color.join(',')});z-index:2147483647;pointer-events:none;`; + el.append(chip); + }, + {id, color: [...rgb], size, inset, mode}, + ); + return async () => { + await locator.page().evaluate(markerId => { + const el = document.querySelector( + `[data-harness-marker="${markerId}"]`, + ); + if (el?.hasAttribute('data-harness-marker-position')) { + el.style.position = + el.getAttribute('data-harness-marker-position') ?? ''; + el.removeAttribute('data-harness-marker-position'); + } + el?.removeAttribute('data-harness-marker'); + document + .querySelector(`[data-harness-marker-chip="${markerId}"]`) + ?.remove(); + document + .querySelector(`[data-harness-marker-style="${markerId}"]`) + ?.remove(); + }, id); + }; +} + +export interface Box { + readonly x: number; + readonly y: number; + readonly width: number; + readonly height: number; +} + +export interface TrackedFrame { + /** Epoch ms. */ + readonly t: number; + /** The marker's bounding box in CSS px, or null when it is not on screen. */ + readonly box: Box | null; + /** Marker pixels found (a fading element finds fewer at the same tolerance). */ + readonly pixels: number; +} + +/** + * Finds the marker in every frame. Decoding runs in a scratch page of the same + * browser, so neither the page under test nor Node needs an image decoder. + */ +export async function trackMarker( + page: Page, + frames: readonly CastFrame[], + { + rgb = MARKER_RGB, + tolerance = 40, + }: {rgb?: readonly [number, number, number]; tolerance?: number} = {}, +): Promise { + const scratch = await page.context().newPage(); + try { + const out: TrackedFrame[] = []; + // Batches keep each evaluate payload moderate. + for (let i = 0; i < frames.length; i += 12) { + const batch = frames.slice(i, i + 12); + const boxes = await scratch.evaluate( + async ({batch: items, color, tol}) => { + const results: { + box: {x: number; y: number; width: number; height: number} | null; + pixels: number; + }[] = []; + for (const item of items) { + const blob = await ( + await fetch(`data:image/${item.format};base64,${item.data}`) + ).blob(); + const bitmap = await createImageBitmap(blob); + const canvas = new OffscreenCanvas(bitmap.width, bitmap.height); + const context = canvas.getContext('2d'); + if (context == null) { + throw new Error('no 2d context'); + } + context.drawImage(bitmap, 0, 0); + const {data, width, height} = context.getImageData( + 0, + 0, + bitmap.width, + bitmap.height, + ); + let minX = Infinity; + let minY = Infinity; + let maxX = -Infinity; + let maxY = -Infinity; + let pixels = 0; + for (let y = 0; y < height; y += 1) { + for (let x = 0; x < width; x += 1) { + const o = (y * width + x) * 4; + if ( + Math.abs(data[o] - color[0]) <= tol && + Math.abs(data[o + 1] - color[1]) <= tol && + Math.abs(data[o + 2] - color[2]) <= tol + ) { + pixels += 1; + if (x < minX) { + minX = x; + } + if (x > maxX) { + maxX = x; + } + if (y < minY) { + minY = y; + } + if (y > maxY) { + maxY = y; + } + } + } + } + const scale = width / item.viewportWidth; + results.push({ + box: + pixels === 0 + ? null + : { + x: minX / scale, + y: minY / scale, + width: (maxX - minX + 1) / scale, + height: (maxY - minY + 1) / scale, + }, + pixels, + }); + } + return results; + }, + { + batch: batch.map(f => ({ + data: f.data, + format: f.format, + viewportWidth: f.viewportWidth, + })), + color: [...rgb], + tol: tolerance, + }, + ); + boxes.forEach((result, k) => + out.push({t: batch[k].t, box: result.box, pixels: result.pixels}), + ); + } + return out; + } finally { + await scratch.close(); + } +} + +export interface RectSample { + /** Epoch ms of the rAF callback. */ + readonly t: number; + readonly box: Box | null; + readonly transform: string; + readonly opacity: number; +} + +/** + * Samples the element's rect, transform and opacity on every animation frame + * for `durationMs`. Main-thread sampling: it stops while the main thread is + * blocked, which is the point of the screencast path above. + */ +export async function sampleRects( + locator: Locator, + durationMs: number, +): Promise { + return locator.evaluate( + (element, duration) => + new Promise< + { + t: number; + box: {x: number; y: number; width: number; height: number} | null; + transform: string; + opacity: number; + }[] + >(resolve => { + const samples: { + t: number; + box: {x: number; y: number; width: number; height: number} | null; + transform: string; + opacity: number; + }[] = []; + const end = performance.now() + duration; + const tick = (now: number) => { + const rect = element.getBoundingClientRect(); + const style = getComputedStyle(element); + samples.push({ + t: performance.timeOrigin + now, + box: {x: rect.x, y: rect.y, width: rect.width, height: rect.height}, + transform: style.transform, + opacity: Number(style.opacity), + }); + if (now < end) { + requestAnimationFrame(tick); + } else { + resolve(samples); + } + }; + requestAnimationFrame(tick); + }), + durationMs, + ); +} + +export interface FrameStats { + readonly frames: number; + /** Frames whose tracked value differs from the frame before. */ + readonly changed: number; + /** Longest stretch without a change inside the window (an open stall at its end included), ms. */ + readonly longestStallMs: number; + /** Median spacing between frames, ms. */ + readonly medianFrameMs: number; + readonly monotonic: boolean; + readonly first: number | null; + readonly last: number | null; +} + +/** Summarises per-frame values (e.g. the marker's y) inside a time window. */ +export function frameStats( + samples: readonly {readonly t: number; readonly value: number | null}[], + window: {readonly from: number; readonly to: number} = { + from: -Infinity, + to: Infinity, + }, + epsilon = 0.25, +): FrameStats { + const inside = samples.filter( + s => s.t >= window.from && s.t <= window.to && s.value != null, + ) as {t: number; value: number}[]; + let changed = 0; + let increasing = 0; + let decreasing = 0; + let stallStart = inside[0]?.t ?? 0; + let longestStallMs = 0; + for (let i = 1; i < inside.length; i += 1) { + const delta = inside[i].value - inside[i - 1].value; + if (Math.abs(delta) > epsilon) { + changed += 1; + if (delta > 0) { + increasing += 1; + } else { + decreasing += 1; + } + longestStallMs = Math.max(longestStallMs, inside[i].t - stallStart); + stallStart = inside[i].t; + } + } + // A stall still open when the window closes counts too: a frozen animation never changes at all. + if (inside.length > 0) { + longestStallMs = Math.max( + longestStallMs, + inside[inside.length - 1].t - stallStart, + ); + } + const gaps = inside + .slice(1) + .map((s, i) => s.t - inside[i].t) + .sort((a, b) => a - b); + return { + frames: inside.length, + changed, + longestStallMs, + medianFrameMs: gaps.length ? gaps[Math.floor(gaps.length / 2)] : 0, + monotonic: increasing === 0 || decreasing === 0, + first: inside[0]?.value ?? null, + last: inside[inside.length - 1]?.value ?? null, + }; +} diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index e85e5fb8bfe6c..71c3ba17ea28e 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -3,20 +3,24 @@ /** * @file harness.interaction.browser.spec.ts * @input Fixture pages written inline (no Storybook), real Chromium/WebKit - * @output Proof that each harness instrument measures what it claims: - * scripted timestamps reach the page; a flick flings natively (Chromium); - * a second finger and a pinch arrive as distinct pointers; sampled curves - * match the oracles; the animation-state waits report a running transition - * and its end, and judge rest by presented values even while an animation - * holds its end; individual transform properties are sampled; a sample at - * an end reports the time it was read at; holding takes only running - * document-timeline animations, each at its own offset, adds a second hold - * to the held set, and releases only them; storyboards write a sheet and a - * trace that follows the curve, and an endless animation does not set their - * length, and a touch filmstrip keeps scripted event times while a handler - * clock sees its pacing; the timer gate holds a component timer until the - * filmstrip is done, drops callbacks the page cancels, stamps held frames - * with the time they run, and leaves the harness its own frames. + * @output Proof that each harness instrument measures what it claims, with a + * negative control beside every positive one: scripted timestamps reach the + * page; a flick flings natively (Chromium); a second finger and a pinch + * arrive as distinct pointers; the screencast shows a compositor animation + * advancing through a blocked main thread while a rAF animation freezes; + * the trace counts zero Layout for a transform animation and one per frame + * for a `top` animation; sampled curves match the oracles, individual + * transform properties included; a sample at an end reports the time it was + * read at; holding takes only running document-timeline animations, each at + * its own offset, adds a second hold to the held set, and releases only + * them; the animation-state waits report a running transition and its end, + * and judge rest by presented values even while an animation holds its end; + * storyboards write a sheet and a trace that follows the curve, and an + * endless animation does not set their length; a touch filmstrip keeps the + * scripted event times while a handler clock sees its pacing; the timer + * gate holds a component timer until the filmstrip is done, drops callbacks + * the page cancels, stamps held frames with the time they run, and leaves + * the harness its own frames. * @position The harness's own test. A component spec trusts these * instruments; this file is why it may. */ @@ -24,7 +28,10 @@ import {expect, test, type Page} from '@playwright/test'; import { animationStoryboard, + blockMainThread, + busyWindowProof, compareCurve, + countBetween, cubicBezier, curveOf, drag, @@ -34,6 +41,7 @@ import { holdAnimations, installInputLog, installTimerGate, + markElement, merge, parseTransform, PARTIAL_LIFT_SKIP, @@ -41,13 +49,17 @@ import { readInputLog, releaseAnimations, sampleCurve, + sampleRects, seekAnimations, springStep, + startScreencast, tap, thawTimers, timeline, touchDriver, touchStoryboard, + traceRendering, + trackMarker, waitForAnimation, waitForRest, watchAnimations, @@ -237,6 +249,127 @@ test.describe('touch input', () => { }); }); +test.describe('frame evidence', () => { + test('a compositor animation advances every frame through a blocked main thread; a rAF one freezes', async ({ + page, + }) => { + await fixture(page); + const box = page.locator('#box'); + await markElement(box, {mode: 'outline'}); + + const run = async (driver: 'compositor' | 'raf') => { + await box.evaluate((el, how) => { + el.getAnimations().forEach(a => a.cancel()); + el.style.transform = 'none'; + if (how === 'compositor') { + el.animate( + [{transform: 'translateX(0px)'}, {transform: 'translateX(280px)'}], + {duration: 1600, easing: 'linear'}, + ); + } else { + const start = performance.now(); + const step = (now: number) => { + const p = Math.min(1, (now - start) / 1600); + el.style.transform = `translateX(${280 * p}px)`; + if (p < 1) { + requestAnimationFrame(step); + } + }; + requestAnimationFrame(step); + } + }, driver); + const cast = await startScreencast(page); + await page.waitForTimeout(200); + const window = await blockMainThread(page, 700); + await page.waitForTimeout(200); + const tracked = await trackMarker(page, await cast.stop()); + return busyWindowProof(tracked, window, {axis: 'x'}); + }; + + const compositor = await run('compositor'); + expect(compositor.medianFrameMs).toBeLessThan(20); + expect(compositor.frames).toBeGreaterThanOrEqual( + compositor.expectedFrames - 2, + ); + expect(compositor.changed).toBeGreaterThanOrEqual(compositor.frames - 3); + expect(compositor.longestStallMs).toBeLessThan(40); + expect(compositor.monotonic).toBe(true); + + const raf = await run('raf'); + // Frozen for the whole block: no presented frame moved (the compositor + // sends none at all when nothing on screen changes). + expect(raf.changed).toBe(0); + expect(raf.longestStallMs).toBeGreaterThan(600); + }); + + test('rect sampling reads one rect per animation frame while the main thread is free', async ({ + page, + browserName, + }) => { + await fixture(page); + const box = page.locator('#box'); + await box.evaluate(el => { + el.animate( + [{transform: 'translateX(0px)'}, {transform: 'translateX(200px)'}], + {duration: 600, easing: 'linear'}, + ); + }); + const samples = await sampleRects(box, 400); + const xs = samples.map(s => s.box?.x ?? NaN); + const gaps = samples + .slice(1) + .map((s, i) => s.t - samples[i].t) + .sort((a, b) => a - b); + const cadence = gaps[Math.floor(gaps.length / 2)]; + expect(cadence).toBeLessThan(34); + expect(samples.length).toBeGreaterThanOrEqual(10); + if (browserName === 'chromium') { + // One sample per presented frame at 60 Hz. (Headless WebKit's animation + // frames are irregular, so only the cadence is checked there.) + expect(samples.length).toBeGreaterThanOrEqual(20); + } + expect(xs.every((x, i) => i === 0 || x >= xs[i - 1])).toBe(true); + expect(xs.at(-1) ?? 0).toBeGreaterThan(xs[0] + 80); + }); + + test('the trace counts no Layout for a transform animation and one per frame for `top`', async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'rendering traces are Chromium-only (README: engine limits)', + ); + await fixture(page); + const box = page.locator('#box'); + // Count the animation's steady middle: starting and ending it may lay out once each. + const animate = async (property: 'transform' | 'top') => { + let started = 0; + const trace = await traceRendering(page, async () => { + started = await box.evaluate((el, prop) => { + const keyframes = + prop === 'transform' + ? [ + {transform: 'translateY(0px)'}, + {transform: 'translateY(200px)'}, + ] + : [{top: '360px'}, {top: '560px'}]; + el.animate(keyframes, {duration: 600}); + return performance.timeOrigin + performance.now(); + }, property); + await page.waitForTimeout(700); + }); + return countBetween(trace, started + 100, started + 500); + }; + const transform = await animate('transform'); + const top = await animate('top'); + expect(transform.Layout ?? 0).toBe(0); + expect(transform.Paint ?? 0).toBe(0); + // ~24 frames in 400 ms: one layout per frame. + expect(top.Layout ?? 0).toBeGreaterThan(18); + }); +}); + test.describe('curve sampling', () => { test('a CSS transition samples onto its cubic-bezier to 0.05 px', async ({ page, diff --git a/internal/interaction-harness/src/index.ts b/internal/interaction-harness/src/index.ts index 537bfc4d4beca..7a45ff45dfe41 100644 --- a/internal/interaction-harness/src/index.ts +++ b/internal/interaction-harness/src/index.ts @@ -12,10 +12,13 @@ * SYNC: When a module is added, export it here and describe it in README.md. */ +export * from './busy'; export * from './curves'; +export * from './frames'; export * from './math'; export * from './output'; export * from './scripts'; export * from './story'; export * from './storyboard'; export * from './touch'; +export * from './trace'; diff --git a/internal/interaction-harness/src/storyboard.ts b/internal/interaction-harness/src/storyboard.ts index 8386edaf86323..a8be8b5c4e844 100644 --- a/internal/interaction-harness/src/storyboard.ts +++ b/internal/interaction-harness/src/storyboard.ts @@ -334,3 +334,64 @@ export async function touchStoryboard( }); return storyboard; } + +/** + * A filmstrip of screencast frames (real presented frames, e.g. the frames of + * a busy-main-thread window), labelled with their time relative to `zero`. + * Evidence for what the compositor presented, not a deterministic storyboard. + */ +export async function castStoryboard( + page: Page, + frames: readonly { + readonly t: number; + readonly data: string; + readonly format: 'png' | 'jpeg'; + }[], + options: StoryboardOptions & { + readonly zero: number; + readonly from?: number; + readonly to?: number; + readonly data?: (t: number) => Readonly>; + }, +): Promise { + const storyboard = new Storyboard(page, options); + for (const frame of frames) { + if (options.from != null && frame.t < options.from) { + continue; + } + if (options.to != null && frame.t > options.to) { + continue; + } + const png = + frame.format === 'png' + ? Buffer.from(frame.data, 'base64') + : await toPng(page, frame.data); + const t = frame.t - options.zero; + storyboard.frames.push({ + label: `t=${t.toFixed(1)} ms`, + t, + data: options.data?.(frame.t) ?? {}, + png, + }); + } + return storyboard; +} + +async function toPng(page: Page, jpegBase64: string): Promise { + const scratch = await page.context().newPage(); + try { + const url = await scratch.evaluate(async data => { + const bitmap = await createImageBitmap( + await (await fetch(`data:image/jpeg;base64,${data}`)).blob(), + ); + const canvas = document.createElement('canvas'); + canvas.width = bitmap.width; + canvas.height = bitmap.height; + canvas.getContext('2d')?.drawImage(bitmap, 0, 0); + return canvas.toDataURL('image/png'); + }, jpegBase64); + return Buffer.from(url.slice(url.indexOf(',') + 1), 'base64'); + } finally { + await scratch.close(); + } +} diff --git a/internal/interaction-harness/src/trace.ts b/internal/interaction-harness/src/trace.ts new file mode 100644 index 0000000000000..773445067dd2f --- /dev/null +++ b/internal/interaction-harness/src/trace.ts @@ -0,0 +1,203 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file trace.ts + * @input Uses a Playwright `Page` (Chromium: CDP `Tracing`) and the page's + * PerformanceObserver for long tasks + * @output `traceRendering` — counts of the main thread's rendering work + * (Layout, Paint, UpdateLayoutTree, PrePaint, …) between two marks — and + * `installLongTaskLog` / `readLongTasks`. + * @position The "no Layout/Paint during a transform/opacity animation" proof. + * A compositor animation costs the main thread nothing per frame; one that + * animates a layout property, or writes styles from JavaScript each frame, + * shows a Layout (and usually a Paint) per frame. Chromium only: WebKit + * exposes no comparable trace to Playwright. + * + * Counting is scoped to the renderer main thread that ran the page's own + * `performance.mark` calls, and to the time between the start and end marks, + * so other pages, the browser process and setup work are excluded. + * + * SYNC: Keep RENDERING_EVENTS equal to the README's list. + */ + +import type {CDPSession, Page} from '@playwright/test'; + +/** Main-thread trace events that are rendering work, by their trace names. */ +export const RENDERING_EVENTS = [ + 'UpdateLayoutTree', // style recalculation + 'Layout', + 'PrePaint', + 'Paint', + 'Layerize', + 'UpdateLayer', + 'ScrollLayer', + 'HitTest', +] as const; + +export interface RenderingTrace { + /** Count per trace event name inside the window, on the page's main thread. */ + readonly counts: Readonly>; + /** Shorthand for the four that matter most. */ + readonly layout: number; + readonly paint: number; + readonly styleRecalc: number; + readonly prePaint: number; + /** Window length, ms. */ + readonly durationMs: number; + /** Main-thread `RunTask`s longer than 50 ms inside the window (long tasks), epoch ms. */ + readonly longTasks: readonly { + readonly start: number; + readonly durationMs: number; + }[]; + /** Every counted rendering event with its epoch-ms time, for sub-window counts. */ + readonly events: readonly {readonly name: string; readonly t: number}[]; + /** The window, epoch ms on the page's clock. */ + readonly window: {readonly from: number; readonly to: number}; +} + +/** Counts rendering events by name inside `[from, to]` (epoch ms). */ +export function countBetween( + trace: RenderingTrace, + from: number, + to: number, +): Record { + const counts: Record = {}; + for (const e of trace.events) { + if (e.t >= from && e.t <= to) { + counts[e.name] = (counts[e.name] ?? 0) + 1; + } + } + return counts; +} + +interface TraceEvent { + name: string; + cat: string; + ph: string; + ts: number; + dur?: number; + pid: number; + tid: number; + args?: Record; +} + +/** + * Traces the page while `body` runs and counts the rendering work its main + * thread did between a start mark (placed just before `body`) and an end mark + * (just after). + */ +export async function traceRendering( + page: Page, + body: () => Promise, +): Promise { + const cdp: CDPSession = await page.context().newCDPSession(page); + const events: TraceEvent[] = []; + cdp.on('Tracing.dataCollected', ({value}) => { + events.push(...(value as unknown as TraceEvent[])); + }); + const complete = new Promise(resolve => + cdp.once('Tracing.tracingComplete', () => resolve()), + ); + await cdp.send('Tracing.start', { + transferMode: 'ReportEvents', + traceConfig: { + includedCategories: [ + 'devtools.timeline', + 'disabled-by-default-devtools.timeline', + 'blink.user_timing', + 'toplevel', + ], + excludedCategories: ['*'], + }, + }); + const tag = `harness-${Math.random().toString(36).slice(2, 8)}`; + const mark = (name: string) => + page.evaluate( + n => performance.timeOrigin + performance.mark(n).startTime, + name, + ); + const fromEpoch = await mark(`${tag}:start`); + await body(); + const toEpoch = await mark(`${tag}:end`); + await cdp.send('Tracing.end'); + await complete; + await cdp.detach().catch(() => {}); + + const start = events.find(e => e.name === `${tag}:start`); + const end = events.find(e => e.name === `${tag}:end`); + if (start == null || end == null) { + throw new Error('traceRendering: the window marks were not traced'); + } + const onMain = (e: TraceEvent) => + e.pid === start.pid && + e.tid === start.tid && + e.ts >= start.ts && + e.ts <= end.ts; + // Trace time (µs, monotonic) -> epoch ms, anchored on the start mark. + const toEpochMs = (ts: number) => fromEpoch + (ts - start.ts) / 1000; + const counts: Record = {}; + const counted: {name: string; t: number}[] = []; + for (const e of events) { + if (!onMain(e) || e.ph === 'E') { + continue; + } + if ((RENDERING_EVENTS as readonly string[]).includes(e.name)) { + counts[e.name] = (counts[e.name] ?? 0) + 1; + counted.push({name: e.name, t: toEpochMs(e.ts)}); + } + } + const longTasks = events + .filter(e => onMain(e) && e.name === 'RunTask' && (e.dur ?? 0) > 50_000) + .map(e => ({start: toEpochMs(e.ts), durationMs: (e.dur ?? 0) / 1000})); + return { + counts, + layout: counts.Layout ?? 0, + paint: counts.Paint ?? 0, + styleRecalc: counts.UpdateLayoutTree ?? 0, + prePaint: counts.PrePaint ?? 0, + durationMs: (end.ts - start.ts) / 1000, + longTasks, + events: counted, + window: {from: fromEpoch, to: toEpoch}, + }; +} + +const LONG_TASK_SCRIPT = `(() => { + if (window.__harnessLongTasks || typeof PerformanceObserver === 'undefined') return; + const tasks = []; + window.__harnessLongTasks = tasks; + try { + new PerformanceObserver(list => { + for (const entry of list.getEntries()) tasks.push({start: performance.timeOrigin + entry.startTime, durationMs: entry.duration, name: entry.name}); + }).observe({type: 'longtask', buffered: true}); + } catch {} +})()`; + +/** Records every long task (> 50 ms) the page runs (install before navigation; Chromium). */ +export async function installLongTaskLog(page: Page): Promise { + await page.addInitScript(LONG_TASK_SCRIPT); +} + +export interface LongTask { + /** Epoch ms. */ + readonly start: number; + readonly durationMs: number; + readonly name: string; +} + +/** Long tasks recorded since navigation, optionally only those overlapping a window. */ +export async function readLongTasks( + page: Page, + range?: {from: number; to: number}, +): Promise { + const tasks = await page.evaluate( + () => + (window as unknown as {__harnessLongTasks?: LongTask[]}) + .__harnessLongTasks ?? [], + ); + return range == null + ? tasks + : tasks.filter( + t => t.start < range.to && t.start + t.durationMs > range.from, + ); +} diff --git a/packages/core/package.json b/packages/core/package.json index 8f02333465c35..491ba2c352ce6 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -808,6 +808,7 @@ }, "devDependencies": { "@astryxdesign/a11y-spec": "workspace:*", + "@astryxdesign/interaction-harness": "workspace:*", "@astryxdesign/cli": "workspace:*", "@babel/cli": "catalog:", "@babel/core": "catalog:", diff --git a/packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts b/packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts new file mode 100644 index 0000000000000..7fb6e3459d27d --- /dev/null +++ b/packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts @@ -0,0 +1,353 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file BottomSheetDetent.interaction.browser.spec.ts + * @input The built "Snap points" BottomSheet story, real touch input + * (Chromium CDP / WebKit's input command) and the interaction harness + * @output Interaction and animation proofs for dragging the sheet to a + * detent: the drag follows the finger and a slow release settles on the + * nearest detent; a flick dismisses; the settle's sampled curve matches its + * timing function; the settle keeps advancing every frame through a blocked + * main thread; the settle does no Layout or Paint per frame; a storyboard of + * the drag and the settle. + * @position The exemplar for `*.interaction.browser.spec.ts`: component lanes + * copy this shape. Each test names the instrument it uses from + * @astryxdesign/interaction-harness, so the evidence layer is explicit. + * + * The expected curve here is today's: the panel's `--ease-standard` CSS + * transition (cubic-bezier(0.24, 1, 0.4, 1), `--duration-medium` = 410 ms). + * When the settle moves to the UIKit spring, `expectedSettle` becomes the + * spring from the motion constants, and the velocity hand-off reported below + * becomes an assertion. + */ + +import {expect, test, type Locator, type Page} from '@playwright/test'; +import { + animationStoryboard, + blockMainThread, + busyWindowProof, + castStoryboard, + compareCurve, + countBetween, + curveOf, + drag, + flick, + frames, + freezeTimers, + holdAnimations, + installLongTaskLog, + installTimerGate, + markElement, + openStory, + readLongTasks, + releaseAnimations, + releaseVelocity, + sampleCurve, + startScreencast, + thawTimers, + timeline, + timingFunction, + touchDriver, + toTouchEvents, + touchStoryboard, + traceRendering, + trackMarker, + withStorybook, + waitForAnimation, + waitForRest, + writeEvidence, + type TouchDriver, +} from '@astryxdesign/interaction-harness'; + +const storybook = withStorybook(test); + +interface OpenSheet { + driver: TouchDriver; + sheet: Locator; + /** Top of the sheet at its tall detent and at the half-viewport detent, CSS px. */ + tallTop: number; + halfTop: number; + handle: {x: number; y: number}; +} + +async function openSheet(page: Page): Promise { + await installLongTaskLog(page); + await openStory(page, storybook(), 'core-bottomsheet--snap-points'); + const driver = await touchDriver(page); + const trigger = page.getByRole('button', {name: 'Open nearby places'}); + const box = await trigger.boundingBox(); + if (box == null) { + throw new Error('no trigger'); + } + await driver.tap({x: box.x + box.width / 2, y: box.y + box.height / 2}); + const sheet = page + .getByRole('dialog', {name: 'Nearby places'}) + .locator('.astryx-bottom-sheet') + .first(); + await expect(sheet).toBeVisible(); + await waitForRest(sheet); + const rect = await sheet.boundingBox(); + if (rect == null) { + throw new Error('no sheet'); + } + const viewport = page.viewportSize(); + if (viewport == null) { + throw new Error('no viewport'); + } + return { + driver, + sheet, + tallTop: rect.y, + // snapPoints={[0.5]}: the extra stop shows half the viewport. + halfTop: viewport.height * 0.5, + handle: {x: rect.x + rect.width / 2, y: rect.y + 12}, + }; +} + +// A plain evaluate, not boundingBox(): it must also work while the page's timers are frozen. +const sheetTop = async (sheet: Locator) => + sheet.evaluate(el => el.getBoundingClientRect().y); + +/** A slow drag that rests before the lift: release velocity 0, so the release projects onto itself. */ +function slowDragTo(from: {x: number; y: number}, dy: number) { + return drag( + from, + {x: from.x, y: from.y + dy}, + {durationMs: 400, hz: 60, restMs: 150}, + ); +} + +test('a slow drag follows the finger and settles on the nearest detent', async ({ + page, +}) => { + const {driver, sheet, tallTop, halfTop, handle} = await openSheet(page); + // Two thirds of the way to the half detent: nearer to it than to tall. + const dy = Math.round((halfTop - tallTop) * 0.66); + const script = slowDragTo(handle, dy); + const lastMove = + toTouchEvents(script).filter(e => e.type === 'touchMove').length - 1; + const tops: number[] = []; + let moves = 0; + let restingTop = NaN; + const started = Date.now(); + await driver.play(script, { + onEvent: async step => { + if (step.type !== 'touchMove') { + return; + } + // Read right away (moves are paced at 60 Hz), and once more after the + // last move has been presented: the finger rests 150 ms before lifting. + tops.push(await sheetTop(sheet)); + if (moves++ === lastMove) { + await frames(page, 1); + restingTop = await sheetTop(sheet); + } + }, + }); + // The sheet tracked the finger: monotonic, and 1:1 once out of the + // starting detent's 40 px magnet. + expect(tops.every((top, i) => i === 0 || top >= tops[i - 1] - 0.5)).toBe( + true, + ); + expect(restingTop).toBeCloseTo(tallTop + dy, 0); + await waitForRest(sheet); + expect(await sheetTop(sheet)).toBeCloseTo(halfTop, 0); + // No long task from the gesture or the settle (the story's own load is excluded). + expect(await readLongTasks(page, {from: started, to: Date.now()})).toEqual( + [], + ); +}); + +test('a downward flick dismisses the sheet', async ({page}) => { + const {driver, handle} = await openSheet(page); + const script = flick(handle, { + velocity: {x: 0, y: 2000}, + durationMs: 80, + hz: 120, + }); + expect(releaseVelocity(script).y).toBeCloseTo(2000, -1); + await driver.play(script); + await expect(page.getByRole('dialog', {name: 'Nearby places'})).toBeHidden(); +}); + +test('the settle follows its timing function, sampled frame by frame', async ({ + page, +}, testInfo) => { + const {driver, sheet, tallTop, halfTop, handle} = await openSheet(page); + const dy = Math.round((halfTop - tallTop) * 0.66); + await driver.play(slowDragTo(handle, dy)); + const [settle] = await waitForAnimation(sheet); + const held = await holdAnimations(page, sheet); + const transition = held.find(a => a.name === 'transform'); + if (transition == null) { + throw new Error(`no transform transition held: ${JSON.stringify(held)}`); + } + // Seek the held settle to every frame time and read the presented transform. + const samples = curveOf( + await sampleCurve(sheet, timeline(transition.durationMs)), + 'translateY', + ); + const from = samples[0].value; + const to = samples[samples.length - 1].value; + const ease = timingFunction(transition.easing); + const expectedSettle = (t: number) => + from + (to - from) * ease(Math.min(1, t / transition.durationMs)); + const result = compareCurve(samples, expectedSettle, 0.5); + await releaseAnimations(page); + // UIKit parity metric, reported rather than asserted until the spring lands: + // the settle's initial velocity vs the finger's at release (here 0). + const initialVelocity = ((expectedSettle(1) - expectedSettle(0)) / 1) * 1000; // px/s over the first ms + testInfo.annotations.push({ + type: 'uikit-parity', + description: `settle starts at ${initialVelocity.toFixed(0)} px/s for a release at 0 px/s (UIKit hands the release velocity to the spring)`, + }); + await writeEvidence( + 'curves/bottomsheet-settle', + { + [`${testInfo.project.name}.json`]: JSON.stringify( + {settle, transition, comparison: result}, + null, + 2, + ), + }, + testInfo, + ); + expect(result.maxError, JSON.stringify(result.worst)).toBeLessThanOrEqual( + 0.5, + ); + expect(samples.at(-1)?.value).toBeCloseTo(halfTop - tallTop, 0); +}); + +test('the settle keeps advancing every frame while the main thread is blocked', async ({ + page, +}, testInfo) => { + const {driver, sheet, tallTop, halfTop, handle} = await openSheet(page); + await markElement(sheet); + const cast = await startScreencast(page); + await driver.play(slowDragTo(handle, Math.round((halfTop - tallTop) * 0.66))); + // The settle must be running (and composited) before the block: an + // animation cannot start while the main thread is held. + const [settle] = await waitForAnimation(sheet, {minProgressMs: 16}); + const block = await blockMainThread(page, 700); + await page.waitForTimeout(150); + const castFrames = await cast.stop(); + const tracked = await trackMarker(page, castFrames); + // Judge the part of the block during which the settle was still moving. + // (until the last pixel: the easing tail moves less than a pixel per frame) + const proof = busyWindowProof( + tracked, + {from: block.from, to: Math.min(block.to, settle.to)}, + {settledWithin: 1}, + ); + await writeEvidence( + 'busy-main-thread/bottomsheet-settle', + { + [`${testInfo.project.name}.json`]: JSON.stringify( + { + block, + settle, + proof, + frames: tracked.map(f => ({ + t: +(f.t - block.from).toFixed(1), + y: f.box?.y ?? null, + h: f.box?.height ?? null, + })), + }, + null, + 1, + ), + }, + testInfo, + ); + const film = await castStoryboard(page, castFrames, { + name: 'bottomsheet-settle-busy-main-thread', + zero: block.from, + from: block.from - 40, + to: Math.min(block.to, settle.to) + 40, + columns: 8, + cellScale: 0.35, + data: t => ({y: tracked.find(f => f.t === t)?.box?.y ?? -1}), + }); + await film.write(testInfo, {block, settle}); + expect(proof.medianFrameMs).toBeLessThan(20); + expect(proof.frames).toBeGreaterThanOrEqual(proof.expectedFrames - 2); + expect(proof.changed).toBeGreaterThanOrEqual(proof.frames - 3); + expect(proof.longestStallMs).toBeLessThan(40); + expect(proof.monotonic).toBe(true); +}); + +test('the settle does no Layout or Paint per frame', async ({ + page, + browserName, +}) => { + test.skip(browserName !== 'chromium', 'rendering traces are Chromium-only'); + const {driver, sheet, tallTop, halfTop, handle} = await openSheet(page); + let settle = {from: 0, to: 0}; + const trace = await traceRendering(page, async () => { + await driver.play( + slowDragTo(handle, Math.round((halfTop - tallTop) * 0.66)), + ); + [settle] = await waitForAnimation(sheet); + await waitForRest(sheet); + }); + // The steady part of the settle: its first and last frames may commit. + const during = countBetween(trace, settle.from + 50, settle.to - 50); + expect(during.Layout ?? 0, JSON.stringify(during)).toBe(0); + expect(during.Paint ?? 0, JSON.stringify(during)).toBe(0); + expect(trace.longTasks).toEqual([]); +}); + +test( + 'storyboard: drag to the half detent and settle', + {tag: '@storyboard'}, + async ({page}, testInfo) => { + const {driver, sheet, tallTop, halfTop, handle} = await openSheet(page); + // Gate the page's timers from here on, so the settle's completion timer can + // be held with its animation while the filmstrip is photographed. + await installTimerGate(page); + const viewport = page.viewportSize() ?? {width: 402, height: 714}; + const clip = {x: 0, y: 0, width: viewport.width, height: viewport.height}; + const dy = Math.round((halfTop - tallTop) * 0.66); + const measure = async () => ({top: await sheetTop(sheet)}); + const dragBoard = await touchStoryboard( + page, + driver, + drag( + handle, + {x: handle.x, y: handle.y + dy}, + {durationMs: 200, hz: 30, restMs: 150}, + ), + { + name: 'bottomsheet-drag-to-detent', + clip, + measure, + columns: 5, + cellScale: 0.4, + }, + ); + await waitForAnimation(sheet); + await freezeTimers(page); + const {storyboard: settleBoard} = await animationStoryboard(page, { + name: 'bottomsheet-settle', + clip, + scope: sheet, + measure, + columns: 7, + cellScale: 0.4, + }); + await releaseAnimations(page, {finish: true}); + await thawTimers(page); + await dragBoard.write(testInfo); + await settleBoard.write(testInfo); + // The trace is the assertion; the sheets are the evidence. + const dragTops = dragBoard.frames.map(f => f.data.top as number); + expect( + dragTops.every((top, i) => i === 0 || top >= dragTops[i - 1] - 0.5), + ).toBe(true); + const settleTops = settleBoard.frames.map(f => f.data.top as number); + expect( + settleTops.every((top, i) => i === 0 || top >= settleTops[i - 1] - 0.5), + ).toBe(true); + expect(settleTops.at(-1)).toBeCloseTo(halfTop, 0); + }, +); diff --git a/playwright.interaction.config.ts b/playwright.interaction.config.ts index f0c61e2e6e563..40dd5653267b6 100644 --- a/playwright.interaction.config.ts +++ b/playwright.interaction.config.ts @@ -15,7 +15,7 @@ * pnpm storybook:build * pnpm exec playwright install chromium webkit * pnpm test:interaction --project chromium - * pnpm test:interaction harness.interaction.browser.spec.ts + * pnpm test:interaction BottomSheetDetent.interaction.browser.spec.ts * * A spec that needs Chromium-only evidence (CDP tracing, scripted touch * timestamps) skips itself in WebKit with the reason; see the harness README. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c93bc6ef3838a..ad5e966730d45 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -882,6 +882,9 @@ importers: '@astryxdesign/cli': specifier: workspace:* version: link:../cli + '@astryxdesign/interaction-harness': + specifier: workspace:* + version: link:../../internal/interaction-harness '@babel/cli': specifier: 'catalog:' version: 7.29.7(@babel/core@7.29.7) From 6be18c6248c4c2e831fcbb1216beb43067cccb78 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Fri, 23 Oct 2026 23:25:27 -0700 Subject: [PATCH 15/93] test(interaction): make the busy-main-thread verdict robust to a loaded machine The proof asserted a stall under 40 ms and near-exact frame counts, so a dropped screencast frame or two on a loaded runner failed it, and frames delivered out of order read as the element stepping backwards. busyWindowProof now returns a verdict that tolerates what load does to a screencast and nothing a frozen animation does: a median frame interval of at most 20 ms, no stall longer than three screencast intervals, and at least 90% of the window's whole frame intervals presenting a new position (one miss always allowed), with the raw numbers in a summary. Screencast frames are sorted by presentation time, the cadence comes from the frames around the window, and blockMainThread can block from inside the page in the frame an animation has run, so a short settle is still judged at length on a busy machine. The exemplar judges its settle until it is within 2 px of rest, since the screencast rounds to whole pixels. A unit test covers the limits; the rAF negative control still fails every criterion. --- internal/interaction-harness/README.md | 16 +- internal/interaction-harness/src/busy.test.ts | 144 ++++++++++++++ internal/interaction-harness/src/busy.ts | 176 ++++++++++++++++-- internal/interaction-harness/src/frames.ts | 19 +- .../src/harness.interaction.browser.spec.ts | 23 ++- ...tomSheetDetent.interaction.browser.spec.ts | 34 ++-- 6 files changed, 364 insertions(+), 48 deletions(-) create mode 100644 internal/interaction-harness/src/busy.test.ts diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 6bb189611db4d..ad3040d447d56 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -45,7 +45,7 @@ committed: a pull request links or attaches it. | `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact release velocity, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. | | `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | | `frames.ts` | `startScreencast` (timestamped composited frames), `markElement` (a marker chip or outline), `trackMarker` (marker box per frame), `sampleRects` (per-rAF rects), `frameStats`. | -| `busy.ts` | `blockMainThread(page, ms)` and `busyWindowProof(tracked, window)`: frames presented, frames that moved, longest stall. | +| `busy.ts` | `blockMainThread(page, ms)` and `busyWindowProof(tracked, window)`: frames presented, frames that moved, longest stall, and a verdict. | | `trace.ts` | `traceRendering(page, body)`: main-thread Layout / Paint / UpdateLayoutTree / PrePaint counts, `countBetween` for a sub-window, long tasks. `installLongTaskLog` / `readLongTasks`. | | `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t), `waitForAnimation`, `watchAnimations`, `waitForRest`, `releaseAnimations`. | | `math.ts` | Oracles independent of the code under test: `cubicBezier`, `timingFunction`, `springStep` (SwiftUI response / dampingFraction, initial velocity), `projectDecay`, `rubberBand`, `parseTransform`, `compareCurve`, `timeline`. | @@ -103,9 +103,17 @@ therefore proven on the iOS simulator, not here. The screencast is captured off the main thread, so a compositor animation shows a new position every frame of the block, and a main-thread animation shows none (nothing changed, so no frame is sent at all: `busyWindowProof` counts the -whole window as one stall). `settledWithin` stops judging at the easing tail -that moves less than a frame pixel per frame. `castStoryboard` turns the frames -of the block into a filmstrip. +whole window as one stall). `settledWithin` (2 px) stops judging at the easing +tail: the screencast rounds to whole pixels, so a tail that still moves shows +plateaus within a pixel or two of rest. The `verdict` tolerates what a +loaded machine does to a screencast and nothing a frozen animation does: a +median frame interval of at most 20 ms, no stall longer than three screencast +intervals (two dropped frames), and at least 90% of the window's whole frame +intervals presenting a new position (one miss is always allowed). +`verdict.failures` names each miss with its numbers, and `summary` carries the +raw numbers for the report. Frames are judged in presentation order: a loaded +machine can deliver them out of order. `castStoryboard` turns the frames of the +block into a filmstrip. **No Layout / Paint.** `traceRendering` records a Chromium trace around the interaction; `countBetween` counts the steady middle of an animation (its diff --git a/internal/interaction-harness/src/busy.test.ts b/internal/interaction-harness/src/busy.test.ts new file mode 100644 index 0000000000000..ad92548c84fc9 --- /dev/null +++ b/internal/interaction-harness/src/busy.test.ts @@ -0,0 +1,144 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file busy.test.ts + * @input Synthetic tracked frames (no browser) + * @output The busy-main-thread verdict's limits: dropped frames a loaded + * machine produces pass, a frozen animation fails every criterion, and + * out-of-order frames are judged in presentation order. + * @position Unit test of ./busy and ./frames' ordering; the browser proof + * with a real compositor animation and a rAF control is in + * harness.interaction.browser.spec.ts. + */ + +import {describe, expect, it} from 'vitest'; +import {busyWindowProof} from './busy'; +import {inPresentationOrder, type TrackedFrame} from './frames'; + +const FRAME = 1000 / 60; +const BLOCK = {from: 1000, to: 1700}; + +/** A marker moving 3 px per frame from t = 800 to 1900, minus `dropped` frame indices. */ +function moving(dropped: readonly number[] = []): TrackedFrame[] { + const frames: TrackedFrame[] = []; + for (let i = 0; 800 + i * FRAME <= 1900; i += 1) { + if (dropped.includes(i)) { + continue; + } + frames.push({ + t: 800 + i * FRAME, + box: {x: 0, y: 3 * i, width: 10, height: 10}, + } as TrackedFrame); + } + return frames; +} + +/** Index of the first frame at or after `t`. */ +const at = (t: number) => Math.ceil((t - 800) / FRAME); + +describe('busyWindowProof verdict', () => { + it('passes a compositor animation with every frame', () => { + const proof = busyWindowProof(moving(), BLOCK); + expect(proof.verdict.failures).toEqual([]); + expect(proof.longestStallFrames).toBe(1); + }); + + it('passes one and two consecutive dropped frames (3 intervals)', () => { + const k = at(1300); + expect(busyWindowProof(moving([k]), BLOCK).verdict.failures).toEqual([]); + const two = busyWindowProof(moving([k, k + 1]), BLOCK); + expect(two.longestStallFrames).toBe(3); + expect(two.verdict.failures).toEqual([]); + }); + + it('fails three consecutive dropped frames', () => { + const k = at(1300); + const proof = busyWindowProof(moving([k, k + 1, k + 2]), BLOCK); + expect(proof.longestStallFrames).toBe(4); + expect(proof.verdict.failures.join('\n')).toMatch(/longest stall/); + }); + + it('fails when more than 10% of the expected frames are missing', () => { + const k = at(1050); + // 39 frames expected: up to 3 may be missing (10% = 3.9). + const scattered = [0, 12, 24].map(d => k + d); + expect(busyWindowProof(moving(scattered), BLOCK).verdict.failures).toEqual( + [], + ); + const many = [0, 6, 12, 18, 24].map(d => k + d); + expect( + busyWindowProof(moving(many), BLOCK).verdict.failures.join('\n'), + ).toMatch(/expected frames moved/); + }); + + it('lets a short window miss a frame, not three', () => { + // 150 ms judged: 8 whole intervals, one miss allowed (10% would be 0.8). + const short = {from: 1000, to: 1190}; + const k = at(1040); + expect(busyWindowProof(moving([k]), short).verdict.failures).toEqual([]); + // Three of nine missing fails, on frame count or cadence. + expect(busyWindowProof(moving([k, k + 3, k + 6]), short).verdict.pass).toBe( + false, + ); + }); + + it('fails a frozen animation on every criterion', () => { + // Frames before and after the block only: nothing changed during it. + const frozen = moving().filter(f => f.t < BLOCK.from || f.t > BLOCK.to); + const proof = busyWindowProof(frozen, BLOCK); + expect(proof.changed).toBe(0); + expect(proof.verdict.pass).toBe(false); + expect(proof.verdict.failures.join('\n')).toMatch(/median frame interval/); + expect(proof.verdict.failures.join('\n')).toMatch(/longest stall/); + expect(proof.verdict.failures.join('\n')).toMatch(/expected frames moved/); + }); + + it('fails a half-rate animation (median interval above display rate)', () => { + const odd = moving().filter((_, i) => i % 2 === 1); + expect(busyWindowProof(odd, BLOCK).verdict.failures.join('\n')).toMatch( + /median frame interval/, + ); + }); +}); + +describe('busyWindowProof settledWithin', () => { + it('ends the judgment at a rounded easing tail, which would read as a stall', () => { + // An ease-out to y = 24 as the screencast reports it: whole pixels, so the + // tail repeats values while it still moves. + const ys = [ + 120, 100, 82, 67, 55, 46, 39, 34, 30, 28, 27, 26, 26, 26, 25, 25, 24, 24, + 24, 24, + ]; + const frames = ys.map( + (y, i) => + ({ + t: 1000 + i * FRAME, + box: {x: 0, y, width: 10, height: 10}, + }) as TrackedFrame, + ); + const window = {from: 1000, to: 1000 + ys.length * FRAME}; + const loose = busyWindowProof(frames, window, { + settledWithin: 2, + marginMs: 0, + }); + expect(loose.verdict.failures).toEqual([]); + const strict = busyWindowProof(frames, window, { + settledWithin: 1, + marginMs: 0, + }); + expect(strict.longestStallFrames).toBeGreaterThanOrEqual(3); + }); +}); + +describe('inPresentationOrder', () => { + it('sorts frames delivered out of order, so motion stays monotonic', () => { + const frames = moving(); + const k = at(1040); + const shuffled = [...frames]; + [shuffled[k], shuffled[k + 1]] = [shuffled[k + 1], shuffled[k]]; + expect(busyWindowProof(shuffled, BLOCK).monotonic).toBe(false); + const proof = busyWindowProof(inPresentationOrder(shuffled), BLOCK); + expect(proof.monotonic).toBe(true); + expect(proof.verdict.failures).toEqual([]); + }); +}); diff --git a/internal/interaction-harness/src/busy.ts b/internal/interaction-harness/src/busy.ts index e8674d07044a5..2f98844603496 100644 --- a/internal/interaction-harness/src/busy.ts +++ b/internal/interaction-harness/src/busy.ts @@ -15,7 +15,7 @@ * SYNC: Keep the README's busy-main-thread section equal to this contract. */ -import type {Page} from '@playwright/test'; +import type {Locator, Page} from '@playwright/test'; import {frameStats, type FrameStats, type TrackedFrame} from './frames'; export interface BusyWindow { @@ -33,25 +33,83 @@ export interface BusyWindow { * headless Chromium, even with only passive listeners, and an animation that * has not started cannot start (see `waitForAnimation`). Finger tracking * under a busy main thread is a simulator proof, not a Playwright one. + * + * With `during`, the page itself waits until an animation on that element (or + * inside it) has run `minProgressMs`, then blocks in the same frame. A round + * trip from the test would start the block later the busier the machine is, + * leaving less of a short animation to judge. */ export async function blockMainThread( page: Page, ms: number, + { + during, + minProgressMs = 16, + timeout = 5_000, + }: {during?: Locator; minProgressMs?: number; timeout?: number} = {}, ): Promise { - return page.evaluate(duration => { - const from = performance.timeOrigin + performance.now(); - const end = performance.now() + duration; - let spins = 0; - while (performance.now() < end) { - spins += 1; - } - return {from, to: performance.timeOrigin + performance.now(), spins}; - }, ms); + if (during == null) { + return page.evaluate(duration => { + const from = performance.timeOrigin + performance.now(); + const end = performance.now() + duration; + let spins = 0; + while (performance.now() < end) { + spins += 1; + } + return {from, to: performance.timeOrigin + performance.now(), spins}; + }, ms); + } + return during.evaluate( + (element, {duration, minProgress, wait}) => + new Promise((resolve, reject) => { + const deadline = performance.now() + wait; + const tick = () => { + const running = element + .getAnimations({subtree: true}) + .some( + a => + a.playState === 'running' && + Number(a.currentTime ?? 0) >= minProgress, + ); + if (running) { + // The same loop as above, in the frame that saw the animation. + const from = performance.timeOrigin + performance.now(); + const end = performance.now() + duration; + while (performance.now() < end) { + // Busy: nothing else runs. + } + resolve({from, to: performance.timeOrigin + performance.now()}); + } else if (performance.now() > deadline) { + reject(new Error('blockMainThread: no animation started')); + } else { + requestAnimationFrame(tick); + } + }; + requestAnimationFrame(tick); + }), + {duration: ms, minProgress: minProgressMs, wait: timeout}, + ); +} + +export interface BusyVerdict { + readonly pass: boolean; + /** Each criterion the window missed, with its measured value and limit. */ + readonly failures: readonly string[]; } export interface BusyProof extends FrameStats { readonly window: BusyWindow; - /** Frames the display should have shown in the window at the measured cadence. */ + /** The part of the window judged: inside the margins, up to `settledWithin`. */ + readonly judged: {readonly from: number; readonly to: number}; + /** + * The screencast's frame interval, from the frames around the window as well + * as inside it (a short window has few intervals; a frozen one has none). + */ + readonly cadenceMs: number; + /** + * Whole frame intervals in the window at the measured cadence: the frames + * the display shows there at least (one more, depending on phase). + */ readonly expectedFrames: number; /** * Longest time inside the window with no presented change, the window's own @@ -60,6 +118,22 @@ export interface BusyProof extends FrameStats { * window. */ readonly longestStallMs: number; + /** + * `longestStallMs` in screencast intervals, rounded: 1 is no frame missed, + * 2 is one dropped frame. Rounding absorbs timestamp jitter. + */ + readonly longestStallFrames: number; + /** + * Frames in the window that presented a new position (the first compared + * with the last frame before the window). + */ + readonly moved: number; + /** `moved` over `expectedFrames`. */ + readonly movedFraction: number; + /** The judgment against the limits (see `busyWindowProof`). */ + readonly verdict: BusyVerdict; + /** One line with the raw numbers, for an annotation or a failure message. */ + readonly summary: string; } /** @@ -67,6 +141,15 @@ export interface BusyProof extends FrameStats { * inside it, how many of them moved the tracked axis, and the longest stall. * The window is trimmed by `marginMs` at both ends so the frame the block * started on and the one it ended on are not counted. + * + * The verdict tolerates what a loaded machine does to a screencast (a dropped + * frame or two) and nothing a main-thread animation does (it presents no + * frame for the whole block). It passes when: + * - the median interval between presented frames is at most + * `maxMedianFrameMs` (display rate); + * - no stall spans more than `maxStallFrames` screencast intervals; + * - at least `minMovedFraction` of the frames the window should present at + * that cadence moved the tracked axis (a short window may still miss one). */ export function busyWindowProof( tracked: readonly TrackedFrame[], @@ -75,13 +158,27 @@ export function busyWindowProof( axis = 'y', marginMs = 20, settledWithin, + maxMedianFrameMs = 20, + maxStallFrames = 3, + minMovedFraction = 0.9, }: { axis?: 'x' | 'y' | 'width' | 'height'; marginMs?: number; + /** Display rate: the most the median frame interval may be, ms. */ + maxMedianFrameMs?: number; + /** The most screencast intervals one stall may span (3 = two dropped frames). */ + maxStallFrames?: number; /** - * Stop judging once the value is within this many px of where it ends: - * an easing tail moves less than a frame pixel per frame, and the - * screencast resolves whole CSS pixels. + * The least fraction of the expected frames that must move. One missed + * frame is always allowed, so a window of under ten frames is not judged + * stricter than a long one. + */ + minMovedFraction?: number; + /** + * Stop judging once the value is within this many px of where it ends. + * The screencast reports whole CSS pixels, so an easing tail that still + * moves shows plateaus (26, 26, 26 against a final 24) that would read + * as a stall: 2 is the usual value. */ settledWithin?: number; } = {}, @@ -129,11 +226,58 @@ export function busyWindowProof( const longestStallMs = Math.max( ...edges.slice(1).map((t, i) => t - edges[i]), ); - const cadence = stats.medianFrameMs || 1000 / 60; + const around = tracked + .filter(f => f.t >= inner.from - 250 && f.t <= inner.to + 250) + .map(f => f.t); + const intervals = around + .slice(1) + .map((t, i) => t - around[i]) + .sort((a, b) => a - b); + const cadence = + intervals.length >= 4 + ? intervals[Math.floor(intervals.length / 2)] + : stats.medianFrameMs || 1000 / 60; + const expectedFrames = Math.floor((inner.to - inner.from) / cadence); + const longestStallFrames = Math.round(longestStallMs / cadence); + const moved = changes.length; + const movedFraction = moved / Math.max(1, expectedFrames); + const failures: string[] = []; + if (stats.frames === 0 || stats.medianFrameMs > maxMedianFrameMs) { + failures.push( + `median frame interval ${stats.medianFrameMs.toFixed(1)} ms > ${maxMedianFrameMs} ms (${stats.frames} frames presented)`, + ); + } + if (longestStallFrames > maxStallFrames) { + failures.push( + `longest stall ${longestStallMs.toFixed(1)} ms = ${longestStallFrames} intervals > ${maxStallFrames}`, + ); + } + const allowedMisses = Math.max( + 1, + Math.floor((1 - minMovedFraction) * expectedFrames + 1e-9), + ); + if (expectedFrames - moved > allowedMisses) { + failures.push( + `${moved} of ${expectedFrames} expected frames moved (${(movedFraction * 100).toFixed(0)}% < ${(minMovedFraction * 100).toFixed(0)}%)`, + ); + } + const summary = + `${(inner.to - inner.from).toFixed(0)} ms judged at ${cadence.toFixed(1)} ms/frame: ${stats.frames} frames presented, ` + + `${moved}/${expectedFrames} moved (${(movedFraction * 100).toFixed(0)}%), ` + + `median ${stats.medianFrameMs.toFixed(1)} ms, ` + + `longest stall ${longestStallMs.toFixed(1)} ms (${longestStallFrames} intervals), ` + + `${stats.monotonic ? 'monotonic' : 'not monotonic'}`; return { ...stats, window, + judged: inner, + cadenceMs: cadence, longestStallMs, - expectedFrames: Math.floor((inner.to - inner.from) / cadence), + expectedFrames, + longestStallFrames, + moved, + movedFraction, + verdict: {pass: failures.length === 0, failures}, + summary, }; } diff --git a/internal/interaction-harness/src/frames.ts b/internal/interaction-harness/src/frames.ts index 927a4e12dd6a5..f7497e211029c 100644 --- a/internal/interaction-harness/src/frames.ts +++ b/internal/interaction-harness/src/frames.ts @@ -108,7 +108,7 @@ async function chromiumScreencast( await cdp.send('Page.stopScreencast'); cdp.off('Page.screencastFrame', onFrame); await cdp.detach().catch(() => {}); - return frames; + return inPresentationOrder(frames); }, }; } @@ -149,13 +149,26 @@ async function webkitScreencast( if (frames.length === 0) { return frames; } + const ordered = inPresentationOrder(frames); // Rebase the monotonic stamps onto the wall clock (first frame ≈ start). - const offset = wallAtStart - frames[0].t; - return frames.map(f => ({...f, t: f.t + offset})); + const offset = wallAtStart - ordered[0].t; + return ordered.map(f => ({...f, t: f.t + offset})); }, }; } +/** + * Frames sorted by their presentation timestamp. A loaded machine can deliver + * screencast frames out of order (a frame stamped 35 ms arriving after one + * stamped 46 ms); judged in arrival order, a steadily moving element would + * appear to step backwards. + */ +export function inPresentationOrder( + frames: readonly T[], +): T[] { + return [...frames].sort((a, b) => a.t - b.t); +} + /** The marker colour: magenta never occurs in Astryx themes. */ export const MARKER_RGB: readonly [number, number, number] = [255, 0, 255]; diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 71c3ba17ea28e..8f5487f3c9f68 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -252,7 +252,7 @@ test.describe('touch input', () => { test.describe('frame evidence', () => { test('a compositor animation advances every frame through a blocked main thread; a rAF one freezes', async ({ page, - }) => { + }, testInfo) => { await fixture(page); const box = page.locator('#box'); await markElement(box, {mode: 'outline'}); @@ -287,19 +287,26 @@ test.describe('frame evidence', () => { }; const compositor = await run('compositor'); - expect(compositor.medianFrameMs).toBeLessThan(20); - expect(compositor.frames).toBeGreaterThanOrEqual( - compositor.expectedFrames - 2, - ); - expect(compositor.changed).toBeGreaterThanOrEqual(compositor.frames - 3); - expect(compositor.longestStallMs).toBeLessThan(40); + testInfo.annotations.push({ + type: 'busy-main-thread compositor', + description: compositor.summary, + }); + expect(compositor.verdict.failures, compositor.summary).toEqual([]); expect(compositor.monotonic).toBe(true); const raf = await run('raf'); + testInfo.annotations.push({ + type: 'busy-main-thread rAF control', + description: raf.summary, + }); // Frozen for the whole block: no presented frame moved (the compositor - // sends none at all when nothing on screen changes). + // sends none at all when nothing on screen changes), and the verdict that + // passes the compositor animation fails it on every criterion. expect(raf.changed).toBe(0); expect(raf.longestStallMs).toBeGreaterThan(600); + expect(raf.verdict.pass).toBe(false); + expect(raf.verdict.failures.join('\n')).toMatch(/longest stall/); + expect(raf.verdict.failures.join('\n')).toMatch(/expected frames moved/); }); test('rect sampling reads one rect per animation frame while the main thread is free', async ({ diff --git a/packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts b/packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts index 7fb6e3459d27d..61d4b7ed5aa0d 100644 --- a/packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts +++ b/packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts @@ -226,26 +226,25 @@ test('the settle keeps advancing every frame while the main thread is blocked', const cast = await startScreencast(page); await driver.play(slowDragTo(handle, Math.round((halfTop - tallTop) * 0.66))); // The settle must be running (and composited) before the block: an - // animation cannot start while the main thread is held. - const [settle] = await waitForAnimation(sheet, {minProgressMs: 16}); - const block = await blockMainThread(page, 700); + // animation cannot start while the main thread is held. The page blocks + // itself in the frame the settle has run 16 ms, however busy the machine. + const block = await blockMainThread(page, 700, { + during: sheet, + minProgressMs: 16, + }); await page.waitForTimeout(150); const castFrames = await cast.stop(); const tracked = await trackMarker(page, castFrames); - // Judge the part of the block during which the settle was still moving. - // (until the last pixel: the easing tail moves less than a pixel per frame) - const proof = busyWindowProof( - tracked, - {from: block.from, to: Math.min(block.to, settle.to)}, - {settledWithin: 1}, - ); + // Judge the part of the block during which the settle was still moving: + // until it is within 2 px of rest (the screencast rounds to whole pixels, + // so the easing tail shows plateaus there). + const proof = busyWindowProof(tracked, block, {settledWithin: 2}); await writeEvidence( 'busy-main-thread/bottomsheet-settle', { [`${testInfo.project.name}.json`]: JSON.stringify( { block, - settle, proof, frames: tracked.map(f => ({ t: +(f.t - block.from).toFixed(1), @@ -263,16 +262,17 @@ test('the settle keeps advancing every frame while the main thread is blocked', name: 'bottomsheet-settle-busy-main-thread', zero: block.from, from: block.from - 40, - to: Math.min(block.to, settle.to) + 40, + to: proof.judged.to + 40, columns: 8, cellScale: 0.35, data: t => ({y: tracked.find(f => f.t === t)?.box?.y ?? -1}), }); - await film.write(testInfo, {block, settle}); - expect(proof.medianFrameMs).toBeLessThan(20); - expect(proof.frames).toBeGreaterThanOrEqual(proof.expectedFrames - 2); - expect(proof.changed).toBeGreaterThanOrEqual(proof.frames - 3); - expect(proof.longestStallMs).toBeLessThan(40); + await film.write(testInfo, {block, judged: proof.judged}); + testInfo.annotations.push({ + type: 'busy-main-thread', + description: proof.summary, + }); + expect(proof.verdict.failures, proof.summary).toEqual([]); expect(proof.monotonic).toBe(true); }); From 3f39b21d2baee0fa34317fe8157fa76d134bed35 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sat, 24 Oct 2026 15:09:37 -0700 Subject: [PATCH 16/93] test(BottomSheet): pin the settle's easing and duration instead of reading them from the animation The settle-curve oracle took its easing and duration from the animation under test, so it passed a 900 ms ease-in or a 150 ms linear settle as readily as the real one. The spec now asserts the theme's values (the default --ease-standard, cubic-bezier(0.24, 1, 0.4, 1), over the neutral theme's --duration-medium, 300 ms) and builds the oracle from them; the header no longer claims 410 ms. --- ...tomSheetDetent.interaction.browser.spec.ts | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts b/packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts index 61d4b7ed5aa0d..201c0d2b671f1 100644 --- a/packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts +++ b/packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts @@ -15,10 +15,11 @@ * @astryxdesign/interaction-harness, so the evidence layer is explicit. * * The expected curve here is today's: the panel's `--ease-standard` CSS - * transition (cubic-bezier(0.24, 1, 0.4, 1), `--duration-medium` = 410 ms). - * When the settle moves to the UIKit spring, `expectedSettle` becomes the - * spring from the motion constants, and the velocity hand-off reported below - * becomes an assertion. + * transition over `--duration-medium`, pinned below as SETTLE_EASING and + * SETTLE_MS rather than read from the animation under test. When the settle + * moves to the UIKit spring, `expectedSettle` becomes the spring from the + * motion constants, and the velocity hand-off reported below becomes an + * assertion. */ import {expect, test, type Locator, type Page} from '@playwright/test'; @@ -61,6 +62,12 @@ import { const storybook = withStorybook(test); +// The settle's timing, from the theme the story renders in: `--ease-standard` +// is the default cubic-bezier(0.24, 1, 0.4, 1), and `--duration-medium` is the +// neutral theme's motion `medium` (300 ms; the base theme's is 410 ms). +const SETTLE_EASING = 'cubic-bezier(0.24, 1, 0.4, 1)'; +const SETTLE_MS = 300; + interface OpenSheet { driver: TouchDriver; sheet: Locator; @@ -182,16 +189,22 @@ test('the settle follows its timing function, sampled frame by frame', async ({ if (transition == null) { throw new Error(`no transform transition held: ${JSON.stringify(held)}`); } + // The settle runs the pinned timing: a changed curve or duration fails here, + // and the oracle below is built from the pinned values, not from it. + expect(transition.easing.replace(/\s+/g, '')).toBe( + SETTLE_EASING.replace(/\s+/g, ''), + ); + expect(transition.durationMs).toBe(SETTLE_MS); // Seek the held settle to every frame time and read the presented transform. const samples = curveOf( - await sampleCurve(sheet, timeline(transition.durationMs)), + await sampleCurve(sheet, timeline(SETTLE_MS)), 'translateY', ); const from = samples[0].value; const to = samples[samples.length - 1].value; - const ease = timingFunction(transition.easing); + const ease = timingFunction(SETTLE_EASING); const expectedSettle = (t: number) => - from + (to - from) * ease(Math.min(1, t / transition.durationMs)); + from + (to - from) * ease(Math.min(1, t / SETTLE_MS)); const result = compareCurve(samples, expectedSettle, 0.5); await releaseAnimations(page); // UIKit parity metric, reported rather than asserted until the spring lands: From 03eadf934ce4377f2056061bf249118269e9b299 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sat, 24 Oct 2026 21:14:26 -0700 Subject: [PATCH 17/93] test(interaction): block after the frame's rendering update; rebase WebKit frames on its clock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `blockMainThread`'s `during` path ran its loop inside the rAF callback of the frame that saw the animation. WebKit hands a newly started animation to its compositor in that frame's rendering update, which runs after rAF callbacks, so the loop held the hand-off and even a compositor animation froze (2 of 29 frames moved). The loop now runs from a task queued in that frame. - WebKit's screencast frames were rebased onto the wall clock by taking the first frame as the start, which shifted every frame earlier by however long that frame took to come (60–100 ms). They are now rebased by the measured offset between the wall clock and CLOCK_MONOTONIC, which WebKit stamps them on. - `busyWindowProof` fails a judged part shorter than four frames (the animation under test was not moving during the block) instead of passing it, takes the end value from `settledAt` when given (WebKit can present a stale position once an accelerated animation ends while the main thread is blocked), and reports where the longest stall starts. - The BottomSheet busy case arms the block before the release and reads where the settle ends from the page. --- internal/interaction-harness/README.md | 9 ++- internal/interaction-harness/src/busy.test.ts | 59 +++++++++++++++- internal/interaction-harness/src/busy.ts | 69 ++++++++++++++----- internal/interaction-harness/src/frames.ts | 19 +++-- .../src/harness.interaction.browser.spec.ts | 32 +++++++++ ...tomSheetDetent.interaction.browser.spec.ts | 20 ++++-- 6 files changed, 176 insertions(+), 32 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index ad3040d447d56..2ec2f56d2074d 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -105,7 +105,14 @@ new position every frame of the block, and a main-thread animation shows none (nothing changed, so no frame is sent at all: `busyWindowProof` counts the whole window as one stall). `settledWithin` (2 px) stops judging at the easing tail: the screencast rounds to whole pixels, so a tail that still moves shows -plateaus within a pixel or two of rest. The `verdict` tolerates what a +plateaus within a pixel or two of rest. A judged part shorter than four frames +fails: the animation under test was not moving during the block. With +`during`, `blockMainThread` starts the loop from a task queued in the frame +that saw the animation, after that frame's rendering update: WebKit hands a +newly started animation to its compositor there, and a loop inside the rAF +callback would freeze even a compositor animation. Call it before the release +that starts a settle and await it after, so the page is already waiting when +the settle begins. The `verdict` tolerates what a loaded machine does to a screencast and nothing a frozen animation does: a median frame interval of at most 20 ms, no stall longer than three screencast intervals (two dropped frames), and at least 90% of the window's whole frame diff --git a/internal/interaction-harness/src/busy.test.ts b/internal/interaction-harness/src/busy.test.ts index ad92548c84fc9..3debc8abd7739 100644 --- a/internal/interaction-harness/src/busy.test.ts +++ b/internal/interaction-harness/src/busy.test.ts @@ -4,8 +4,9 @@ * @file busy.test.ts * @input Synthetic tracked frames (no browser) * @output The busy-main-thread verdict's limits: dropped frames a loaded - * machine produces pass, a frozen animation fails every criterion, and - * out-of-order frames are judged in presentation order. + * machine produces pass, a frozen animation fails every criterion, a window + * with nothing moving to judge fails, and out-of-order frames are judged in + * presentation order. * @position Unit test of ./busy and ./frames' ordering; the browser proof * with a real compositor animation and a rAF control is in * harness.interaction.browser.spec.ts. @@ -93,6 +94,31 @@ describe('busyWindowProof verdict', () => { expect(proof.verdict.failures.join('\n')).toMatch(/expected frames moved/); }); + it('fails a window in which the animation under test was not moving', () => { + // Settled before the block: the judgment ends where it starts. + const settled = moving().map(f => + f.t < 950 + ? f + : ({...f, box: {...f.box!, y: 3 * at(950)}} as TrackedFrame), + ); + const before = busyWindowProof(settled, BLOCK, {settledWithin: 2}); + expect(before.verdict.pass).toBe(false); + expect(before.verdict.failures.join('\n')).toMatch( + /judged = -?\d+ frames < 4: the animation under test was not moving/, + ); + // Settled 40 ms into the block: two frames, too few to judge. + const early = moving().map(f => + f.t < 1040 + ? f + : ({...f, box: {...f.box!, y: 3 * at(1040)}} as TrackedFrame), + ); + expect( + busyWindowProof(early, BLOCK, {settledWithin: 2}).verdict.failures.join( + '\n', + ), + ).toMatch(/frames < 4: the animation under test was not moving/); + }); + it('fails a half-rate animation (median interval above display rate)', () => { const odd = moving().filter((_, i) => i % 2 === 1); expect(busyWindowProof(odd, BLOCK).verdict.failures.join('\n')).toMatch( @@ -128,6 +154,35 @@ describe('busyWindowProof settledWithin', () => { }); expect(strict.longestStallFrames).toBeGreaterThanOrEqual(3); }); + + it('takes the end value from settledAt over a stale last frame', () => { + // The tail reaches y = 24, then the last frame shows a stale 100 (WebKit + // presents the last committed position once an accelerated animation + // ends while the main thread is still blocked). + const ys = [ + 120, 100, 82, 67, 55, 46, 39, 34, 30, 28, 27, 26, 25, 24, 24, 24, 100, + ]; + const frames = ys.map( + (y, i) => + ({ + t: 1000 + i * FRAME, + box: {x: 0, y, width: 10, height: 10}, + }) as TrackedFrame, + ); + const window = {from: 1000, to: 1000 + ys.length * FRAME}; + const stale = busyWindowProof(frames, window, { + settledWithin: 2, + marginMs: 0, + }); + expect(stale.verdict.pass).toBe(false); + const fromPage = busyWindowProof(frames, window, { + settledWithin: 2, + settledAt: 24, + marginMs: 0, + }); + expect(fromPage.verdict.failures).toEqual([]); + expect(fromPage.judged.to).toBe(frames[11].t); + }); }); describe('inPresentationOrder', () => { diff --git a/internal/interaction-harness/src/busy.ts b/internal/interaction-harness/src/busy.ts index 2f98844603496..b7a60b7ac837e 100644 --- a/internal/interaction-harness/src/busy.ts +++ b/internal/interaction-harness/src/busy.ts @@ -35,9 +35,16 @@ export interface BusyWindow { * under a busy main thread is a simulator proof, not a Playwright one. * * With `during`, the page itself waits until an animation on that element (or - * inside it) has run `minProgressMs`, then blocks in the same frame. A round - * trip from the test would start the block later the busier the machine is, - * leaving less of a short animation to judge. + * inside it) has run `minProgressMs`, then blocks from a task it queues in + * that frame. A round trip from the test would start the block later the + * busier the machine is, leaving less of a short animation to judge. The task + * runs after the frame's rendering update, which is where WebKit hands a newly + * started animation to its compositor: a loop inside the rAF callback itself + * holds that hand-off, and even a compositor animation freezes. To catch a + * settle near its start, call it before the release that starts the settle and + * await it after: the page is already waiting when the animation begins. Any + * animation inside `during` counts, so pick an element that runs nothing + * else meanwhile. */ export async function blockMainThread( page: Page, @@ -72,13 +79,17 @@ export async function blockMainThread( Number(a.currentTime ?? 0) >= minProgress, ); if (running) { - // The same loop as above, in the frame that saw the animation. - const from = performance.timeOrigin + performance.now(); - const end = performance.now() + duration; - while (performance.now() < end) { - // Busy: nothing else runs. - } - resolve({from, to: performance.timeOrigin + performance.now()}); + // The same loop as above, from a task queued in the frame that saw + // the animation: it runs once that frame's rendering update has + // handed the animation to the compositor. + setTimeout(() => { + const from = performance.timeOrigin + performance.now(); + const end = performance.now() + duration; + while (performance.now() < end) { + // Busy: nothing else runs. + } + resolve({from, to: performance.timeOrigin + performance.now()}); + }, 0); } else if (performance.now() > deadline) { reject(new Error('blockMainThread: no animation started')); } else { @@ -118,6 +129,8 @@ export interface BusyProof extends FrameStats { * window. */ readonly longestStallMs: number; + /** Where the longest stall starts, ms after the window's start. */ + readonly longestStallAtMs: number; /** * `longestStallMs` in screencast intervals, rounded: 1 is no frame missed, * 2 is one dropped frame. Rounding absorbs timestamp jitter. @@ -149,7 +162,11 @@ export interface BusyProof extends FrameStats { * `maxMedianFrameMs` (display rate); * - no stall spans more than `maxStallFrames` screencast intervals; * - at least `minMovedFraction` of the frames the window should present at - * that cadence moved the tracked axis (a short window may still miss one). + * that cadence moved the tracked axis (a short window may still miss one); + * - the judged part spans at least `minJudgedFrames` frames. Fewer means the + * animation under test was not moving during the block (it settled before + * the block, or the block caught another animation), so there is nothing to + * judge, and that fails rather than passes. */ export function busyWindowProof( tracked: readonly TrackedFrame[], @@ -158,9 +175,11 @@ export function busyWindowProof( axis = 'y', marginMs = 20, settledWithin, + settledAt, maxMedianFrameMs = 20, maxStallFrames = 3, minMovedFraction = 0.9, + minJudgedFrames = 4, }: { axis?: 'x' | 'y' | 'width' | 'height'; marginMs?: number; @@ -174,6 +193,8 @@ export function busyWindowProof( * stricter than a long one. */ minMovedFraction?: number; + /** The fewest whole frame intervals the judged part of the window may span. */ + minJudgedFrames?: number; /** * Stop judging once the value is within this many px of where it ends. * The screencast reports whole CSS pixels, so an easing tail that still @@ -181,6 +202,14 @@ export function busyWindowProof( * as a stall: 2 is the usual value. */ settledWithin?: number; + /** + * Where the value ends, for `settledWithin`: read it from the page once the + * animation has ended (`waitForRest`). Defaults to the last frame's value, + * which can be stale: once an accelerated animation ends while the main + * thread is still blocked, WebKit presents the layer's last committed + * position until the main thread commits the end state. + */ + settledAt?: number; } = {}, ): BusyProof { let end = window.to - marginMs; @@ -188,7 +217,8 @@ export function busyWindowProof( const valued = tracked.flatMap(f => f.box == null ? [] : [{t: f.t, value: f.box[axis]}], ); - const final = valued.length > 0 ? valued[valued.length - 1].value : null; + const final = + settledAt ?? (valued.length > 0 ? valued[valued.length - 1].value : null); const arrived = valued.find( f => f.t >= window.from && @@ -223,9 +253,10 @@ export function busyWindowProof( } } const edges = [inner.from, ...changes, inner.to]; - const longestStallMs = Math.max( - ...edges.slice(1).map((t, i) => t - edges[i]), - ); + const stalls = edges.slice(1).map((t, i) => t - edges[i]); + const longestStallMs = Math.max(...stalls); + // Where the longest stall starts, from the start of the window. + const longestStallAtMs = edges[stalls.indexOf(longestStallMs)] - window.from; const around = tracked .filter(f => f.t >= inner.from - 250 && f.t <= inner.to + 250) .map(f => f.t); @@ -252,6 +283,11 @@ export function busyWindowProof( `longest stall ${longestStallMs.toFixed(1)} ms = ${longestStallFrames} intervals > ${maxStallFrames}`, ); } + if (expectedFrames < minJudgedFrames) { + failures.push( + `${(inner.to - inner.from).toFixed(0)} ms judged = ${expectedFrames} frames < ${minJudgedFrames}: the animation under test was not moving during the block`, + ); + } const allowedMisses = Math.max( 1, Math.floor((1 - minMovedFraction) * expectedFrames + 1e-9), @@ -265,7 +301,7 @@ export function busyWindowProof( `${(inner.to - inner.from).toFixed(0)} ms judged at ${cadence.toFixed(1)} ms/frame: ${stats.frames} frames presented, ` + `${moved}/${expectedFrames} moved (${(movedFraction * 100).toFixed(0)}%), ` + `median ${stats.medianFrameMs.toFixed(1)} ms, ` + - `longest stall ${longestStallMs.toFixed(1)} ms (${longestStallFrames} intervals), ` + + `longest stall ${longestStallMs.toFixed(1)} ms (${longestStallFrames} intervals) at +${longestStallAtMs.toFixed(0)} ms, ` + `${stats.monotonic ? 'monotonic' : 'not monotonic'}`; return { ...stats, @@ -273,6 +309,7 @@ export function busyWindowProof( judged: inner, cadenceMs: cadence, longestStallMs, + longestStallAtMs, expectedFrames, longestStallFrames, moved, diff --git a/internal/interaction-harness/src/frames.ts b/internal/interaction-harness/src/frames.ts index f7497e211029c..fd0ed34c4c7f8 100644 --- a/internal/interaction-harness/src/frames.ts +++ b/internal/interaction-harness/src/frames.ts @@ -50,8 +50,9 @@ export interface ScreencastOptions { * Starts capturing composited frames. Chromium delivers one per presented * frame (16.7 ms at 60 Hz), timestamped by the compositor; WebKit's own * screencast delivers one per composited update (about every 14 ms in the - * Linux build), stamped on its monotonic clock and rebased here onto the wall - * clock. Both keep delivering while the page's main thread is blocked, and + * Linux build), stamped on CLOCK_MONOTONIC and rebased here onto the wall + * clock by that clock's measured offset, so both engines share the page's + * timeline. Both keep delivering while the page's main thread is blocked, and * neither sends a frame when nothing on screen changed. */ export async function startScreencast( @@ -141,7 +142,11 @@ async function webkitScreencast( quality, })) as {generation: number}; generation = started.generation; - const wallAtStart = Date.now(); + // WebKit stamps frames in seconds on CLOCK_MONOTONIC, which this process + // shares: rebase them by the wall clock's offset from that clock, sampled + // together. (Taking the first frame as the start shifted every frame earlier + // by however long that frame took to come, 60–100 ms on a busy page.) + const wallMinusMonotonic = Date.now() - Number(process.hrtime.bigint()) / 1e6; return { async stop() { await session.send('Screencast.stopScreencast'); @@ -149,10 +154,10 @@ async function webkitScreencast( if (frames.length === 0) { return frames; } - const ordered = inPresentationOrder(frames); - // Rebase the monotonic stamps onto the wall clock (first frame ≈ start). - const offset = wallAtStart - ordered[0].t; - return ordered.map(f => ({...f, t: f.t + offset})); + return inPresentationOrder(frames).map(f => ({ + ...f, + t: f.t + wallMinusMonotonic, + })); }, }; } diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 8f5487f3c9f68..79936623d7ff8 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -309,6 +309,38 @@ test.describe('frame evidence', () => { expect(raf.verdict.failures.join('\n')).toMatch(/expected frames moved/); }); + test('a block that starts during a just-started transition still lets the compositor advance it', async ({ + page, + }, testInfo) => { + // `during` blocks in the frame the transition has run 16 ms. In WebKit + // the transition reaches the compositor in that frame's rendering update, + // so a loop inside the rAF callback froze it (2 of 29 frames moved). + // Linear and longer than the block, so every frame of the block moves a + // whole pixel or more and the transition is still running when it ends. + await fixture(page); + const box = page.locator('#box'); + await markElement(box, {mode: 'outline'}); + const cast = await startScreencast(page); + await box.evaluate(el => { + el.style.transition = 'transform 1600ms linear'; + el.getBoundingClientRect(); + el.style.transform = 'translateX(280px)'; + }); + const window = await blockMainThread(page, 700, { + during: box, + minProgressMs: 16, + }); + await waitForRest(box); + const tracked = await trackMarker(page, await cast.stop()); + const proof = busyWindowProof(tracked, window, {axis: 'x'}); + testInfo.annotations.push({ + type: 'busy-main-thread transition', + description: proof.summary, + }); + expect(proof.verdict.failures, proof.summary).toEqual([]); + expect(proof.monotonic).toBe(true); + }); + test('rect sampling reads one rect per animation frame while the main thread is free', async ({ page, browserName, diff --git a/packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts b/packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts index 201c0d2b671f1..cdf43e69ce38b 100644 --- a/packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts +++ b/packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts @@ -237,27 +237,35 @@ test('the settle keeps advancing every frame while the main thread is blocked', const {driver, sheet, tallTop, halfTop, handle} = await openSheet(page); await markElement(sheet); const cast = await startScreencast(page); - await driver.play(slowDragTo(handle, Math.round((halfTop - tallTop) * 0.66))); // The settle must be running (and composited) before the block: an - // animation cannot start while the main thread is held. The page blocks - // itself in the frame the settle has run 16 ms, however busy the machine. - const block = await blockMainThread(page, 700, { + // animation cannot start while the main thread is held. Armed before the + // release, the page waits for the settle itself (the sheet runs no other + // animation during the drag) and blocks once it has run 16 ms, however busy + // the machine; armed after, the round trip would leave little of the settle + // to judge. + const blocking = blockMainThread(page, 700, { during: sheet, minProgressMs: 16, }); - await page.waitForTimeout(150); + await driver.play(slowDragTo(handle, Math.round((halfTop - tallTop) * 0.66))); + const block = await blocking; + // Where the settle ends, from the page: a frame after the block can still + // show a stale position (see `settledAt`). + await waitForRest(sheet); + const settledAt = await sheet.evaluate(el => el.getBoundingClientRect().y); const castFrames = await cast.stop(); const tracked = await trackMarker(page, castFrames); // Judge the part of the block during which the settle was still moving: // until it is within 2 px of rest (the screencast rounds to whole pixels, // so the easing tail shows plateaus there). - const proof = busyWindowProof(tracked, block, {settledWithin: 2}); + const proof = busyWindowProof(tracked, block, {settledWithin: 2, settledAt}); await writeEvidence( 'busy-main-thread/bottomsheet-settle', { [`${testInfo.project.name}.json`]: JSON.stringify( { block, + settledAt, proof, frames: tracked.map(f => ({ t: +(f.t - block.from).toFixed(1), From 8d0a8297f687fe56437175fd6be2a1e2cc058898 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sat, 24 Oct 2026 22:52:42 -0700 Subject: [PATCH 18/93] test(interaction): a held animation the page cancels leaves the held set Retargeting a CSS transition cancels it. If that happened while it was held, `seekAnimations` and `sampleCurve` set its time again, which revived it over the page's own new transition, and `releaseAnimations` replayed it from the start. Seeking, sampling and releasing now drop an `idle` animation from the held set first, and times count from a fixed origin (the earliest start held until the release), so dropping one never moves the others. The self-test retargets a held transition, seeks, releases and waits for rest: the cancelled transition stays idle and the box ends where the page sent it. On the old code the cancelled transition is still held after the seek. --- internal/interaction-harness/src/curves.ts | 60 +++++++++++++++---- .../src/harness.interaction.browser.spec.ts | 42 ++++++++++++- 2 files changed, 90 insertions(+), 12 deletions(-) diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index ed4fe0df69a84..be5e6100f0c88 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -190,6 +190,9 @@ export interface HeldAnimation { * each by its offset from the earliest start, and `releaseAnimations` resumes * only it. A second call adds to the held set: what the first held stays held * until the release, and offsets count from the earliest start of all of it. + * An animation the page cancels after the hold (retargeting a CSS transition + * cancels it) leaves the held set: seeking, sampling and releasing skip it + * rather than revive it. */ export async function holdAnimations( page: Page, @@ -236,10 +239,14 @@ export async function holdAnimations( const w = window as unknown as { __harnessHeld?: Animation[]; __harnessHeldStarts?: number[]; + __harnessHeldOrigin?: number; }; w.__harnessHeld = [...(w.__harnessHeld ?? []), ...held]; w.__harnessHeldStarts = [...(w.__harnessHeldStarts ?? []), ...starts]; - const earliest = Math.min(...w.__harnessHeldStarts); + // Times count from the earliest start of everything held until the + // release, so dropping one later never moves the others. + const earliest = Math.min(w.__harnessHeldOrigin ?? Infinity, ...starts); + w.__harnessHeldOrigin = earliest; const offsets = starts.map(start => start - earliest); return held.map((animation, index) => { const effect = animation.effect as KeyframeEffect | null; @@ -290,12 +297,22 @@ export async function seekAnimations( const w = window as unknown as { __harnessHeld?: Animation[]; __harnessHeldStarts?: number[]; + __harnessHeldOrigin?: number; }; - const starts = w.__harnessHeldStarts ?? []; - const earliest = Math.min(...starts); + // An animation the page cancelled since the hold (a CSS transition is + // cancelled whenever the page retargets its property) leaves the held set: + // applying or playing it again would revive what the page dropped. + const kept = (w.__harnessHeld ?? []).flatMap((animation, index) => + animation.playState === 'idle' + ? [] + : [{animation, start: (w.__harnessHeldStarts ?? [])[index] ?? 0}], + ); + w.__harnessHeld = kept.map(k => k.animation); + w.__harnessHeldStarts = kept.map(k => k.start); + const origin = w.__harnessHeldOrigin ?? 0; let presented = -Infinity; - (w.__harnessHeld ?? []).forEach((animation, index) => { - const offset = (starts[index] ?? earliest) - earliest; + kept.forEach(({animation, start}) => { + const offset = start - origin; const end = Number( animation.effect?.getComputedTiming().endTime ?? Infinity, ); @@ -323,10 +340,20 @@ export async function releaseAnimations( const w = window as unknown as { __harnessHeld?: Animation[]; __harnessHeldStarts?: number[]; + __harnessHeldOrigin?: number; }; - const held = w.__harnessHeld ?? []; + // An animation the page cancelled since the hold (a CSS transition is + // cancelled whenever the page retargets its property) leaves the held set: + // applying or playing it again would revive what the page dropped. + const kept = (w.__harnessHeld ?? []).flatMap((animation, index) => + animation.playState === 'idle' + ? [] + : [{animation, start: (w.__harnessHeldStarts ?? [])[index] ?? 0}], + ); + w.__harnessHeld = kept.map(k => k.animation); + w.__harnessHeldStarts = kept.map(k => k.start); try { - for (const animation of held) { + for (const {animation} of kept) { const end = Number( animation.effect?.getComputedTiming().endTime ?? Infinity, ); @@ -339,6 +366,7 @@ export async function releaseAnimations( } finally { w.__harnessHeld = []; w.__harnessHeldStarts = []; + w.__harnessHeldOrigin = undefined; } }, finish); } @@ -394,10 +422,20 @@ export async function sampleCurve( const w = window as unknown as { __harnessHeld?: Animation[]; __harnessHeldStarts?: number[]; + __harnessHeldOrigin?: number; }; - const held = w.__harnessHeld ?? []; - const starts = w.__harnessHeldStarts ?? []; - const earliest = Math.min(...starts); + // An animation the page cancelled since the hold (a CSS transition is + // cancelled whenever the page retargets its property) leaves the held set: + // applying or playing it again would revive what the page dropped. + const kept = (w.__harnessHeld ?? []).flatMap((animation, index) => + animation.playState === 'idle' + ? [] + : [{animation, start: (w.__harnessHeldStarts ?? [])[index] ?? 0}], + ); + w.__harnessHeld = kept.map(k => k.animation); + w.__harnessHeldStarts = kept.map(k => k.start); + const held = kept.map(k => k.animation); + const origin = w.__harnessHeldOrigin ?? 0; if (held.length === 0) { throw new Error( 'sampleCurve: no held animations (call holdAnimations first)', @@ -425,7 +463,7 @@ export async function sampleCurve( let presentedOpacity = -Infinity; held.forEach((animation, index) => { // Each at its own offset, just inside its end: see seekAnimations. - const offset = (starts[index] ?? earliest) - earliest; + const offset = kept[index].start - origin; const end = Number( animation.effect?.getComputedTiming().endTime ?? Infinity, ); diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 0caeb7665653a..cf314dfdf6102 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -11,7 +11,7 @@ * holds its end; individual transform properties are sampled; a sample at * an end reports the time it was read at; holding takes only running * document-timeline animations, each at its own offset, adds a second hold - * to the held set, and releases only them. + * to the held set, drops one the page cancels, and releases only them. * @position The harness's own test. A component spec trusts these * instruments; this file is why it may. */ @@ -471,6 +471,46 @@ test.describe('holding', () => { ).toBe(0); }); + test('a held transition the page retargets is dropped, not revived', async ({ + page, + }) => { + await fixture(page); + const box = page.locator('#box'); + await box.evaluate(el => { + el.style.transition = 'transform 300ms linear'; + el.getBoundingClientRect(); + el.style.transform = 'translateX(300px)'; + }); + await waitForAnimation(box); + expect(await holdAnimations(page, box)).toHaveLength(1); + // The page sends the box back: the held transition is cancelled, and a + // new one runs from where the box is. + await box.evaluate(el => { + const w = window as unknown as {__cancelled: Animation}; + w.__cancelled = el.getAnimations()[0]; + el.style.transform = 'none'; + }); + const state = () => + page.evaluate(() => { + const w = window as unknown as { + __cancelled: Animation; + __harnessHeld: Animation[]; + }; + return { + cancelled: w.__cancelled.playState, + held: w.__harnessHeld.length, + }; + }); + await seekAnimations(page, 150); + expect(await state()).toEqual({cancelled: 'idle', held: 0}); + await releaseAnimations(page); + await waitForRest(box); + expect(await state()).toEqual({cancelled: 'idle', held: 0}); + expect(await box.evaluate(el => getComputedStyle(el).transform)).toBe( + 'none', + ); + }); + test('finishing resumes an animation that repeats forever and still clears the held set', async ({ page, }) => { From a4998609228c826cdd775bf88b009608b45f2bc9 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 00:08:16 -0700 Subject: [PATCH 19/93] docs(interaction): the touch filmstrip's clock limit names no missing instrument `touchStoryboard`'s doc and the README pointed a clock-timed gesture at `startScreencast` and `castStoryboard`, which this branch does not have. They now say a storyboard cannot show such a gesture and to record it in real time. --- internal/interaction-harness/README.md | 4 ++-- internal/interaction-harness/src/storyboard.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 47d373b4f72c7..d851fa2135649 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -97,8 +97,8 @@ An animation that repeats forever does not set a storyboard's default length; with nothing else to film, pass `times`. A touch filmstrip keeps the scripted event `timeStamp`s (Chromium), but a handler that reads `performance.now()` or `Date.now()` sees one event per photograph, about 100 ms apart, so a -clock-timed release flings slower when filmed than when played: film such a -gesture from a screencast. +clock-timed release flings slower when filmed than when played: a storyboard +cannot show such a gesture; record it in real time instead. ## Engine limits (measured with Playwright 1.61) diff --git a/internal/interaction-harness/src/storyboard.ts b/internal/interaction-harness/src/storyboard.ts index 8386edaf86323..ead43b713fbcd 100644 --- a/internal/interaction-harness/src/storyboard.ts +++ b/internal/interaction-harness/src/storyboard.ts @@ -305,8 +305,8 @@ export interface TouchStoryboardOptions extends StoryboardOptions { * sees the filmstrip's pacing in both engines: two frames, `measure` and a * photograph per event, about 100 ms between moves instead of the script's * 16.7. Its release velocity is that much lower, so a flick that dismisses - * when played can settle back when filmed: film such a gesture from a - * screencast (`startScreencast`, `castStoryboard`) instead. + * when played can settle back when filmed: a storyboard cannot show such a + * gesture; record it in real time instead. */ export async function touchStoryboard( page: Page, From b36bf63e16d3f5ef997258ebbe96935d82b8744e Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 00:08:52 -0700 Subject: [PATCH 20/93] docs(interaction): the touch filmstrip's clock limit names the screencast instruments With frame evidence in the harness, a clock-timed gesture a storyboard cannot show is recorded with `startScreencast` and `castStoryboard`: the doc and the README name them. --- internal/interaction-harness/README.md | 3 ++- internal/interaction-harness/src/storyboard.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 2171dfd4d1b11..a40c8755f3a72 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -140,7 +140,8 @@ with nothing else to film, pass `times`. A touch filmstrip keeps the scripted event `timeStamp`s (Chromium), but a handler that reads `performance.now()` or `Date.now()` sees one event per photograph, about 100 ms apart, so a clock-timed release flings slower when filmed than when played: a storyboard -cannot show such a gesture; record it in real time instead. +cannot show such a gesture; record it in real time instead, with +`startScreencast` and `castStoryboard`. ## Engine limits (measured with Playwright 1.61) diff --git a/internal/interaction-harness/src/storyboard.ts b/internal/interaction-harness/src/storyboard.ts index e1208a0f83aad..7bbcf533c3e6f 100644 --- a/internal/interaction-harness/src/storyboard.ts +++ b/internal/interaction-harness/src/storyboard.ts @@ -306,7 +306,8 @@ export interface TouchStoryboardOptions extends StoryboardOptions { * photograph per event, about 100 ms between moves instead of the script's * 16.7. Its release velocity is that much lower, so a flick that dismisses * when played can settle back when filmed: a storyboard cannot show such a - * gesture; record it in real time instead. + * gesture; record it in real time instead (`startScreencast`, + * `castStoryboard`). */ export async function touchStoryboard( page: Page, From 91f8ddd5fe9e6b032145386dfa0753a55ba6c29e Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 01:44:46 -0700 Subject: [PATCH 21/93] test(interaction): a hold stays where it was read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `pause()` only schedules the pause: the engine takes the hold time at its next frame, so a held animation stayed held a frame past `heldAtMs`, and a read right after the hold varied by a frame between runs. Setting the current time right after `pause()` completes a pending pause at once, where `heldAtMs` read it. The self-test holds a 1 s move and checks it is still at `heldAtMs` two frames later; on the previous code it reads one frame later in Chromium (16.6–16.7 ms) and 15–70 ms later in WebKit. --- internal/interaction-harness/src/curves.ts | 5 +++ .../src/harness.interaction.browser.spec.ts | 34 +++++++++++++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index be5e6100f0c88..4f1312cf434e1 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -232,6 +232,11 @@ export async function holdAnimations( ); const held = all.map(animation => { animation.pause(); + // `pause()` only schedules the pause: the engine takes the hold time at + // its next frame, a frame past where this read it. Setting the current + // time completes a pending pause at once, here. + // eslint-disable-next-line no-self-assign + animation.currentTime = animation.currentTime; return animation; }); // A later hold adds to the held set: what an earlier one held stays held diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index cf314dfdf6102..64546515d058f 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -10,8 +10,9 @@ * and its end, and judge rest by presented values even while an animation * holds its end; individual transform properties are sampled; a sample at * an end reports the time it was read at; holding takes only running - * document-timeline animations, each at its own offset, adds a second hold - * to the held set, drops one the page cancels, and releases only them. + * document-timeline animations, each at its own offset, holds each where it + * was read, adds a second hold to the held set, drops one the page cancels, + * and releases only them. * @position The harness's own test. A component spec trusts these * instruments; this file is why it may. */ @@ -434,6 +435,35 @@ test.describe('holding', () => { await releaseAnimations(page); }); + test('a hold stays where it was read, not a frame later', async ({page}) => { + await fixture(page); + await page.evaluate(() => { + document + .getElementById('box') + ?.animate( + [{transform: 'translateX(0px)'}, {transform: 'translateX(300px)'}], + {duration: 1000, easing: 'linear'}, + ); + }); + const box = page.locator('#box'); + await waitForAnimation(box, {minProgressMs: 50}); + const [held] = await holdAnimations(page, box); + // Two frames later, the held animation is still where the hold read it. + const after = await box.evaluate( + el => + new Promise(resolve => + requestAnimationFrame(() => + requestAnimationFrame(() => + resolve(Number(el.getAnimations()[0]?.currentTime)), + ), + ), + ), + ); + expect(held.heldAtMs).not.toBeNull(); + expect(Math.abs(after - (held.heldAtMs ?? 0))).toBeLessThan(0.5); + await releaseAnimations(page); + }); + test('a second hold adds to the held set, and releasing resumes both', async ({ page, }) => { From 3f44fe980b89f5cf8cb892696e03acb300029ff1 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 02:57:02 -0700 Subject: [PATCH 22/93] test(interaction): a block never starts on a pending animation `blockMainThread({during})` counted an animation as running as soon as its current time reached `minProgressMs`. An animation created in the frame the page looks is still pending, not on the compositor yet, and at `minProgressMs: 0` the block started on it and froze it: in WebKit deterministically, and in Chromium on a Lightbox page while the block still ran inside the frame's callback. A pending animation no longer counts, and `minProgressMs` is raised to one frame (16 ms), since in WebKit an animation's start time resolves before its layer is flushed. The self-test creates the animation in the frame the helper looks and blocks at `minProgressMs: 0`. On the previous code it fails 6/6 in WebKit (0 frames presented through the block); now it passes 10/10 in WebKit and 6/6 in Chromium. --- internal/interaction-harness/README.md | 4 +- internal/interaction-harness/src/busy.ts | 13 +++++- .../src/harness.interaction.browser.spec.ts | 46 +++++++++++++++++++ 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index a40c8755f3a72..0d9eef08f35c1 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -112,7 +112,9 @@ that saw the animation, after that frame's rendering update: WebKit hands a newly started animation to its compositor there, and a loop inside the rAF callback would freeze even a compositor animation. Call it before the release that starts a settle and await it after, so the page is already waiting when -the settle begins. The `verdict` tolerates what a +the settle begins. An animation counts once it is no longer pending and has +run at least one frame: a `minProgressMs` under 16 is raised to 16, since one +created in the frame the page looks is not on the compositor yet. The `verdict` tolerates what a loaded machine does to a screencast and nothing a frozen animation does: a median frame interval of at most 20 ms, no stall longer than three screencast intervals (two dropped frames), and at least 90% of the window's whole frame diff --git a/internal/interaction-harness/src/busy.ts b/internal/interaction-harness/src/busy.ts index b7a60b7ac837e..78d456ec97522 100644 --- a/internal/interaction-harness/src/busy.ts +++ b/internal/interaction-harness/src/busy.ts @@ -45,7 +45,16 @@ export interface BusyWindow { * await it after: the page is already waiting when the animation begins. Any * animation inside `during` counts, so pick an element that runs nothing * else meanwhile. + * + * An animation counts once it is no longer pending and has run at least one + * frame (`minProgressMs` below 16 ms is raised to 16). One created in the + * frame the page looks is still pending, not yet on the compositor + * (Chromium), and in WebKit its start time resolves before its layer is + * flushed: a block that started on it froze it. */ +/** One display frame, ms: the least progress `blockMainThread` waits for. */ +const ONE_FRAME_MS = 16; + export async function blockMainThread( page: Page, ms: number, @@ -55,6 +64,7 @@ export async function blockMainThread( timeout = 5_000, }: {during?: Locator; minProgressMs?: number; timeout?: number} = {}, ): Promise { + const minProgress = Math.max(minProgressMs, ONE_FRAME_MS); if (during == null) { return page.evaluate(duration => { const from = performance.timeOrigin + performance.now(); @@ -76,6 +86,7 @@ export async function blockMainThread( .some( a => a.playState === 'running' && + !a.pending && Number(a.currentTime ?? 0) >= minProgress, ); if (running) { @@ -98,7 +109,7 @@ export async function blockMainThread( }; requestAnimationFrame(tick); }), - {duration: ms, minProgress: minProgressMs, wait: timeout}, + {duration: ms, minProgress, wait: timeout}, ); } diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 10080cefc7e16..ba3a98b4e6767 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -342,6 +342,52 @@ test.describe('frame evidence', () => { expect(proof.monotonic).toBe(true); }); + test('a block never starts on an animation that is not on the compositor yet', async ({ + page, + }, testInfo) => { + // An animation created in the frame the helper looks is still pending (not + // committed to the compositor). A block that started on it then froze it + // in Chromium: no presented frame moved. + await fixture(page); + const box = page.locator('#box'); + await markElement(box, {mode: 'outline'}); + await box.evaluate(el => { + el.style.willChange = 'transform'; + }); + const cast = await startScreencast(page); + // A rAF loop registered before the helper's tick runs ahead of it in every + // frame; on its 12th frame it creates the animation, and the helper's tick + // looks in that same frame. + await box.evaluate(el => { + let n = 0; + const loop = () => { + n += 1; + if (n === 12) { + el.animate( + [{transform: 'translateX(0px)'}, {transform: 'translateX(280px)'}], + {duration: 1600, easing: 'linear'}, + ); + return; + } + requestAnimationFrame(loop); + }; + requestAnimationFrame(loop); + }); + const window = await blockMainThread(page, 700, { + during: box, + minProgressMs: 0, + }); + await waitForRest(box); + const tracked = await trackMarker(page, await cast.stop()); + const proof = busyWindowProof(tracked, window, {axis: 'x'}); + testInfo.annotations.push({ + type: 'busy-main-thread pending animation', + description: proof.summary, + }); + expect(proof.verdict.failures, proof.summary).toEqual([]); + expect(proof.monotonic).toBe(true); + }); + test('rect sampling reads one rect per animation frame while the main thread is free', async ({ page, browserName, From bc744cba80e03ce9ee90a353726bdafbe735a137 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 04:53:56 -0700 Subject: [PATCH 23/93] test(interaction): the WebKit container keeps compiled shaders between runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WebKit composites through Mesa's software GL in the Playwright image, which compiles each shader variant the first time a page draws it. Mesa caches what it compiles, but the cache lived inside the container, so every run started cold: its first tests paid 550–680 ms for the first frame of a blurred or newly animated layer, and a 683 ms animation could end before it was ever presented or held. The script now keeps the cache on the host (per image, under ~/.cache/astryx-webkit-shaders, or ASTRYX_WEBKIT_SHADER_CACHE) and points MESA_SHADER_CACHE_DIR at it. The first frame of a blurred minimize in a fresh container drops to 75–147 ms. The self-test checks, inside the container, that the cache is the host's and that Mesa writes to it; with the previous script it fails. --- internal/interaction-harness/README.md | 6 ++++ .../scripts/webkit-container.mjs | 26 ++++++++++++++ .../src/harness.interaction.browser.spec.ts | 34 +++++++++++++++++++ 3 files changed, 66 insertions(+) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 0d9eef08f35c1..af04199e0237a 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -33,6 +33,12 @@ image (podman or docker; the repository is mounted at the same path): node internal/interaction-harness/scripts/webkit-container.mjs [playwright test args…] ``` +WebKit composites through Mesa's software GL there, which compiles each shader +the first time a page draws it (450–700 ms for the first frame of a blurred or +newly animated layer). The script keeps Mesa's shader cache on the host +(`~/.cache/astryx-webkit-shaders/`, or `ASTRYX_WEBKIT_SHADER_CACHE`), so +only the first run on a machine pays for them. + Evidence (storyboards, curve traces, busy-main-thread frames) is written to `test-results/interaction-evidence/` (override with `ASTRYX_INTERACTION_EVIDENCE`) and attached to the test report. It is never diff --git a/internal/interaction-harness/scripts/webkit-container.mjs b/internal/interaction-harness/scripts/webkit-container.mjs index 32cc0452396df..6f4fb988df593 100644 --- a/internal/interaction-harness/scripts/webkit-container.mjs +++ b/internal/interaction-harness/scripts/webkit-container.mjs @@ -13,10 +13,19 @@ * matches the installed playwright, so its browsers match the client. * Uses podman when present, else docker. `--network=host` lets the specs' * local Storybook server and the browser reach each other. + * + * WebKit composites through Mesa's software GL in the image, which compiles + * each shader variant the first time a page draws it: 450–700 ms for the + * first frame of a blurred or newly animated layer, long enough for a short + * animation to end before it is ever presented. Mesa caches what it compiles, + * and the cache is kept on the host (`~/.cache/astryx-webkit-shaders/`, + * or `ASTRYX_WEBKIT_SHADER_CACHE`), so a run starts with every shader an + * earlier run compiled instead of paying for them in its first tests. */ import {spawnSync} from 'node:child_process'; import fs from 'node:fs'; import {createRequire} from 'node:module'; +import os from 'node:os'; import path from 'node:path'; import {fileURLToPath} from 'node:url'; @@ -39,6 +48,19 @@ if (engine == null) { process.exit(1); } const args = process.argv.slice(2); +// Where the container sees the host's shader cache (see the header). +const containerShaderCache = '/astryx-shader-cache'; +const shaderCache = + process.env.ASTRYX_WEBKIT_SHADER_CACHE ?? + path.join( + os.homedir(), + '.cache', + 'astryx-webkit-shaders', + image.replace(/[^\w.-]+/g, '_'), + ); +fs.mkdirSync(shaderCache, {recursive: true}); +// The harness self-test reads this marker inside the container. +fs.writeFileSync(path.join(shaderCache, 'ASTRYX-SHADER-CACHE'), `${image}\n`); const env = [ 'ASTRYX_STORYBOOK_DIR', 'ASTRYX_INTERACTION_EVIDENCE', @@ -54,6 +76,10 @@ const command = [ '--ipc=host', '-v', `${root}:${root}`, + '-v', + `${shaderCache}:${containerShaderCache}`, + '-e', + `MESA_SHADER_CACHE_DIR=${containerShaderCache}`, '-w', root, ...env, diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index ba3a98b4e6767..31b500967ca03 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -26,6 +26,8 @@ * instruments; this file is why it may. */ +import fs from 'node:fs'; +import path from 'node:path'; import {expect, test, type Page} from '@playwright/test'; import { animationStoryboard, @@ -388,6 +390,38 @@ test.describe('frame evidence', () => { expect(proof.monotonic).toBe(true); }); + test('in the WebKit container, compiled shaders outlive the run', async ({ + page, + }, testInfo) => { + // The official Playwright image keeps its browsers here. + test.skip( + testInfo.project.name !== 'webkit' || + process.env.PLAYWRIGHT_BROWSERS_PATH !== '/ms-playwright', + 'runs inside webkit-container.mjs', + ); + // A cache inside the container dies with it: every run then compiled each + // shader again, 450–700 ms in the first frame of a blurred layer. + const dir = process.env.MESA_SHADER_CACHE_DIR; + expect(dir, 'MESA_SHADER_CACHE_DIR names the host-kept cache').toBeTruthy(); + // Written on the host by the script before the container starts. + expect( + fs.readFileSync(path.join(dir ?? '', 'ASTRYX-SHADER-CACHE'), 'utf8'), + ).toContain('playwright'); + // Mesa writes what it compiles there once a page draws a blurred layer. + await page.setContent( + '
', + ); + await page.evaluate( + () => + document + .getElementById('glass') + ?.animate([{scale: '1'}, {scale: '0.9'}], {duration: 200}).finished, + ); + expect( + fs.readdirSync(dir ?? '').filter(name => name !== 'ASTRYX-SHADER-CACHE'), + ).not.toEqual([]); + }); + test('rect sampling reads one rect per animation frame while the main thread is free', async ({ page, browserName, From 45385723a6e863b24114eeefa4feecbb8d8cad30 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 05:43:08 -0700 Subject: [PATCH 24/93] ci: the harness's typecheck runs through pnpm typecheck, once Since main's root `pnpm typecheck` (#8390) runs every workspace package's own typecheck script, the separate "Typecheck interaction harness" steps in ci.yml and deploy.yml ran it a second time, and their comment ("the only gate over it") no longer held. Both steps go. The routing test that pinned them becomes one more directory in "keeps every typecheck gate that guarded main": internal/interaction-harness keeps the typecheck script that pnpm typecheck reaches. --- .github/scripts/ci-test-routing.test.mjs | 9 +-------- .github/workflows/ci.yml | 6 ------ .github/workflows/deploy.yml | 3 --- 3 files changed, 1 insertion(+), 17 deletions(-) diff --git a/.github/scripts/ci-test-routing.test.mjs b/.github/scripts/ci-test-routing.test.mjs index 02e66cabd0df1..09c4f6252fd49 100644 --- a/.github/scripts/ci-test-routing.test.mjs +++ b/.github/scripts/ci-test-routing.test.mjs @@ -630,6 +630,7 @@ describe('deploy.yml push gating', () => { 'packages/cli', 'apps/storybook', 'internal/a11y-spec', + 'internal/interaction-harness', ]) { const {typecheck} = JSON.parse(read(`${dir}/package.json`)).scripts; expect(typecheck, `${dir} lost its typecheck`).toBeTruthy(); @@ -699,12 +700,4 @@ describe('ci.yml interaction proofs', () => { // Unknown scope (exit 2) runs the proofs; only "out of scope" skips them. expect(scope).toContain('if [ "$scope" = 1 ]; then'); }); - - it('typechecks the harness in both workflows', () => { - for (const file of ['ci.yml', 'deploy.yml']) { - expect(runLines(load(file).jobs.typecheck), file).toContain( - '@astryxdesign/interaction-harness typecheck', - ); - } - }); }); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0cf77b251cf4..7df2ea07b73c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1216,12 +1216,6 @@ jobs: bash "$legacy" fi - # The interaction harness is consumed as TypeScript source by Vitest and - # Playwright, which do not typecheck; this is the only gate over it. - - name: Typecheck interaction harness - if: needs.check-scope.outputs.docsite_only != 'true' && needs.check-scope.outputs.spec_only != 'true' && needs.check-scope.outputs.tooling_only != 'true' - run: pnpm -F @astryxdesign/interaction-harness typecheck - # Sandbox preview build — the single slowest piece of PR CI (~4min), run as # its own job so it overlaps build-storybook instead of serializing after it. # PR-only: merge_group runs never deploy previews, and docsite-only PRs skip diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 5aa1c2c672b48..2e92172d0185f 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -100,9 +100,6 @@ jobs: - name: Typecheck every package run: pnpm typecheck - - name: Typecheck interaction harness - run: pnpm -F @astryxdesign/interaction-harness typecheck - # Stable join for the three post-merge validation lanes. test: needs: [test-ui, test-node, typecheck] From 26f2b9823300c38b72c5aed551e403a4c7b03829 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 06:17:46 -0700 Subject: [PATCH 25/93] test(interaction): a watch on first contact ignores what already runs The recorder's first look at the document came a frame after it started, so on a page the harness touched for the first time, watchAnimations armed before the recorder had seen an animation already running there (a spinner). That animation then read as started by the watched action: started() resolved with it with no action at all, or returned it instead of the animation the action started. The recorder now records once as it starts, before any watch arms. The self-test starts an endless spinner before the harness touches the page, then checks that the watch reports nothing until a 300 ms animation starts, and then only that one. --- internal/interaction-harness/src/curves.ts | 4 +++ .../src/harness.interaction.browser.spec.ts | 26 ++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index 4f1312cf434e1..fecf367572855 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -79,6 +79,10 @@ function animationRecorder(): void { record(); frame(tick); }; + // What already runs when the recorder starts (a spinner on a page the + // harness touches for the first time) is seen now, before any watch arms; + // first seen a frame later, it would read as started by the watched action. + record(); frame(tick); for (const type of ['animationstart', 'transitionrun', 'transitionstart']) { document.addEventListener(type, record, true); diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 64546515d058f..ef8a6607c45c5 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -8,7 +8,8 @@ * a second finger and a pinch arrive as distinct pointers; sampled curves * match the oracles; the animation-state waits report a running transition * and its end, and judge rest by presented values even while an animation - * holds its end; individual transform properties are sampled; a sample at + * holds its end, and a watch taken on first contact ignores an animation + * already running; individual transform properties are sampled; a sample at * an end reports the time it was read at; holding takes only running * document-timeline animations, each at its own offset, holds each where it * was read, adds a second hold to the held set, drops one the page cancels, @@ -630,6 +631,29 @@ test.describe('animation state', () => { expect(entrance.finishedAt).not.toBeNull(); }); + test("a watch taken on the page's first contact ignores an animation already running", async ({ + page, + }) => { + await fixture(page); + const box = page.locator('#box'); + // An endless spinner, started before the harness has touched the page. + await box.evaluate(el => + el.animate([{rotate: '0deg'}, {rotate: '360deg'}], { + duration: 1000, + iterations: Infinity, + }), + ); + const watch = await watchAnimations(box); + await expect(watch.started({timeout: 300})).rejects.toThrow( + /no animation started/, + ); + await box.evaluate(el => + el.animate([{opacity: 1}, {opacity: 0.5}], {duration: 300}), + ); + const started = await watch.started(); + expect(started.map(a => a.durationMs)).toEqual([300]); + }); + test('a watch reports only what started after it', async ({page}) => { await fixture(page); const box = page.locator('#box'); From 5850e7afe3ec35094a49667a088a05ac0724cfa8 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 06:54:50 -0700 Subject: [PATCH 26/93] test(interaction): a later hold never moves the first hold's times Held times count from an origin, the earliest start among the held animations. A second holdAnimations that took an animation which had started earlier than the first set moved that origin, so the offsetMs the first hold had returned went stale, and seeking or sampling its animations read them early. The origin is now the one the first hold set, until the release; a later hold's earlier-started animation gets a negative offsetMs. The self-test starts a 600 ms linear move 100 ms after another animation, holds the move first, then the other, and samples the move at 0, 150 and 300 ms: 0, 75 and 150 px. On the previous code it read 0, 33 and 108. --- internal/interaction-harness/README.md | 2 +- internal/interaction-harness/src/curves.ts | 27 ++++++----- .../src/harness.interaction.browser.spec.ts | 45 ++++++++++++++++++- 3 files changed, 61 insertions(+), 13 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 4cf4778d5a5cd..7bde09afd686f 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -54,7 +54,7 @@ the limits below. Read positions a frame after a move: Chromium delivers animations are Web Animations too), seek to a time, read `getComputedStyle`: the presented value at that exact time, independent of frame timing. Finished animations, ones the page paused and ones on a scroll or view timeline are left -alone; held animations keep their offsets from the earliest start, and release +alone; held animations keep their offsets from the first hold's earliest start, and release resumes only them. Compare it with an oracle from `math.ts` through `compareCurve`; a tolerance states the error budget (for a spring baked into `linear()` stops of `h` ms, chords err by up to diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index fecf367572855..3fea45055fecf 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -180,8 +180,10 @@ export interface HeldAnimation { /** Where it was when held, ms. */ readonly heldAtMs: number | null; /** - * How much later than the earliest held animation it started, ms: a seek to - * t moves it to its own t - offsetMs, so staggered starts stay staggered. + * How much later than the held origin (the earliest start among the + * animations the first hold took) it started, ms; negative for an earlier + * start held by a later hold. A seek to t moves it to its own t - offsetMs, + * so staggered starts stay staggered. */ readonly offsetMs: number; } @@ -191,9 +193,10 @@ export interface HeldAnimation { * target is inside `scope`) and returns what was held. Finished animations, * animations the page paused, and animations on a scroll or view timeline are * left as they are. Later `seekAnimations` calls move exactly the held set, - * each by its offset from the earliest start, and `releaseAnimations` resumes + * each by its offset from the held origin, and `releaseAnimations` resumes * only it. A second call adds to the held set: what the first held stays held - * until the release, and offsets count from the earliest start of all of it. + * until the release, and offsets keep counting from the origin the first call + * set (the earliest start it took), so offsets already returned stay true. * An animation the page cancels after the hold (retargeting a CSS transition * cancels it) leaves the held set: seeking, sampling and releasing skip it * rather than revive it. @@ -252,11 +255,15 @@ export async function holdAnimations( }; w.__harnessHeld = [...(w.__harnessHeld ?? []), ...held]; w.__harnessHeldStarts = [...(w.__harnessHeldStarts ?? []), ...starts]; - // Times count from the earliest start of everything held until the - // release, so dropping one later never moves the others. - const earliest = Math.min(w.__harnessHeldOrigin ?? Infinity, ...starts); - w.__harnessHeldOrigin = earliest; - const offsets = starts.map(start => start - earliest); + // Times count from the origin the first hold set (its earliest start) + // until the release: a later hold, even of an animation that started + // earlier (its offset is then negative), and dropping one never move it, + // so every offsetMs already returned stays true. + const origin = Number.isFinite(w.__harnessHeldOrigin ?? Number.NaN) + ? (w.__harnessHeldOrigin as number) + : Math.min(...starts); + w.__harnessHeldOrigin = origin; + const offsets = starts.map(start => start - origin); return held.map((animation, index) => { const effect = animation.effect as KeyframeEffect | null; const timing = effect?.getComputedTiming(); @@ -288,7 +295,7 @@ export async function holdAnimations( } /** - * Moves every held animation to `timeMs` from the earliest held start (delay + * Moves every held animation to `timeMs` from the held origin (delay * included): each to its own `timeMs - offsetMs`. A time at or past an * animation's end is clamped to END_INSET_MS before it: an animation without * `fill` stops applying its effect exactly at the end, and the frame wanted diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index ef8a6607c45c5..b10578c39e662 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -12,8 +12,8 @@ * already running; individual transform properties are sampled; a sample at * an end reports the time it was read at; holding takes only running * document-timeline animations, each at its own offset, holds each where it - * was read, adds a second hold to the held set, drops one the page cancels, - * and releases only them. + * was read, adds a second hold to the held set without moving the first + * hold's times, drops one the page cancels, and releases only them. * @position The harness's own test. A component spec trusts these * instruments; this file is why it may. */ @@ -502,6 +502,47 @@ test.describe('holding', () => { ).toBe(0); }); + test("a later hold of an earlier-started animation keeps the first hold's times", async ({ + page, + }) => { + await fixture(page); + // #early starts 100 ms before #box's 600 ms linear move to 300 px. + await page.evaluate( + () => + new Promise(resolve => { + const early = document.createElement('div'); + early.id = 'early'; + document.body.append(early); + early.animate([{opacity: 0}, {opacity: 1}], {duration: 2000}); + setTimeout(() => { + document + .getElementById('box') + ?.animate( + [ + {transform: 'translateX(0px)'}, + {transform: 'translateX(300px)'}, + ], + {duration: 600, easing: 'linear'}, + ); + resolve(); + }, 100); + }), + ); + const box = page.locator('#box'); + await waitForAnimation(box); + // The later-started element is held first. + const [first] = await holdAnimations(page, box); + const [later] = await holdAnimations(page, page.locator('#early')); + const samples = await sampleCurve(box, [0, 150, 300]); + expect( + samples.map(sample => Math.round(sample.transform.translateX)), + ).toEqual([0, 75, 150]); + // A later hold's earlier start has a negative offset; the origin stays. + expect(first.offsetMs).toBe(0); + expect(later.offsetMs).toBeLessThan(-50); + await releaseAnimations(page); + }); + test('a held transition the page retargets is dropped, not revived', async ({ page, }) => { From c6d91e8814d8a150237def05ae181f9b28d95252 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 07:02:55 -0700 Subject: [PATCH 27/93] ci: pin the interaction routing's fail-closed edges by running them AST-030 FR9: removing a fail-closed edge must fail a test. The routing test only read the decide script's text, so turning its unknown default into "out of scope", or recording every helper failure as out of scope, or a guard step's `exit 1` into `exit 0`, left it green. It now runs the decide script under bash -eo pipefail with a stub git: a helper that answers in scope, out of scope or unknown, a base that cannot be fetched, a helper missing at the base, no base, a docsite-only change and a merge group, and checks the `applies` output of each. It also runs each guard step and checks that it fails. Each of those five mutations now fails it. flick's doc and the README said a velocity tracker reads exactly `velocity` for any profile ending at 1. That holds for the script's last move (what releaseVelocity reads); a tracker that blends recent intervals (INV10) reads it only when the profile is flat over them, as the default is. --- .github/scripts/ci-interaction.test.mjs | 85 +++++++++++++++++++++ internal/interaction-harness/README.md | 2 +- internal/interaction-harness/src/scripts.ts | 10 ++- 3 files changed, 92 insertions(+), 5 deletions(-) diff --git a/.github/scripts/ci-interaction.test.mjs b/.github/scripts/ci-interaction.test.mjs index e8b393ffbf486..860d11c879c78 100644 --- a/.github/scripts/ci-interaction.test.mjs +++ b/.github/scripts/ci-interaction.test.mjs @@ -10,7 +10,9 @@ * 4-core runner, and otherwise it passes on Linux without work. */ +import {spawnSync} from 'node:child_process'; import fs from 'node:fs'; +import os from 'node:os'; import path from 'node:path'; import {describe, expect, it} from 'vitest'; @@ -27,6 +29,54 @@ const step = (job, name) => const decide = () => step(scope, 'Decide whether the interaction proofs apply'); const applies = "needs.interaction-scope.outputs.applies == 'true'"; +// Runs a step's script as Actions runs it (bash -eo pipefail) with a stub +// `git` first on PATH: `fetch` exits FETCH, and `show` prints a helper that +// exits HELPER (or fails when HELPER is 'missing'). Returns the exit status +// and the outputs the script wrote. +function runStep(script, {env = {}, fetch = 0, helper = 0} = {}) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ci-interaction-')); + try { + const bin = path.join(dir, 'bin'); + fs.mkdirSync(bin); + fs.writeFileSync( + path.join(bin, 'git'), + [ + '#!/usr/bin/env bash', + 'case "$1" in', + ` fetch) exit ${fetch} ;;`, + helper === 'missing' + ? ' show) exit 128 ;;' + : ` show) echo 'exit ${helper}' ;;`, + ' *) exit 3 ;;', + 'esac', + '', + ].join('\n'), + {mode: 0o755}, + ); + const output = path.join(dir, 'output'); + fs.writeFileSync(output, ''); + const result = spawnSync( + 'bash', + ['--noprofile', '--norc', '-eo', 'pipefail', '-c', script], + { + encoding: 'utf8', + env: { + PATH: `${bin}:${process.env.PATH}`, + GITHUB_OUTPUT: output, + EVENT_NAME: 'pull_request', + DOCSITE_ONLY: 'false', + TOOLING_ONLY: 'false', + BASE_SHA: '0123456789abcdef0123456789abcdef01234567', + ...env, + }, + }, + ); + return {status: result.status, output: fs.readFileSync(output, 'utf8')}; + } finally { + fs.rmSync(dir, {recursive: true, force: true}); + } +} + describe('interaction proof routing', () => { it('is joined by test', () => { expect(ci.jobs.test.needs).toContain('pr-interaction'); @@ -84,6 +134,41 @@ describe('interaction proof routing', () => { expect(decide().run).toContain('if [ "$scope" = 1 ]; then'); }); + // AST-030 FR9: removing a fail-closed edge fails a test. Only a helper that + // answers "out of scope" (exit 1) skips the proofs; an unknown answer, a + // failed fetch, a helper missing at the base, or no base runs them. + it('runs the proofs unless the helper answers "out of scope"', () => { + const decided = options => runStep(decide().run, options); + for (const [label, options, expected] of [ + ['in scope (helper exit 0)', {helper: 0}, 'true'], + ['out of scope (helper exit 1)', {helper: 1}, 'false'], + ['unknown (helper exit 2)', {helper: 2}, 'true'], + ['the base cannot be fetched', {fetch: 1}, 'true'], + ['the helper is missing at the base', {helper: 'missing'}, 'true'], + ['no base SHA', {env: {BASE_SHA: ''}}, 'true'], + ['docsite only', {env: {DOCSITE_ONLY: 'true'}, helper: 0}, 'false'], + [ + 'not a pull request', + {env: {EVENT_NAME: 'merge_group'}, helper: 0}, + 'false', + ], + ]) { + const {status, output} = decided(options); + expect(status, label).toBe(0); + expect(output, label).toBe(`applies=${expected}\n`); + } + }); + + it('fails each guard step when it runs', () => { + for (const [job, name] of [ + [scope, 'Require successful scope classification'], + [proofs, 'Require a decided interaction scope'], + [proofs, "Require this pull request's Storybook build"], + ]) { + expect(runStep(step(job, name).run).status, name).toBe(1); + } + }); + it('keeps the harness, components, Storybook and its own setup in scope', () => { const pattern = new RegExp( decide().run.match(/bash "\$helper" "\$BASE_SHA" '([^']+)'/)[1], diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 8139a9fa33348..fad8af855df60 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -32,7 +32,7 @@ node internal/interaction-harness/scripts/webkit-container.mjs [playwright test | Module | What it gives a spec | | ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact release velocity, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. | +| `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact final-move speed, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. | | `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | | `story.ts` | `phone`, `withStorybook`, `openStory`, `frames`. | diff --git a/internal/interaction-harness/src/scripts.ts b/internal/interaction-harness/src/scripts.ts index 79e8744060f78..91d52f9cacf7b 100644 --- a/internal/interaction-harness/src/scripts.ts +++ b/internal/interaction-harness/src/scripts.ts @@ -143,10 +143,12 @@ export interface FlickOptions { /** * A throw: the finger moves with the given velocity profile and lifts while * still moving (the lift lands a quarter frame after the last move, as on a - * device). Its last move runs at `velocity` times `profile(1)`, so with the - * default constant profile, or any profile that ends at 1, the release - * velocity (what `releaseVelocity` and a velocity tracker read from the last - * move) is exactly `velocity`. + * device). Its last move runs at `velocity` times `profile(1)`, so with any + * profile that ends at 1 the last move, which is what `releaseVelocity` + * reads, runs at exactly `velocity`. A tracker that blends recent intervals + * (a Core component's, `architecture:interaction-modality` INV10) reads + * exactly `velocity` only when the profile is flat over those intervals, as + * the default constant profile is. */ export function flick(from: Point, options: FlickOptions): TouchScript { const { From 0a9e7bf55c071f409c7e797681170851a4a63b58 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 07:25:07 -0700 Subject: [PATCH 28/93] docs(interaction): seek and sample times count from the held origin Since a later hold keeps the first hold's origin, a later-held animation that started earlier has a negative offset, so the file header's "each keeping its offset from the earliest" and sampleCurve's "ms from the held animations' start" no longer described it. Both now name the held origin. --- internal/interaction-harness/src/curves.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index 3fea45055fecf..ac42e7e8f5fa7 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -6,8 +6,8 @@ * the Web Animations API in the page (CSS transitions and CSS animations are * Web Animations too) * @output `holdAnimations` (pause every running animation and describe it), - * `seekAnimations` (move them to t ms, each keeping its offset from the - * earliest), `sampleCurve` (the presented + * `seekAnimations` (move them to t ms from the held origin, each keeping + * its offset), `sampleCurve` (the presented * transform/opacity at each t, with the individual `translate`, `rotate` and * `scale` properties composed in), `releaseAnimations`, `curveOf`, which * picks one channel out of the samples for ./math's `compareCurve`, and the @@ -425,7 +425,7 @@ export interface CurveSample { /** * The presented transform and opacity of `locator` at each time (ms from the - * held animations' start). Hold the animations first. An animation of the + * held origin, as for `seekAnimations`). Hold the animations first. An animation of the * individual `translate`, `rotate` or `scale` properties is included: they * are composed with `transform` into one matrix. */ From 2438fc7201717058690c64b13f4e6be807457610 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 07:37:51 -0700 Subject: [PATCH 29/93] ci: a head cut before the interaction proofs passes with a notice pr-interaction runs the proofs on the pull request's head. Once the proofs land on main, an in-scope pull request whose branch predates them has no playwright.interaction.config.ts, and Playwright would fail the required `test` check until it merged main. The scope decision now passes such a head with a notice: the head has no config, and the pull request's own diff does not change it (the base-loaded scope helper says so). A pull request that deletes the config changes it, so the proofs still run and fail; a head that cannot be fetched, an unknown helper answer or an unknown scope also runs them. The routing test runs the decide script with a stub git for each of those cases, and each mutation of the new edges (the head fetch, the HEAD_SHA guard, `touches=0`, `|| touches=$?`, the `= 1` comparison, the in-scope condition) fails it. --- .github/scripts/ci-interaction.test.mjs | 79 ++++++++++++++++++++++--- .github/workflows/ci.yml | 20 ++++++- 2 files changed, 90 insertions(+), 9 deletions(-) diff --git a/.github/scripts/ci-interaction.test.mjs b/.github/scripts/ci-interaction.test.mjs index 860d11c879c78..959a20ce6d35e 100644 --- a/.github/scripts/ci-interaction.test.mjs +++ b/.github/scripts/ci-interaction.test.mjs @@ -7,7 +7,10 @@ * `test` join requires pr-interaction, and a failing proof fails `test` (FR7). * A Linux job decides whether the change touches the harness, a component, * Storybook, or the proofs' own setup; only then does pr-interaction take a - * 4-core runner, and otherwise it passes on Linux without work. + * 4-core runner, and otherwise it passes on Linux without work. The proofs run + * on the pull request's head: a head cut before they existed (no + * playwright.interaction.config.ts) passes with a notice, and a pull request + * that deletes the config still runs them, and fails. */ import {spawnSync} from 'node:child_process'; @@ -29,11 +32,26 @@ const step = (job, name) => const decide = () => step(scope, 'Decide whether the interaction proofs apply'); const applies = "needs.interaction-scope.outputs.applies == 'true'"; +const HEAD_SHA = 'fedcba9876543210fedcba9876543210fedcba98'; +const CONFIG_PATTERN = '^playwright\\.interaction\\.config\\.ts$'; + // Runs a step's script as Actions runs it (bash -eo pipefail) with a stub -// `git` first on PATH: `fetch` exits FETCH, and `show` prints a helper that -// exits HELPER (or fails when HELPER is 'missing'). Returns the exit status -// and the outputs the script wrote. -function runStep(script, {env = {}, fetch = 0, helper = 0} = {}) { +// `git` first on PATH: fetching the base exits FETCH and fetching the head +// FETCH_HEAD; `show` prints a helper that answers HELPER, or TOUCHES for the +// config pattern (or fails when HELPER is 'missing'); `cat-file` finds the +// config in the head when HEAD_HAS_CONFIG. Returns the exit status and the +// outputs the script wrote. +function runStep( + script, + { + env = {}, + fetch = 0, + fetchHead = 0, + helper = 0, + touches = 1, + headHasConfig = true, + } = {}, +) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ci-interaction-')); try { const bin = path.join(dir, 'bin'); @@ -43,10 +61,11 @@ function runStep(script, {env = {}, fetch = 0, helper = 0} = {}) { [ '#!/usr/bin/env bash', 'case "$1" in', - ` fetch) exit ${fetch} ;;`, + ` fetch) case "$*" in *${HEAD_SHA}*) exit ${fetchHead} ;; *) exit ${fetch} ;; esac ;;`, helper === 'missing' ? ' show) exit 128 ;;' - : ` show) echo 'exit ${helper}' ;;`, + : ` show) printf '%s\\n' 'if [ "$2" = "${CONFIG_PATTERN}" ]; then exit ${touches}; fi' 'exit ${helper}' ;;`, + ` cat-file) exit ${headHasConfig ? 0 : 1} ;;`, ' *) exit 3 ;;', 'esac', '', @@ -67,6 +86,7 @@ function runStep(script, {env = {}, fetch = 0, helper = 0} = {}) { DOCSITE_ONLY: 'false', TOOLING_ONLY: 'false', BASE_SHA: '0123456789abcdef0123456789abcdef01234567', + HEAD_SHA, ...env, }, }, @@ -159,6 +179,50 @@ describe('interaction proof routing', () => { } }); + it('passes a head cut before the proofs existed, and still fails one that deletes their config', () => { + const decided = options => runStep(decide().run, options); + for (const [label, options, expected] of [ + ['the head has the config', {headHasConfig: true}, 'true'], + [ + 'the branch predates the proofs (no config, not changed by the PR)', + {headHasConfig: false, touches: 1}, + 'false', + ], + [ + 'the pull request deletes the config', + {headHasConfig: false, touches: 0}, + 'true', + ], + [ + 'whether the PR changes the config is unknown', + {headHasConfig: false, touches: 2}, + 'true', + ], + [ + 'the head cannot be fetched', + {headHasConfig: false, touches: 1, fetchHead: 1}, + 'true', + ], + [ + 'no head SHA', + {headHasConfig: false, touches: 1, env: {HEAD_SHA: ''}}, + 'true', + ], + [ + 'the scope itself is unknown', + {headHasConfig: false, touches: 1, helper: 2}, + 'true', + ], + ]) { + const {status, output} = decided(options); + expect(status, label).toBe(0); + expect(output, label).toBe(`applies=${expected}\n`); + } + expect(decide().run).toContain( + '::notice title=Interaction proofs predate this branch::', + ); + }); + it('fails each guard step when it runs', () => { for (const [job, name] of [ [scope, 'Require successful scope classification'], @@ -197,6 +261,7 @@ describe('interaction proof routing', () => { it('passes without work on docsite-only, tooling-only and merge-group runs', () => { expect(decide().env).toMatchObject({ + HEAD_SHA: '${{ github.event.pull_request.head.sha }}', EVENT_NAME: '${{ github.event_name }}', DOCSITE_ONLY: '${{ needs.check-scope.outputs.docsite_only }}', TOOLING_ONLY: '${{ needs.check-scope.outputs.tooling_only }}', diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7df2ea07b73c9..77620b07de97e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3784,6 +3784,10 @@ jobs: # The helper comes from the base, since an older head lacks it. Exit 1 # means out of scope; 0 in scope; 2 unknown, so the proofs run. + # pr-interaction runs the proofs on the PR head. A head cut before they + # existed has no playwright.interaction.config.ts and passes with a + # notice; a PR that deletes the config (its diff changes it) still runs + # them, and fails. Anything undecided runs them. - name: Decide whether the interaction proofs apply id: scope env: @@ -3791,6 +3795,7 @@ jobs: DOCSITE_ONLY: ${{ needs.check-scope.outputs.docsite_only }} TOOLING_ONLY: ${{ needs.check-scope.outputs.tooling_only }} BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | if [ "$DOCSITE_ONLY" = true ] || [ "$TOOLING_ONLY" = true ]; then echo 'Specialized lane; no interaction proofs.' @@ -3813,9 +3818,20 @@ jobs: if [ "$scope" = 1 ]; then echo 'No harness, component, Storybook or proof-setup change; skipping the interaction proofs.' echo 'applies=false' >> "$GITHUB_OUTPUT" - else - echo 'applies=true' >> "$GITHUB_OUTPUT" + exit 0 + fi + if [ "$scope" = 0 ] && [ -n "$HEAD_SHA" ] \ + && git fetch --quiet --no-tags --depth=1 origin "$HEAD_SHA" \ + && ! git cat-file -e "$HEAD_SHA:playwright.interaction.config.ts" 2>/dev/null; then + touches=0 + bash "$helper" "$BASE_SHA" '^playwright\.interaction\.config\.ts$' || touches=$? + if [ "$touches" = 1 ]; then + echo "::notice title=Interaction proofs predate this branch::This pull request's head has no playwright.interaction.config.ts; merge main to run the interaction proofs." + echo 'applies=false' >> "$GITHUB_OUTPUT" + exit 0 + fi fi + echo 'applies=true' >> "$GITHUB_OUTPUT" pr-interaction: needs: [check-scope, interaction-scope, build-storybook] From eeef165d69af2adde1ac0c93a9fb537720f11290 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 07:37:51 -0700 Subject: [PATCH 30/93] ci: a head cut before the interaction proofs passes with a notice pr-interaction runs the proofs on the pull request's head. Once the proofs land on main, an in-scope pull request whose branch predates them has no playwright.interaction.config.ts, and Playwright would fail the required `test` check until it merged main. The scope decision now passes such a head with a notice: the head has no config, and the pull request's own diff does not change it (the base-loaded scope helper says so). A pull request that deletes the config changes it, so the proofs still run and fail; a head that cannot be fetched, an unknown helper answer or an unknown scope also runs them. The routing test runs the decide script with a stub git for each of those cases, and each mutation of the new edges (the head fetch, the HEAD_SHA guard, `touches=0`, `|| touches=$?`, the `= 1` comparison, the in-scope condition) fails it. CI-Rerun: main's gate test (fixed in #8406) failed this head's test-node --- .github/scripts/ci-interaction.test.mjs | 79 ++++++++++++++++++++++--- .github/workflows/ci.yml | 20 ++++++- 2 files changed, 90 insertions(+), 9 deletions(-) diff --git a/.github/scripts/ci-interaction.test.mjs b/.github/scripts/ci-interaction.test.mjs index 860d11c879c78..959a20ce6d35e 100644 --- a/.github/scripts/ci-interaction.test.mjs +++ b/.github/scripts/ci-interaction.test.mjs @@ -7,7 +7,10 @@ * `test` join requires pr-interaction, and a failing proof fails `test` (FR7). * A Linux job decides whether the change touches the harness, a component, * Storybook, or the proofs' own setup; only then does pr-interaction take a - * 4-core runner, and otherwise it passes on Linux without work. + * 4-core runner, and otherwise it passes on Linux without work. The proofs run + * on the pull request's head: a head cut before they existed (no + * playwright.interaction.config.ts) passes with a notice, and a pull request + * that deletes the config still runs them, and fails. */ import {spawnSync} from 'node:child_process'; @@ -29,11 +32,26 @@ const step = (job, name) => const decide = () => step(scope, 'Decide whether the interaction proofs apply'); const applies = "needs.interaction-scope.outputs.applies == 'true'"; +const HEAD_SHA = 'fedcba9876543210fedcba9876543210fedcba98'; +const CONFIG_PATTERN = '^playwright\\.interaction\\.config\\.ts$'; + // Runs a step's script as Actions runs it (bash -eo pipefail) with a stub -// `git` first on PATH: `fetch` exits FETCH, and `show` prints a helper that -// exits HELPER (or fails when HELPER is 'missing'). Returns the exit status -// and the outputs the script wrote. -function runStep(script, {env = {}, fetch = 0, helper = 0} = {}) { +// `git` first on PATH: fetching the base exits FETCH and fetching the head +// FETCH_HEAD; `show` prints a helper that answers HELPER, or TOUCHES for the +// config pattern (or fails when HELPER is 'missing'); `cat-file` finds the +// config in the head when HEAD_HAS_CONFIG. Returns the exit status and the +// outputs the script wrote. +function runStep( + script, + { + env = {}, + fetch = 0, + fetchHead = 0, + helper = 0, + touches = 1, + headHasConfig = true, + } = {}, +) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ci-interaction-')); try { const bin = path.join(dir, 'bin'); @@ -43,10 +61,11 @@ function runStep(script, {env = {}, fetch = 0, helper = 0} = {}) { [ '#!/usr/bin/env bash', 'case "$1" in', - ` fetch) exit ${fetch} ;;`, + ` fetch) case "$*" in *${HEAD_SHA}*) exit ${fetchHead} ;; *) exit ${fetch} ;; esac ;;`, helper === 'missing' ? ' show) exit 128 ;;' - : ` show) echo 'exit ${helper}' ;;`, + : ` show) printf '%s\\n' 'if [ "$2" = "${CONFIG_PATTERN}" ]; then exit ${touches}; fi' 'exit ${helper}' ;;`, + ` cat-file) exit ${headHasConfig ? 0 : 1} ;;`, ' *) exit 3 ;;', 'esac', '', @@ -67,6 +86,7 @@ function runStep(script, {env = {}, fetch = 0, helper = 0} = {}) { DOCSITE_ONLY: 'false', TOOLING_ONLY: 'false', BASE_SHA: '0123456789abcdef0123456789abcdef01234567', + HEAD_SHA, ...env, }, }, @@ -159,6 +179,50 @@ describe('interaction proof routing', () => { } }); + it('passes a head cut before the proofs existed, and still fails one that deletes their config', () => { + const decided = options => runStep(decide().run, options); + for (const [label, options, expected] of [ + ['the head has the config', {headHasConfig: true}, 'true'], + [ + 'the branch predates the proofs (no config, not changed by the PR)', + {headHasConfig: false, touches: 1}, + 'false', + ], + [ + 'the pull request deletes the config', + {headHasConfig: false, touches: 0}, + 'true', + ], + [ + 'whether the PR changes the config is unknown', + {headHasConfig: false, touches: 2}, + 'true', + ], + [ + 'the head cannot be fetched', + {headHasConfig: false, touches: 1, fetchHead: 1}, + 'true', + ], + [ + 'no head SHA', + {headHasConfig: false, touches: 1, env: {HEAD_SHA: ''}}, + 'true', + ], + [ + 'the scope itself is unknown', + {headHasConfig: false, touches: 1, helper: 2}, + 'true', + ], + ]) { + const {status, output} = decided(options); + expect(status, label).toBe(0); + expect(output, label).toBe(`applies=${expected}\n`); + } + expect(decide().run).toContain( + '::notice title=Interaction proofs predate this branch::', + ); + }); + it('fails each guard step when it runs', () => { for (const [job, name] of [ [scope, 'Require successful scope classification'], @@ -197,6 +261,7 @@ describe('interaction proof routing', () => { it('passes without work on docsite-only, tooling-only and merge-group runs', () => { expect(decide().env).toMatchObject({ + HEAD_SHA: '${{ github.event.pull_request.head.sha }}', EVENT_NAME: '${{ github.event_name }}', DOCSITE_ONLY: '${{ needs.check-scope.outputs.docsite_only }}', TOOLING_ONLY: '${{ needs.check-scope.outputs.tooling_only }}', diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7df2ea07b73c9..77620b07de97e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3784,6 +3784,10 @@ jobs: # The helper comes from the base, since an older head lacks it. Exit 1 # means out of scope; 0 in scope; 2 unknown, so the proofs run. + # pr-interaction runs the proofs on the PR head. A head cut before they + # existed has no playwright.interaction.config.ts and passes with a + # notice; a PR that deletes the config (its diff changes it) still runs + # them, and fails. Anything undecided runs them. - name: Decide whether the interaction proofs apply id: scope env: @@ -3791,6 +3795,7 @@ jobs: DOCSITE_ONLY: ${{ needs.check-scope.outputs.docsite_only }} TOOLING_ONLY: ${{ needs.check-scope.outputs.tooling_only }} BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | if [ "$DOCSITE_ONLY" = true ] || [ "$TOOLING_ONLY" = true ]; then echo 'Specialized lane; no interaction proofs.' @@ -3813,9 +3818,20 @@ jobs: if [ "$scope" = 1 ]; then echo 'No harness, component, Storybook or proof-setup change; skipping the interaction proofs.' echo 'applies=false' >> "$GITHUB_OUTPUT" - else - echo 'applies=true' >> "$GITHUB_OUTPUT" + exit 0 + fi + if [ "$scope" = 0 ] && [ -n "$HEAD_SHA" ] \ + && git fetch --quiet --no-tags --depth=1 origin "$HEAD_SHA" \ + && ! git cat-file -e "$HEAD_SHA:playwright.interaction.config.ts" 2>/dev/null; then + touches=0 + bash "$helper" "$BASE_SHA" '^playwright\.interaction\.config\.ts$' || touches=$? + if [ "$touches" = 1 ]; then + echo "::notice title=Interaction proofs predate this branch::This pull request's head has no playwright.interaction.config.ts; merge main to run the interaction proofs." + echo 'applies=false' >> "$GITHUB_OUTPUT" + exit 0 + fi fi + echo 'applies=true' >> "$GITHUB_OUTPUT" pr-interaction: needs: [check-scope, interaction-scope, build-storybook] From a9b2acf7fe982f0b0c7a60a915e58cf325ccb0e0 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 09:04:57 -0700 Subject: [PATCH 31/93] fix(interaction): a watch keeps the start of an animation the page cancels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit waitForAnimation and watchAnimations(…).started() built an animation's active interval from its startTime. A page that cancels a finished animation (an entrance whose class is removed on animationend) nulls that, so the entrance was reported as starting at the page's time origin, before its trigger. The recorder now keeps each animation's start time in its entry: from startTime once it resolves (read on every recorder pass), or, for one cancelled before any pass read it, back from where it finished. The interval falls back to it when startTime is null. The new self-test removes an entrance's class on animationend, 200 ms after the page loaded, and checks that the reported start follows the trigger. On the previous code it read about 250 ms before the trigger in Chromium and WebKit. --- internal/interaction-harness/src/curves.ts | 49 ++++++++++++++++--- .../src/harness.interaction.browser.spec.ts | 40 ++++++++++++++- 2 files changed, 82 insertions(+), 7 deletions(-) diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index ac42e7e8f5fa7..a63cfffc81169 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -35,7 +35,10 @@ export const END_INSET_MS = 0.5; * Page side: records every animation the document runs, from its first frame * or its creation (a frame loop on the page's real requestAnimationFrame, and * the CSS animation and transition events), so an animation that starts and - * ends before a test looks is still seen, with the time it finished. + * ends before a test looks is still seen, with the time it finished. Each + * entry keeps the animation's start time too: a page that cancels a finished + * animation (an entrance whose class is removed on `animationend`) nulls its + * `startTime`. */ function animationRecorder(): void { const w = window as unknown as {__harnessAnimations?: unknown}; @@ -48,7 +51,10 @@ function animationRecorder(): void { animation: Animation; seenAt: number; finishedAt: number | null; + startedAt: number | null; }[] = []; + // Entries whose start time has not resolved yet (still pending when seen). + let unstarted: (typeof log)[number][] = []; let armedAt = performance.now(); const remember = (animation: Animation) => { // A running animation is recorded while still pending: on a cold page the @@ -61,9 +67,26 @@ function animationRecorder(): void { animation, seenAt: performance.now(), finishedAt: null as number | null, + startedAt: + animation.startTime == null ? null : Number(animation.startTime), }; - animation.addEventListener('finish', () => { + if (entry.startedAt == null) { + unstarted.push(entry); + } + animation.addEventListener('finish', event => { entry.finishedAt = performance.now(); + if (entry.startedAt == null) { + // Cancelled before a frame read its start: back from where it ended. + const end = Number( + (animation.effect as KeyframeEffect | null)?.getComputedTiming() + .endTime ?? 0, + ); + const at = (event as AnimationPlaybackEvent).timelineTime; + entry.startedAt = + animation.startTime != null + ? Number(animation.startTime) + : (at == null ? entry.finishedAt : Number(at)) - end; + } }); log.push(entry); if (log.length > 5000) { @@ -74,6 +97,12 @@ function animationRecorder(): void { for (const animation of document.getAnimations()) { remember(animation); } + unstarted = unstarted.filter(entry => { + if (entry.startedAt == null && entry.animation.startTime != null) { + entry.startedAt = Number(entry.animation.startTime); + } + return entry.startedAt == null && entry.animation.playState !== 'idle'; + }); }; const tick = () => { record(); @@ -646,6 +675,7 @@ export async function waitForAnimation( animation: Animation; seenAt: number; finishedAt: number | null; + startedAt: number | null; }[]; record: () => void; armedAt: () => number; @@ -654,10 +684,17 @@ export async function waitForAnimation( const curveEnd = w.__harnessCurveEnd; const recorder = w.__harnessAnimations; recorder.record(); - const describe = (a: Animation, finishedAt: number | null) => { + const describe = ( + a: Animation, + finishedAt: number | null, + recordedStart: number | null, + ) => { const effect = a.effect as KeyframeEffect | null; const timing = effect?.getComputedTiming(); - const startedAt = performance.timeOrigin + Number(a.startTime); + // The recorded start stands in once the page has cancelled it. + const startedAt = + performance.timeOrigin + + Number(a.startTime ?? recordedStart ?? Number.NaN); const name = 'transitionProperty' in a ? (a as unknown as {transitionProperty: string}) @@ -699,7 +736,7 @@ export async function waitForAnimation( a => Number(a.currentTime ?? 0) >= minProgress, ); if (progressed.length > 0) { - return progressed.map(a => describe(a, null)); + return progressed.map(a => describe(a, null, null)); } if (running.length > 0) { return []; @@ -718,7 +755,7 @@ export async function waitForAnimation( : null); return recorded .filter(e => finishedAt(e) != null) - .map(e => describe(e.animation, finishedAt(e))); + .map(e => describe(e.animation, finishedAt(e), e.startedAt)); }, {minProgress: minProgressMs, since: since ?? null}, ); diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index b10578c39e662..bdf70296dab12 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -9,7 +9,7 @@ * match the oracles; the animation-state waits report a running transition * and its end, and judge rest by presented values even while an animation * holds its end, and a watch taken on first contact ignores an animation - * already running; individual transform properties are sampled; a sample at + * already running and keeps the start of one the page cancels when it ends; individual transform properties are sampled; a sample at * an end reports the time it was read at; holding takes only running * document-timeline animations, each at its own offset, holds each where it * was read, adds a second hold to the held set without moving the first @@ -713,6 +713,44 @@ test.describe('animation state', () => { expect(started.map(a => a.durationMs)).toEqual([70]); }); + test('an entrance the page cancels when it ends keeps its start', async ({ + page, + }) => { + await fixture(page); + const box = page.locator('#box'); + await page.addStyleTag({ + content: + '@keyframes enter { from { opacity: 0 } to { opacity: 1 } }' + + ' .entering { animation: enter 60ms linear }', + }); + const watch = await watchAnimations(box); + // The page's time origin, where a lost start lands, is well before the + // trigger. + await page.waitForTimeout(200); + // Removing the class on animationend cancels the finished CSS animation, + // which nulls its startTime. + const triggeredAt = await box.evaluate(el => { + el.addEventListener( + 'animationend', + () => el.classList.remove('entering'), + {once: true}, + ); + el.classList.add('entering'); + return performance.timeOrigin + performance.now(); + }); + await expect + .poll(() => box.evaluate(el => el.classList.contains('entering'))) + .toBe(false); + const [entrance] = await watch.started(); + expect(entrance.kind).toBe('CSSAnimation'); + expect(entrance.finishedAt).not.toBeNull(); + // It started on a frame after the trigger, not at the page's time origin. + expect(entrance.from).toBeGreaterThan(triggeredAt - 17); + expect(entrance.from).toBeLessThan(triggeredAt + 500); + expect(entrance.to - entrance.from).toBeCloseTo(60, 3); + expect(entrance.finishedAt ?? 0).toBeGreaterThanOrEqual(entrance.to - 1); + }); + test('an end value held by the keyframes is not part of the curve, and individual transform properties are sampled', async ({ page, }) => { From cf0ad08299c5261ebb729db218b4048cedda60b9 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 09:07:05 -0700 Subject: [PATCH 32/93] chore(interaction): the WebKit container script moves up the stack Nothing in this change runs internal/interaction-harness/scripts/ webkit-container.mjs: CI runs the Chromium project, and the script only runs the WebKit project locally inside the Playwright image on hosts whose system libraries are too old for Playwright's WebKit build. The script, its README paragraph and the config comment that names it move to the stacked change whose WebKit evidence first runs through it, which keeps this one under the 2,000-line limit for a pull request. --- internal/interaction-harness/README.md | 8 --- .../scripts/webkit-container.mjs | 72 ------------------- playwright.interaction.config.ts | 3 - 3 files changed, 83 deletions(-) delete mode 100644 internal/interaction-harness/scripts/webkit-container.mjs diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index fad8af855df60..0edec528d0477 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -20,14 +20,6 @@ pnpm test:interaction --project chromium # everything, Chromium pnpm test:interaction # one spec, both engines ``` -On Linux distributions whose system libraries are older than Playwright's -WebKit build needs, run the WebKit project inside the official Playwright -image (podman or docker; the repository is mounted at the same path): - -```sh -node internal/interaction-harness/scripts/webkit-container.mjs [playwright test args…] -``` - ## Instruments | Module | What it gives a spec | diff --git a/internal/interaction-harness/scripts/webkit-container.mjs b/internal/interaction-harness/scripts/webkit-container.mjs deleted file mode 100644 index 32cc0452396df..0000000000000 --- a/internal/interaction-harness/scripts/webkit-container.mjs +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env node -// Copyright (c) Meta Platforms, Inc. and affiliates. - -/** - * Runs the interaction proofs' WebKit project inside the official Playwright - * container image, for hosts where WebKit cannot launch (Playwright's Linux - * WebKit build needs newer system libraries than some distributions ship). - * - * node internal/interaction-harness/scripts/webkit-container.mjs [playwright test args…] - * - * The repository is mounted at the same path, so node_modules, the built - * Storybook and the evidence directory are the host's own. The image tag - * matches the installed playwright, so its browsers match the client. - * Uses podman when present, else docker. `--network=host` lets the specs' - * local Storybook server and the browser reach each other. - */ -import {spawnSync} from 'node:child_process'; -import fs from 'node:fs'; -import {createRequire} from 'node:module'; -import path from 'node:path'; -import {fileURLToPath} from 'node:url'; - -const root = path.resolve( - path.dirname(fileURLToPath(import.meta.url)), - '../../..', -); -const require = createRequire(path.join(root, 'package.json')); -const {version} = JSON.parse( - fs.readFileSync(require.resolve('playwright/package.json'), 'utf8'), -); -const image = - process.env.PLAYWRIGHT_IMAGE ?? - `mcr.microsoft.com/playwright:v${version}-noble`; -const engine = ['podman', 'docker'].find( - cmd => spawnSync(cmd, ['--version'], {stdio: 'ignore'}).status === 0, -); -if (engine == null) { - console.error('webkit-container: neither podman nor docker is installed'); - process.exit(1); -} -const args = process.argv.slice(2); -const env = [ - 'ASTRYX_STORYBOOK_DIR', - 'ASTRYX_INTERACTION_EVIDENCE', - 'INTERACTION_WORKERS', - 'CI', -] - .filter(name => process.env[name] != null) - .flatMap(name => ['-e', `${name}=${process.env[name]}`]); -const command = [ - 'run', - '--rm', - '--network=host', - '--ipc=host', - '-v', - `${root}:${root}`, - '-w', - root, - ...env, - image, - 'node', - path.join(root, 'node_modules/@playwright/test/cli.js'), - 'test', - '--config', - 'playwright.interaction.config.ts', - '--project', - 'webkit', - ...args, -]; -console.log(`${engine} ${command.join(' ')}`); -const result = spawnSync(engine, command, {stdio: 'inherit'}); -process.exit(result.status ?? 1); diff --git a/playwright.interaction.config.ts b/playwright.interaction.config.ts index 357a5dc3b7642..b3dfd65472ea7 100644 --- a/playwright.interaction.config.ts +++ b/playwright.interaction.config.ts @@ -19,9 +19,6 @@ * * A spec that needs Chromium-only evidence (CDP tracing, scripted touch * timestamps) skips itself in WebKit with the reason; see the harness README. - * Where WebKit cannot launch on the host (missing system libraries on some - * Linux distributions), internal/interaction-harness/scripts/webkit-container.mjs - * runs the WebKit project inside the official Playwright image. * * SYNC: Keep the phone profile here equal to `phone` in * internal/interaction-harness/src/story.ts. From 990fa369316a0e6c7f0aa6bfba301c7f5de7919c Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 09:17:39 -0700 Subject: [PATCH 33/93] chore(interaction): the BottomSheet detent exemplar leaves the harness change The exemplar spec pins BottomSheet outcomes that the component's contract does not state yet, which is a decision for BottomSheet's owner, and it took this change past the 2,000-line limit for a pull request. It moves, with the core package's development dependency on the harness and its README and config mentions, to a pull request of its own stacked on this one, so this change is the frame-evidence instruments alone. --- internal/interaction-harness/README.md | 4 +- packages/core/package.json | 1 - ...tomSheetDetent.interaction.browser.spec.ts | 374 ------------------ playwright.interaction.config.ts | 2 +- pnpm-lock.yaml | 3 - 5 files changed, 2 insertions(+), 382 deletions(-) delete mode 100644 packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index af04199e0237a..8b123e574e846 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -8,8 +8,6 @@ unpublished.** Specs live next to their component as a phone-sized touch screen (402 × 714 CSS px page, 402 × 874 screen, 3 device pixels per CSS px, touch, mobile viewport rules), in Chromium and WebKit. -The exemplar to copy is -[`BottomSheetDetent.interaction.browser.spec.ts`](../../packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts). The harness proves its own instruments in [`src/harness.interaction.browser.spec.ts`](src/harness.interaction.browser.spec.ts): every positive check has a negative control beside it (a rAF animation that @@ -21,7 +19,7 @@ freezes, a `top` animation that lays out every frame). pnpm storybook:build # the specs read stories from the built Storybook pnpm exec playwright install chromium webkit pnpm test:interaction --project chromium # everything, Chromium -pnpm test:interaction BottomSheetDetent # one spec, both engines +pnpm test:interaction # one spec, both engines pnpm storyboards # only the tests tagged @storyboard ``` diff --git a/packages/core/package.json b/packages/core/package.json index 6246323d18ab9..6e9a4db109641 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -808,7 +808,6 @@ }, "devDependencies": { "@astryxdesign/a11y-spec": "workspace:*", - "@astryxdesign/interaction-harness": "workspace:*", "@astryxdesign/cli": "workspace:*", "@babel/cli": "catalog:", "@babel/core": "catalog:", diff --git a/packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts b/packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts deleted file mode 100644 index cdf43e69ce38b..0000000000000 --- a/packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts +++ /dev/null @@ -1,374 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. - -/** - * @file BottomSheetDetent.interaction.browser.spec.ts - * @input The built "Snap points" BottomSheet story, real touch input - * (Chromium CDP / WebKit's input command) and the interaction harness - * @output Interaction and animation proofs for dragging the sheet to a - * detent: the drag follows the finger and a slow release settles on the - * nearest detent; a flick dismisses; the settle's sampled curve matches its - * timing function; the settle keeps advancing every frame through a blocked - * main thread; the settle does no Layout or Paint per frame; a storyboard of - * the drag and the settle. - * @position The exemplar for `*.interaction.browser.spec.ts`: component lanes - * copy this shape. Each test names the instrument it uses from - * @astryxdesign/interaction-harness, so the evidence layer is explicit. - * - * The expected curve here is today's: the panel's `--ease-standard` CSS - * transition over `--duration-medium`, pinned below as SETTLE_EASING and - * SETTLE_MS rather than read from the animation under test. When the settle - * moves to the UIKit spring, `expectedSettle` becomes the spring from the - * motion constants, and the velocity hand-off reported below becomes an - * assertion. - */ - -import {expect, test, type Locator, type Page} from '@playwright/test'; -import { - animationStoryboard, - blockMainThread, - busyWindowProof, - castStoryboard, - compareCurve, - countBetween, - curveOf, - drag, - flick, - frames, - freezeTimers, - holdAnimations, - installLongTaskLog, - installTimerGate, - markElement, - openStory, - readLongTasks, - releaseAnimations, - releaseVelocity, - sampleCurve, - startScreencast, - thawTimers, - timeline, - timingFunction, - touchDriver, - toTouchEvents, - touchStoryboard, - traceRendering, - trackMarker, - withStorybook, - waitForAnimation, - waitForRest, - writeEvidence, - type TouchDriver, -} from '@astryxdesign/interaction-harness'; - -const storybook = withStorybook(test); - -// The settle's timing, from the theme the story renders in: `--ease-standard` -// is the default cubic-bezier(0.24, 1, 0.4, 1), and `--duration-medium` is the -// neutral theme's motion `medium` (300 ms; the base theme's is 410 ms). -const SETTLE_EASING = 'cubic-bezier(0.24, 1, 0.4, 1)'; -const SETTLE_MS = 300; - -interface OpenSheet { - driver: TouchDriver; - sheet: Locator; - /** Top of the sheet at its tall detent and at the half-viewport detent, CSS px. */ - tallTop: number; - halfTop: number; - handle: {x: number; y: number}; -} - -async function openSheet(page: Page): Promise { - await installLongTaskLog(page); - await openStory(page, storybook(), 'core-bottomsheet--snap-points'); - const driver = await touchDriver(page); - const trigger = page.getByRole('button', {name: 'Open nearby places'}); - const box = await trigger.boundingBox(); - if (box == null) { - throw new Error('no trigger'); - } - await driver.tap({x: box.x + box.width / 2, y: box.y + box.height / 2}); - const sheet = page - .getByRole('dialog', {name: 'Nearby places'}) - .locator('.astryx-bottom-sheet') - .first(); - await expect(sheet).toBeVisible(); - await waitForRest(sheet); - const rect = await sheet.boundingBox(); - if (rect == null) { - throw new Error('no sheet'); - } - const viewport = page.viewportSize(); - if (viewport == null) { - throw new Error('no viewport'); - } - return { - driver, - sheet, - tallTop: rect.y, - // snapPoints={[0.5]}: the extra stop shows half the viewport. - halfTop: viewport.height * 0.5, - handle: {x: rect.x + rect.width / 2, y: rect.y + 12}, - }; -} - -// A plain evaluate, not boundingBox(): it must also work while the page's timers are frozen. -const sheetTop = async (sheet: Locator) => - sheet.evaluate(el => el.getBoundingClientRect().y); - -/** A slow drag that rests before the lift: release velocity 0, so the release projects onto itself. */ -function slowDragTo(from: {x: number; y: number}, dy: number) { - return drag( - from, - {x: from.x, y: from.y + dy}, - {durationMs: 400, hz: 60, restMs: 150}, - ); -} - -test('a slow drag follows the finger and settles on the nearest detent', async ({ - page, -}) => { - const {driver, sheet, tallTop, halfTop, handle} = await openSheet(page); - // Two thirds of the way to the half detent: nearer to it than to tall. - const dy = Math.round((halfTop - tallTop) * 0.66); - const script = slowDragTo(handle, dy); - const lastMove = - toTouchEvents(script).filter(e => e.type === 'touchMove').length - 1; - const tops: number[] = []; - let moves = 0; - let restingTop = NaN; - const started = Date.now(); - await driver.play(script, { - onEvent: async step => { - if (step.type !== 'touchMove') { - return; - } - // Read right away (moves are paced at 60 Hz), and once more after the - // last move has been presented: the finger rests 150 ms before lifting. - tops.push(await sheetTop(sheet)); - if (moves++ === lastMove) { - await frames(page, 1); - restingTop = await sheetTop(sheet); - } - }, - }); - // The sheet tracked the finger: monotonic, and 1:1 once out of the - // starting detent's 40 px magnet. - expect(tops.every((top, i) => i === 0 || top >= tops[i - 1] - 0.5)).toBe( - true, - ); - expect(restingTop).toBeCloseTo(tallTop + dy, 0); - await waitForRest(sheet); - expect(await sheetTop(sheet)).toBeCloseTo(halfTop, 0); - // No long task from the gesture or the settle (the story's own load is excluded). - expect(await readLongTasks(page, {from: started, to: Date.now()})).toEqual( - [], - ); -}); - -test('a downward flick dismisses the sheet', async ({page}) => { - const {driver, handle} = await openSheet(page); - const script = flick(handle, { - velocity: {x: 0, y: 2000}, - durationMs: 80, - hz: 120, - }); - expect(releaseVelocity(script).y).toBeCloseTo(2000, -1); - await driver.play(script); - await expect(page.getByRole('dialog', {name: 'Nearby places'})).toBeHidden(); -}); - -test('the settle follows its timing function, sampled frame by frame', async ({ - page, -}, testInfo) => { - const {driver, sheet, tallTop, halfTop, handle} = await openSheet(page); - const dy = Math.round((halfTop - tallTop) * 0.66); - await driver.play(slowDragTo(handle, dy)); - const [settle] = await waitForAnimation(sheet); - const held = await holdAnimations(page, sheet); - const transition = held.find(a => a.name === 'transform'); - if (transition == null) { - throw new Error(`no transform transition held: ${JSON.stringify(held)}`); - } - // The settle runs the pinned timing: a changed curve or duration fails here, - // and the oracle below is built from the pinned values, not from it. - expect(transition.easing.replace(/\s+/g, '')).toBe( - SETTLE_EASING.replace(/\s+/g, ''), - ); - expect(transition.durationMs).toBe(SETTLE_MS); - // Seek the held settle to every frame time and read the presented transform. - const samples = curveOf( - await sampleCurve(sheet, timeline(SETTLE_MS)), - 'translateY', - ); - const from = samples[0].value; - const to = samples[samples.length - 1].value; - const ease = timingFunction(SETTLE_EASING); - const expectedSettle = (t: number) => - from + (to - from) * ease(Math.min(1, t / SETTLE_MS)); - const result = compareCurve(samples, expectedSettle, 0.5); - await releaseAnimations(page); - // UIKit parity metric, reported rather than asserted until the spring lands: - // the settle's initial velocity vs the finger's at release (here 0). - const initialVelocity = ((expectedSettle(1) - expectedSettle(0)) / 1) * 1000; // px/s over the first ms - testInfo.annotations.push({ - type: 'uikit-parity', - description: `settle starts at ${initialVelocity.toFixed(0)} px/s for a release at 0 px/s (UIKit hands the release velocity to the spring)`, - }); - await writeEvidence( - 'curves/bottomsheet-settle', - { - [`${testInfo.project.name}.json`]: JSON.stringify( - {settle, transition, comparison: result}, - null, - 2, - ), - }, - testInfo, - ); - expect(result.maxError, JSON.stringify(result.worst)).toBeLessThanOrEqual( - 0.5, - ); - expect(samples.at(-1)?.value).toBeCloseTo(halfTop - tallTop, 0); -}); - -test('the settle keeps advancing every frame while the main thread is blocked', async ({ - page, -}, testInfo) => { - const {driver, sheet, tallTop, halfTop, handle} = await openSheet(page); - await markElement(sheet); - const cast = await startScreencast(page); - // The settle must be running (and composited) before the block: an - // animation cannot start while the main thread is held. Armed before the - // release, the page waits for the settle itself (the sheet runs no other - // animation during the drag) and blocks once it has run 16 ms, however busy - // the machine; armed after, the round trip would leave little of the settle - // to judge. - const blocking = blockMainThread(page, 700, { - during: sheet, - minProgressMs: 16, - }); - await driver.play(slowDragTo(handle, Math.round((halfTop - tallTop) * 0.66))); - const block = await blocking; - // Where the settle ends, from the page: a frame after the block can still - // show a stale position (see `settledAt`). - await waitForRest(sheet); - const settledAt = await sheet.evaluate(el => el.getBoundingClientRect().y); - const castFrames = await cast.stop(); - const tracked = await trackMarker(page, castFrames); - // Judge the part of the block during which the settle was still moving: - // until it is within 2 px of rest (the screencast rounds to whole pixels, - // so the easing tail shows plateaus there). - const proof = busyWindowProof(tracked, block, {settledWithin: 2, settledAt}); - await writeEvidence( - 'busy-main-thread/bottomsheet-settle', - { - [`${testInfo.project.name}.json`]: JSON.stringify( - { - block, - settledAt, - proof, - frames: tracked.map(f => ({ - t: +(f.t - block.from).toFixed(1), - y: f.box?.y ?? null, - h: f.box?.height ?? null, - })), - }, - null, - 1, - ), - }, - testInfo, - ); - const film = await castStoryboard(page, castFrames, { - name: 'bottomsheet-settle-busy-main-thread', - zero: block.from, - from: block.from - 40, - to: proof.judged.to + 40, - columns: 8, - cellScale: 0.35, - data: t => ({y: tracked.find(f => f.t === t)?.box?.y ?? -1}), - }); - await film.write(testInfo, {block, judged: proof.judged}); - testInfo.annotations.push({ - type: 'busy-main-thread', - description: proof.summary, - }); - expect(proof.verdict.failures, proof.summary).toEqual([]); - expect(proof.monotonic).toBe(true); -}); - -test('the settle does no Layout or Paint per frame', async ({ - page, - browserName, -}) => { - test.skip(browserName !== 'chromium', 'rendering traces are Chromium-only'); - const {driver, sheet, tallTop, halfTop, handle} = await openSheet(page); - let settle = {from: 0, to: 0}; - const trace = await traceRendering(page, async () => { - await driver.play( - slowDragTo(handle, Math.round((halfTop - tallTop) * 0.66)), - ); - [settle] = await waitForAnimation(sheet); - await waitForRest(sheet); - }); - // The steady part of the settle: its first and last frames may commit. - const during = countBetween(trace, settle.from + 50, settle.to - 50); - expect(during.Layout ?? 0, JSON.stringify(during)).toBe(0); - expect(during.Paint ?? 0, JSON.stringify(during)).toBe(0); - expect(trace.longTasks).toEqual([]); -}); - -test( - 'storyboard: drag to the half detent and settle', - {tag: '@storyboard'}, - async ({page}, testInfo) => { - const {driver, sheet, tallTop, halfTop, handle} = await openSheet(page); - // Gate the page's timers from here on, so the settle's completion timer can - // be held with its animation while the filmstrip is photographed. - await installTimerGate(page); - const viewport = page.viewportSize() ?? {width: 402, height: 714}; - const clip = {x: 0, y: 0, width: viewport.width, height: viewport.height}; - const dy = Math.round((halfTop - tallTop) * 0.66); - const measure = async () => ({top: await sheetTop(sheet)}); - const dragBoard = await touchStoryboard( - page, - driver, - drag( - handle, - {x: handle.x, y: handle.y + dy}, - {durationMs: 200, hz: 30, restMs: 150}, - ), - { - name: 'bottomsheet-drag-to-detent', - clip, - measure, - columns: 5, - cellScale: 0.4, - }, - ); - await waitForAnimation(sheet); - await freezeTimers(page); - const {storyboard: settleBoard} = await animationStoryboard(page, { - name: 'bottomsheet-settle', - clip, - scope: sheet, - measure, - columns: 7, - cellScale: 0.4, - }); - await releaseAnimations(page, {finish: true}); - await thawTimers(page); - await dragBoard.write(testInfo); - await settleBoard.write(testInfo); - // The trace is the assertion; the sheets are the evidence. - const dragTops = dragBoard.frames.map(f => f.data.top as number); - expect( - dragTops.every((top, i) => i === 0 || top >= dragTops[i - 1] - 0.5), - ).toBe(true); - const settleTops = settleBoard.frames.map(f => f.data.top as number); - expect( - settleTops.every((top, i) => i === 0 || top >= settleTops[i - 1] - 0.5), - ).toBe(true); - expect(settleTops.at(-1)).toBeCloseTo(halfTop, 0); - }, -); diff --git a/playwright.interaction.config.ts b/playwright.interaction.config.ts index 40dd5653267b6..f0c61e2e6e563 100644 --- a/playwright.interaction.config.ts +++ b/playwright.interaction.config.ts @@ -15,7 +15,7 @@ * pnpm storybook:build * pnpm exec playwright install chromium webkit * pnpm test:interaction --project chromium - * pnpm test:interaction BottomSheetDetent.interaction.browser.spec.ts + * pnpm test:interaction harness.interaction.browser.spec.ts * * A spec that needs Chromium-only evidence (CDP tracing, scripted touch * timestamps) skips itself in WebKit with the reason; see the harness README. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ad5e966730d45..c93bc6ef3838a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -882,9 +882,6 @@ importers: '@astryxdesign/cli': specifier: workspace:* version: link:../cli - '@astryxdesign/interaction-harness': - specifier: workspace:* - version: link:../../internal/interaction-harness '@babel/cli': specifier: 'catalog:' version: 7.29.7(@babel/core@7.29.7) From 2642e064611560c05e23775c7878c4ae033a0e08 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 09:39:18 -0700 Subject: [PATCH 34/93] fix(interaction): a storyboard retakes a photograph Chromium returns empty Chromium photographs a page by copying its compositor surface, and answers "Unable to capture screenshot" when that copy comes back empty: right after the page's surface changes (a navigation, a resize) the view can have no surface to copy. Chromium retries five times itself, but without waiting for a new frame, so a gap longer than a few tasks exhausts them. One CI run failed the storyboard self-test that way on its first photograph, about 100 ms after the page loaded. Storyboard.capture now takes such a photograph again once the page has presented two more frames, up to three attempts in all, and lists each retake under `recaptures` in the trace. Any other error, or a third empty copy, still throws. The new self-test makes the page's screenshot fail once with Chromium's message: the previous code threw, in Chromium and WebKit; now the second capture comes at least two frames later and the trace records it, and captures that keep failing still throw. --- internal/interaction-harness/README.md | 5 +- .../src/harness.interaction.browser.spec.ts | 51 +++++++++++++++++++ .../interaction-harness/src/storyboard.ts | 46 ++++++++++++++--- 3 files changed, 93 insertions(+), 9 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index d851fa2135649..bf6d040dea957 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -98,7 +98,10 @@ with nothing else to film, pass `times`. A touch filmstrip keeps the scripted event `timeStamp`s (Chromium), but a handler that reads `performance.now()` or `Date.now()` sees one event per photograph, about 100 ms apart, so a clock-timed release flings slower when filmed than when played: a storyboard -cannot show such a gesture; record it in real time instead. +cannot show such a gesture; record it in real time instead. Chromium can return +an empty copy of the page's surface right after it changes ("Unable to capture +screenshot"); a storyboard takes that photograph again two frames later, up to +three times, and lists each retake under `recaptures` in the trace. ## Engine limits (measured with Playwright 1.61) diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index bec81428da5b7..1f5a180ed7064 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -44,6 +44,7 @@ import { sampleCurve, seekAnimations, springStep, + Storyboard, tap, thawTimers, timeline, @@ -730,6 +731,56 @@ test.describe('storyboards', {tag: '@storyboard'}, () => { expect(files['sheet.png']).toMatch(/sheet\.png$/); }); + test('a capture Chromium returns empty is taken again a frame later', async ({ + page, + }) => { + await fixture(page); + // Counts the page's frames, so the wait before the second capture shows. + await page.evaluate(() => { + const w = window as unknown as {__frames: number}; + w.__frames = 0; + const tick = () => { + w.__frames += 1; + requestAnimationFrame(tick); + }; + requestAnimationFrame(tick); + }); + const screenshot = page.screenshot.bind(page); + const calledAt: number[] = []; + let failures = 1; + // What Chromium answers when its surface copy comes back empty. + page.screenshot = (async (options?: Parameters[0]) => { + calledAt.push( + await page.evaluate( + () => (window as unknown as {__frames: number}).__frames, + ), + ); + if (failures-- > 0) { + throw new Error( + 'page.screenshot: Protocol error (Page.captureScreenshot): Unable to capture screenshot', + ); + } + return screenshot(options); + }) as Page['screenshot']; + const storyboard = new Storyboard(page, { + name: 'harness-self-test-recapture', + clip: {x: 0, y: 340, width: 402, height: 120}, + }); + await storyboard.capture('t=0 ms', 0); + expect(storyboard.frames).toHaveLength(1); + expect(calledAt).toHaveLength(2); + expect(calledAt[1] - calledAt[0]).toBeGreaterThanOrEqual(2); + expect(storyboard.trace().recaptures).toEqual([ + {label: 't=0 ms', error: expect.stringContaining('Unable to capture')}, + ]); + // Captures that keep failing still fail, with Chromium's error. + failures = Infinity; + await expect(storyboard.capture('t=1 ms', 1)).rejects.toThrow( + /Unable to capture screenshot/, + ); + expect(storyboard.frames).toHaveLength(1); + }); + test('a touch filmstrip photographs every scripted event', async ({ page, }, testInfo) => { diff --git a/internal/interaction-harness/src/storyboard.ts b/internal/interaction-harness/src/storyboard.ts index ead43b713fbcd..da5941124b9fa 100644 --- a/internal/interaction-harness/src/storyboard.ts +++ b/internal/interaction-harness/src/storyboard.ts @@ -55,8 +55,21 @@ export interface StoryboardOptions { readonly cellScale?: number; } +/** + * Chromium photographs the page by copying its compositor surface. Right after + * the page's surface changes (a navigation, a resize) the copy can come back + * empty, and Chromium answers "Unable to capture screenshot" once its own five + * retries, issued without waiting for a new frame, have all found no surface. + * A capture that fails that way is taken again after the page has presented + * two more frames, at most this many times in all. + */ +const CAPTURE_ATTEMPTS = 3; +const EMPTY_SURFACE_COPY = /Unable to capture screenshot/; + export class Storyboard { readonly frames: StoryboardFrame[] = []; + /** Captures Chromium returned empty and that were taken again a frame later. */ + readonly recaptures: {label: string; error: string}[] = []; private clipBox: ClipBox | undefined; constructor( @@ -64,20 +77,36 @@ export class Storyboard { readonly options: StoryboardOptions, ) {} - /** Photographs the clip now and records `data` with it. */ + /** + * Photographs the clip now and records `data` with it. An empty surface + * copy (see CAPTURE_ATTEMPTS) is taken again once the page has presented two + * more frames; any other failure, or a third empty copy, throws. + */ async capture( label: string, t: number, data: Readonly> = {}, ): Promise { const clip = await this.resolveClip(); - const png = await this.page.screenshot({ - clip, - scale: 'css', - animations: 'allow', - caret: 'hide', - }); - this.frames.push({label, t, data, png}); + for (let attempt = 1; ; attempt += 1) { + try { + const png = await this.page.screenshot({ + clip, + scale: 'css', + animations: 'allow', + caret: 'hide', + }); + this.frames.push({label, t, data, png}); + return; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (attempt >= CAPTURE_ATTEMPTS || !EMPTY_SURFACE_COPY.test(message)) { + throw error; + } + this.recaptures.push({label, error: message.split('\n')[0]}); + await frames(this.page, 2); + } + } } private async resolveClip(): Promise { @@ -123,6 +152,7 @@ export class Storyboard { browser: this.page.context().browser()?.browserType().name() ?? null, viewport: this.page.viewportSize(), clip: this.clipBox ?? null, + ...(this.recaptures.length > 0 ? {recaptures: this.recaptures} : {}), ...extra, frames: this.frames.map(({label, t, data}) => ({label, t, data})), }; From ca4fee0145830e24370ff652405036ad761359a6 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 10:07:27 -0700 Subject: [PATCH 35/93] test(interaction): pin what the long-task log sees; the cache test watches its own run The long-task log reads the Long Tasks API, which reports the tasks of the page's own event loop. A DevTools page.evaluate, such as blockMainThread without `during`, is not one of them: a 300 ms block of that kind left the log empty while the trace counted a 300 ms task. installLongTaskLog now says so, and a self-test pins both sides: a 120 ms timer task is in the log, and the 300 ms evaluate is in the trace's long tasks but not in the log. Without the log installed, the test fails. The shader-cache test passed from a warm cache even if Mesa stopped writing there. It now launches its own browser with an empty directory inside the host-kept mount and expects this run's shaders to appear in it; with Mesa's cache disabled it fails. --- .../src/harness.interaction.browser.spec.ts | 92 ++++++++++++++++--- internal/interaction-harness/src/trace.ts | 9 +- 2 files changed, 86 insertions(+), 15 deletions(-) diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 31b500967ca03..28a5310295389 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -9,7 +9,8 @@ * arrive as distinct pointers; the screencast shows a compositor animation * advancing through a blocked main thread while a rAF animation freezes; * the trace counts zero Layout for a transform animation and one per frame - * for a `top` animation; sampled curves match the oracles, individual + * for a `top` animation; the long-task log records the page's own long + * task but not a DevTools evaluate, which the trace counts; sampled curves match the oracles, individual * transform properties included; a sample at an end reports the time it was * read at; holding takes only running document-timeline animations, each at * its own offset, holds each where it was read, adds a second hold to the @@ -43,6 +44,7 @@ import { freezeTimers, holdAnimations, installInputLog, + installLongTaskLog, installTimerGate, markElement, merge, @@ -50,6 +52,7 @@ import { PARTIAL_LIFT_SKIP, pinch, readInputLog, + readLongTasks, releaseAnimations, sampleCurve, sampleRects, @@ -391,7 +394,7 @@ test.describe('frame evidence', () => { }); test('in the WebKit container, compiled shaders outlive the run', async ({ - page, + browser, }, testInfo) => { // The official Playwright image keeps its browsers here. test.skip( @@ -408,18 +411,33 @@ test.describe('frame evidence', () => { fs.readFileSync(path.join(dir ?? '', 'ASTRYX-SHADER-CACHE'), 'utf8'), ).toContain('playwright'); // Mesa writes what it compiles there once a page draws a blurred layer. - await page.setContent( - '
', - ); - await page.evaluate( - () => - document - .getElementById('glass') - ?.animate([{scale: '1'}, {scale: '0.9'}], {duration: 200}).finished, - ); - expect( - fs.readdirSync(dir ?? '').filter(name => name !== 'ASTRYX-SHADER-CACHE'), - ).not.toEqual([]); + // Earlier runs have filled the cache already, so this run's browser gets + // an empty directory beside them, in the same host-kept mount, and the + // test watches its own writes arrive. + const fresh = fs.mkdtempSync(path.join(dir ?? '', 'self-test-')); + try { + const own = await browser.browserType().launch({ + env: {...process.env, MESA_SHADER_CACHE_DIR: fresh}, + }); + try { + const page = await own.newPage(); + await page.setContent( + '
', + ); + await page.evaluate( + () => + document + .getElementById('glass') + ?.animate([{scale: '1'}, {scale: '0.9'}], {duration: 200}) + .finished, + ); + } finally { + await own.close(); + } + expect(fs.readdirSync(fresh, {recursive: true})).not.toEqual([]); + } finally { + fs.rmSync(fresh, {recursive: true, force: true}); + } }); test('rect sampling reads one rect per animation frame while the main thread is free', async ({ @@ -488,6 +506,52 @@ test.describe('frame evidence', () => { // ~24 frames in 400 ms: one layout per frame. expect(top.Layout ?? 0).toBeGreaterThan(18); }); + + test("the long-task log records the page's own long task but not a DevTools evaluate, which the trace counts", async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'the Long Tasks API and rendering traces are Chromium-only', + ); + await installLongTaskLog(page); + await fixture(page); + // The page's own task: a timer that spins 120 ms. + const own = await page.evaluate( + () => + new Promise<{from: number; to: number}>(resolve => { + setTimeout(() => { + const from = performance.timeOrigin + performance.now(); + const end = performance.now() + 120; + while (performance.now() < end) { + // spin + } + resolve({from, to: performance.timeOrigin + performance.now()}); + }, 0); + }), + ); + await frames(page, 2); + const recorded = await readLongTasks(page, own); + expect(recorded).toHaveLength(1); + expect(recorded[0].durationMs).toBeGreaterThanOrEqual(110); + // A DevTools evaluate that spins 300 ms: the trace counts it, the log + // never sees it. + let block = {from: 0, to: 0}; + const trace = await traceRendering(page, async () => { + block = await blockMainThread(page, 300); + }); + await frames(page, 2); + expect( + trace.longTasks.some( + t => + t.durationMs >= 290 && + t.start < block.to && + t.start >= block.from - 20, + ), + ).toBe(true); + expect(await readLongTasks(page, block)).toEqual([]); + }); }); test.describe('curve sampling', () => { diff --git a/internal/interaction-harness/src/trace.ts b/internal/interaction-harness/src/trace.ts index 773445067dd2f..91566bb93473d 100644 --- a/internal/interaction-harness/src/trace.ts +++ b/internal/interaction-harness/src/trace.ts @@ -173,7 +173,14 @@ const LONG_TASK_SCRIPT = `(() => { } catch {} })()`; -/** Records every long task (> 50 ms) the page runs (install before navigation; Chromium). */ +/** + * Records every long task (> 50 ms) the page runs (install before navigation; + * Chromium), through the Long Tasks API. That sees the tasks of the page's own + * event loop: a timer, an event handler, `blockMainThread(page, ms, {during})`. + * A DevTools `page.evaluate`, such as `blockMainThread(page, ms)` without + * `during`, is not one of them and is never recorded here; the trace's + * `longTasks` (`traceRendering`) counts every main-thread task over 50 ms. + */ export async function installLongTaskLog(page: Page): Promise { await page.addInitScript(LONG_TASK_SCRIPT); } From 18b3bd38b30b2231efe4ca6080f6275ba17fbc0c Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 10:17:44 -0700 Subject: [PATCH 36/93] docs(interaction): the storyboard retake waits two frames, three attempts in all The recaptures doc and the self-test's title said a retake comes "a frame later", and the README said "up to three times"; capture waits two frames and makes at most three attempts in all. The wording now says so. --- internal/interaction-harness/README.md | 2 +- .../src/harness.interaction.browser.spec.ts | 2 +- internal/interaction-harness/src/storyboard.ts | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index bf6d040dea957..52489f7251178 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -101,7 +101,7 @@ clock-timed release flings slower when filmed than when played: a storyboard cannot show such a gesture; record it in real time instead. Chromium can return an empty copy of the page's surface right after it changes ("Unable to capture screenshot"); a storyboard takes that photograph again two frames later, up to -three times, and lists each retake under `recaptures` in the trace. +three attempts in all, and lists each retake under `recaptures` in the trace. ## Engine limits (measured with Playwright 1.61) diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 1f5a180ed7064..06203df025557 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -731,7 +731,7 @@ test.describe('storyboards', {tag: '@storyboard'}, () => { expect(files['sheet.png']).toMatch(/sheet\.png$/); }); - test('a capture Chromium returns empty is taken again a frame later', async ({ + test('a capture Chromium returns empty is taken again two frames later', async ({ page, }) => { await fixture(page); diff --git a/internal/interaction-harness/src/storyboard.ts b/internal/interaction-harness/src/storyboard.ts index da5941124b9fa..2dc22fdc8cf04 100644 --- a/internal/interaction-harness/src/storyboard.ts +++ b/internal/interaction-harness/src/storyboard.ts @@ -61,14 +61,14 @@ export interface StoryboardOptions { * empty, and Chromium answers "Unable to capture screenshot" once its own five * retries, issued without waiting for a new frame, have all found no surface. * A capture that fails that way is taken again after the page has presented - * two more frames, at most this many times in all. + * two more frames, at most this many attempts in all. */ const CAPTURE_ATTEMPTS = 3; const EMPTY_SURFACE_COPY = /Unable to capture screenshot/; export class Storyboard { readonly frames: StoryboardFrame[] = []; - /** Captures Chromium returned empty and that were taken again a frame later. */ + /** Captures Chromium returned empty and that were taken again two frames later. */ readonly recaptures: {label: string; error: string}[] = []; private clipBox: ClipBox | undefined; From a68b352dd6386595e57cf6026fe844f49dcfd34a Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 10:46:04 -0700 Subject: [PATCH 37/93] test(interaction): the self-test films a cast window castStoryboard was the one instrument the self-test did not exercise. The new test records a screencast of a running animation as JPEG (WebKit's frames always are), films a window of its middle frames zeroed on an earlier one, and checks that the filmstrip keeps exactly the presented frames inside the window, labels each from the zero, hands the data callback each frame's presentation time, and stores every cell as a PNG; a window with no presented frame in it films nothing, and writing it fails. Dropping the window filter, labelling from the raw time, keeping the JPEG bytes, or passing the zeroed time to the callback each fails it. --- .../src/harness.interaction.browser.spec.ts | 58 ++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 28a5310295389..d9afa2a2e1d6b 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -22,7 +22,8 @@ * scripted event times while a handler clock sees its pacing; the timer * gate holds a component timer until the filmstrip is done, drops callbacks * the page cancels, stamps held frames with the time they run, and leaves - * the harness its own frames. + * the harness its own frames; a cast filmstrip keeps exactly the presented + * frames of its window, labelled from its zero, as PNG. * @position The harness's own test. A component spec trusts these * instruments; this file is why it may. */ @@ -32,6 +33,7 @@ import path from 'node:path'; import {expect, test, type Page} from '@playwright/test'; import { animationStoryboard, + castStoryboard, blockMainThread, busyWindowProof, compareCurve, @@ -1313,4 +1315,58 @@ test.describe('storyboards', {tag: '@storyboard'}, () => { expect(state).toMatchObject({timer: false, frame: false}); expect(state.heldFor).toBeGreaterThanOrEqual(140); }); + + test('a cast filmstrip keeps exactly the presented frames of its window, labelled from its zero', async ({ + page, + }, testInfo) => { + await fixture(page); + const box = page.locator('#box'); + // JPEG frames (WebKit's are always JPEG) take the conversion path. + const cast = await startScreencast(page, {format: 'jpeg'}); + await box.evaluate(el => + el.animate( + [{transform: 'translateX(0px)'}, {transform: 'translateX(200px)'}], + {duration: 600}, + ), + ); + await page.waitForTimeout(500); + const presented = await cast.stop(); + expect(presented.length).toBeGreaterThan(8); + // A window of the middle frames, zeroed on one of them. + const zero = presented[2].t; + const from = presented[3].t; + const to = presented[presented.length - 3].t; + const inside = presented.filter(f => f.t >= from && f.t <= to); + const seen: number[] = []; + const film = await castStoryboard(page, presented, { + name: 'harness-self-test-cast', + zero, + from, + to, + data: t => { + seen.push(t); + return {t}; + }, + }); + expect(film.frames.map(f => f.t)).toEqual(inside.map(f => f.t - zero)); + expect(film.frames.map(f => f.label)).toEqual( + inside.map(f => `t=${(f.t - zero).toFixed(1)} ms`), + ); + expect(seen).toEqual(inside.map(f => f.t)); + // Every cell is a PNG, whatever the screencast delivered. + for (const frame of film.frames) { + expect([...frame.png.subarray(0, 4)]).toEqual([0x89, 0x50, 0x4e, 0x47]); + } + const files = await film.write(testInfo); + expect(files['sheet.png']).toMatch(/sheet\.png$/); + // A window with no presented frame in it films nothing. + const empty = await castStoryboard(page, presented, { + name: 'harness-self-test-cast-empty', + zero, + from: to + 1, + to: to + 2, + }); + expect(empty.frames).toEqual([]); + await expect(empty.write(testInfo)).rejects.toThrow(/no frames/); + }); }); From 56d5de0aaa3254398435c4438c091595d6d2665a Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 12:32:18 -0700 Subject: [PATCH 38/93] chore(interaction): the WebKit container script lands with the storyboards internal/interaction-harness/scripts/webkit-container.mjs left the touch-input change to keep it under the size limit, since nothing there runs it. It returns here unchanged, with its README paragraph and the config comment that names it: the storyboard evidence is the first that runs the WebKit project through it on hosts whose system libraries are too old for Playwright's WebKit build. --- internal/interaction-harness/README.md | 8 +++ .../scripts/webkit-container.mjs | 72 +++++++++++++++++++ playwright.interaction.config.ts | 3 + 3 files changed, 83 insertions(+) create mode 100644 internal/interaction-harness/scripts/webkit-container.mjs diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index aaf1d65708e6d..40ed28ceeb014 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -21,6 +21,14 @@ pnpm test:interaction # one spec, both engines pnpm storyboards # only the tests tagged @storyboard ``` +On Linux distributions whose system libraries are older than Playwright's +WebKit build needs, run the WebKit project inside the official Playwright +image (podman or docker; the repository is mounted at the same path): + +```sh +node internal/interaction-harness/scripts/webkit-container.mjs [playwright test args…] +``` + Evidence (storyboards, curve traces) is written to `test-results/interaction-evidence/` (override with `ASTRYX_INTERACTION_EVIDENCE`) and attached to the test report. It is never diff --git a/internal/interaction-harness/scripts/webkit-container.mjs b/internal/interaction-harness/scripts/webkit-container.mjs new file mode 100644 index 0000000000000..32cc0452396df --- /dev/null +++ b/internal/interaction-harness/scripts/webkit-container.mjs @@ -0,0 +1,72 @@ +#!/usr/bin/env node +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * Runs the interaction proofs' WebKit project inside the official Playwright + * container image, for hosts where WebKit cannot launch (Playwright's Linux + * WebKit build needs newer system libraries than some distributions ship). + * + * node internal/interaction-harness/scripts/webkit-container.mjs [playwright test args…] + * + * The repository is mounted at the same path, so node_modules, the built + * Storybook and the evidence directory are the host's own. The image tag + * matches the installed playwright, so its browsers match the client. + * Uses podman when present, else docker. `--network=host` lets the specs' + * local Storybook server and the browser reach each other. + */ +import {spawnSync} from 'node:child_process'; +import fs from 'node:fs'; +import {createRequire} from 'node:module'; +import path from 'node:path'; +import {fileURLToPath} from 'node:url'; + +const root = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '../../..', +); +const require = createRequire(path.join(root, 'package.json')); +const {version} = JSON.parse( + fs.readFileSync(require.resolve('playwright/package.json'), 'utf8'), +); +const image = + process.env.PLAYWRIGHT_IMAGE ?? + `mcr.microsoft.com/playwright:v${version}-noble`; +const engine = ['podman', 'docker'].find( + cmd => spawnSync(cmd, ['--version'], {stdio: 'ignore'}).status === 0, +); +if (engine == null) { + console.error('webkit-container: neither podman nor docker is installed'); + process.exit(1); +} +const args = process.argv.slice(2); +const env = [ + 'ASTRYX_STORYBOOK_DIR', + 'ASTRYX_INTERACTION_EVIDENCE', + 'INTERACTION_WORKERS', + 'CI', +] + .filter(name => process.env[name] != null) + .flatMap(name => ['-e', `${name}=${process.env[name]}`]); +const command = [ + 'run', + '--rm', + '--network=host', + '--ipc=host', + '-v', + `${root}:${root}`, + '-w', + root, + ...env, + image, + 'node', + path.join(root, 'node_modules/@playwright/test/cli.js'), + 'test', + '--config', + 'playwright.interaction.config.ts', + '--project', + 'webkit', + ...args, +]; +console.log(`${engine} ${command.join(' ')}`); +const result = spawnSync(engine, command, {stdio: 'inherit'}); +process.exit(result.status ?? 1); diff --git a/playwright.interaction.config.ts b/playwright.interaction.config.ts index f47d26982cbe3..f0c61e2e6e563 100644 --- a/playwright.interaction.config.ts +++ b/playwright.interaction.config.ts @@ -19,6 +19,9 @@ * * A spec that needs Chromium-only evidence (CDP tracing, scripted touch * timestamps) skips itself in WebKit with the reason; see the harness README. + * Where WebKit cannot launch on the host (missing system libraries on some + * Linux distributions), internal/interaction-harness/scripts/webkit-container.mjs + * runs the WebKit project inside the official Playwright image. * * SYNC: Keep the phone profile here equal to `phone` in * internal/interaction-harness/src/story.ts. From 37050d36b3bd4deee67b5904fc009ae692e0d729 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 12:47:14 -0700 Subject: [PATCH 39/93] ci: the interaction jobs keep to the 15-minute budget and report their durations Since #8427 every job that can run on a pull request carries a `timeout-minutes` of at most 15, and `ci-duration` waits on every one of them to report run times. `pr-interaction` gets the budget's 15 minutes, like the other proof jobs, and `interaction-scope` 5, like the other jobs that only classify; both join `ci-duration`'s needs. `ci-duration-guard.test.mjs` failed three ways on this branch merged with main (no timeout on either job, 26 jobs expected in the report against 24 listed) and passes now. --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f0c2602ecd1d..a462cbf7e9aee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1720,6 +1720,7 @@ jobs: needs: [check-scope] if: ${{ github.event_name != 'workflow_dispatch' && always() && !cancelled() && needs.check-scope.outputs.spec_only != 'true' }} runs-on: ubuntu-slim + timeout-minutes: 5 permissions: contents: read outputs: @@ -1790,6 +1791,7 @@ jobs: if: ${{ github.event_name != 'workflow_dispatch' && always() && !cancelled() && needs.check-scope.outputs.spec_only != 'true' }} # Frame evidence measures timing: four cores, one Playwright worker. runs-on: ${{ needs.interaction-scope.outputs.applies == 'true' && '4-core-ubuntu' || 'ubuntu-slim' }} + timeout-minutes: 15 permissions: contents: read steps: @@ -2551,6 +2553,8 @@ jobs: build-sandbox, build, pr-a11y, + interaction-scope, + pr-interaction, test-build, fixture-contrast, visual-scout, From 71827fe711f9b85876811a054b648c0b8817d8d9 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 13:00:28 -0700 Subject: [PATCH 40/93] fix(interaction): the animation recorder stands down while a trace runs The animation recorder runs a callback every frame, which makes every frame a main-thread frame: Blink then restyles each running animation's target, a composited one included, and lays out an anchored one, and each of its reads of document.getAnimations() flushes style. A rendering trace counts exactly that work, so a "no Layout" window counted one Layout per frame that the page under test never caused. traceRendering now sets `__harnessRenderingTrace` before its start mark. While it is set, the recorder runs no frame callback and does not read the page's animations on CSS animation events; its animate() wrapper still records script animations, reading no style. After the end mark, traceRendering clears the flag and calls `__harnessAnimations.resume()`, which catches up on what started meanwhile, outside the counted window, and runs the recorder's frames again. A recorder that starts during a trace still reads once, at its start, so what already runs is never taken for something the watched action started. Two self-tests. With a trace flagged, a CSS transition and a script animation start: the recorder reads nothing (the previous one read 16-17 times in Chromium and 11-12 in WebKit in the same 200 ms), reads every frame again once resumed, and the watch reports both. And an anchored element's composited animation, traced with the recorder running, lays out 0 times in its steady middle, where the previous code counted 24; a page that asks for a frame every frame still lays it out in each. Found and patched by the menus lane; adapted to the recorder's current first-contact read. --- internal/interaction-harness/README.md | 5 +- internal/interaction-harness/src/curves.ts | 40 +++++- .../src/harness.interaction.browser.spec.ts | 128 +++++++++++++++++- internal/interaction-harness/src/trace.ts | 32 ++++- 4 files changed, 196 insertions(+), 9 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index a95bcef132cfe..4457d4febe51a 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -131,7 +131,10 @@ block into a filmstrip. **No Layout / Paint.** `traceRendering` records a Chromium trace around the interaction; `countBetween` counts the steady middle of an animation (its first and last frames may commit). A transform/opacity animation does no -Layout or Paint there; a layout-property animation does one per frame. +Layout or Paint there; a layout-property animation does one per frame. The +harness's own animation recorder stands down while the trace runs, since its +frame loop would make every frame a main-thread one, restyling each running +animation's target and laying out an anchored one, and catches up afterwards. **Storyboards.** Deterministic: frames come from seeking held animations or from stepping scripted input, never from wall-clock capture. The JSON trace diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index a542156527312..9136e6f383b9a 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -113,17 +113,48 @@ function animationRecorder(): void { return entry.startedAt == null && entry.animation.playState !== 'idle'; }); }; + // While a rendering trace runs (traceRendering sets + // `__harnessRenderingTrace`), the recorder stands down. Its per-frame + // callback would make every frame a main-thread frame, in which Blink + // restyles each running animation's target, a composited one included (and + // lays out an anchored one), and document.getAnimations() flushes style: + // exactly what the trace counts. The animate() wrapper below still records + // script animations (it reads no style), and traceRendering resumes the + // recorder once the trace ends. + const tracing = () => + (window as unknown as {__harnessRenderingTrace?: boolean}) + .__harnessRenderingTrace === true; + let looping = false; const tick = () => { + if (tracing()) { + looping = false; + return; + } record(); frame(tick); }; + const loop = () => { + if (!looping) { + looping = true; + frame(tick); + } + }; // What already runs when the recorder starts (a spinner on a page the // harness touches for the first time) is seen now, before any watch arms; // first seen a frame later, it would read as started by the watched action. + // (A recorder that starts during a trace reads once here, then waits.) record(); - frame(tick); + loop(); for (const type of ['animationstart', 'transitionrun', 'transitionstart']) { - document.addEventListener(type, record, true); + document.addEventListener( + type, + () => { + if (!tracing()) { + record(); + } + }, + true, + ); } // A script animation is recorded as it is made: WebKit can start and finish // a short one between two frames of a fresh page. @@ -139,6 +170,11 @@ function animationRecorder(): void { w.__harnessAnimations = { log, record, + // Catches up on what started during a trace, then runs its frames again. + resume: () => { + record(); + loop(); + }, arm: () => (armedAt = performance.now()), armedAt: () => armedAt, }; diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index d04a77d300f34..6e56b1010bacd 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -9,8 +9,11 @@ * arrive as distinct pointers; the screencast shows a compositor animation * advancing through a blocked main thread while a rAF animation freezes; * the trace counts zero Layout for a transform animation and one per frame - * for a `top` animation; the long-task log records the page's own long - * task but not a DevTools evaluate, which the trace counts; sampled curves + * for a `top` animation, and none from the harness's own animation recorder + * while an anchored element's composited animation runs (a page that asks + * for a frame every frame lays it out in each); the long-task log records + * the page's own long task but not a DevTools evaluate, which the trace + * counts; sampled curves * match the oracles, individual transform properties included; a sample at * an end reports the time it was read at; holding takes only running * document-timeline animations, each at its own offset, holds each where it @@ -19,7 +22,9 @@ * animation-state waits report a running transition and its end, and judge * rest by presented values even while an animation holds its end, and a * watch taken on first contact ignores an animation already running and - * keeps the start of one the page cancels when it ends; storyboards write a + * keeps the start of one the page cancels when it ends, and the recorder + * reads nothing while a rendering trace runs and catches up after it; + * storyboards write a * sheet and a trace that follows the curve, an endless animation does not * set their length, and a photograph Chromium returns empty is taken again * two frames later; a touch filmstrip keeps scripted event times while a @@ -79,6 +84,16 @@ import { withFinger, } from './index'; +// An element positioned against an anchor: a composited animation of it lays +// out in every main-thread frame, so it shows any per-frame main-thread work. +const ANCHORED = ` + +
`; + const FIXTURE = ` +
`; + +async function busyFixture(page: Page) { + await page.route('http://busy.test/', route => + route.fulfill({contentType: 'text/html', body: BUSY_FIXTURE}), + ); + await page.goto('http://busy.test/'); +} + // A real navigation (not setContent, whose document.open() erases the window // listeners the input log installs). async function fixture(page: Page) { @@ -266,7 +284,7 @@ test.describe('frame evidence', () => { test('a compositor animation advances every frame through a blocked main thread; a rAF one freezes, whatever the screencast rate', async ({ page, }, testInfo) => { - await fixture(page); + await busyFixture(page); const box = page.locator('#box'); await markElement(box, {mode: 'outline'}); @@ -276,14 +294,14 @@ test.describe('frame evidence', () => { el.style.transform = 'none'; if (how === 'compositor') { el.animate( - [{transform: 'translateX(0px)'}, {transform: 'translateX(280px)'}], + [{transform: 'translateY(0px)'}, {transform: 'translateY(560px)'}], {duration: 1600, easing: 'linear'}, ); } else { const start = performance.now(); const step = (now: number) => { const p = Math.min(1, (now - start) / 1600); - el.style.transform = `translateX(${280 * p}px)`; + el.style.transform = `translateY(${560 * p}px)`; if (p < 1) { requestAnimationFrame(step); } @@ -296,7 +314,7 @@ test.describe('frame evidence', () => { const window = await blockMainThread(page, 700); await page.waitForTimeout(200); const tracked = await trackMarker(page, await cast.stop()); - return busyWindowProof(tracked, window, {axis: 'x'}); + return busyWindowProof(tracked, window, {axis: 'y'}); }; const compositor = await run('compositor'); @@ -306,6 +324,9 @@ test.describe('frame evidence', () => { }); expect(compositor.verdict.failures, compositor.summary).toEqual([]); expect(compositor.monotonic).toBe(true); + // Resolvable on the cast's grid: two position steps or more per frame. + expect(compositor.notes, compositor.summary).toEqual([]); + expect(compositor.pxPerFrame).toBeGreaterThanOrEqual(2 * compositor.stepPx); const raf = await run('raf'); testInfo.annotations.push({ @@ -338,16 +359,19 @@ test.describe('frame evidence', () => { // `during` blocks in the frame the transition has run 16 ms. In WebKit // the transition reaches the compositor in that frame's rendering update, // so a loop inside the rAF callback froze it (2 of 29 frames moved). - // Linear and longer than the block, so every frame of the block moves a - // whole pixel or more and the transition is still running when it ends. - await fixture(page); + // Linear and longer than the block, so every frame of the block moves two + // position steps or more and the transition is still running when it + // ends. The cast shows the box at rest first, so a transition that reaches + // the screen after the block began is judged from there (onScreenFromMs). + await busyFixture(page); const box = page.locator('#box'); await markElement(box, {mode: 'outline'}); const cast = await startScreencast(page); + await expect.poll(() => cast.frameCount()).toBeGreaterThan(0); await box.evaluate(el => { el.style.transition = 'transform 1600ms linear'; el.getBoundingClientRect(); - el.style.transform = 'translateX(280px)'; + el.style.transform = 'translateY(560px)'; }); const window = await blockMainThread(page, 700, { during: box, @@ -355,7 +379,7 @@ test.describe('frame evidence', () => { }); await waitForRest(box); const tracked = await trackMarker(page, await cast.stop()); - const proof = busyWindowProof(tracked, window, {axis: 'x'}); + const proof = busyWindowProof(tracked, window, {axis: 'y'}); testInfo.annotations.push({ type: 'busy-main-thread transition', description: proof.summary, @@ -370,7 +394,7 @@ test.describe('frame evidence', () => { // An animation created in the frame the helper looks is still pending (not // committed to the compositor). A block that started on it then froze it // in Chromium: no presented frame moved. - await fixture(page); + await busyFixture(page); const box = page.locator('#box'); await markElement(box, {mode: 'outline'}); await box.evaluate(el => { @@ -386,7 +410,7 @@ test.describe('frame evidence', () => { n += 1; if (n === 12) { el.animate( - [{transform: 'translateX(0px)'}, {transform: 'translateX(280px)'}], + [{transform: 'translateY(0px)'}, {transform: 'translateY(560px)'}], {duration: 1600, easing: 'linear'}, ); return; @@ -401,7 +425,7 @@ test.describe('frame evidence', () => { }); await waitForRest(box); const tracked = await trackMarker(page, await cast.stop()); - const proof = busyWindowProof(tracked, window, {axis: 'x'}); + const proof = busyWindowProof(tracked, window, {axis: 'y'}); testInfo.annotations.push({ type: 'busy-main-thread pending animation', description: proof.summary, From fb3423ff220f82b1f1499be6de3fb12aeb765d5f Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Sun, 25 Oct 2026 18:59:04 -0700 Subject: [PATCH 49/93] fix(interaction): an animation reversed by reverse() reads to its full end A negative playback rate runs the keyframes backward like a reverse direction: its held run is where the motion starts, so curveEndMs (and waitForRest and waitForAnimation) read to the full end. Self-test added. --- internal/interaction-harness/src/curves.ts | 8 ++++++-- .../src/harness.interaction.browser.spec.ts | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index 9d2a083ccb69b..9366213fbd56c 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -182,8 +182,12 @@ async function installCurveEnd(page: Page): Promise { return delay + active; } // A held end is read only for an animation that plays forward: run - // backward (or alternating), the held run is where its motion starts. - if ((timing?.direction ?? 'normal') !== 'normal') { + // backward (a reverse or alternating direction, or `reverse()`'s + // negative playback rate), the held run is where its motion starts. + if ( + (timing?.direction ?? 'normal') !== 'normal' || + animation.playbackRate < 0 + ) { return delay + active; } const frames = effect.getKeyframes(); diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index e8e2f76dca50a..6f5dbd4601b5d 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -548,6 +548,21 @@ test.describe('holding', () => { expect(reversed.curveEndMs).toBeCloseTo(1000, 0); await releaseAnimations(page, {finish: true}); + // So does one reversed by `reverse()` (a negative playback rate). + await box.evaluate(el => { + el.animate( + [ + {translate: '0px 0px', offset: 0}, + {translate: '100px 0px', offset: 0.5}, + {translate: '100px 0px', offset: 1}, + ], + {duration: 1000, easing: 'linear'}, + ).reverse(); + }); + const [played] = await holdAnimations(page, box); + expect(played.curveEndMs).toBeCloseTo(1000, 0); + await releaseAnimations(page, {finish: true}); + await box.evaluate(el => { el.animate([{rotate: 'x 0deg'}, {rotate: 'x 60deg'}], { duration: 300, From e2e775a49e072ad43ce47eb40099b4251500941e Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Sun, 25 Oct 2026 19:47:01 -0700 Subject: [PATCH 50/93] fix(interaction): a reverse()d animation is reported from its real start and refused by a hold Chromium gives an animation running backward a startTime where it would reach 0 (a full duration in the future), so waitForAnimation reported it a duration late. It now dates such an animation from when its current time stood at its end. holdAnimations refuses it with a clear error: seeking and sampling run a curve forward, so it cannot be read. The self-test awaits the animation's ready promise (reverse() is pending until then), checks the reported interval, and expects the refusal; the README says so. --- internal/interaction-harness/README.md | 5 +++- internal/interaction-harness/src/curves.ts | 23 ++++++++++++++++--- .../src/harness.interaction.browser.spec.ts | 21 +++++++++++------ 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 70a3b56b809cd..33d2f6eaf0508 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -62,7 +62,10 @@ than `playState`, does not wait the hold out. That holds for an animation that plays forward under a linear or cubic-bézier effect easing (the time the run starts is solved from the curve); under any other effect easing (steps, `linear()` stops, an overshoot) or another direction the curve is read to its -full end, and `waitForRest` waits the hold out. +full end, and `waitForRest` waits the hold out. An animation running backward (`reverse()`, a +negative playback rate) is reported by `waitForAnimation` from when it +started, but `holdAnimations` refuses it: seeking and sampling run a curve +forward. **Waiting for an animation.** The page records every animation from the first call into `curves.ts` on it (and on every later document): a frame loop on the diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index 9366213fbd56c..a4997343118ae 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -314,6 +314,15 @@ export async function holdAnimations( (target != null && (root === target || root.contains(target)))) ); }); + // Seeking and sampling run a curve forward: an animation running + // backward (`reverse()`, a negative playback rate) cannot be held and + // read, so it is refused rather than misread. + const backward = all.find(animation => animation.playbackRate < 0); + if (backward != null) { + throw new Error( + `holdAnimations: ${describe((backward.effect as KeyframeEffect | null)?.target ?? null) ?? 'an animation'} runs backward (reverse() or a negative playback rate); hold and sample a forward run instead`, + ); + } // Each animation's start on the document timeline, read before the pause // clears it (a pending play has no start time yet: it starts now). const starts = all.map(animation => @@ -802,10 +811,18 @@ export async function waitForAnimation( ) => { const effect = a.effect as KeyframeEffect | null; const timing = effect?.getComputedTiming(); - // The recorded start stands in once the page has cancelled it. + // The recorded start stands in once the page has cancelled it. An + // animation running backward started when its current time last + // stood at its end: its startTime is where it would reach 0. + const rate = a.playbackRate; const startedAt = - performance.timeOrigin + - Number(a.startTime ?? recordedStart ?? Number.NaN); + rate < 0 && a.currentTime != null + ? performance.timeOrigin + + Number(document.timeline.currentTime ?? performance.now()) - + (Number(timing?.endTime ?? 0) - Number(a.currentTime)) / + Math.abs(rate) + : performance.timeOrigin + + Number(a.startTime ?? recordedStart ?? Number.NaN); const name = 'transitionProperty' in a ? (a as unknown as {transitionProperty: string}) diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 6f5dbd4601b5d..3d7ea367c5391 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -548,20 +548,27 @@ test.describe('holding', () => { expect(reversed.curveEndMs).toBeCloseTo(1000, 0); await releaseAnimations(page, {finish: true}); - // So does one reversed by `reverse()` (a negative playback rate). - await box.evaluate(el => { - el.animate( + // One reversed by `reverse()` (a negative playback rate) is reported as + // starting when it did, running its whole curve, and is refused by a + // hold, which seeks curves forward only. + const readyAt = await box.evaluate(async el => { + const animation = el.animate( [ {translate: '0px 0px', offset: 0}, {translate: '100px 0px', offset: 0.5}, {translate: '100px 0px', offset: 1}, ], {duration: 1000, easing: 'linear'}, - ).reverse(); + ); + animation.reverse(); + await animation.ready; + return performance.timeOrigin + performance.now(); }); - const [played] = await holdAnimations(page, box); - expect(played.curveEndMs).toBeCloseTo(1000, 0); - await releaseAnimations(page, {finish: true}); + const [running] = await waitForAnimation(box); + expect(Math.abs(running.from - readyAt)).toBeLessThan(100); + expect(running.to - running.from).toBeCloseTo(1000, -1); + await expect(holdAnimations(page, box)).rejects.toThrow(/runs backward/); + await box.evaluate(el => el.getAnimations().forEach(a => a.finish())); await box.evaluate(el => { el.animate([{rotate: 'x 0deg'}, {rotate: 'x 60deg'}], { From 6ef5fc380a98fcd40f2cb61ef398e96a9bd4d9d7 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 19:50:41 -0700 Subject: [PATCH 51/93] fix(interaction): a traced waitForRest waits on the real timers, which a frozen gate cannot hold Inside a traced body waitForRest scheduled its wait with the page's setTimeout, which installTimerGate replaces and holds while freezeTimers is on: with the gate frozen it neither resolved nor reached its deadline until thawed. The gate now keeps its real setTimeout as `timeout` beside its real `frame`, and the traced wait uses it. Its self-test, beside the anchored one, freezes the gate around a traced wait: "held" on the previous code, resting now. --- internal/interaction-harness/src/curves.ts | 21 ++++++---- .../src/harness.interaction.browser.spec.ts | 38 +++++++++++++++++++ internal/interaction-harness/src/story.ts | 7 ++-- 3 files changed, 55 insertions(+), 11 deletions(-) diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index b31eecc4b5fdc..e6a592935f5de 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -979,13 +979,18 @@ export async function waitForRest( backdrop, ].join(' | '); }; - // The page's real frames, even while a timer gate holds its callbacks. - const frame = - ( - window as unknown as { - __harnessTimerGate?: {frame: typeof requestAnimationFrame}; - } - ).__harnessTimerGate?.frame ?? requestAnimationFrame; + // The page's real frames and timers, even while a timer gate holds + // its callbacks: a frozen gate never holds the wait itself. + const gate = ( + window as unknown as { + __harnessTimerGate?: { + frame: typeof requestAnimationFrame; + timeout?: typeof setTimeout; + }; + } + ).__harnessTimerGate; + const frame = gate?.frame ?? requestAnimationFrame; + const timeout = gate?.timeout ?? setTimeout; // Set by traceRendering while it traces (see the recorder above). const tracing = () => (window as unknown as {__harnessRenderingTrace?: boolean}) @@ -1027,7 +1032,7 @@ export async function waitForRest( return; } const room = deadline - performance.now(); - setTimeout( + timeout( () => { if (performance.now() > deadline) { reject(new Error('waitForRest: still moving at the deadline')); diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index fc50c2367fd09..b25419975f6c6 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -655,6 +655,44 @@ test.describe('frame evidence', () => { expect(trace.window.to - started).toBeGreaterThanOrEqual(590); }); + test('a waitForRest awaited inside a traced body runs on the real timers, so a frozen timer gate cannot hold it', async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'rendering traces are Chromium-only (README: engine limits)', + ); + await page.route('http://anchored.test/', route => + route.fulfill({contentType: 'text/html', body: ANCHORED}), + ); + await page.goto('http://anchored.test/'); + const pop = page.locator('#pop'); + await waitForRest(pop); + await installTimerGate(page); + await freezeTimers(page); + let outcome = ''; + await traceRendering(page, async () => { + await pop.evaluate(el => { + el.animate( + [{transform: 'translateY(0px)'}, {transform: 'translateY(100px)'}], + {duration: 300}, + ); + }); + // Held by the gate, the wait would neither resolve nor reach its + // deadline: the race reads that as "held". + outcome = await Promise.race([ + waitForRest(pop, {timeout: 2000}).then( + () => 'rested', + () => 'rejected', + ), + new Promise(resolve => setTimeout(() => resolve('held'), 4000)), + ]); + }); + await thawTimers(page); + expect(outcome).toBe('rested'); + }); + test('a CSS transition and animation that start and end inside a trace are reported, and lay out nothing in between', async ({ page, browserName, diff --git a/internal/interaction-harness/src/story.ts b/internal/interaction-harness/src/story.ts index 598f75d852cfe..a1bbb8d09864c 100644 --- a/internal/interaction-harness/src/story.ts +++ b/internal/interaction-harness/src/story.ts @@ -110,8 +110,9 @@ export async function frames(page: Page, count = 2): Promise { // A held callback is queued under its timer or frame id, so the page's own // clearTimeout and cancelAnimationFrame still cancel it while frozen; a held -// frame callback gets the time it actually runs at. `frame` stays the real -// requestAnimationFrame, for the harness's own waits. +// frame callback gets the time it actually runs at. `frame` and `timeout` +// stay the real requestAnimationFrame and setTimeout, for the harness's own +// waits. const TIMER_GATE = `(() => { if (window.__harnessTimerGate) return; const realTimeout = window.setTimeout.bind(window); @@ -119,7 +120,7 @@ const TIMER_GATE = `(() => { const realInterval = window.setInterval.bind(window); const realFrame = window.requestAnimationFrame.bind(window); const realCancelFrame = window.cancelAnimationFrame.bind(window); - const gate = {frozen: false, queue: [], frame: realFrame}; + const gate = {frozen: false, queue: [], frame: realFrame, timeout: realTimeout}; window.__harnessTimerGate = gate; const drop = key => { gate.queue = gate.queue.filter(entry => entry.key !== key); }; window.setTimeout = (fn, ms, ...args) => { From 2efaf04232cf1fcebc4ca9e22d76520e15d91842 Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Sun, 25 Oct 2026 20:56:34 -0700 Subject: [PATCH 52/93] fix(interaction): a held run counts only when it reaches offset 1 and keys every property A property keyed only before the held run, or a last keyframe short of offset 1, gets an implicit end frame, so the element keeps changing after the run starts; curveEndMs now reads such an animation to its full end, with a self-test for each case. The reverse() self-test takes its watch before the animation starts, so it reads that animation and not earlier ones. The README lists math.ts once. --- internal/interaction-harness/README.md | 1 - internal/interaction-harness/src/curves.ts | 18 ++++++++++++- .../src/harness.interaction.browser.spec.ts | 25 ++++++++++++++++++- 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 33d2f6eaf0508..a55cc41603655 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -29,7 +29,6 @@ pnpm test:interaction # one spec, both engines | `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t), `waitForAnimation`, `watchAnimations`, `waitForRest`, `releaseAnimations`. | | `math.ts` | Oracles independent of the code under test: `cubicBezier`, `timingFunction`, `springStep` (SwiftUI response / dampingFraction, initial velocity), `projectDecay`, `rubberBand`, `parseTransform`, `compareCurve`, `timeline`. | | `story.ts` | `phone`, `withStorybook`, `openStory`, `frames`. | -| `math.ts` | Oracles independent of the code under test: `cubicBezier`, `timingFunction`, `springStep` (SwiftUI response / dampingFraction, initial velocity), `projectDecay`, `rubberBand`, `parseTransform`, `compareCurve`, `timeline`. | ## The proofs diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index a4997343118ae..52d871f6adb3f 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -191,8 +191,24 @@ async function installCurveEnd(page: Page): Promise { return delay + active; } const frames = effect.getKeyframes(); + // A run holds only when it reaches offset 1 and every frame in it keys + // every property the animation keys: a property left out, or a last + // frame before offset 1, gets an implicit end frame and keeps moving. + const keyed = new Set( + frames.flatMap(k => Object.keys(k).filter(p => !timingKeys.has(p))), + ); + const keysAll = (k: ComputedKeyframe) => + [...keyed].every(p => k[p] !== undefined); + const last = frames[frames.length - 1]; + if (last == null || Number(last.computedOffset) !== 1 || !keysAll(last)) { + return delay + active; + } let first = frames.length - 1; - while (first > 0 && values(frames[first - 1]) === values(frames[first])) { + while ( + first > 0 && + keysAll(frames[first - 1]) && + values(frames[first - 1]) === values(frames[first]) + ) { first -= 1; } const offset = Number(frames[first]?.computedOffset ?? 1); diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 3d7ea367c5391..fa81621cc4eae 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -551,6 +551,7 @@ test.describe('holding', () => { // One reversed by `reverse()` (a negative playback rate) is reported as // starting when it did, running its whole curve, and is refused by a // hold, which seeks curves forward only. + const watch = await watchAnimations(box); const readyAt = await box.evaluate(async el => { const animation = el.animate( [ @@ -564,12 +565,34 @@ test.describe('holding', () => { await animation.ready; return performance.timeOrigin + performance.now(); }); - const [running] = await waitForAnimation(box); + const [running] = await watch.started(); expect(Math.abs(running.from - readyAt)).toBeLessThan(100); expect(running.to - running.from).toBeCloseTo(1000, -1); await expect(holdAnimations(page, box)).rejects.toThrow(/runs backward/); await box.evaluate(el => el.getAnimations().forEach(a => a.finish())); + // A property keyed only before the held run, and a last keyframe short of + // offset 1, each get an implicit end frame: the curve runs to its end. + for (const keyframes of [ + [ + {translate: '0px 0px', opacity: 0, offset: 0}, + {translate: '100px 0px', offset: 0.5}, + {translate: '100px 0px', offset: 1}, + ], + [ + {translate: '0px 0px', offset: 0}, + {translate: '100px 0px', offset: 0.5}, + {translate: '100px 0px', offset: 0.8}, + ], + ]) { + await box.evaluate((el, frames) => { + el.animate(frames, {duration: 1000, easing: 'linear'}); + }, keyframes); + const [moving] = await holdAnimations(page, box); + expect(moving.curveEndMs).toBeCloseTo(1000, 0); + await releaseAnimations(page, {finish: true}); + } + await box.evaluate(el => { el.animate([{rotate: 'x 0deg'}, {rotate: 'x 60deg'}], { duration: 300, From 7c8ab74748260a24878a7c27a1b6cb9333b850cf Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 21:22:04 -0700 Subject: [PATCH 53/93] fix(interaction): the busy verdict reads the compositor's own frames where the screencast skipped them Under parallel load Chromium's screencast skipped frames at fixed points of a block (about +365 and +465 ms) while the compositor kept presenting every 16.7 ms and the marker kept moving, so the verdict counted the skipped frames as not moving and measured a 66.9 ms stall: 4 of 19 recorded runs failed. startCompositorTrace records when the compositor presented each frame (Display::FrameDisplayed, Chromium; [] elsewhere). Passed as `presented`, a gap between two moved frames with nothing delivered inside it is measured between the compositor's own frames: frames it presented there count as presented (recorderSkipped, in the summary), and a gap it did not present through is a stall as before. Gaps at the judged part's edges, or holding a delivered frame that did not move, are judged from the screencast. No limit changes. A recorded run is replayed as a unit test (fails without the record, passes with it), beside controls the record never excuses: a frozen element while the compositor draws, a compositor stall, an unmoved delivered frame inside a gap, repeated real drops, an edge gap, an empty record. In Chromium the self-test skips four screencast frames mid-block (passes with the record), and its rAF control still fails with it: the compositor presents nothing during that block. --- internal/interaction-harness/README.md | 18 ++- .../src/busy.skip.fixture.ts | 125 ++++++++++++++++ internal/interaction-harness/src/busy.test.ts | 134 ++++++++++++++++++ internal/interaction-harness/src/busy.ts | 117 +++++++++++++-- .../src/harness.interaction.browser.spec.ts | 80 +++++++++++ internal/interaction-harness/src/trace.ts | 57 ++++++++ 6 files changed, 519 insertions(+), 12 deletions(-) create mode 100644 internal/interaction-harness/src/busy.skip.fixture.ts diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index f0fe5978005e1..4963512792a3b 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -49,8 +49,8 @@ committed: a pull request links or attaches it. | `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact final-move speed, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. | | `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | | `frames.ts` | `startScreencast` (timestamped composited frames), `markElement` (a marker chip or outline), `trackMarker` (marker box per frame), `sampleRects` (per-rAF rects), `frameStats`. | -| `busy.ts` | `blockMainThread(page, ms)` and `busyWindowProof(tracked, window)`: frames presented, frames that moved, longest stall, and a verdict. | -| `trace.ts` | `traceRendering(page, body)`: main-thread Layout / Paint / UpdateLayoutTree / PrePaint counts, `countBetween` for a sub-window, long tasks. `installLongTaskLog` / `readLongTasks`. | +| `busy.ts` | `blockMainThread(page, ms)` and `busyWindowProof(tracked, window)`: frames presented, frames that moved, longest stall, a verdict; `presented` adds the compositor's own frames. | +| `trace.ts` | `traceRendering(page, body)`: main-thread Layout / Paint / UpdateLayoutTree / PrePaint counts, `countBetween`, long tasks; `installLongTaskLog` / `readLongTasks`; `startCompositorTrace`: when the compositor presents. | | `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t), `waitForAnimation`, `watchAnimations`, `waitForRest`, `releaseAnimations`. | | `math.ts` | Oracles independent of the code under test: `cubicBezier`, `timingFunction`, `springStep` (SwiftUI response / dampingFraction, initial velocity), `projectDecay`, `rubberBand`, `parseTransform`, `compareCurve`, `timeline`. | | `storyboard.ts` | `animationStoryboard` (seek to t = 0, 16.7, 33.3 … ms), `touchStoryboard` (a frame per scripted event), `castStoryboard` (real screencast frames): PNG contact sheet + JSON trace. | @@ -145,6 +145,19 @@ not moved. The proof reports `stepPx` and `pxPerFrame` and notes motion under two steps per frame, which never fails it. Move a proof's element at least two steps a frame: the self-tests move 560 px in 1,600 ms, about 4.8 px a frame. +On a loaded machine Chromium's screencast can skip frames the compositor +still presents. `startCompositorTrace(page)`, started with the screencast and +stopped after the block, records when the compositor presented each frame +(`Display::FrameDisplayed`); pass it as `presented`. A gap between two moved +frames with nothing delivered inside it is then measured between the +compositor's own frames: the frames it presented there count as presented +(`recorderSkipped`, in the summary), and a gap it did not present through is +a stall as before. A gap that opens or closes the judged part, or holds a +delivered frame that did not move, is judged from the screencast, so a +frozen element fails whatever else the compositor draws (a rAF animation's +block presents no compositor frame at all). WebKit has no such trace: +`stop()` resolves to `[]` and the screencast is judged alone. + **No Layout / Paint.** `traceRendering` records a Chromium trace around the interaction; `countBetween` counts the steady middle of an animation (its first and last frames may commit). A transform/opacity animation does no @@ -182,6 +195,7 @@ in all, and lists each retake under `recaptures` in the trace. | Curve sampling, storyboards, input log | yes | yes | | Screencast frame evidence (busy main thread) | yes | yes (WebKit's own screencast; frames keep coming while the main thread is blocked) | | Rendering trace, long tasks | yes | no (Playwright exposes neither) | +| Compositor presentation record | yes | no: `stop()` resolves to `[]`; the busy verdict judges the screencast alone | What WebKit cannot show here — native scroll hand-off, flings, a finger lifting while another stays down, finger tracking under a busy main thread, and diff --git a/internal/interaction-harness/src/busy.skip.fixture.ts b/internal/interaction-harness/src/busy.skip.fixture.ts new file mode 100644 index 0000000000000..409886f12e6f8 --- /dev/null +++ b/internal/interaction-harness/src/busy.skip.fixture.ts @@ -0,0 +1,125 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file busy.skip.fixture.ts + * @input One recorded busy-main-thread run in headless Chromium under + * parallel load: a compositor coast through a 566 ms block + * @output `frames`: the screencast frames the marker was tracked in, as + * [ms after the block started, y in CSS px, or null where the marker was not + * found]; `presented`: when the compositor presented each frame + * (`Display::FrameDisplayed`), ms after the block started; `block`: the + * block's span on the same clock. The screencast skipped frames at about + * +366 and +466 ms while the compositor kept presenting every 16.7 ms. + * @position Fixture for busy.test.ts + */ + +export const block = {from: 0, to: 566.1}; + +export const frames: ReadonlyArray = [ + [-1145.8, null], + [-1114, null], + [-1063.9, null], + [-1031.1, null], + [-996.6, null], + [-964.3, null], + [-930.7, null], + [-896.7, null], + [-864, null], + [-830.6, null], + [-797.8, null], + [-762.7, null], + [-731.3, null], + [-686.5, null], + [-680.8, null], + [-651, null], + [-634.1, null], + [-620.7, null], + [-615.8, null], + [-514.4, null], + [-495, null], + [-472.4, null], + [-465.3, null], + [-448.5, null], + [-430.3, null], + [-414.8, null], + [-398.3, null], + [-379.5, null], + [-364.2, null], + [-348.3, null], + [-330.6, null], + [-315, null], + [-297.6, null], + [-281.9, null], + [-265.1, null], + [-247.6, null], + [-230.8, null], + [-215.3, null], + [-198.6, null], + [-181, null], + [-165.3, null], + [-147.7, null], + [-132, null], + [-115.4, null], + [-98.5, null], + [-81.2, 504], + [-65.2, 500], + [-48.5, 496], + [-30.9, 491], + [-15.2, 487], + [1.3, 484], + [32.8, 480], + [49.8, 476], + [66.3, 473], + [82.8, 469], + [99.7, 466], + [116.1, 463], + [132.9, 460], + [149.5, 457], + [166.3, 454], + [182.9, 451], + [199.4, 448], + [216.2, 446], + [232.8, 443], + [249.4, 441], + [266.1, 438], + [282.7, 436], + [299.5, 434], + [316, 432], + [332.7, 430], + [349.5, 428], + [366.1, 426], + [433.1, 418], + [449.7, 417], + [466.2, 415], + [516.2, 411], + [532.8, 409], + [549.5, 408], + [566.2, 407], + [650.6, 401], + [666.2, 400], + [682.9, 399], + [699.6, 398], + [716.3, 397], + [816.1, 391], + [832.8, 391], + [849.5, 390], + [866.2, 389], + [882.9, 388], + [899.5, 388], + [916.1, 387], + [1033.8, 383], + [1050.1, 383], + [1066.7, 382], + [1083.3, 382], + [1099.5, 381], + [1117.3, 381], + [1134.2, 380], + [1149.5, 380], +]; + +export const presented: readonly number[] = [ + -98.7, -81.3, -65.3, -48.7, -31.1, -15.3, 1.3, 32.5, 49.6, 66.1, 82.7, 99.6, + 115.9, 132.6, 149.3, 166.1, 182.6, 199.3, 216, 232.6, 249.3, 266, 282.6, + 299.2, 315.8, 332.6, 349.3, 366, 381.5, 398.3, 415.1, 432.9, 449.6, 466.1, + 481.5, 498.3, 516, 532.6, 549.4, 566.1, 581.7, 598.4, 615, 631.6, 650.3, 666, +]; diff --git a/internal/interaction-harness/src/busy.test.ts b/internal/interaction-harness/src/busy.test.ts index 21ec486dbc363..37a22cf1638a8 100644 --- a/internal/interaction-harness/src/busy.test.ts +++ b/internal/interaction-harness/src/busy.test.ts @@ -14,6 +14,7 @@ import {describe, expect, it} from 'vitest'; import {busyWindowProof} from './busy'; +import * as skip from './busy.skip.fixture'; import {inPresentationOrder, type TrackedFrame} from './frames'; const FRAME = 1000 / 60; @@ -275,6 +276,139 @@ describe('busyWindowProof position step', () => { }); }); +/** The compositor presenting every frame from `from` to `to`, minus `gaps` ([from, to] spans). */ +function drawn( + gaps: ReadonlyArray = [], + from = 700, + to = 2000, +): number[] { + const draws: number[] = []; + for (let t = from; t <= to; t += FRAME) { + if (!gaps.some(([a, b]) => t > a && t < b)) { + draws.push(t); + } + } + return draws; +} + +const skipFrames = (): TrackedFrame[] => + skip.frames.map( + ([t, y]) => + ({ + t, + box: y == null ? null : {x: 0, y, width: 10, height: 10}, + }) as TrackedFrame, + ); + +describe('busyWindowProof with the compositor record', () => { + it('reads frames the screencast skipped while the compositor presented them as presented: a recorded run that fails without the record', () => { + const frames = skipFrames(); + const alone = busyWindowProof(frames, skip.block, {settledWithin: 1}); + expect(alone.verdict.failures.join('\n')).toMatch( + /longest stall 6\d\.\d ms = 4 intervals > 3/, + ); + expect(alone.verdict.failures.join('\n')).toMatch( + /26 of 31 expected frames moved/, + ); + const proof = busyWindowProof(frames, skip.block, { + settledWithin: 1, + presented: skip.presented, + }); + expect(proof.verdict.failures).toEqual([]); + expect(proof.recorderSkipped).toBe(5); + expect(proof.longestStallFrames).toBeLessThanOrEqual(2); + expect(proof.summary).toMatch( + /the screencast skipped 5 frames the compositor presented \(at \+3\d\d ms, \+4\d\d ms\)/, + ); + }); + + it('counts a gap the compositor drew through as skipped, and one it did not draw through as a stall', () => { + const k = at(1300); + const gap: [number, number] = [ + 800 + (k - 1) * FRAME, + 800 + (k + 4) * FRAME, + ]; + const frames = moving([k, k + 1, k + 2, k + 3]); + expect(busyWindowProof(frames, BLOCK).verdict.pass).toBe(false); + const skipped = busyWindowProof(frames, BLOCK, {presented: drawn()}); + expect(skipped.verdict.failures).toEqual([]); + expect(skipped.recorderSkipped).toBe(4); + const stalled = busyWindowProof(frames, BLOCK, {presented: drawn([gap])}); + expect(stalled.recorderSkipped).toBe(0); + expect(stalled.verdict.failures.join('\n')).toMatch(/longest stall/); + }); + + it('still fails an element that does not move while the compositor presents for something else', () => { + const frozen = moving().map(f => + f.t >= BLOCK.from && f.t <= BLOCK.to + ? ({...f, box: {...f.box!, y: 3 * at(BLOCK.from)}} as TrackedFrame) + : f, + ); + const proof = busyWindowProof(frozen, BLOCK, {presented: drawn()}); + expect(proof.verdict.failures.join('\n')).toMatch(/longest stall/); + expect(proof.verdict.failures.join('\n')).toMatch(/expected frames moved/); + }); + + it('still fails a frozen animation, whether or not the compositor presents during the block', () => { + const frozen = moving().filter(f => f.t < BLOCK.from || f.t > BLOCK.to); + for (const presented of [drawn([[BLOCK.from, BLOCK.to]]), drawn()]) { + const proof = busyWindowProof(frozen, BLOCK, {presented}); + expect(proof.verdict.failures.join('\n')).toMatch(/no frame presented/); + expect(proof.verdict.failures.join('\n')).toMatch(/longest stall/); + } + }); + + it('keeps a gap a stall when the screencast delivered a frame inside it that did not move', () => { + const k = at(1300); + const frames = moving([k, k + 1, k + 3]).map(f => + f.t > 800 + (k + 1.5) * FRAME && f.t < 800 + (k + 2.5) * FRAME + ? ({...f, box: {...f.box!, y: 3 * (k - 1)}} as TrackedFrame) + : f, + ); + const proof = busyWindowProof(frames, BLOCK, {presented: drawn()}); + expect(proof.recorderSkipped).toBe(0); + expect(proof.verdict.failures.join('\n')).toMatch(/longest stall/); + }); + + it('still counts single dropped frames the compositor did not present as missed', () => { + const k = at(1050); + const drops = [0, 6, 12, 18, 24, 30].map(d => k + d); + const frames = moving(drops); + const gaps = drops.map( + i => + [800 + (i - 0.5) * FRAME, 800 + (i + 0.5) * FRAME] as [number, number], + ); + expect( + busyWindowProof(frames, BLOCK, { + presented: drawn(gaps), + }).verdict.failures.join('\n'), + ).toMatch(/expected frames moved/); + expect( + busyWindowProof(frames, BLOCK, {presented: drawn()}).verdict.failures, + ).toEqual([]); + }); + + it('judges a gap at the edge of the judged part from the screencast alone', () => { + // Nothing delivered for the block's first 90 ms: a stall at its start, + // whatever the compositor drew, since no moved frame opens the gap. + const frames = moving().filter( + f => f.t < BLOCK.from || f.t > BLOCK.from + 90, + ); + const proof = busyWindowProof(frames, BLOCK, {presented: drawn()}); + expect(proof.recorderSkipped).toBe(0); + expect(proof.verdict.failures.join('\n')).toMatch(/longest stall/); + }); + + it('treats an empty record as no record, and says so', () => { + const frames = moving([at(1300), at(1300) + 1, at(1300) + 2, at(1300) + 3]); + const proof = busyWindowProof(frames, BLOCK, {presented: []}); + expect(proof.verdict.failures).toEqual( + busyWindowProof(frames, BLOCK).verdict.failures, + ); + expect(proof.summary).toMatch(/no compositor frames recorded/); + }); +}); + describe('inPresentationOrder', () => { it('sorts frames delivered out of order, so motion stays monotonic', () => { const frames = moving(); diff --git a/internal/interaction-harness/src/busy.ts b/internal/interaction-harness/src/busy.ts index 01e43786d58d4..0c93ab15387cf 100644 --- a/internal/interaction-harness/src/busy.ts +++ b/internal/interaction-harness/src/busy.ts @@ -142,6 +142,17 @@ export interface BusyProof extends FrameStats { readonly pxPerFrame: number; /** Facts worth reading beside the verdict, which never fail it. */ readonly notes: readonly string[]; + /** + * With a compositor record (`presented`): frames the screencast skipped + * between two moved frames while the compositor presented them, which are + * neither stalls nor misses; 0 without one. + */ + readonly recorderSkipped: number; + /** + * The compositor's frame interval around the window, ms, from `presented`; + * null without a usable record (the screencast alone is judged). + */ + readonly compositorFrameMs: number | null; /** * The screencast's frame interval, from the frames around the window as well * as inside it (a short window has few intervals; a frozen one has none). @@ -213,6 +224,16 @@ export interface BusyProof extends FrameStats { * `stepPx` and `pxPerFrame` say whether a frame's motion is resolvable at * all: under two position steps per frame, a frame delivered early can show * no step and read as not moved, which the proof notes (never a failure). + * + * A loaded machine's screencast can skip frames the compositor still presents + * (Chromium). Given the compositor's record (`presented`, from + * `startCompositorTrace`), a gap between two moved frames with no frame + * delivered inside it is measured between the compositor's own frames: it is + * a stall only where the compositor presented nothing, and the frames it did + * present there are neither stalls nor misses (`recorderSkipped`). A gap + * that opens or closes the judged part, or holds a delivered frame that did + * not move, is judged from the screencast as before, so a frozen element + * fails whatever else the compositor draws. */ export function busyWindowProof( tracked: readonly TrackedFrame[], @@ -226,6 +247,7 @@ export function busyWindowProof( maxStallFrames = 3, minMovedFraction = 0.9, minJudgedFrames = 4, + presented, }: { axis?: 'x' | 'y' | 'width' | 'height'; marginMs?: number; @@ -260,6 +282,12 @@ export function busyWindowProof( * position until the main thread commits the end state. */ settledAt?: number; + /** + * When the compositor presented each frame, epoch ms on the page clock + * (`startCompositorTrace`; Chromium). Empty or absent, the screencast is + * judged alone. + */ + presented?: readonly number[]; } = {}, ): BusyProof { let end = window.to - marginMs; @@ -320,10 +348,6 @@ export function busyWindowProof( const judged = {from: onScreenFrom ?? inner.from, to: inner.to}; const changes = inWindow.filter(t => t > judged.from || onScreenFrom == null); const edges = [judged.from, ...changes, judged.to]; - const stalls = edges.slice(1).map((t, i) => t - edges[i]); - const longestStallMs = Math.max(...stalls); - // Where the longest stall starts, from the start of the window. - const longestStallAtMs = edges[stalls.indexOf(longestStallMs)] - window.from; const around = tracked .filter(f => f.t >= inner.from - 250 && f.t <= inner.to + 250) .map(f => f.t); @@ -339,10 +363,68 @@ export function busyWindowProof( intervals.length >= 4 ? intervals[Math.floor(intervals.length / 2)] : stats.medianFrameMs || 1000 / 60; - const expectedFrames = Math.floor((judged.to - judged.from) / cadence); - const longestStallFrames = Math.round(longestStallMs / cadence); + // The compositor's own frames, when recorded: its interval, from the frames + // around the judged part. + const draws = [...(presented ?? [])] + .filter(t => t >= judged.from - 250 && t <= judged.to + 250) + .sort((a, b) => a - b); + const drawIntervals = draws + .slice(1) + .map((t, i) => t - draws[i]) + .sort((a, b) => a - b); + const compositorFrameMs = + drawIntervals.length >= 4 + ? drawIntervals[Math.floor(drawIntervals.length / 2)] + : null; + const changed = new Set(inWindow); + const delivered = tracked + .filter(f => f.box != null && f.t > judged.from && f.t < judged.to) + .map(f => f.t); + // Each gap between presented changes, in intervals of the channel that saw + // it: the screencast's, or, for a gap between two moved frames with no + // frame delivered inside it, the compositor's own longest interval there. + const gaps = edges.slice(1).map((to, i) => { + const from = edges[i]; + const ms = to - from; + if ( + compositorFrameMs == null || + !changed.has(from) || + !changed.has(to) || + delivered.some(t => t > from && t < to) + ) { + return {from, ms, frames: Math.round(ms / cadence), skipped: 0}; + } + const span = draws.filter( + t => t > from - compositorFrameMs / 2 && t < to + compositorFrameMs / 2, + ); + const longest = + span.length >= 2 + ? Math.min(ms, Math.max(...span.slice(1).map((t, j) => t - span[j]))) + : ms; + const frames = Math.round(longest / compositorFrameMs); + return { + from, + ms: longest, + frames, + skipped: Math.max(0, Math.round(ms / compositorFrameMs) - frames), + }; + }); + const longestGap = gaps.reduce((a, b) => + b.frames > a.frames || (b.frames === a.frames && b.ms > a.ms) ? b : a, + ); + const longestStallMs = longestGap.ms; + // Where the longest stall starts, from the start of the window. + const longestStallAtMs = longestGap.from - window.from; + const longestStallFrames = longestGap.frames; + const skips = gaps.filter(g => g.skipped > 0); + const recorderSkipped = skips.reduce((sum, g) => sum + g.skipped, 0); + // With a compositor record, frames count in its intervals, and the frames + // it presented where the screencast skipped are presented frames. + const expectedFrames = Math.floor( + (judged.to - judged.from) / (compositorFrameMs ?? cadence), + ); const moved = changes.length; - const movedFraction = moved / Math.max(1, expectedFrames); + const movedFraction = (moved + recorderSkipped) / Math.max(1, expectedFrames); // The cast's position step, and the motion per interval it has to resolve. const positions = tracked.flatMap(f => (f.box == null ? [] : [f.box[axis]])); const stepPx = @@ -361,6 +443,16 @@ export function busyWindowProof( ? 0 : Math.abs(last.value - first.value) / ((last.t - first.t) / cadence); const notes: string[] = []; + if (presented != null && compositorFrameMs == null) { + notes.push( + 'no compositor frames recorded around the window: judged from the screencast alone', + ); + } + if (recorderSkipped > 0) { + notes.push( + `the screencast skipped ${recorderSkipped} frames the compositor presented (at ${skips.map(g => `+${(g.from - window.from).toFixed(0)} ms`).join(', ')})`, + ); + } if (pxPerFrame > 0 && pxPerFrame < 2 * stepPx) { notes.push( `${pxPerFrame.toFixed(1)} px/frame is under two of the cast's ${stepPx} px position steps: a frame delivered early can show no step and read as not moved`, @@ -388,20 +480,23 @@ export function busyWindowProof( 1, Math.floor((1 - minMovedFraction) * expectedFrames + 1e-9), ); - if (expectedFrames - moved > allowedMisses) { + if (expectedFrames - moved - recorderSkipped > allowedMisses) { failures.push( - `${moved} of ${expectedFrames} expected frames moved (${(movedFraction * 100).toFixed(0)}% < ${(minMovedFraction * 100).toFixed(0)}%)`, + `${moved} of ${expectedFrames} expected frames moved${recorderSkipped > 0 ? ` (and ${recorderSkipped} the screencast skipped)` : ''} (${(movedFraction * 100).toFixed(0)}% < ${(minMovedFraction * 100).toFixed(0)}%)`, ); } const onScreenFromMs = onScreenFrom == null ? null : onScreenFrom - window.from; const summary = `${(judged.to - judged.from).toFixed(0)} ms judged at ${cadence.toFixed(1)} ms/frame: ${stats.frames} frames presented, ` + - `${moved}/${expectedFrames} moved (${(movedFraction * 100).toFixed(0)}%), ` + + `${moved}/${expectedFrames} moved${recorderSkipped > 0 ? ` and ${recorderSkipped} skipped by the screencast` : ''} (${(movedFraction * 100).toFixed(0)}%), ` + `median ${stats.medianFrameMs.toFixed(1)} ms, ` + `longest stall ${longestStallMs.toFixed(1)} ms (${longestStallFrames} intervals) at +${longestStallAtMs.toFixed(0)} ms, ` + `${stats.monotonic ? 'monotonic' : 'not monotonic'}, ` + `${pxPerFrame.toFixed(1)} px/frame in ${stepPx} px steps` + + (compositorFrameMs == null + ? '' + : `, compositor presenting every ${compositorFrameMs.toFixed(1)} ms`) + (onScreenFromMs == null ? '' : `, on screen from +${onScreenFromMs.toFixed(0)} ms (nothing moved before the block)`) + @@ -414,6 +509,8 @@ export function busyWindowProof( stepPx, pxPerFrame, notes, + recorderSkipped, + compositorFrameMs, cadenceMs: cadence, longestStallMs, longestStallAtMs, diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index f5d5f65a785c4..7fa5c08e529f6 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -64,6 +64,7 @@ import { sampleRects, seekAnimations, springStep, + startCompositorTrace, startScreencast, Storyboard, tap, @@ -353,6 +354,85 @@ test.describe('frame evidence', () => { expect(slowRaf.verdict.failures.join('\n')).toMatch(/longest stall/); }); + test('frames the screencast skips while the compositor presents them are not a stall; a frozen animation still fails with the compositor record', async ({ + page, + browserName, + }, testInfo) => { + test.skip( + browserName !== 'chromium', + 'compositor traces are Chromium-only (README: engine limits)', + ); + await busyFixture(page); + const box = page.locator('#box'); + await markElement(box, {mode: 'outline'}); + const run = async (driver: 'compositor' | 'raf') => { + await box.evaluate((el, how) => { + el.getAnimations().forEach(a => a.cancel()); + el.style.transform = 'none'; + if (how === 'compositor') { + el.animate( + [{transform: 'translateY(0px)'}, {transform: 'translateY(560px)'}], + {duration: 1600, easing: 'linear'}, + ); + } else { + const start = performance.now(); + const step = (now: number) => { + const p = Math.min(1, (now - start) / 1600); + el.style.transform = `translateY(${560 * p}px)`; + if (p < 1) { + requestAnimationFrame(step); + } + }; + requestAnimationFrame(step); + } + }, driver); + const cast = await startScreencast(page); + const compositor = await startCompositorTrace(page); + await page.waitForTimeout(200); + const window = await blockMainThread(page, 700); + await page.waitForTimeout(200); + const presented = await compositor.stop(); + const tracked = await trackMarker(page, await cast.stop()); + return {window, tracked, presented}; + }; + + const moving = await run('compositor'); + expect(moving.presented.length).toBeGreaterThan(0); + // The screencast as a loaded machine delivers it: four frames in a row + // skipped mid-block, which the compositor still presented. + const mid = moving.tracked.findIndex(f => f.t > moving.window.from + 350); + const skipped = moving.tracked.filter((_, i) => i < mid || i >= mid + 4); + const alone = busyWindowProof(skipped, moving.window, {axis: 'y'}); + expect(alone.verdict.failures.join('\n'), alone.summary).toMatch( + /longest stall/, + ); + const proof = busyWindowProof(skipped, moving.window, { + axis: 'y', + presented: moving.presented, + }); + testInfo.annotations.push({ + type: 'busy-main-thread compositor record', + description: proof.summary, + }); + expect(proof.verdict.failures, proof.summary).toEqual([]); + expect(proof.recorderSkipped).toBeGreaterThanOrEqual(4); + + // A rAF animation presents no change for the whole block: the record + // excuses nothing, whatever the compositor drew meanwhile. + const frozen = await run('raf'); + const still = busyWindowProof(frozen.tracked, frozen.window, { + axis: 'y', + presented: frozen.presented, + }); + testInfo.annotations.push({ + type: 'busy-main-thread rAF control, compositor record', + description: `${still.summary}; compositor frames inside the block: ${frozen.presented.filter(t => t > frozen.window.from && t < frozen.window.to).length}`, + }); + expect(still.verdict.pass).toBe(false); + expect(still.recorderSkipped).toBe(0); + expect(still.verdict.failures.join('\n')).toMatch(/longest stall/); + }); + test('a block that starts during a just-started transition still lets the compositor advance it', async ({ page, }, testInfo) => { diff --git a/internal/interaction-harness/src/trace.ts b/internal/interaction-harness/src/trace.ts index 91566bb93473d..dcf608e5d0814 100644 --- a/internal/interaction-harness/src/trace.ts +++ b/internal/interaction-harness/src/trace.ts @@ -162,6 +162,63 @@ export async function traceRendering( }; } +/** + * Records when the compositor presents each frame, for `busyWindowProof`'s + * `presented`: a Chromium trace of `Display::FrameDisplayed`, on the page + * clock. Start it with the screencast, before the block, and stop it after. + * Engines without a CDP trace (WebKit) resolve `stop()` to [], and the + * verdict then judges the screencast alone. It runs its own trace, so not + * inside `traceRendering`. + */ +export async function startCompositorTrace( + page: Page, +): Promise<{stop(): Promise}> { + let cdp: CDPSession; + try { + cdp = await page.context().newCDPSession(page); + } catch { + return {stop: async () => []}; + } + const events: TraceEvent[] = []; + cdp.on('Tracing.dataCollected', ({value}) => { + events.push(...(value as unknown as TraceEvent[])); + }); + const complete = new Promise(resolve => + cdp.once('Tracing.tracingComplete', () => resolve()), + ); + await cdp.send('Tracing.start', { + transferMode: 'ReportEvents', + traceConfig: { + includedCategories: [ + 'disabled-by-default-display.framedisplayed', + 'blink.user_timing', + ], + excludedCategories: ['*'], + }, + }); + const tag = `harness-compositor-${Math.random().toString(36).slice(2, 8)}`; + const epoch = await page.evaluate( + n => performance.timeOrigin + performance.mark(n).startTime, + tag, + ); + return { + async stop() { + await cdp.send('Tracing.end'); + await complete; + await cdp.detach().catch(() => {}); + const mark = events.find(e => e.name === tag); + if (mark == null) { + throw new Error('startCompositorTrace: the clock mark was not traced'); + } + // Trace time (µs, monotonic) -> epoch ms, anchored on the mark. + return events + .filter(e => e.name === 'Display::FrameDisplayed') + .map(e => epoch + (e.ts - mark.ts) / 1000) + .sort((a, b) => a - b); + }, + }; +} + const LONG_TASK_SCRIPT = `(() => { if (window.__harnessLongTasks || typeof PerformanceObserver === 'undefined') return; const tasks = []; From 93d2efe4cd497d2e59a88ceb4bdf1418a47e57a7 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 21:26:35 -0700 Subject: [PATCH 54/93] fix(interaction): a traced waitForRest looks for a few frames before settling on no animation Inside a trace, waitForRest settled on its first look when the element had no animation yet, where the per-frame check needs `frames` still frames: an animation the action started two frames later began 8-16 ms after the traced window had closed, so a "no Layout / Paint" count read 0 for motion the window never held. Finding nothing, the traced wait now looks again `frames` times a frame apart (on the gate's real timer) before settling. Its self-test starts the animation two frames after the action and checks the traced window holds all of it. --- internal/interaction-harness/README.md | 6 ++- internal/interaction-harness/src/curves.ts | 18 ++++++-- .../src/harness.interaction.browser.spec.ts | 42 +++++++++++++++++++ 3 files changed, 61 insertions(+), 5 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 585c621e23351..52f65cd58d318 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -167,8 +167,10 @@ frame loop would make every frame a main-thread one, restyling each running animation's target and laying out an anchored one: it reads only the element a CSS animation or transition event names, once, when that animation starts, and catches up afterwards. A `waitForRest` awaited inside the traced body stands -down too: it waits on timers for the scheduled ends of the element's curves and -reads no style until the trace ends (in the anchored fixture's steady middle, +down too: it waits on timers for the scheduled ends of the element's curves +(looking `frames` times a frame apart before settling on none, so an +animation the action starts a frame or two later is waited for) and reads no +style until the trace ends (in the anchored fixture's steady middle, 24 Layouts before, 0 now). **Storyboards.** Deterministic: frames come from seeking held animations or diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index e6a592935f5de..c82c4f536b62f 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -953,7 +953,9 @@ export async function watchAnimations( * main-thread frame, which the trace counts. It then waits on timers for the * scheduled ends of the curves on the element instead, reading no style, so * motion that no animation drives (a script writing styles) is not seen until - * the trace ends. + * the trace ends. Finding no animation, it looks again `frames` times a frame + * apart before settling, so one the action starts a few frames later is + * waited for. */ export async function waitForRest( locator: Locator, @@ -1012,6 +1014,10 @@ export async function waitForRest( // While traced: one look at the animations, then a timer to the latest // scheduled curve end. One not started yet is looked at again a frame // later; a trace that has ended hands over to the per-frame check. + // Finding nothing settles only after `frames` looks a frame apart, as + // the per-frame check needs `frames` still frames: an animation the + // action starts a frame or two later is still waited for. + let idle = 0; const scheduled = () => { let wait = 0; for (const a of element.getAnimations({subtree: true})) { @@ -1028,8 +1034,14 @@ export async function waitForRest( } } if (wait <= 0) { - settle(); - return; + idle += 1; + if (idle >= need) { + settle(); + return; + } + wait = 16; + } else { + idle = 0; } const room = deadline - performance.now(); timeout( diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 88bf1e77a7f34..55d691afa1f77 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -735,6 +735,48 @@ test.describe('frame evidence', () => { expect(trace.window.to - started).toBeGreaterThanOrEqual(590); }); + test('a waitForRest awaited inside a traced body waits for an animation the action starts two frames later', async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'rendering traces are Chromium-only (README: engine limits)', + ); + await page.route('http://anchored.test/', route => + route.fulfill({contentType: 'text/html', body: ANCHORED}), + ); + await page.goto('http://anchored.test/'); + const pop = page.locator('#pop'); + await waitForRest(pop); + const trace = await traceRendering(page, async () => { + // The action starts the animation two frames later, as a component's + // effect often does. + await pop.evaluate(el => { + requestAnimationFrame(() => + requestAnimationFrame(() => { + (window as unknown as {__started: number}).__started = + performance.timeOrigin + performance.now(); + el.animate( + [ + {transform: 'translateY(0px)'}, + {transform: 'translateY(100px)'}, + ], + {duration: 600}, + ); + }), + ); + }); + await waitForRest(pop); + }); + const started = await page.evaluate( + () => (window as unknown as {__started?: number}).__started ?? null, + ); + // The traced window holds the whole animation. + expect(started).not.toBeNull(); + expect(trace.window.to - (started as number)).toBeGreaterThanOrEqual(590); + }); + test('a waitForRest awaited inside a traced body runs on the real timers, so a frozen timer gate cannot hold it', async ({ page, browserName, From ad21d1415f8b71b967ce98c22fc8e4c62e85b280 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 18:53:04 -0700 Subject: [PATCH 55/93] test(interaction): held offsets against the runners' own start times The holding self-test asserted the second runner's held offset was over 80 ms because it started 100 ms after the first. A start time resolves at the animation's first committed frame, so on a loaded machine the first runner's start landed late and the offset read 75-78 ms. It now reads both start times from the page and asserts the held offset equals their difference; the first runner's first frame is held back 60 ms, as load does, so the old assertion fails every run. --- .../src/harness.interaction.browser.spec.ts | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index fa81621cc4eae..f11f3ec60d54e 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -324,7 +324,7 @@ test.describe('holding', () => { await fixture(page); // Besides #box: a finished fade, an animation the page paused, one on a // scroll timeline, and a second runner started 100 ms after #box's. - await page.evaluate(async () => { + const startGap = await page.evaluate(async () => { const make = (id: string) => { const el = document.createElement('div'); el.id = id; @@ -360,15 +360,26 @@ test.describe('holding', () => { [{transform: 'translateX(0px)'}, {transform: 'translateX(300px)'}], {duration: 600, easing: 'linear'}, ); - run(document.getElementById('box')); - await new Promise(resolve => setTimeout(resolve, 100)); - run(make('later')); + const box = run(document.getElementById('box')); + const gap = new Promise(resolve => setTimeout(resolve, 100)); + // #box's first frame comes late, as on a loaded machine: its start time + // resolves then, so the runners' offset is the difference of their start + // times, not the timer's 100 ms. + const until = performance.now() + 60; + while (performance.now() < until) { + // Busy: no frame runs. + } + await gap; + const later = run(make('later')); + await Promise.all([box?.ready, later?.ready]); + return Number(later?.startTime) - Number(box?.startTime); }); const held = await holdAnimations(page); expect(held.map(a => a.target)).toEqual(['div#box', 'div#later']); const [first, later] = held; expect(first.offsetMs).toBe(0); - expect(later.offsetMs).toBeGreaterThan(80); + expect(startGap).toBeGreaterThan(0); + expect(later.offsetMs).toBeCloseTo(startGap, 0); const x = (id: string) => page .locator(`#${id}`) From 35b0f22e964b1e59595eac1976642ef43ffbb0a9 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 22:03:47 -0700 Subject: [PATCH 56/93] fix(interaction): a reversed run's progress counts from its end; waits, seeks and samples go through the playback rate The review's items on 2efaf04232: - waitForAnimation counted a run's progress as currentTime minus its delay, which a run reversed by reverse() meets in the frame it starts (its current time starts at its end), so a wait could return before Chromium lists the run and the hold resolved [] instead of refusing it. A backward run now progresses from its end, and progress is measured on the timeline. - At a playback rate other than 1, waitForAnimation's from and to, and seekAnimations' and sampleCurve's local times, read the effect clock: a 1,000 ms curve at rate 2 reported a 1,000 ms interval and a seek to t presented the curve at t. They now go through the rate (a hold still refuses a backward run). A self-test pins each: a reversed run's 100 ms wait returns once it has presented 100 ms (16.7 ms in Chromium and 84 ms in WebKit before), and at rate 2 the interval is 500 ms and a seek or sample at t reads the curve at 2t. --- internal/interaction-harness/src/curves.ts | 49 ++++++++++++------ .../src/harness.interaction.browser.spec.ts | 50 +++++++++++++++++++ 2 files changed, 83 insertions(+), 16 deletions(-) diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index 52d871f6adb3f..bc1bbdc6765c8 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -441,9 +441,11 @@ export async function seekAnimations( const end = Number( animation.effect?.getComputedTiming().endTime ?? Infinity, ); - const local = Math.min(t - offset, end - inset); + // A faster rate runs more curve per timeline ms. + const rate = animation.playbackRate || 1; + const local = Math.min((t - offset) * rate, end - inset); animation.currentTime = local; - presented = Math.max(presented, local + offset); + presented = Math.max(presented, local / rate + offset); }); return Number.isFinite(presented) ? presented : t; }, @@ -594,19 +596,27 @@ export async function sampleCurve( const end = Number( animation.effect?.getComputedTiming().endTime ?? Infinity, ); - const local = Math.min(t - offset, end - inset); + // A faster rate runs more curve per timeline ms. + const rate = animation.playbackRate || 1; + const local = Math.min((t - offset) * rate, end - inset); animation.currentTime = local; - presented = Math.max(presented, local + offset); + presented = Math.max(presented, local / rate + offset); const properties = own[index]; if (properties == null) { return; } - presentedOwn = Math.max(presentedOwn, local + offset); + presentedOwn = Math.max(presentedOwn, local / rate + offset); if (TRANSFORM.some(p => properties.has(p))) { - presentedTransform = Math.max(presentedTransform, local + offset); + presentedTransform = Math.max( + presentedTransform, + local / rate + offset, + ); } if (properties.has('opacity')) { - presentedOpacity = Math.max(presentedOpacity, local + offset); + presentedOpacity = Math.max( + presentedOpacity, + local / rate + offset, + ); } }); const or = (value: number, fallback: number) => @@ -849,8 +859,10 @@ export async function waitForAnimation( return { kind: a.constructor.name, name, - from: startedAt + Number(timing?.delay ?? 0), - to: startedAt + curveEnd(a), + // Through the playback rate's speed: at rate 2 a curve runs in half + // its duration. + from: startedAt + Number(timing?.delay ?? 0) / Math.abs(rate || 1), + to: startedAt + curveEnd(a) / Math.abs(rate || 1), durationMs: Number(timing?.duration ?? 0), easing: String(effect?.getTiming().easing ?? 'linear'), keyframes: (effect?.getKeyframes() ?? []).map( @@ -876,13 +888,18 @@ export async function waitForAnimation( ? element.getAnimations({subtree: true}) : recorded.filter(e => e.finishedAt == null).map(e => e.animation) ).filter(a => a.playState === 'running' && a.startTime != null); - // Progress along the curve: a transition's delay presents nothing. - const progressed = running.filter( - a => - Number(a.currentTime ?? 0) - - Number(a.effect?.getComputedTiming().delay ?? 0) >= - minProgress, - ); + // Progress along the curve, on the timeline: a transition's delay + // presents nothing, a run going backward (`reverse()`) progresses from + // its end, and a faster rate presents more curve per ms. + const progressed = running.filter(a => { + const timing = a.effect?.getComputedTiming(); + const time = Number(a.currentTime ?? 0); + const local = + a.playbackRate < 0 + ? Number(timing?.endTime ?? 0) - time + : time - Number(timing?.delay ?? 0); + return local / Math.abs(a.playbackRate || 1) >= minProgress; + }); if (progressed.length > 0) { return progressed.map(a => describe(a, null, null)); } diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index f11f3ec60d54e..7c17d5e2ac91b 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -510,6 +510,56 @@ test.describe('holding', () => { await releaseAnimations(page); }); + test('a reversed run counts its progress from its end, and a faster playback rate is read on the timeline by waits, seeks and samples', async ({ + page, + }) => { + await fixture(page); + const box = page.locator('#box'); + // Reversed, the run starts at its end: progress counts down from there, + // so a wait for 100 ms of it returns once the run has presented that much. + const watch = await watchAnimations(box); + await box.evaluate(el => { + el.animate([{translate: '0px 0px'}, {translate: '100px 0px'}], { + duration: 1000, + easing: 'linear', + }).reverse(); + }); + await watch.started({minProgressMs: 100}); + const presented = await box.evaluate(el => { + const [animation] = el.getAnimations(); + return ( + Number(animation.effect?.getComputedTiming().endTime ?? 0) - + Number(animation.currentTime ?? 0) + ); + }); + expect(presented).toBeGreaterThanOrEqual(100); + await box.evaluate(el => el.getAnimations().forEach(a => a.finish())); + + // At playback rate 2 a 1,000 ms curve runs 500 ms: its interval, and a + // seek or sample at t reads the curve at 2t. + const fast = await watchAnimations(box); + await box.evaluate(el => { + const animation = el.animate( + [{translate: '0px 0px'}, {translate: '100px 0px'}], + {duration: 1000, easing: 'linear'}, + ); + animation.playbackRate = 2; + }); + const [running] = await fast.started(); + expect(running.to - running.from).toBeCloseTo(500, 0); + await holdAnimations(page, box); + await seekAnimations(page, 125); + const x = await box.evaluate( + el => + new DOMMatrix(getComputedStyle(el).transform).m41 + + Number.parseFloat(getComputedStyle(el).translate || '0'), + ); + expect(x).toBeCloseTo(25, 0); + const [sample] = await sampleCurve(box, [250]); + expect(sample.transform.translateX).toBeCloseTo(50, 0); + await releaseAnimations(page, {finish: true}); + }); + test('a held end is solved through a cubic-bézier effect easing, a reversed one reads to the full end, and a rotate about x composes on its axis', async ({ page, }) => { From a2613849a20b17740a8b786ccdaa1ae282f12177 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Sun, 25 Oct 2026 23:49:56 -0700 Subject: [PATCH 57/93] fix(interaction): the busy verdict counts the frames a screencast delivered, not its median interval Without a compositor record (every WebKit run), the moved criterion counted the frames the judged part should hold at the median interval of the frames around the block. WebKit's casts mix short and long intervals, so the median lands on either: the same 46 moved frames were judged against 40 to 50 expected frames across six runs, and a cast cycling 11.7, 11.7 and 17 ms with every frame moving failed at 49 of 56. The criterion now counts the frames the screencast delivered in the judged part: a delivered frame that shows no new position is a miss, and a frame it did not deliver is a gap, judged by the stall check. With a compositor record it still counts the compositor's frames, and a part that delivered nothing is still judged against its span, so a frozen animation fails every criterion. deliveredFrames reports the count. No limit changes. --- internal/interaction-harness/README.md | 8 +- internal/interaction-harness/src/busy.test.ts | 89 ++++++++++++++++--- internal/interaction-harness/src/busy.ts | 44 ++++++--- 3 files changed, 114 insertions(+), 27 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 4963512792a3b..85d5821068f74 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -123,8 +123,9 @@ created in the frame the page looks is not on the compositor yet. The `verdict` frames the screencast delivered, whatever its rate (a loaded machine's can deliver one frame in five or six), and tolerates nothing a frozen animation does: at least one frame in the judged part, no stall longer than three of the -screencast's intervals (two dropped frames), and at least 90% of the window's -whole intervals presenting a new position (one miss is always allowed). Pass +screencast's intervals (two dropped frames), and at least 90% of the frames it +delivered in the judged part presenting a new position (one miss is always +allowed; a frame it did not deliver is a gap, for the stall check). Pass `maxMedianFrameMs` (20 at 60 Hz) only in a proof that claims display rate. `verdict.failures` names each miss with its numbers, and `summary` carries the raw numbers for the report. Frames are judged in presentation order: a loaded @@ -155,7 +156,8 @@ compositor's own frames: the frames it presented there count as presented a stall as before. A gap that opens or closes the judged part, or holds a delivered frame that did not move, is judged from the screencast, so a frozen element fails whatever else the compositor draws (a rAF animation's -block presents no compositor frame at all). WebKit has no such trace: +block presents no compositor frame at all). With the record, the 90% counts the +compositor's frames in the judged part. WebKit has no such trace: `stop()` resolves to `[]` and the screencast is judged alone. **No Layout / Paint.** `traceRendering` records a Chromium trace around the diff --git a/internal/interaction-harness/src/busy.test.ts b/internal/interaction-harness/src/busy.test.ts index 37a22cf1638a8..6d25a3c30e328 100644 --- a/internal/interaction-harness/src/busy.test.ts +++ b/internal/interaction-harness/src/busy.test.ts @@ -38,6 +38,28 @@ function moving(dropped: readonly number[] = []): TrackedFrame[] { /** Index of the first frame at or after `t`. */ const at = (t: number) => Math.ceil((t - 800) / FRAME); +/** `moving()` with each frame in `repeats` presenting the previous frame's position. */ +function repeating(repeats: readonly number[]): TrackedFrame[] { + return moving().map((f, i) => + repeats.includes(i) + ? ({...f, box: {...f.box!, y: 3 * (i - 1)}} as TrackedFrame) + : f, + ); +} + +/** A marker moving 3 px per frame from t = 800 to 1900, frames `intervals` apart in turn. */ +function cadenced(intervals: readonly number[]): TrackedFrame[] { + const frames: TrackedFrame[] = []; + for (let i = 0, t = 800; t <= 1900; i += 1) { + frames.push({ + t, + box: {x: 0, y: 3 * i, width: 10, height: 10}, + } as TrackedFrame); + t += intervals[i % intervals.length]; + } + return frames; +} + describe('busyWindowProof verdict', () => { it('passes a compositor animation with every frame', () => { const proof = busyWindowProof(moving(), BLOCK); @@ -60,28 +82,65 @@ describe('busyWindowProof verdict', () => { expect(proof.verdict.failures.join('\n')).toMatch(/longest stall/); }); - it('fails when more than 10% of the expected frames are missing', () => { + it('fails when more than 10% of the delivered frames do not move', () => { const k = at(1050); - // 39 frames expected: up to 3 may be missing (10% = 3.9). + // 39 frames delivered: up to 3 may show no new position (10% = 3.9). const scattered = [0, 12, 24].map(d => k + d); - expect(busyWindowProof(moving(scattered), BLOCK).verdict.failures).toEqual( - [], - ); + expect( + busyWindowProof(repeating(scattered), BLOCK).verdict.failures, + ).toEqual([]); const many = [0, 6, 12, 18, 24].map(d => k + d); expect( - busyWindowProof(moving(many), BLOCK).verdict.failures.join('\n'), + busyWindowProof(repeating(many), BLOCK).verdict.failures.join('\n'), ).toMatch(/expected frames moved/); }); + it('judges a frame the screencast did not deliver as a gap for the stall check, not a miss', () => { + // The same five frames, not delivered: two-interval gaps, and every + // delivered frame moved. + const k = at(1050); + const many = [0, 6, 12, 18, 24].map(d => k + d); + const proof = busyWindowProof(moving(many), BLOCK); + expect(proof.verdict.failures).toEqual([]); + expect(proof.deliveredFrames).toBe(proof.moved); + }); + + it('judges an irregular cast on the frames it delivered', () => { + // A cast cycling 11.7, 11.7 and 17 ms: its median interval would expect + // 56 frames where it delivered 49, every one moved. + const proof = busyWindowProof(cadenced([11.7, 11.7, 17]), BLOCK); + expect(proof.verdict.failures).toEqual([]); + expect(proof.deliveredFrames).toBe(49); + expect(proof.moved).toBe(49); + // Still a stall where it delivers nothing for four frames, and still a + // miss where five delivered frames show no new position. + const gap = cadenced([11.7, 11.7, 17]).filter( + f => f.t < 1300 || f.t > 1360, + ); + expect(busyWindowProof(gap, BLOCK).verdict.failures.join('\n')).toMatch( + /longest stall/, + ); + const unmoved = cadenced([11.7, 11.7, 17]).map((f, i, all) => + [20, 26, 32, 38, 44].includes(i) + ? ({...f, box: {...all[i - 1].box!}} as TrackedFrame) + : f, + ); + expect(busyWindowProof(unmoved, BLOCK).verdict.failures.join('\n')).toMatch( + /expected frames moved/, + ); + }); + it('lets a short window miss a frame, not three', () => { - // 150 ms judged: 8 whole intervals, one miss allowed (10% would be 0.8). + // 150 ms judged: nine frames delivered, one miss allowed (10% would be 0.9). const short = {from: 1000, to: 1190}; const k = at(1040); - expect(busyWindowProof(moving([k]), short).verdict.failures).toEqual([]); - // Three of nine missing fails, on frame count or cadence. - expect(busyWindowProof(moving([k, k + 3, k + 6]), short).verdict.pass).toBe( - false, - ); + expect(busyWindowProof(repeating([k]), short).verdict.failures).toEqual([]); + expect( + busyWindowProof( + repeating([k, k + 3, k + 6]), + short, + ).verdict.failures.join('\n'), + ).toMatch(/expected frames moved/); }); it('fails a frozen animation on every criterion', () => { @@ -307,8 +366,10 @@ describe('busyWindowProof with the compositor record', () => { expect(alone.verdict.failures.join('\n')).toMatch( /longest stall 6\d\.\d ms = 4 intervals > 3/, ); - expect(alone.verdict.failures.join('\n')).toMatch( - /26 of 31 expected frames moved/, + // Every frame the screencast delivered moved: only the stall remains, + // which the record resolves. + expect(alone.verdict.failures.join('\n')).not.toMatch( + /expected frames moved/, ); const proof = busyWindowProof(frames, skip.block, { settledWithin: 1, diff --git a/internal/interaction-harness/src/busy.ts b/internal/interaction-harness/src/busy.ts index 0c93ab15387cf..161edd91e209f 100644 --- a/internal/interaction-harness/src/busy.ts +++ b/internal/interaction-harness/src/busy.ts @@ -159,10 +159,12 @@ export interface BusyProof extends FrameStats { */ readonly cadenceMs: number; /** - * Whole frame intervals in the window at the measured cadence: the frames - * the display shows there at least (one more, depending on phase). + * Whole frame intervals in the judged part at the measured cadence (the + * compositor's, when recorded): the `minJudgedFrames` floor reads it. */ readonly expectedFrames: number; + /** Frames the screencast delivered in the judged part, with the marker found. */ + readonly deliveredFrames: number; /** * Longest time inside the window with no presented change, the window's own * edges included. A frozen animation presents no new frame at all (nothing @@ -182,7 +184,11 @@ export interface BusyProof extends FrameStats { * with the last frame before the window). */ readonly moved: number; - /** `moved` over `expectedFrames`. */ + /** + * `moved` (and `recorderSkipped`) over the frames judged: the frames the + * screencast delivered in the judged part, or with a compositor record the + * compositor's frames there; `expectedFrames` when nothing was delivered. + */ readonly movedFraction: number; /** The judgment against the limits (see `busyWindowProof`). */ readonly verdict: BusyVerdict; @@ -204,8 +210,11 @@ export interface BusyProof extends FrameStats { * claims display rate and passes `maxMedianFrameMs`, their median interval * is at most that; * - no stall spans more than `maxStallFrames` screencast intervals; - * - at least `minMovedFraction` of the frames the window should present at - * that cadence moved the tracked axis (a short window may still miss one); + * - at least `minMovedFraction` of the frames judged moved the tracked axis + * (a short window may still miss one): the frames the screencast delivered + * in the judged part, so an irregular cast is judged on what it delivered + * and a frame it did not deliver is a gap for the stall check, not a miss; + * with a compositor record, the compositor's frames there; * - the judged part spans at least `minJudgedFrames` frames. Fewer means the * animation under test was not moving during the block (it settled before * the block, or the block caught another animation), so there is nothing to @@ -424,7 +433,21 @@ export function busyWindowProof( (judged.to - judged.from) / (compositorFrameMs ?? cadence), ); const moved = changes.length; - const movedFraction = (moved + recorderSkipped) / Math.max(1, expectedFrames); + // The frames judged for motion: the screencast's own frames in the judged + // part (each `changes` entry is one of them), or the compositor's frames + // when recorded. A part that delivered nothing is judged against its span, + // so a frozen animation misses here too. + const deliveredFrames = tracked.filter( + f => + f.box != null && + (onScreenFrom == null ? f.t >= judged.from : f.t > judged.from) && + f.t <= judged.to, + ).length; + const judgedFrames = + compositorFrameMs != null || deliveredFrames === 0 + ? expectedFrames + : deliveredFrames; + const movedFraction = (moved + recorderSkipped) / Math.max(1, judgedFrames); // The cast's position step, and the motion per interval it has to resolve. const positions = tracked.flatMap(f => (f.box == null ? [] : [f.box[axis]])); const stepPx = @@ -478,18 +501,18 @@ export function busyWindowProof( } const allowedMisses = Math.max( 1, - Math.floor((1 - minMovedFraction) * expectedFrames + 1e-9), + Math.floor((1 - minMovedFraction) * judgedFrames + 1e-9), ); - if (expectedFrames - moved - recorderSkipped > allowedMisses) { + if (judgedFrames - moved - recorderSkipped > allowedMisses) { failures.push( - `${moved} of ${expectedFrames} expected frames moved${recorderSkipped > 0 ? ` (and ${recorderSkipped} the screencast skipped)` : ''} (${(movedFraction * 100).toFixed(0)}% < ${(minMovedFraction * 100).toFixed(0)}%)`, + `${moved} of ${judgedFrames} expected frames moved${recorderSkipped > 0 ? ` (and ${recorderSkipped} the screencast skipped)` : ''} (${(movedFraction * 100).toFixed(0)}% < ${(minMovedFraction * 100).toFixed(0)}%)`, ); } const onScreenFromMs = onScreenFrom == null ? null : onScreenFrom - window.from; const summary = `${(judged.to - judged.from).toFixed(0)} ms judged at ${cadence.toFixed(1)} ms/frame: ${stats.frames} frames presented, ` + - `${moved}/${expectedFrames} moved${recorderSkipped > 0 ? ` and ${recorderSkipped} skipped by the screencast` : ''} (${(movedFraction * 100).toFixed(0)}%), ` + + `${moved}/${judgedFrames} moved${recorderSkipped > 0 ? ` and ${recorderSkipped} skipped by the screencast` : ''} (${(movedFraction * 100).toFixed(0)}%), ` + `median ${stats.medianFrameMs.toFixed(1)} ms, ` + `longest stall ${longestStallMs.toFixed(1)} ms (${longestStallFrames} intervals) at +${longestStallAtMs.toFixed(0)} ms, ` + `${stats.monotonic ? 'monotonic' : 'not monotonic'}, ` + @@ -515,6 +538,7 @@ export function busyWindowProof( longestStallMs, longestStallAtMs, expectedFrames, + deliveredFrames, longestStallFrames, moved, movedFraction, From 10ba1660de418f53127837a06a529890ad06e405 Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Sun, 25 Oct 2026 23:59:43 -0700 Subject: [PATCH 58/93] fix(interaction): a run going backward is dated from when it began Chromium gives a run going backward a startTime where its current time would reach 0, a whole run after it began. waitForAnimation and watch.started() now anchor it at startTime minus endTime over the speed, in describe and in the recorder's saved start (so one the page cancels when it ends keeps its real start), and take from/to from the end delay and endTime minus the delay, which play first and last going backward. A self-test covers a delayed reverse(), one that ended before the look, and one cancelled when it ends; it failed by 296 ms before (Chromium, 27/27 and 5x repeat pass). --- internal/interaction-harness/src/curves.ts | 64 +++++++++++++------ .../src/harness.interaction.browser.spec.ts | 54 ++++++++++++++++ 2 files changed, 99 insertions(+), 19 deletions(-) diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index bc1bbdc6765c8..fc7c5ef642bc7 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -57,6 +57,23 @@ function animationRecorder(): void { // Entries whose start time has not resolved yet (still pending when seen). let unstarted: (typeof log)[number][] = []; let armedAt = performance.now(); + // When the animation began to play, on the document timeline. A run going + // backward reports as its startTime the moment its current time would + // reach 0, a whole run after it began (it begins at its end). + const playStart = (animation: Animation): number | null => { + if (animation.startTime == null) { + return null; + } + const rate = animation.playbackRate; + if (rate >= 0) { + return Number(animation.startTime); + } + const end = Number( + (animation.effect as KeyframeEffect | null)?.getComputedTiming() + .endTime ?? 0, + ); + return Number(animation.startTime) - end / Math.abs(rate); + }; const remember = (animation: Animation) => { // A running animation is recorded while still pending: on a cold page the // next frame can come only after a short animation has finished. @@ -68,8 +85,7 @@ function animationRecorder(): void { animation, seenAt: performance.now(), finishedAt: null as number | null, - startedAt: - animation.startTime == null ? null : Number(animation.startTime), + startedAt: playStart(animation), }; if (entry.startedAt == null) { unstarted.push(entry); @@ -84,9 +100,9 @@ function animationRecorder(): void { ); const at = (event as AnimationPlaybackEvent).timelineTime; entry.startedAt = - animation.startTime != null - ? Number(animation.startTime) - : (at == null ? entry.finishedAt : Number(at)) - end; + playStart(animation) ?? + (at == null ? entry.finishedAt : Number(at)) - + end / Math.abs(animation.playbackRate || 1); } }); log.push(entry); @@ -100,7 +116,7 @@ function animationRecorder(): void { } unstarted = unstarted.filter(entry => { if (entry.startedAt == null && entry.animation.startTime != null) { - entry.startedAt = Number(entry.animation.startTime); + entry.startedAt = playStart(entry.animation); } return entry.startedAt == null && entry.animation.playState !== 'idle'; }); @@ -837,18 +853,21 @@ export async function waitForAnimation( ) => { const effect = a.effect as KeyframeEffect | null; const timing = effect?.getComputedTiming(); - // The recorded start stands in once the page has cancelled it. An - // animation running backward started when its current time last - // stood at its end: its startTime is where it would reach 0. + // The recorded start stands in once the page has cancelled it. A + // run going backward began at its end: its startTime is where its + // current time would reach 0, a whole run later. const rate = a.playbackRate; + const speed = Math.abs(rate || 1); + const endTime = Number(timing?.endTime ?? 0); + const delay = Number(timing?.delay ?? 0); + const start = + a.startTime == null + ? recordedStart + : rate < 0 + ? Number(a.startTime) - endTime / speed + : Number(a.startTime); const startedAt = - rate < 0 && a.currentTime != null - ? performance.timeOrigin + - Number(document.timeline.currentTime ?? performance.now()) - - (Number(timing?.endTime ?? 0) - Number(a.currentTime)) / - Math.abs(rate) - : performance.timeOrigin + - Number(a.startTime ?? recordedStart ?? Number.NaN); + performance.timeOrigin + Number(start ?? Number.NaN); const name = 'transitionProperty' in a ? (a as unknown as {transitionProperty: string}) @@ -860,9 +879,16 @@ export async function waitForAnimation( kind: a.constructor.name, name, // Through the playback rate's speed: at rate 2 a curve runs in half - // its duration. - from: startedAt + Number(timing?.delay ?? 0) / Math.abs(rate || 1), - to: startedAt + curveEnd(a) / Math.abs(rate || 1), + // its duration. Backward, the end delay plays first and the start + // delay last. + from: + rate < 0 + ? startedAt + Number(timing?.endDelay ?? 0) / speed + : startedAt + delay / speed, + to: + rate < 0 + ? startedAt + (endTime - delay) / speed + : startedAt + curveEnd(a) / speed, durationMs: Number(timing?.duration ?? 0), easing: String(effect?.getTiming().easing ?? 'linear'), keyframes: (effect?.getKeyframes() ?? []).map( diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 7c17d5e2ac91b..ed46369fab0f8 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -1003,6 +1003,60 @@ test.describe('animation state', () => { expect(entrance.finishedAt ?? 0).toBeGreaterThanOrEqual(entrance.to - 1); }); + test('a run going backward is dated from when it began: with a delay, ended before the look, or cancelled when it ends', async ({ + page, + }) => { + await fixture(page); + const box = page.locator('#box'); + const frames = [ + {translate: '0px 0px', offset: 0}, + {translate: '100px 0px', offset: 1}, + ]; + + // A 300 ms delay plays last when going backward: the curve starts at once. + let watch = await watchAnimations(box); + let readyAt = await box.evaluate(async (el, keyframes) => { + const animation = el.animate(keyframes, {duration: 400, delay: 300}); + animation.reverse(); + await animation.ready; + return performance.timeOrigin + performance.now(); + }, frames); + const [delayed] = await watch.started(); + expect(Math.abs(delayed.from - readyAt)).toBeLessThan(50); + expect(delayed.to - delayed.from).toBeCloseTo(400, 0); + await box.evaluate(el => el.getAnimations().forEach(a => a.finish())); + + // A short one that ended before the look keeps its curve inside its life. + watch = await watchAnimations(box); + readyAt = await box.evaluate(async (el, keyframes) => { + const animation = el.animate(keyframes, {duration: 60}); + animation.reverse(); + await animation.ready; + return performance.timeOrigin + performance.now(); + }, frames); + await page.waitForTimeout(200); + const [ended] = await watch.started(); + expect(ended.finishedAt).not.toBeNull(); + expect(Math.abs(ended.from - readyAt)).toBeLessThan(50); + expect(ended.to).toBeLessThanOrEqual((ended.finishedAt ?? 0) + 1); + + // One the page cancels when it ends keeps its start. + watch = await watchAnimations(box); + readyAt = await box.evaluate(async (el, keyframes) => { + const animation = el.animate(keyframes, {duration: 60}); + animation.addEventListener('finish', () => animation.cancel(), { + once: true, + }); + animation.reverse(); + await animation.ready; + return performance.timeOrigin + performance.now(); + }, frames); + await page.waitForTimeout(200); + const [cancelled] = await watch.started(); + expect(Math.abs(cancelled.from - readyAt)).toBeLessThan(50); + expect(cancelled.to - cancelled.from).toBeCloseTo(60, 0); + }); + test('an end value held by the keyframes is not part of the curve, and individual transform properties are sampled', async ({ page, }) => { From 92bc9c2f4ceb417c5b03998cc2cb742cc8b0d62f Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Mon, 26 Oct 2026 00:30:21 -0700 Subject: [PATCH 59/93] fix(interaction): a finish read before its event goes through the rate, and a backward end delay is not progress waitForAnimation's fallback finish time (read before the finish event is delivered, right after finish()) now goes through the playback rate's speed, and for a run going backward is its startTime; it was a whole run late for a reverse()d run and 500 ms late at rate 2. minProgressMs no longer counts a backward run's end delay, which plays first and presents nothing. Self-tests for both; the finish case failed by 1,000 ms before (Chromium, 27/27 and 5x repeat pass). --- internal/interaction-harness/src/curves.ts | 41 ++++++++++++------- .../src/harness.interaction.browser.spec.ts | 31 ++++++++++++++ 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index fc7c5ef642bc7..60f5c4ec41fc9 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -914,15 +914,18 @@ export async function waitForAnimation( ? element.getAnimations({subtree: true}) : recorded.filter(e => e.finishedAt == null).map(e => e.animation) ).filter(a => a.playState === 'running' && a.startTime != null); - // Progress along the curve, on the timeline: a transition's delay - // presents nothing, a run going backward (`reverse()`) progresses from - // its end, and a faster rate presents more curve per ms. + // Progress along the curve, on the timeline: a delay presents nothing + // (going backward, the end delay plays first), a run going backward + // (`reverse()`) progresses from its end, and a faster rate presents + // more curve per ms. const progressed = running.filter(a => { const timing = a.effect?.getComputedTiming(); const time = Number(a.currentTime ?? 0); const local = a.playbackRate < 0 - ? Number(timing?.endTime ?? 0) - time + ? Number(timing?.endTime ?? 0) - + Number(timing?.endDelay ?? 0) - + time : time - Number(timing?.delay ?? 0); return local / Math.abs(a.playbackRate || 1) >= minProgress; }); @@ -933,17 +936,25 @@ export async function waitForAnimation( return []; } // Finished: by its finish event, or by its state when the event has not - // been delivered yet (then at its scheduled end). - const finishedAt = (e: (typeof recorded)[number]) => - e.finishedAt ?? - (e.animation.playState === 'finished' && e.animation.startTime != null - ? Number(e.animation.startTime) + - Number( - ( - e.animation.effect as KeyframeEffect | null - )?.getComputedTiming().endTime ?? 0, - ) - : null); + // been delivered yet (then at its scheduled end, through the rate's + // speed; a run going backward ends at its startTime, where its current + // time reaches 0). + const finishedAt = (e: (typeof recorded)[number]) => { + if (e.finishedAt != null) { + return e.finishedAt; + } + const a = e.animation; + if (a.playState !== 'finished' || a.startTime == null) { + return null; + } + const end = Number( + (a.effect as KeyframeEffect | null)?.getComputedTiming().endTime ?? + 0, + ); + return a.playbackRate < 0 + ? Number(a.startTime) + : Number(a.startTime) + end / Math.abs(a.playbackRate || 1); + }; return recorded .filter(e => finishedAt(e) != null) .map(e => describe(e.animation, finishedAt(e), e.startedAt)); diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index ed46369fab0f8..a680ce3121fed 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -1055,6 +1055,37 @@ test.describe('animation state', () => { const [cancelled] = await watch.started(); expect(Math.abs(cancelled.from - readyAt)).toBeLessThan(50); expect(cancelled.to - cancelled.from).toBeCloseTo(60, 0); + + // Read right after finish(), before the finish event is delivered: the + // scheduled end goes through the rate (backward, it is the startTime). + for (const rate of [-1, 2]) { + watch = await watchAnimations(box); + await box.evaluate( + async (el, [keyframes, playbackRate]) => { + const animation = el.animate(keyframes as Keyframe[], { + duration: 1000, + }); + animation.playbackRate = playbackRate as number; + await animation.ready; + animation.finish(); + }, + [frames, rate] as const, + ); + const [read] = await watch.started(); + expect(read.finishedAt).not.toBeNull(); + expect(Math.abs((read.finishedAt ?? 0) - read.to)).toBeLessThan(50); + } + + // Going backward, an end delay presents nothing: a run still inside it + // has not progressed. + await box.evaluate((el, keyframes) => { + const animation = el.animate(keyframes, {duration: 400, endDelay: 500}); + animation.reverse(); + }, frames); + await expect( + waitForAnimation(box, {minProgressMs: 100, timeout: 300}), + ).rejects.toThrow(); + await box.evaluate(el => el.getAnimations().forEach(a => a.finish())); }); test('an end value held by the keyframes is not part of the curve, and individual transform properties are sampled', async ({ From 66d1919d58a59b79dbe0bc92d1b432607559764e Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Mon, 26 Oct 2026 00:20:59 -0700 Subject: [PATCH 60/93] fix(interaction): a traced waitForRest waits for a run going backward until it finishes While a rendering trace runs, waitForRest waits on timers for each curve's time left, computed as its end less its current time. Going backward (reverse(), a negative playback rate) the current time falls from the end time to 0, so that difference is the time already played: a reversed move with a 200 ms end delay settled within 50 ms, before it began to move, and one without waited about twice its length. Backward, the time left is now the current time over the rate's magnitude: the run is waited for until it finishes, as the untraced path waits it out. A self-test pins both: the delayed reversed move holds the traced wait for its 800 ms, and a reversed move is waited for no longer than the same move forward. --- internal/interaction-harness/src/curves.ts | 12 +++-- .../src/harness.interaction.browser.spec.ts | 50 +++++++++++++++++++ 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index c82c4f536b62f..f120c558a2f0b 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -951,7 +951,8 @@ export async function watchAnimations( * While a rendering trace runs (`traceRendering`), it stands down as the * animation recorder does: a per-frame callback would make every frame a * main-thread frame, which the trace counts. It then waits on timers for the - * scheduled ends of the curves on the element instead, reading no style, so + * scheduled ends of the curves on the element instead (a run going backward, + * until it finishes), reading no style, so * motion that no animation drives (a script writing styles) is not seen until * the trace ends. Finding no animation, it looks again `frames` times a frame * apart before settling, so one the action starts a few frames later is @@ -1027,9 +1028,12 @@ export async function waitForRest( ) { wait = Math.max(wait, 16); } else if (a.playState === 'running') { - const left = - (curveEnd(a) - Number(a.currentTime ?? 0)) / - Math.abs(a.playbackRate || 1); + // The curve's time left through the playback rate: forward, to + // its (held) end; backward (`reverse()`), to local time 0, + // where it finishes, as the untraced path waits it out. + const rate = a.playbackRate || 1; + const at = Number(a.currentTime ?? 0); + const left = rate > 0 ? (curveEnd(a) - at) / rate : at / -rate; wait = Math.max(wait, left); } } diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 55d691afa1f77..5bf3b7a851a0d 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -777,6 +777,56 @@ test.describe('frame evidence', () => { expect(trace.window.to - (started as number)).toBeGreaterThanOrEqual(590); }); + test('a waitForRest awaited inside a traced body waits for a run going backward to its end, and no longer', async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'rendering traces are Chromium-only (README: engine limits)', + ); + await page.route('http://anchored.test/', route => + route.fulfill({contentType: 'text/html', body: ANCHORED}), + ); + await page.goto('http://anchored.test/'); + const pop = page.locator('#pop'); + await waitForRest(pop); + // How long a traced waitForRest holds after a 600 ms move starts. + const traced = async (reverse: boolean, endDelay = 0) => { + let started = 0; + const trace = await traceRendering(page, async () => { + started = await pop.evaluate( + (el, {reverse, endDelay}) => { + const run = el.animate( + [ + {transform: 'translateY(0px)'}, + {transform: 'translateY(100px)'}, + ], + {duration: 600, endDelay}, + ); + if (reverse) { + // Its current time falls from its end time to 0. + run.reverse(); + } + return performance.timeOrigin + performance.now(); + }, + {reverse, endDelay}, + ); + await waitForRest(pop); + }); + await waitForRest(pop); + return trace.window.to - started; + }; + // Reversed with a 200 ms end delay: the move plays from 200 to 800 ms. + expect(await traced(true, 200)).toBeGreaterThanOrEqual(790); + // A reversed run is waited for as long as the same run forward, not + // twice as long. + const forward = await traced(false); + const backward = await traced(true); + expect(backward).toBeGreaterThanOrEqual(590); + expect(backward).toBeLessThan(forward * 1.5); + }); + test('a waitForRest awaited inside a traced body runs on the real timers, so a frozen timer gate cannot hold it', async ({ page, browserName, From a2ac6f9f0f6c6715fdf60523fae2afaa18b4789a Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Mon, 26 Oct 2026 01:03:40 -0700 Subject: [PATCH 61/93] fix(interaction): waitForRest waits out a backward run's end delay and curve A run going backward plays its end delay first, then its curve down to the start delay; waitForRest counted it as done once its current time passed curveEnd, so it resolved inside the end delay (47 ms into a 600 ms run with a 500 ms end delay) and the element then moved. It now waits until the current time reaches the start delay. Self-test added (Chromium 27/27, 5x repeat). --- internal/interaction-harness/src/curves.ts | 19 +++++++++++-------- .../src/harness.interaction.browser.spec.ts | 9 +++++++++ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index 60f5c4ec41fc9..90522f5d7a42c 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -1060,14 +1060,17 @@ export async function waitForRest( let still = 0; const tick = () => { const now = read(); - const moving = element - .getAnimations({subtree: true}) - .some( - a => - a.pending || - (a.playState === 'running' && - Number(a.currentTime ?? 0) < curveEnd(a)), - ); + const moving = element.getAnimations({subtree: true}).some( + a => + a.pending || + (a.playState === 'running' && + // Going backward the curve ends at the start delay, after + // the end delay (which plays first) and the curve itself. + (a.playbackRate < 0 + ? Number(a.currentTime ?? 0) > + Number(a.effect?.getComputedTiming().delay ?? 0) + : Number(a.currentTime ?? 0) < curveEnd(a))), + ); still = !moving && now === last ? still + 1 : 0; last = now; if (still >= need) { diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index a680ce3121fed..6e806d693488d 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -1086,6 +1086,15 @@ test.describe('animation state', () => { waitForAnimation(box, {minProgressMs: 100, timeout: 300}), ).rejects.toThrow(); await box.evaluate(el => el.getAnimations().forEach(a => a.finish())); + + // Nor is it rest: waitForRest waits out the end delay and the curve. + const restFrom = await box.evaluate((el, keyframes) => { + el.animate(keyframes, {duration: 600, endDelay: 500}).reverse(); + return performance.now(); + }, frames); + await waitForRest(box, {timeout: 5_000}); + const restAt = await box.evaluate(() => performance.now()); + expect(restAt - restFrom).toBeGreaterThan(1_000); }); test('an end value held by the keyframes is not part of the curve, and individual transform properties are sampled', async ({ From 1e242bed3ff02263f58be1488898c799b8cd6957 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Mon, 26 Oct 2026 01:20:36 -0700 Subject: [PATCH 62/93] fix(interaction): a rendering trace reports DevTools-run tasks apart from the page's long tasks traceRendering's longTasks counted every main-thread task over 50 ms, including tasks that ran a DevTools command. A harness read inside the body (waitForAnimation describing every animation's keyframes: 232 ms for 1,200 animated rows) was reported as the page's long task and failed proofs that the page has none. A task that ran a DevTools command is recognized in the trace by the DevTools session's message that opens it. Such a task is now reported in devtoolsTasks, unless page script ran inside it (an event listener, timer or frame callback the evaluate set off), which keeps it a long task: an evaluate that clicks a button whose handler spins 60 ms is still the page's. blockMainThread(page, ms) without during is reported in devtoolsTasks; with during it stays a page task, as the Long Tasks log sees it. No limit or rendering count changes. --- internal/interaction-harness/README.md | 6 +- .../src/harness.interaction.browser.spec.ts | 83 +++++++++++++++++-- internal/interaction-harness/src/trace.ts | 62 ++++++++++++-- 3 files changed, 135 insertions(+), 16 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 85d5821068f74..7529c21ebf99f 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -163,7 +163,11 @@ compositor's frames in the judged part. WebKit has no such trace: **No Layout / Paint.** `traceRendering` records a Chromium trace around the interaction; `countBetween` counts the steady middle of an animation (its first and last frames may commit). A transform/opacity animation does no -Layout or Paint there; a layout-property animation does one per frame. +Layout or Paint there; a layout-property animation does one per frame. Its +`longTasks` are the page's own: a task that ran a DevTools command (a harness +read such as `waitForAnimation` describing keyframes, or `blockMainThread` +without `during`) is in `devtoolsTasks` instead, unless page script ran +inside it (a listener or timer the evaluate set off). **Storyboards.** Deterministic: frames come from seeking held animations or from stepping scripted input, never from wall-clock capture. The JSON trace diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 7fa5c08e529f6..f061ff66dcda3 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -628,7 +628,7 @@ test.describe('frame evidence', () => { expect(top.Layout ?? 0).toBeGreaterThan(18); }); - test("the long-task log records the page's own long task but not a DevTools evaluate, which the trace counts", async ({ + test("the long-task log records the page's own long task but not a DevTools evaluate, which the trace reports apart", async ({ page, browserName, }) => { @@ -656,23 +656,88 @@ test.describe('frame evidence', () => { const recorded = await readLongTasks(page, own); expect(recorded).toHaveLength(1); expect(recorded[0].durationMs).toBeGreaterThanOrEqual(110); - // A DevTools evaluate that spins 300 ms: the trace counts it, the log - // never sees it. + // A DevTools evaluate that spins 300 ms: the trace reports it apart from + // the page's long tasks, and the log never sees it. let block = {from: 0, to: 0}; const trace = await traceRendering(page, async () => { block = await blockMainThread(page, 300); }); await frames(page, 2); + const inBlock = (t: {start: number}) => + t.start < block.to && t.start >= block.from - 20; expect( - trace.longTasks.some( - t => - t.durationMs >= 290 && - t.start < block.to && - t.start >= block.from - 20, - ), + trace.devtoolsTasks.some(t => t.durationMs >= 290 && inBlock(t)), ).toBe(true); + expect(trace.longTasks.filter(inBlock)).toEqual([]); expect(await readLongTasks(page, block)).toEqual([]); }); + + test("a harness read inside a trace is not the page's long task, while page script an evaluate sets off is", async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'rendering traces are Chromium-only (README: engine limits)', + ); + // Enough animated rows that describing their keyframes takes a DevTools + // task over 50 ms. + const rows = Array.from( + {length: 1200}, + (_, i) => `
row ${i}
`, + ).join(''); + await page.setContent( + `
${rows}
`, + ); + const slow = page.locator('#slow'); + await slow.evaluate(el => + el.addEventListener('click', () => { + const end = performance.now() + 60; + while (performance.now() < end) { + // spin + } + }), + ); + await frames(page, 3); + const now = () => + page.evaluate(() => performance.timeOrigin + performance.now()); + const read = {from: 0, to: 0}; + const click = {from: 0, to: 0}; + let timer = {from: 0, to: 0}; + const trace = await traceRendering(page, async () => { + read.from = await now(); + await waitForAnimation(page.locator('#list')); + read.to = await now(); + // The page's own task: a timer that spins 60 ms. + timer = await page.evaluate( + () => + new Promise<{from: number; to: number}>(resolve => { + setTimeout(() => { + const from = performance.timeOrigin + performance.now(); + const end = performance.now() + 60; + while (performance.now() < end) { + // spin + } + resolve({from, to: performance.timeOrigin + performance.now()}); + }, 0); + }), + ); + // Page script run inside a DevTools evaluate: a 60 ms click handler. + click.from = await now(); + await slow.evaluate(el => (el as HTMLElement).click()); + click.to = await now(); + }); + const within = (w: {from: number; to: number}) => (t: {start: number}) => + t.start >= w.from - 20 && t.start < w.to; + expect(trace.devtoolsTasks.some(within(read))).toBe(true); + expect(trace.longTasks.filter(within(read))).toEqual([]); + expect( + trace.longTasks.some(t => within(timer)(t) && t.durationMs >= 55), + ).toBe(true); + expect( + trace.longTasks.some(t => within(click)(t) && t.durationMs >= 55), + ).toBe(true); + }); }); test.describe('curve sampling', () => { diff --git a/internal/interaction-harness/src/trace.ts b/internal/interaction-harness/src/trace.ts index dcf608e5d0814..63dd620ee8f2c 100644 --- a/internal/interaction-harness/src/trace.ts +++ b/internal/interaction-harness/src/trace.ts @@ -22,6 +22,15 @@ import type {CDPSession, Page} from '@playwright/test'; +/** Trace events that run page script: a listener, a timer, a frame callback. */ +const PAGE_SCRIPT_EVENTS = new Set([ + 'EventDispatch', + 'FunctionCall', + 'TimerFire', + 'FireAnimationFrame', + 'FireIdleCallback', +]); + /** Main-thread trace events that are rendering work, by their trace names. */ export const RENDERING_EVENTS = [ 'UpdateLayoutTree', // style recalculation @@ -44,11 +53,26 @@ export interface RenderingTrace { readonly prePaint: number; /** Window length, ms. */ readonly durationMs: number; - /** Main-thread `RunTask`s longer than 50 ms inside the window (long tasks), epoch ms. */ + /** + * The page's own main-thread tasks longer than 50 ms inside the window (long + * tasks), epoch ms. A task that ran a DevTools command (a `page.evaluate`) + * is one only if page script ran inside it (an event listener, a timer, a + * frame callback it set off); otherwise it is in `devtoolsTasks`. + */ readonly longTasks: readonly { readonly start: number; readonly durationMs: number; }[]; + /** + * Main-thread tasks longer than 50 ms inside the window that ran a DevTools + * command and no page script, epoch ms: a harness read inside the body + * (`waitForAnimation` describing every animation's keyframes) or + * `blockMainThread(page, ms)` without `during`. + */ + readonly devtoolsTasks: readonly { + readonly start: number; + readonly durationMs: number; + }[]; /** Every counted rendering event with its epoch-ms time, for sub-window counts. */ readonly events: readonly {readonly name: string; readonly t: number}[]; /** The window, epoch ms on the page's clock. */ @@ -146,9 +170,34 @@ export async function traceRendering( counted.push({name: e.name, t: toEpochMs(e.ts)}); } } - const longTasks = events - .filter(e => onMain(e) && e.name === 'RunTask' && (e.dur ?? 0) > 50_000) - .map(e => ({start: toEpochMs(e.ts), durationMs: (e.dur ?? 0) / 1000})); + // A task that ran a DevTools command received it over the DevTools + // session's pipe; page script it set off shows as its listener, timer or + // frame callback. + const main = events.filter(e => onMain(e) && e.ph !== 'E'); + const longTasks: {start: number; durationMs: number}[] = []; + const devtoolsTasks: {start: number; durationMs: number}[] = []; + for (const task of main) { + if (task.name !== 'RunTask' || (task.dur ?? 0) <= 50_000) { + continue; + } + const end = task.ts + (task.dur ?? 0); + const inside = main.filter( + e => e !== task && e.ts >= task.ts && e.ts <= end, + ); + const devtools = inside.some( + e => + e.name === 'Receive mojo message' && + ( + e.args?.chrome_mojo_event_info as + {mojo_interface_tag?: unknown} | undefined + )?.mojo_interface_tag === 'blink.mojom.DevToolsSession', + ); + const pageScript = inside.some(e => PAGE_SCRIPT_EVENTS.has(e.name)); + (devtools && !pageScript ? devtoolsTasks : longTasks).push({ + start: toEpochMs(task.ts), + durationMs: (task.dur ?? 0) / 1000, + }); + } return { counts, layout: counts.Layout ?? 0, @@ -157,6 +206,7 @@ export async function traceRendering( prePaint: counts.PrePaint ?? 0, durationMs: (end.ts - start.ts) / 1000, longTasks, + devtoolsTasks, events: counted, window: {from: fromEpoch, to: toEpoch}, }; @@ -235,8 +285,8 @@ const LONG_TASK_SCRIPT = `(() => { * Chromium), through the Long Tasks API. That sees the tasks of the page's own * event loop: a timer, an event handler, `blockMainThread(page, ms, {during})`. * A DevTools `page.evaluate`, such as `blockMainThread(page, ms)` without - * `during`, is not one of them and is never recorded here; the trace's - * `longTasks` (`traceRendering`) counts every main-thread task over 50 ms. + * `during`, is not one of them and is never recorded here; `traceRendering` + * reports it in `devtoolsTasks`, apart from the page's `longTasks`. */ export async function installLongTaskLog(page: Page): Promise { await page.addInitScript(LONG_TASK_SCRIPT); From efdffbed37188d9970d8436d7f241ef233297383 Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Mon, 26 Oct 2026 02:11:10 -0700 Subject: [PATCH 63/93] refactor(interaction): the animation-state waits move to their own pull request This pull request keeps curve sampling (hold, seek, sample, release, the curve end); the recorder, waitForAnimation, watchAnimations and waitForRest, with their self-tests, land separately on top. The holding self-tests that waited with them now poll the page for a running or idle animation. Chromium: 19 tests, 5x repeat, pass. --- internal/interaction-harness/README.md | 28 +- internal/interaction-harness/src/curves.ts | 449 +----------------- .../src/harness.interaction.browser.spec.ts | 363 ++------------ 3 files changed, 55 insertions(+), 785 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index a55cc41603655..cd53a4b9ca8d8 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -26,7 +26,7 @@ pnpm test:interaction # one spec, both engines | ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact final-move speed, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. | | `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | -| `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t), `waitForAnimation`, `watchAnimations`, `waitForRest`, `releaseAnimations`. | +| `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t), `releaseAnimations`. | | `math.ts` | Oracles independent of the code under test: `cubicBezier`, `timingFunction`, `springStep` (SwiftUI response / dampingFraction, initial velocity), `projectDecay`, `rubberBand`, `parseTransform`, `compareCurve`, `timeline`. | | `story.ts` | `phone`, `withStorybook`, `openStory`, `frames`. | @@ -54,28 +54,14 @@ budget (for a spring baked into `linear()` stops of `h` ms, chords err by up to inside it, the last frame it presents, and the sample says so (`presentedAt`, where oracles are read). An animation whose keyframes end in a run of identical values (an end value held so a later interruption can read it) -ends where that run starts: `curveEndMs` and `waitForAnimation`'s interval -stop there, and `waitForRest`, which judges the -presented `transform`, `translate`, `rotate`, `scale` and `opacity` rather -than `playState`, does not wait the hold out. That holds for an animation that -plays forward under a linear or cubic-bézier effect easing (the time the run -starts is solved from the curve); under any other effect easing (steps, -`linear()` stops, an overshoot) or another direction the curve is read to its -full end, and `waitForRest` waits the hold out. An animation running backward (`reverse()`, a -negative playback rate) is reported by `waitForAnimation` from when it -started, but `holdAnimations` refuses it: seeking and sampling run a curve +ends where that run starts: `curveEndMs` stops there. That holds for an +animation that plays forward under a linear or cubic-bézier effect easing (the +time the run starts is solved from the curve); under any other effect easing +(steps, `linear()` stops, an overshoot) or another direction the curve is read +to its full end. `holdAnimations` refuses an animation running backward +(`reverse()`, a negative playback rate): seeking and sampling run a curve forward. -**Waiting for an animation.** The page records every animation from the first -call into `curves.ts` on it (and on every later document): a frame loop on the -page's own `requestAnimationFrame`, the CSS animation and transition events, -and `Element.animate` as it is called, pending animations included. So -`waitForAnimation` also reports one that started after the last -`waitForRest` or `waitForAnimation` and has already finished (`finishedAt` -set): a short entrance no longer races the first look on a cold page. When the -action may be the page's first contact with the harness, take -`watchAnimations(locator)` before it and read `watch.started()` after. - ## Engine limits (measured with Playwright 1.61) | | Chromium 149 | WebKit 26.5 (Linux build) | diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index 90522f5d7a42c..394e285915ba9 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -10,8 +10,7 @@ * its offset), `sampleCurve` (the presented * transform/opacity at each t, with the individual `translate`, `rotate` and * `scale` properties composed in), `releaseAnimations`, `curveOf`, which - * picks one channel out of the samples for ./math's `compareCurve`, and the - * animation-state waits `waitForAnimation` and `waitForRest`. + * picks one channel out of the samples for ./math's `compareCurve`. * @position Curve sampling: the exact presented value of an animation at a * fixed time, independent of frame timing, so a curve can be compared with * the UIKit reference to a fraction of a pixel. A paused animation's effect @@ -32,139 +31,7 @@ export const END_INSET_MS = 0.5; * forward under a linear or cubic-bézier effect easing (the time the run * starts is solved from the curve); any other ends at its end time. */ -/** - * Page side: records every animation the document runs, from its first frame - * or its creation (a frame loop on the page's real requestAnimationFrame, and - * the CSS animation and transition events), so an animation that starts and - * ends before a test looks is still seen, with the time it finished. Each - * entry keeps the animation's start time too: a page that cancels a finished - * animation (an entrance whose class is removed on `animationend`) nulls its - * `startTime`. - */ -function animationRecorder(): void { - const w = window as unknown as {__harnessAnimations?: unknown}; - if (w.__harnessAnimations) { - return; - } - const frame = window.requestAnimationFrame.bind(window); - const seen = new WeakSet(); - const log: { - animation: Animation; - seenAt: number; - finishedAt: number | null; - startedAt: number | null; - }[] = []; - // Entries whose start time has not resolved yet (still pending when seen). - let unstarted: (typeof log)[number][] = []; - let armedAt = performance.now(); - // When the animation began to play, on the document timeline. A run going - // backward reports as its startTime the moment its current time would - // reach 0, a whole run after it began (it begins at its end). - const playStart = (animation: Animation): number | null => { - if (animation.startTime == null) { - return null; - } - const rate = animation.playbackRate; - if (rate >= 0) { - return Number(animation.startTime); - } - const end = Number( - (animation.effect as KeyframeEffect | null)?.getComputedTiming() - .endTime ?? 0, - ); - return Number(animation.startTime) - end / Math.abs(rate); - }; - const remember = (animation: Animation) => { - // A running animation is recorded while still pending: on a cold page the - // next frame can come only after a short animation has finished. - if (seen.has(animation) || animation.playState !== 'running') { - return; - } - seen.add(animation); - const entry = { - animation, - seenAt: performance.now(), - finishedAt: null as number | null, - startedAt: playStart(animation), - }; - if (entry.startedAt == null) { - unstarted.push(entry); - } - animation.addEventListener('finish', event => { - entry.finishedAt = performance.now(); - if (entry.startedAt == null) { - // Cancelled before a frame read its start: back from where it ended. - const end = Number( - (animation.effect as KeyframeEffect | null)?.getComputedTiming() - .endTime ?? 0, - ); - const at = (event as AnimationPlaybackEvent).timelineTime; - entry.startedAt = - playStart(animation) ?? - (at == null ? entry.finishedAt : Number(at)) - - end / Math.abs(animation.playbackRate || 1); - } - }); - log.push(entry); - if (log.length > 5000) { - log.splice(0, log.length - 5000); - } - }; - const record = () => { - for (const animation of document.getAnimations()) { - remember(animation); - } - unstarted = unstarted.filter(entry => { - if (entry.startedAt == null && entry.animation.startTime != null) { - entry.startedAt = playStart(entry.animation); - } - return entry.startedAt == null && entry.animation.playState !== 'idle'; - }); - }; - const tick = () => { - record(); - frame(tick); - }; - // What already runs when the recorder starts (a spinner on a page the - // harness touches for the first time) is seen now, before any watch arms; - // first seen a frame later, it would read as started by the watched action. - record(); - frame(tick); - for (const type of ['animationstart', 'transitionrun', 'transitionstart']) { - document.addEventListener(type, record, true); - } - // A script animation is recorded as it is made: WebKit can start and finish - // a short one between two frames of a fresh page. - const animate = Element.prototype.animate; - Element.prototype.animate = function ( - this: Element, - ...args: Parameters - ) { - const animation = animate.apply(this, args); - remember(animation); - return animation; - }; - w.__harnessAnimations = { - log, - record, - arm: () => (armedAt = performance.now()), - armedAt: () => armedAt, - }; -} - -const recordingPages = new WeakSet(); - -/** Starts the recorder now and on every later document of the page. */ -async function installRecorder(page: Page): Promise { - if (!recordingPages.has(page)) { - recordingPages.add(page); - await page.addInitScript(animationRecorder); - } - await page.evaluate(animationRecorder); -} - async function installCurveEnd(page: Page): Promise { - await installRecorder(page); await page.evaluate(() => { const w = window as unknown as { __harnessCurveEnd?: (animation: Animation) => number; @@ -776,317 +643,3 @@ export function curveOf( value: channel === 'opacity' ? s.opacity : s.transform[channel], })); } - -export interface ActiveAnimation { - readonly kind: string; - readonly name: string | null; - /** - * Epoch ms (page clock) when the animation's curve starts and ends. A held - * end value (see `HeldAnimation.curveEndMs`, which says when one is read) - * is not part of the curve. - */ - readonly from: number; - readonly to: number; - readonly durationMs: number; - readonly easing: string; - readonly keyframes: readonly Record[]; - /** - * Epoch ms (page clock) when it finished, if it had already finished when - * it was reported; null while it runs. - */ - readonly finishedAt: number | null; -} - -/** - * Waits until an animation is running on `locator`'s element (or inside it) - * and has presented `minProgressMs` of its curve (after any delay), and - * returns its active - * interval on the page clock. Use it before blocking the main thread: an - * animation that has not started yet cannot start while the thread is blocked. - * - * It does not race a short animation: the page records every animation from - * its first frame (from the first call into this module on the page: this, - * `waitForRest`, `watchAnimations`, sampling or holding), so one that started - * after the last `waitForRest` or `waitForAnimation` and has already - * finished is reported too, with `finishedAt` set. For an action that may be - * the page's first contact with this module, take `watchAnimations(locator)` - * before the action instead. - */ -export async function waitForAnimation( - locator: Locator, - { - timeout = 5_000, - minProgressMs = 1, - since, - }: { - timeout?: number; - minProgressMs?: number; - /** Page-clock ms (performance.now): only animations first seen after it. */ - since?: number; - } = {}, -): Promise { - const deadline = Date.now() + timeout; - await installCurveEnd(locator.page()); - for (;;) { - const found = await locator.evaluate( - (element, {minProgress, since: watchedFrom}) => { - const w = window as unknown as { - __harnessCurveEnd: (a: Animation) => number; - __harnessAnimations: { - log: { - animation: Animation; - seenAt: number; - finishedAt: number | null; - startedAt: number | null; - }[]; - record: () => void; - armedAt: () => number; - }; - }; - const curveEnd = w.__harnessCurveEnd; - const recorder = w.__harnessAnimations; - recorder.record(); - const describe = ( - a: Animation, - finishedAt: number | null, - recordedStart: number | null, - ) => { - const effect = a.effect as KeyframeEffect | null; - const timing = effect?.getComputedTiming(); - // The recorded start stands in once the page has cancelled it. A - // run going backward began at its end: its startTime is where its - // current time would reach 0, a whole run later. - const rate = a.playbackRate; - const speed = Math.abs(rate || 1); - const endTime = Number(timing?.endTime ?? 0); - const delay = Number(timing?.delay ?? 0); - const start = - a.startTime == null - ? recordedStart - : rate < 0 - ? Number(a.startTime) - endTime / speed - : Number(a.startTime); - const startedAt = - performance.timeOrigin + Number(start ?? Number.NaN); - const name = - 'transitionProperty' in a - ? (a as unknown as {transitionProperty: string}) - .transitionProperty - : 'animationName' in a - ? (a as unknown as {animationName: string}).animationName - : null; - return { - kind: a.constructor.name, - name, - // Through the playback rate's speed: at rate 2 a curve runs in half - // its duration. Backward, the end delay plays first and the start - // delay last. - from: - rate < 0 - ? startedAt + Number(timing?.endDelay ?? 0) / speed - : startedAt + delay / speed, - to: - rate < 0 - ? startedAt + (endTime - delay) / speed - : startedAt + curveEnd(a) / speed, - durationMs: Number(timing?.duration ?? 0), - easing: String(effect?.getTiming().easing ?? 'linear'), - keyframes: (effect?.getKeyframes() ?? []).map( - (k): Record => ({...k}), - ), - finishedAt: - finishedAt == null ? null : performance.timeOrigin + finishedAt, - }; - }; - const inside = (a: Animation) => { - const target = (a.effect as KeyframeEffect | null)?.target ?? null; - return ( - target != null && (target === element || element.contains(target)) - ); - }; - const after = watchedFrom ?? recorder.armedAt(); - const recorded = recorder.log.filter( - e => e.seenAt >= after && inside(e.animation), - ); - // Running: since the watch, or (without one) any on the element. - const running = ( - watchedFrom == null - ? element.getAnimations({subtree: true}) - : recorded.filter(e => e.finishedAt == null).map(e => e.animation) - ).filter(a => a.playState === 'running' && a.startTime != null); - // Progress along the curve, on the timeline: a delay presents nothing - // (going backward, the end delay plays first), a run going backward - // (`reverse()`) progresses from its end, and a faster rate presents - // more curve per ms. - const progressed = running.filter(a => { - const timing = a.effect?.getComputedTiming(); - const time = Number(a.currentTime ?? 0); - const local = - a.playbackRate < 0 - ? Number(timing?.endTime ?? 0) - - Number(timing?.endDelay ?? 0) - - time - : time - Number(timing?.delay ?? 0); - return local / Math.abs(a.playbackRate || 1) >= minProgress; - }); - if (progressed.length > 0) { - return progressed.map(a => describe(a, null, null)); - } - if (running.length > 0) { - return []; - } - // Finished: by its finish event, or by its state when the event has not - // been delivered yet (then at its scheduled end, through the rate's - // speed; a run going backward ends at its startTime, where its current - // time reaches 0). - const finishedAt = (e: (typeof recorded)[number]) => { - if (e.finishedAt != null) { - return e.finishedAt; - } - const a = e.animation; - if (a.playState !== 'finished' || a.startTime == null) { - return null; - } - const end = Number( - (a.effect as KeyframeEffect | null)?.getComputedTiming().endTime ?? - 0, - ); - return a.playbackRate < 0 - ? Number(a.startTime) - : Number(a.startTime) + end / Math.abs(a.playbackRate || 1); - }; - return recorded - .filter(e => finishedAt(e) != null) - .map(e => describe(e.animation, finishedAt(e), e.startedAt)); - }, - {minProgress: minProgressMs, since: since ?? null}, - ); - if (found.length > 0) { - await armRecorder(locator.page()); - return found; - } - if (Date.now() > deadline) { - throw new Error( - 'waitForAnimation: no animation started on the element, none running ' + - 'and none recorded since the last wait. If the action is the ' + - "page's first contact with the harness, take " + - 'watchAnimations(locator) before it.', - ); - } - await new Promise(resolve => setTimeout(resolve, 4)); - } -} - -/** Marks now as the point later waits count animations from. */ -async function armRecorder(page: Page): Promise { - return page.evaluate(() => - ( - window as unknown as {__harnessAnimations: {arm: () => number}} - ).__harnessAnimations.arm(), - ); -} - -export interface AnimationWatch { - /** - * Resolves with the animations on the element that started after the watch - * was taken: running ones once they have presented `minProgressMs`, or, - * when none runs, those that already finished. - */ - started(options?: { - timeout?: number; - minProgressMs?: number; - }): Promise; -} - -/** - * Starts watching `locator`'s element before an action, so the animation the - * action starts is caught even when it ends before the test looks (a short - * entrance on a cold page): - * - * const watch = await watchAnimations(sheet); - * await driver.play(tap(button)); - * const [entrance] = await watch.started(); - */ -export async function watchAnimations( - locator: Locator, -): Promise { - await installCurveEnd(locator.page()); - const since = await armRecorder(locator.page()); - return { - started: options => waitForAnimation(locator, {...options, since}), - }; -} - -/** - * Resolves once `locator`'s element is at rest: no animation on it (or inside - * it) is pending or still before the end of its curve, and its presented - * `transform`, `translate`, `rotate`, `scale` and `opacity` (and `::backdrop` - * opacity) have held still for `frames` consecutive frames. - * - * It judges curves and presented values, not `playState`: an animation that - * holds its end value (a run of identical final keyframes kept so a later - * interruption can read it) still reports `running` while nothing moves. - * A hold is read as such only where `HeldAnimation.curveEndMs` reads it (a - * forward run under a linear or cubic-bézier effect easing); under any other - * easing or direction, waitForRest waits the hold out, and a hold longer than - * `timeout` rejects with "still moving". - */ -export async function waitForRest( - locator: Locator, - {timeout = 5_000, frames = 3}: {timeout?: number; frames?: number} = {}, -): Promise { - await installCurveEnd(locator.page()); - await locator.evaluate( - (element, {ms, need}) => - new Promise((resolve, reject) => { - const deadline = performance.now() + ms; - const curveEnd = ( - window as unknown as {__harnessCurveEnd: (a: Animation) => number} - ).__harnessCurveEnd; - const read = () => { - const style = getComputedStyle(element); - const backdrop = getComputedStyle(element, '::backdrop').opacity; - return [ - style.transform, - style.translate, - style.rotate, - style.scale, - style.opacity, - backdrop, - ].join(' | '); - }; - const frame = requestAnimationFrame; - let last = read(); - let still = 0; - const tick = () => { - const now = read(); - const moving = element.getAnimations({subtree: true}).some( - a => - a.pending || - (a.playState === 'running' && - // Going backward the curve ends at the start delay, after - // the end delay (which plays first) and the curve itself. - (a.playbackRate < 0 - ? Number(a.currentTime ?? 0) > - Number(a.effect?.getComputedTiming().delay ?? 0) - : Number(a.currentTime ?? 0) < curveEnd(a))), - ); - still = !moving && now === last ? still + 1 : 0; - last = now; - if (still >= need) { - // Later waits count animations from here. - ( - window as unknown as {__harnessAnimations?: {arm: () => number}} - ).__harnessAnimations?.arm(); - resolve(); - } else if (performance.now() > deadline) { - reject(new Error(`waitForRest: still moving (${now})`)); - } else { - frame(tick); - } - }; - frame(tick); - }), - {ms: timeout, need: frames}, - ); -} diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 6e806d693488d..21417851b0403 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -6,10 +6,8 @@ * @output Proof that each harness instrument measures what it claims: * scripted timestamps reach the page; a flick flings natively (Chromium); * a second finger and a pinch arrive as distinct pointers; sampled curves - * match the oracles; the animation-state waits report a running transition - * and its end, and judge rest by presented values even while an animation - * holds its end, and a watch taken on first contact ignores an animation - * already running and keeps the start of one the page cancels when it ends; individual transform properties are sampled; a sample at + * match the oracles; a held end is not part of the curve, and individual + * transform properties are sampled; a sample at * an end reports the time it was read at; holding takes only running * document-timeline animations, each at its own offset, holds each where it * was read, adds a second hold to the held set without moving the first @@ -18,7 +16,7 @@ * instruments; this file is why it may. */ -import {expect, test, type Page} from '@playwright/test'; +import {expect, test, type Locator, type Page} from '@playwright/test'; import { compareCurve, cubicBezier, @@ -28,7 +26,6 @@ import { holdAnimations, installInputLog, merge, - parseTransform, PARTIAL_LIFT_SKIP, pinch, readInputLog, @@ -39,9 +36,6 @@ import { tap, timeline, touchDriver, - waitForAnimation, - waitForRest, - watchAnimations, withFinger, } from './index'; @@ -67,6 +61,37 @@ async function fixture(page: Page) { await page.goto('http://harness.test/'); } +/** Until an animation on the element runs with a start time, `ms` into it. */ +async function untilRunning(locator: Locator, ms = 0) { + await expect + .poll(() => + locator.evaluate( + (el, min) => + el + .getAnimations() + .some( + a => + a.playState === 'running' && + a.startTime != null && + Number(a.currentTime ?? 0) >= min, + ), + ms, + ), + ) + .toBe(true); +} + +/** Until nothing on the element is running. */ +async function untilIdle(locator: Locator) { + await expect + .poll(() => + locator.evaluate(el => + el.getAnimations().some(a => a.playState === 'running'), + ), + ) + .toBe(false); +} + test.describe('touch input', () => { test('the page receives trusted touches and pointers at the scripted positions and times', async ({ page, @@ -510,34 +535,12 @@ test.describe('holding', () => { await releaseAnimations(page); }); - test('a reversed run counts its progress from its end, and a faster playback rate is read on the timeline by waits, seeks and samples', async ({ + test('a faster playback rate is read on the timeline by seeks and samples', async ({ page, }) => { await fixture(page); const box = page.locator('#box'); - // Reversed, the run starts at its end: progress counts down from there, - // so a wait for 100 ms of it returns once the run has presented that much. - const watch = await watchAnimations(box); - await box.evaluate(el => { - el.animate([{translate: '0px 0px'}, {translate: '100px 0px'}], { - duration: 1000, - easing: 'linear', - }).reverse(); - }); - await watch.started({minProgressMs: 100}); - const presented = await box.evaluate(el => { - const [animation] = el.getAnimations(); - return ( - Number(animation.effect?.getComputedTiming().endTime ?? 0) - - Number(animation.currentTime ?? 0) - ); - }); - expect(presented).toBeGreaterThanOrEqual(100); - await box.evaluate(el => el.getAnimations().forEach(a => a.finish())); - - // At playback rate 2 a 1,000 ms curve runs 500 ms: its interval, and a - // seek or sample at t reads the curve at 2t. - const fast = await watchAnimations(box); + // At playback rate 2 a seek or sample at t reads the curve at 2t. await box.evaluate(el => { const animation = el.animate( [{translate: '0px 0px'}, {translate: '100px 0px'}], @@ -545,8 +548,6 @@ test.describe('holding', () => { ); animation.playbackRate = 2; }); - const [running] = await fast.started(); - expect(running.to - running.from).toBeCloseTo(500, 0); await holdAnimations(page, box); await seekAnimations(page, 125); const x = await box.evaluate( @@ -609,11 +610,9 @@ test.describe('holding', () => { expect(reversed.curveEndMs).toBeCloseTo(1000, 0); await releaseAnimations(page, {finish: true}); - // One reversed by `reverse()` (a negative playback rate) is reported as - // starting when it did, running its whole curve, and is refused by a - // hold, which seeks curves forward only. - const watch = await watchAnimations(box); - const readyAt = await box.evaluate(async el => { + // One reversed by `reverse()` (a negative playback rate) is refused by + // a hold, which seeks curves forward only. + await box.evaluate(async el => { const animation = el.animate( [ {translate: '0px 0px', offset: 0}, @@ -624,11 +623,9 @@ test.describe('holding', () => { ); animation.reverse(); await animation.ready; - return performance.timeOrigin + performance.now(); }); - const [running] = await watch.started(); - expect(Math.abs(running.from - readyAt)).toBeLessThan(100); - expect(running.to - running.from).toBeCloseTo(1000, -1); + // Chromium lists a reversed run in getAnimations() a frame after ready. + await untilRunning(box); await expect(holdAnimations(page, box)).rejects.toThrow(/runs backward/); await box.evaluate(el => el.getAnimations().forEach(a => a.finish())); @@ -680,7 +677,7 @@ test.describe('holding', () => { ); }); const box = page.locator('#box'); - await waitForAnimation(box, {minProgressMs: 50}); + await untilRunning(box, 50); const [held] = await holdAnimations(page, box); // Two frames later, the held animation is still where the hold read it. const after = await box.evaluate( @@ -723,7 +720,7 @@ test.describe('holding', () => { await releaseAnimations(page); // The first hold was released too: the move runs to its end, and rest is // its end, not the frame it was held on. - await waitForRest(box); + await untilIdle(box); expect(await box.evaluate(el => getComputedStyle(el).transform)).toBe( 'matrix(1, 0, 0, 1, 300, 0)', ); @@ -762,7 +759,7 @@ test.describe('holding', () => { }), ); const box = page.locator('#box'); - await waitForAnimation(box); + await untilRunning(box); // The later-started element is held first. const [first] = await holdAnimations(page, box); const [later] = await holdAnimations(page, page.locator('#early')); @@ -786,7 +783,7 @@ test.describe('holding', () => { el.getBoundingClientRect(); el.style.transform = 'translateX(300px)'; }); - await waitForAnimation(box); + await untilRunning(box); expect(await holdAnimations(page, box)).toHaveLength(1); // The page sends the box back: the held transition is cancelled, and a // new one runs from where the box is. @@ -809,7 +806,7 @@ test.describe('holding', () => { await seekAnimations(page, 150); expect(await state()).toEqual({cancelled: 'idle', held: 0}); await releaseAnimations(page); - await waitForRest(box); + await untilIdle(box); expect(await state()).toEqual({cancelled: 'idle', held: 0}); expect(await box.evaluate(el => getComputedStyle(el).transform)).toBe( 'none', @@ -844,259 +841,7 @@ test.describe('holding', () => { }); }); -test.describe('animation state', () => { - test('waitForAnimation reports a running transition and waitForRest resolves after it', async ({ - page, - }) => { - await fixture(page); - const box = page.locator('#box'); - await box.evaluate(el => { - el.style.transition = 'transform 300ms linear'; - void (el as HTMLElement).offsetWidth; - el.style.transform = 'translateY(100px)'; - }); - const [running] = await waitForAnimation(box); - expect(running.kind).toBe('CSSTransition'); - expect(running.name).toBe('transform'); - expect(running.durationMs).toBe(300); - expect(running.to - running.from).toBeCloseTo(300, 0); - await waitForRest(box); - const settledAt = await page.evaluate( - () => performance.timeOrigin + performance.now(), - ); - expect(settledAt).toBeGreaterThanOrEqual(running.to); - const transform = await box.evaluate(el => getComputedStyle(el).transform); - expect(parseTransform(transform).translateY).toBeCloseTo(100, 1); - }); - - test("waitForAnimation's progress starts after a transition's delay", async ({ - page, - }) => { - await fixture(page); - const box = page.locator('#box'); - await box.evaluate(el => { - el.style.transition = 'transform 300ms linear 200ms'; - void (el as HTMLElement).offsetWidth; - el.style.transform = 'translateY(100px)'; - }); - await waitForAnimation(box, {minProgressMs: 50}); - // 50 ms along the curve: past the 200 ms delay, not inside it. - const currentTime = await box.evaluate(el => - Number(el.getAnimations()[0]?.currentTime), - ); - expect(currentTime).toBeGreaterThanOrEqual(250); - await waitForRest(box); - }); - - test('waitForAnimation reports a short animation that ended before it looked', async ({ - page, - }) => { - await fixture(page); - const box = page.locator('#box'); - await waitForRest(box); - await box.evaluate(el => { - el.style.transition = 'transform 60ms linear'; - void (el as HTMLElement).offsetWidth; - el.style.transform = 'translateY(40px)'; - }); - // A cold page answers late: the transition is over before the first look. - await page.waitForTimeout(250); - const [ended] = await waitForAnimation(box); - expect(ended.kind).toBe('CSSTransition'); - expect(ended.durationMs).toBe(60); - expect(ended.finishedAt).not.toBeNull(); - expect(ended.finishedAt ?? 0).toBeGreaterThanOrEqual(ended.to - 1); - }); - - test("a watch taken before the action catches an entrance on the page's first contact", async ({ - page, - }) => { - await fixture(page); - const box = page.locator('#box'); - const watch = await watchAnimations(box); - await box.evaluate(el => - el.animate([{opacity: 0}, {opacity: 1}], {duration: 50}), - ); - await page.waitForTimeout(250); - const [entrance] = await watch.started(); - expect(entrance.kind).toBe('Animation'); - expect(entrance.durationMs).toBe(50); - expect(entrance.finishedAt).not.toBeNull(); - }); - - test("a watch taken on the page's first contact ignores an animation already running", async ({ - page, - }) => { - await fixture(page); - const box = page.locator('#box'); - // An endless spinner, started before the harness has touched the page. - await box.evaluate(el => - el.animate([{rotate: '0deg'}, {rotate: '360deg'}], { - duration: 1000, - iterations: Infinity, - }), - ); - const watch = await watchAnimations(box); - await expect(watch.started({timeout: 300})).rejects.toThrow( - /no animation started/, - ); - await box.evaluate(el => - el.animate([{opacity: 1}, {opacity: 0.5}], {duration: 300}), - ); - const started = await watch.started(); - expect(started.map(a => a.durationMs)).toEqual([300]); - }); - - test('a watch reports only what started after it', async ({page}) => { - await fixture(page); - const box = page.locator('#box'); - await box.evaluate(el => - el.animate([{opacity: 0}, {opacity: 1}], {duration: 40}), - ); - await page.waitForTimeout(150); - const watch = await watchAnimations(box); - await expect(watch.started({timeout: 300})).rejects.toThrow( - /no animation started/, - ); - await box.evaluate(el => - el.animate([{opacity: 1}, {opacity: 0.5}], {duration: 70}), - ); - const started = await watch.started(); - expect(started.map(a => a.durationMs)).toEqual([70]); - }); - - test('an entrance the page cancels when it ends keeps its start', async ({ - page, - }) => { - await fixture(page); - const box = page.locator('#box'); - await page.addStyleTag({ - content: - '@keyframes enter { from { opacity: 0 } to { opacity: 1 } }' + - ' .entering { animation: enter 60ms linear }', - }); - const watch = await watchAnimations(box); - // The page's time origin, where a lost start lands, is well before the - // trigger. - await page.waitForTimeout(200); - // Removing the class on animationend cancels the finished CSS animation, - // which nulls its startTime. - const triggeredAt = await box.evaluate(el => { - el.addEventListener( - 'animationend', - () => el.classList.remove('entering'), - {once: true}, - ); - el.classList.add('entering'); - return performance.timeOrigin + performance.now(); - }); - await expect - .poll(() => box.evaluate(el => el.classList.contains('entering'))) - .toBe(false); - const [entrance] = await watch.started(); - expect(entrance.kind).toBe('CSSAnimation'); - expect(entrance.finishedAt).not.toBeNull(); - // It started on a frame after the trigger, not at the page's time origin. - expect(entrance.from).toBeGreaterThan(triggeredAt - 17); - expect(entrance.from).toBeLessThan(triggeredAt + 500); - expect(entrance.to - entrance.from).toBeCloseTo(60, 3); - expect(entrance.finishedAt ?? 0).toBeGreaterThanOrEqual(entrance.to - 1); - }); - - test('a run going backward is dated from when it began: with a delay, ended before the look, or cancelled when it ends', async ({ - page, - }) => { - await fixture(page); - const box = page.locator('#box'); - const frames = [ - {translate: '0px 0px', offset: 0}, - {translate: '100px 0px', offset: 1}, - ]; - - // A 300 ms delay plays last when going backward: the curve starts at once. - let watch = await watchAnimations(box); - let readyAt = await box.evaluate(async (el, keyframes) => { - const animation = el.animate(keyframes, {duration: 400, delay: 300}); - animation.reverse(); - await animation.ready; - return performance.timeOrigin + performance.now(); - }, frames); - const [delayed] = await watch.started(); - expect(Math.abs(delayed.from - readyAt)).toBeLessThan(50); - expect(delayed.to - delayed.from).toBeCloseTo(400, 0); - await box.evaluate(el => el.getAnimations().forEach(a => a.finish())); - - // A short one that ended before the look keeps its curve inside its life. - watch = await watchAnimations(box); - readyAt = await box.evaluate(async (el, keyframes) => { - const animation = el.animate(keyframes, {duration: 60}); - animation.reverse(); - await animation.ready; - return performance.timeOrigin + performance.now(); - }, frames); - await page.waitForTimeout(200); - const [ended] = await watch.started(); - expect(ended.finishedAt).not.toBeNull(); - expect(Math.abs(ended.from - readyAt)).toBeLessThan(50); - expect(ended.to).toBeLessThanOrEqual((ended.finishedAt ?? 0) + 1); - - // One the page cancels when it ends keeps its start. - watch = await watchAnimations(box); - readyAt = await box.evaluate(async (el, keyframes) => { - const animation = el.animate(keyframes, {duration: 60}); - animation.addEventListener('finish', () => animation.cancel(), { - once: true, - }); - animation.reverse(); - await animation.ready; - return performance.timeOrigin + performance.now(); - }, frames); - await page.waitForTimeout(200); - const [cancelled] = await watch.started(); - expect(Math.abs(cancelled.from - readyAt)).toBeLessThan(50); - expect(cancelled.to - cancelled.from).toBeCloseTo(60, 0); - - // Read right after finish(), before the finish event is delivered: the - // scheduled end goes through the rate (backward, it is the startTime). - for (const rate of [-1, 2]) { - watch = await watchAnimations(box); - await box.evaluate( - async (el, [keyframes, playbackRate]) => { - const animation = el.animate(keyframes as Keyframe[], { - duration: 1000, - }); - animation.playbackRate = playbackRate as number; - await animation.ready; - animation.finish(); - }, - [frames, rate] as const, - ); - const [read] = await watch.started(); - expect(read.finishedAt).not.toBeNull(); - expect(Math.abs((read.finishedAt ?? 0) - read.to)).toBeLessThan(50); - } - - // Going backward, an end delay presents nothing: a run still inside it - // has not progressed. - await box.evaluate((el, keyframes) => { - const animation = el.animate(keyframes, {duration: 400, endDelay: 500}); - animation.reverse(); - }, frames); - await expect( - waitForAnimation(box, {minProgressMs: 100, timeout: 300}), - ).rejects.toThrow(); - await box.evaluate(el => el.getAnimations().forEach(a => a.finish())); - - // Nor is it rest: waitForRest waits out the end delay and the curve. - const restFrom = await box.evaluate((el, keyframes) => { - el.animate(keyframes, {duration: 600, endDelay: 500}).reverse(); - return performance.now(); - }, frames); - await waitForRest(box, {timeout: 5_000}); - const restAt = await box.evaluate(() => performance.now()); - expect(restAt - restFrom).toBeGreaterThan(1_000); - }); - +test.describe('held ends', () => { test('an end value held by the keyframes is not part of the curve, and individual transform properties are sampled', async ({ page, }) => { @@ -1114,20 +859,6 @@ test.describe('animation state', () => { {duration: 5300, fill: 'forwards'}, ); }); - const startedAt = await page.evaluate(() => performance.now()); - const [active] = await waitForAnimation(box); - expect(active.to - active.from).toBeCloseTo(300, 0); // the curve, not the hold - await waitForRest(box, {timeout: 3_000}); - const after = await box.evaluate(el => ({ - at: performance.now(), - running: el.getAnimations().some(a => a.playState === 'running'), - translate: getComputedStyle(el).translate, - })); - expect(after.running).toBe(true); // still holding its end value - expect(after.at - startedAt).toBeGreaterThanOrEqual(300); - expect(after.at - startedAt).toBeLessThan(1_500); - expect(after.translate).toBe('0px 120px'); - const held = await holdAnimations(page, box); expect(held[0].curveEndMs).toBeCloseTo(300, 0); const [mid, end] = await sampleCurve(box, [150, 300]); From c833823744f9461db425279d54503244af772233 Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Mon, 26 Oct 2026 02:11:45 -0700 Subject: [PATCH 64/93] feat(interaction): animation-state waits for interaction proofs The harness's animation-state instruments, split out of the curve-sampling pull request so each stays reviewable: - a page-side recorder that sees every animation from its first frame or creation, so one that starts and ends before a test looks is still reported, with its finish time and start (kept when the page cancels it); - waitForAnimation: the running animations on an element past a minimum progress, or the ones that finished since the last wait, with their curve interval through the playback rate (a run going backward is dated from when it began); - watchAnimations: a watch taken before an action, for an entrance on a page's first contact; - waitForRest: rest judged by presented transform, translate, rotate, scale and opacity, not playState, so an end value held by the keyframes is not waited out, and a backward run's end delay and curve are. Self-tests cover each in real Chromium. Co-authored-by: Christopher Chedeau --- internal/interaction-harness/README.md | 28 +- internal/interaction-harness/src/curves.ts | 449 +++++++++++++++++- .../src/harness.interaction.browser.spec.ts | 363 ++++++++++++-- 3 files changed, 785 insertions(+), 55 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index cd53a4b9ca8d8..a55cc41603655 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -26,7 +26,7 @@ pnpm test:interaction # one spec, both engines | ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact final-move speed, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. | | `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | -| `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t), `releaseAnimations`. | +| `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t), `waitForAnimation`, `watchAnimations`, `waitForRest`, `releaseAnimations`. | | `math.ts` | Oracles independent of the code under test: `cubicBezier`, `timingFunction`, `springStep` (SwiftUI response / dampingFraction, initial velocity), `projectDecay`, `rubberBand`, `parseTransform`, `compareCurve`, `timeline`. | | `story.ts` | `phone`, `withStorybook`, `openStory`, `frames`. | @@ -54,14 +54,28 @@ budget (for a spring baked into `linear()` stops of `h` ms, chords err by up to inside it, the last frame it presents, and the sample says so (`presentedAt`, where oracles are read). An animation whose keyframes end in a run of identical values (an end value held so a later interruption can read it) -ends where that run starts: `curveEndMs` stops there. That holds for an -animation that plays forward under a linear or cubic-bézier effect easing (the -time the run starts is solved from the curve); under any other effect easing -(steps, `linear()` stops, an overshoot) or another direction the curve is read -to its full end. `holdAnimations` refuses an animation running backward -(`reverse()`, a negative playback rate): seeking and sampling run a curve +ends where that run starts: `curveEndMs` and `waitForAnimation`'s interval +stop there, and `waitForRest`, which judges the +presented `transform`, `translate`, `rotate`, `scale` and `opacity` rather +than `playState`, does not wait the hold out. That holds for an animation that +plays forward under a linear or cubic-bézier effect easing (the time the run +starts is solved from the curve); under any other effect easing (steps, +`linear()` stops, an overshoot) or another direction the curve is read to its +full end, and `waitForRest` waits the hold out. An animation running backward (`reverse()`, a +negative playback rate) is reported by `waitForAnimation` from when it +started, but `holdAnimations` refuses it: seeking and sampling run a curve forward. +**Waiting for an animation.** The page records every animation from the first +call into `curves.ts` on it (and on every later document): a frame loop on the +page's own `requestAnimationFrame`, the CSS animation and transition events, +and `Element.animate` as it is called, pending animations included. So +`waitForAnimation` also reports one that started after the last +`waitForRest` or `waitForAnimation` and has already finished (`finishedAt` +set): a short entrance no longer races the first look on a cold page. When the +action may be the page's first contact with the harness, take +`watchAnimations(locator)` before it and read `watch.started()` after. + ## Engine limits (measured with Playwright 1.61) | | Chromium 149 | WebKit 26.5 (Linux build) | diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index 394e285915ba9..90522f5d7a42c 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -10,7 +10,8 @@ * its offset), `sampleCurve` (the presented * transform/opacity at each t, with the individual `translate`, `rotate` and * `scale` properties composed in), `releaseAnimations`, `curveOf`, which - * picks one channel out of the samples for ./math's `compareCurve`. + * picks one channel out of the samples for ./math's `compareCurve`, and the + * animation-state waits `waitForAnimation` and `waitForRest`. * @position Curve sampling: the exact presented value of an animation at a * fixed time, independent of frame timing, so a curve can be compared with * the UIKit reference to a fraction of a pixel. A paused animation's effect @@ -31,7 +32,139 @@ export const END_INSET_MS = 0.5; * forward under a linear or cubic-bézier effect easing (the time the run * starts is solved from the curve); any other ends at its end time. */ +/** + * Page side: records every animation the document runs, from its first frame + * or its creation (a frame loop on the page's real requestAnimationFrame, and + * the CSS animation and transition events), so an animation that starts and + * ends before a test looks is still seen, with the time it finished. Each + * entry keeps the animation's start time too: a page that cancels a finished + * animation (an entrance whose class is removed on `animationend`) nulls its + * `startTime`. + */ +function animationRecorder(): void { + const w = window as unknown as {__harnessAnimations?: unknown}; + if (w.__harnessAnimations) { + return; + } + const frame = window.requestAnimationFrame.bind(window); + const seen = new WeakSet(); + const log: { + animation: Animation; + seenAt: number; + finishedAt: number | null; + startedAt: number | null; + }[] = []; + // Entries whose start time has not resolved yet (still pending when seen). + let unstarted: (typeof log)[number][] = []; + let armedAt = performance.now(); + // When the animation began to play, on the document timeline. A run going + // backward reports as its startTime the moment its current time would + // reach 0, a whole run after it began (it begins at its end). + const playStart = (animation: Animation): number | null => { + if (animation.startTime == null) { + return null; + } + const rate = animation.playbackRate; + if (rate >= 0) { + return Number(animation.startTime); + } + const end = Number( + (animation.effect as KeyframeEffect | null)?.getComputedTiming() + .endTime ?? 0, + ); + return Number(animation.startTime) - end / Math.abs(rate); + }; + const remember = (animation: Animation) => { + // A running animation is recorded while still pending: on a cold page the + // next frame can come only after a short animation has finished. + if (seen.has(animation) || animation.playState !== 'running') { + return; + } + seen.add(animation); + const entry = { + animation, + seenAt: performance.now(), + finishedAt: null as number | null, + startedAt: playStart(animation), + }; + if (entry.startedAt == null) { + unstarted.push(entry); + } + animation.addEventListener('finish', event => { + entry.finishedAt = performance.now(); + if (entry.startedAt == null) { + // Cancelled before a frame read its start: back from where it ended. + const end = Number( + (animation.effect as KeyframeEffect | null)?.getComputedTiming() + .endTime ?? 0, + ); + const at = (event as AnimationPlaybackEvent).timelineTime; + entry.startedAt = + playStart(animation) ?? + (at == null ? entry.finishedAt : Number(at)) - + end / Math.abs(animation.playbackRate || 1); + } + }); + log.push(entry); + if (log.length > 5000) { + log.splice(0, log.length - 5000); + } + }; + const record = () => { + for (const animation of document.getAnimations()) { + remember(animation); + } + unstarted = unstarted.filter(entry => { + if (entry.startedAt == null && entry.animation.startTime != null) { + entry.startedAt = playStart(entry.animation); + } + return entry.startedAt == null && entry.animation.playState !== 'idle'; + }); + }; + const tick = () => { + record(); + frame(tick); + }; + // What already runs when the recorder starts (a spinner on a page the + // harness touches for the first time) is seen now, before any watch arms; + // first seen a frame later, it would read as started by the watched action. + record(); + frame(tick); + for (const type of ['animationstart', 'transitionrun', 'transitionstart']) { + document.addEventListener(type, record, true); + } + // A script animation is recorded as it is made: WebKit can start and finish + // a short one between two frames of a fresh page. + const animate = Element.prototype.animate; + Element.prototype.animate = function ( + this: Element, + ...args: Parameters + ) { + const animation = animate.apply(this, args); + remember(animation); + return animation; + }; + w.__harnessAnimations = { + log, + record, + arm: () => (armedAt = performance.now()), + armedAt: () => armedAt, + }; +} + +const recordingPages = new WeakSet(); + +/** Starts the recorder now and on every later document of the page. */ +async function installRecorder(page: Page): Promise { + if (!recordingPages.has(page)) { + recordingPages.add(page); + await page.addInitScript(animationRecorder); + } + await page.evaluate(animationRecorder); +} + async function installCurveEnd(page: Page): Promise { + await installRecorder(page); await page.evaluate(() => { const w = window as unknown as { __harnessCurveEnd?: (animation: Animation) => number; @@ -643,3 +776,317 @@ export function curveOf( value: channel === 'opacity' ? s.opacity : s.transform[channel], })); } + +export interface ActiveAnimation { + readonly kind: string; + readonly name: string | null; + /** + * Epoch ms (page clock) when the animation's curve starts and ends. A held + * end value (see `HeldAnimation.curveEndMs`, which says when one is read) + * is not part of the curve. + */ + readonly from: number; + readonly to: number; + readonly durationMs: number; + readonly easing: string; + readonly keyframes: readonly Record[]; + /** + * Epoch ms (page clock) when it finished, if it had already finished when + * it was reported; null while it runs. + */ + readonly finishedAt: number | null; +} + +/** + * Waits until an animation is running on `locator`'s element (or inside it) + * and has presented `minProgressMs` of its curve (after any delay), and + * returns its active + * interval on the page clock. Use it before blocking the main thread: an + * animation that has not started yet cannot start while the thread is blocked. + * + * It does not race a short animation: the page records every animation from + * its first frame (from the first call into this module on the page: this, + * `waitForRest`, `watchAnimations`, sampling or holding), so one that started + * after the last `waitForRest` or `waitForAnimation` and has already + * finished is reported too, with `finishedAt` set. For an action that may be + * the page's first contact with this module, take `watchAnimations(locator)` + * before the action instead. + */ +export async function waitForAnimation( + locator: Locator, + { + timeout = 5_000, + minProgressMs = 1, + since, + }: { + timeout?: number; + minProgressMs?: number; + /** Page-clock ms (performance.now): only animations first seen after it. */ + since?: number; + } = {}, +): Promise { + const deadline = Date.now() + timeout; + await installCurveEnd(locator.page()); + for (;;) { + const found = await locator.evaluate( + (element, {minProgress, since: watchedFrom}) => { + const w = window as unknown as { + __harnessCurveEnd: (a: Animation) => number; + __harnessAnimations: { + log: { + animation: Animation; + seenAt: number; + finishedAt: number | null; + startedAt: number | null; + }[]; + record: () => void; + armedAt: () => number; + }; + }; + const curveEnd = w.__harnessCurveEnd; + const recorder = w.__harnessAnimations; + recorder.record(); + const describe = ( + a: Animation, + finishedAt: number | null, + recordedStart: number | null, + ) => { + const effect = a.effect as KeyframeEffect | null; + const timing = effect?.getComputedTiming(); + // The recorded start stands in once the page has cancelled it. A + // run going backward began at its end: its startTime is where its + // current time would reach 0, a whole run later. + const rate = a.playbackRate; + const speed = Math.abs(rate || 1); + const endTime = Number(timing?.endTime ?? 0); + const delay = Number(timing?.delay ?? 0); + const start = + a.startTime == null + ? recordedStart + : rate < 0 + ? Number(a.startTime) - endTime / speed + : Number(a.startTime); + const startedAt = + performance.timeOrigin + Number(start ?? Number.NaN); + const name = + 'transitionProperty' in a + ? (a as unknown as {transitionProperty: string}) + .transitionProperty + : 'animationName' in a + ? (a as unknown as {animationName: string}).animationName + : null; + return { + kind: a.constructor.name, + name, + // Through the playback rate's speed: at rate 2 a curve runs in half + // its duration. Backward, the end delay plays first and the start + // delay last. + from: + rate < 0 + ? startedAt + Number(timing?.endDelay ?? 0) / speed + : startedAt + delay / speed, + to: + rate < 0 + ? startedAt + (endTime - delay) / speed + : startedAt + curveEnd(a) / speed, + durationMs: Number(timing?.duration ?? 0), + easing: String(effect?.getTiming().easing ?? 'linear'), + keyframes: (effect?.getKeyframes() ?? []).map( + (k): Record => ({...k}), + ), + finishedAt: + finishedAt == null ? null : performance.timeOrigin + finishedAt, + }; + }; + const inside = (a: Animation) => { + const target = (a.effect as KeyframeEffect | null)?.target ?? null; + return ( + target != null && (target === element || element.contains(target)) + ); + }; + const after = watchedFrom ?? recorder.armedAt(); + const recorded = recorder.log.filter( + e => e.seenAt >= after && inside(e.animation), + ); + // Running: since the watch, or (without one) any on the element. + const running = ( + watchedFrom == null + ? element.getAnimations({subtree: true}) + : recorded.filter(e => e.finishedAt == null).map(e => e.animation) + ).filter(a => a.playState === 'running' && a.startTime != null); + // Progress along the curve, on the timeline: a delay presents nothing + // (going backward, the end delay plays first), a run going backward + // (`reverse()`) progresses from its end, and a faster rate presents + // more curve per ms. + const progressed = running.filter(a => { + const timing = a.effect?.getComputedTiming(); + const time = Number(a.currentTime ?? 0); + const local = + a.playbackRate < 0 + ? Number(timing?.endTime ?? 0) - + Number(timing?.endDelay ?? 0) - + time + : time - Number(timing?.delay ?? 0); + return local / Math.abs(a.playbackRate || 1) >= minProgress; + }); + if (progressed.length > 0) { + return progressed.map(a => describe(a, null, null)); + } + if (running.length > 0) { + return []; + } + // Finished: by its finish event, or by its state when the event has not + // been delivered yet (then at its scheduled end, through the rate's + // speed; a run going backward ends at its startTime, where its current + // time reaches 0). + const finishedAt = (e: (typeof recorded)[number]) => { + if (e.finishedAt != null) { + return e.finishedAt; + } + const a = e.animation; + if (a.playState !== 'finished' || a.startTime == null) { + return null; + } + const end = Number( + (a.effect as KeyframeEffect | null)?.getComputedTiming().endTime ?? + 0, + ); + return a.playbackRate < 0 + ? Number(a.startTime) + : Number(a.startTime) + end / Math.abs(a.playbackRate || 1); + }; + return recorded + .filter(e => finishedAt(e) != null) + .map(e => describe(e.animation, finishedAt(e), e.startedAt)); + }, + {minProgress: minProgressMs, since: since ?? null}, + ); + if (found.length > 0) { + await armRecorder(locator.page()); + return found; + } + if (Date.now() > deadline) { + throw new Error( + 'waitForAnimation: no animation started on the element, none running ' + + 'and none recorded since the last wait. If the action is the ' + + "page's first contact with the harness, take " + + 'watchAnimations(locator) before it.', + ); + } + await new Promise(resolve => setTimeout(resolve, 4)); + } +} + +/** Marks now as the point later waits count animations from. */ +async function armRecorder(page: Page): Promise { + return page.evaluate(() => + ( + window as unknown as {__harnessAnimations: {arm: () => number}} + ).__harnessAnimations.arm(), + ); +} + +export interface AnimationWatch { + /** + * Resolves with the animations on the element that started after the watch + * was taken: running ones once they have presented `minProgressMs`, or, + * when none runs, those that already finished. + */ + started(options?: { + timeout?: number; + minProgressMs?: number; + }): Promise; +} + +/** + * Starts watching `locator`'s element before an action, so the animation the + * action starts is caught even when it ends before the test looks (a short + * entrance on a cold page): + * + * const watch = await watchAnimations(sheet); + * await driver.play(tap(button)); + * const [entrance] = await watch.started(); + */ +export async function watchAnimations( + locator: Locator, +): Promise { + await installCurveEnd(locator.page()); + const since = await armRecorder(locator.page()); + return { + started: options => waitForAnimation(locator, {...options, since}), + }; +} + +/** + * Resolves once `locator`'s element is at rest: no animation on it (or inside + * it) is pending or still before the end of its curve, and its presented + * `transform`, `translate`, `rotate`, `scale` and `opacity` (and `::backdrop` + * opacity) have held still for `frames` consecutive frames. + * + * It judges curves and presented values, not `playState`: an animation that + * holds its end value (a run of identical final keyframes kept so a later + * interruption can read it) still reports `running` while nothing moves. + * A hold is read as such only where `HeldAnimation.curveEndMs` reads it (a + * forward run under a linear or cubic-bézier effect easing); under any other + * easing or direction, waitForRest waits the hold out, and a hold longer than + * `timeout` rejects with "still moving". + */ +export async function waitForRest( + locator: Locator, + {timeout = 5_000, frames = 3}: {timeout?: number; frames?: number} = {}, +): Promise { + await installCurveEnd(locator.page()); + await locator.evaluate( + (element, {ms, need}) => + new Promise((resolve, reject) => { + const deadline = performance.now() + ms; + const curveEnd = ( + window as unknown as {__harnessCurveEnd: (a: Animation) => number} + ).__harnessCurveEnd; + const read = () => { + const style = getComputedStyle(element); + const backdrop = getComputedStyle(element, '::backdrop').opacity; + return [ + style.transform, + style.translate, + style.rotate, + style.scale, + style.opacity, + backdrop, + ].join(' | '); + }; + const frame = requestAnimationFrame; + let last = read(); + let still = 0; + const tick = () => { + const now = read(); + const moving = element.getAnimations({subtree: true}).some( + a => + a.pending || + (a.playState === 'running' && + // Going backward the curve ends at the start delay, after + // the end delay (which plays first) and the curve itself. + (a.playbackRate < 0 + ? Number(a.currentTime ?? 0) > + Number(a.effect?.getComputedTiming().delay ?? 0) + : Number(a.currentTime ?? 0) < curveEnd(a))), + ); + still = !moving && now === last ? still + 1 : 0; + last = now; + if (still >= need) { + // Later waits count animations from here. + ( + window as unknown as {__harnessAnimations?: {arm: () => number}} + ).__harnessAnimations?.arm(); + resolve(); + } else if (performance.now() > deadline) { + reject(new Error(`waitForRest: still moving (${now})`)); + } else { + frame(tick); + } + }; + frame(tick); + }), + {ms: timeout, need: frames}, + ); +} diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 21417851b0403..6e806d693488d 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -6,8 +6,10 @@ * @output Proof that each harness instrument measures what it claims: * scripted timestamps reach the page; a flick flings natively (Chromium); * a second finger and a pinch arrive as distinct pointers; sampled curves - * match the oracles; a held end is not part of the curve, and individual - * transform properties are sampled; a sample at + * match the oracles; the animation-state waits report a running transition + * and its end, and judge rest by presented values even while an animation + * holds its end, and a watch taken on first contact ignores an animation + * already running and keeps the start of one the page cancels when it ends; individual transform properties are sampled; a sample at * an end reports the time it was read at; holding takes only running * document-timeline animations, each at its own offset, holds each where it * was read, adds a second hold to the held set without moving the first @@ -16,7 +18,7 @@ * instruments; this file is why it may. */ -import {expect, test, type Locator, type Page} from '@playwright/test'; +import {expect, test, type Page} from '@playwright/test'; import { compareCurve, cubicBezier, @@ -26,6 +28,7 @@ import { holdAnimations, installInputLog, merge, + parseTransform, PARTIAL_LIFT_SKIP, pinch, readInputLog, @@ -36,6 +39,9 @@ import { tap, timeline, touchDriver, + waitForAnimation, + waitForRest, + watchAnimations, withFinger, } from './index'; @@ -61,37 +67,6 @@ async function fixture(page: Page) { await page.goto('http://harness.test/'); } -/** Until an animation on the element runs with a start time, `ms` into it. */ -async function untilRunning(locator: Locator, ms = 0) { - await expect - .poll(() => - locator.evaluate( - (el, min) => - el - .getAnimations() - .some( - a => - a.playState === 'running' && - a.startTime != null && - Number(a.currentTime ?? 0) >= min, - ), - ms, - ), - ) - .toBe(true); -} - -/** Until nothing on the element is running. */ -async function untilIdle(locator: Locator) { - await expect - .poll(() => - locator.evaluate(el => - el.getAnimations().some(a => a.playState === 'running'), - ), - ) - .toBe(false); -} - test.describe('touch input', () => { test('the page receives trusted touches and pointers at the scripted positions and times', async ({ page, @@ -535,12 +510,34 @@ test.describe('holding', () => { await releaseAnimations(page); }); - test('a faster playback rate is read on the timeline by seeks and samples', async ({ + test('a reversed run counts its progress from its end, and a faster playback rate is read on the timeline by waits, seeks and samples', async ({ page, }) => { await fixture(page); const box = page.locator('#box'); - // At playback rate 2 a seek or sample at t reads the curve at 2t. + // Reversed, the run starts at its end: progress counts down from there, + // so a wait for 100 ms of it returns once the run has presented that much. + const watch = await watchAnimations(box); + await box.evaluate(el => { + el.animate([{translate: '0px 0px'}, {translate: '100px 0px'}], { + duration: 1000, + easing: 'linear', + }).reverse(); + }); + await watch.started({minProgressMs: 100}); + const presented = await box.evaluate(el => { + const [animation] = el.getAnimations(); + return ( + Number(animation.effect?.getComputedTiming().endTime ?? 0) - + Number(animation.currentTime ?? 0) + ); + }); + expect(presented).toBeGreaterThanOrEqual(100); + await box.evaluate(el => el.getAnimations().forEach(a => a.finish())); + + // At playback rate 2 a 1,000 ms curve runs 500 ms: its interval, and a + // seek or sample at t reads the curve at 2t. + const fast = await watchAnimations(box); await box.evaluate(el => { const animation = el.animate( [{translate: '0px 0px'}, {translate: '100px 0px'}], @@ -548,6 +545,8 @@ test.describe('holding', () => { ); animation.playbackRate = 2; }); + const [running] = await fast.started(); + expect(running.to - running.from).toBeCloseTo(500, 0); await holdAnimations(page, box); await seekAnimations(page, 125); const x = await box.evaluate( @@ -610,9 +609,11 @@ test.describe('holding', () => { expect(reversed.curveEndMs).toBeCloseTo(1000, 0); await releaseAnimations(page, {finish: true}); - // One reversed by `reverse()` (a negative playback rate) is refused by - // a hold, which seeks curves forward only. - await box.evaluate(async el => { + // One reversed by `reverse()` (a negative playback rate) is reported as + // starting when it did, running its whole curve, and is refused by a + // hold, which seeks curves forward only. + const watch = await watchAnimations(box); + const readyAt = await box.evaluate(async el => { const animation = el.animate( [ {translate: '0px 0px', offset: 0}, @@ -623,9 +624,11 @@ test.describe('holding', () => { ); animation.reverse(); await animation.ready; + return performance.timeOrigin + performance.now(); }); - // Chromium lists a reversed run in getAnimations() a frame after ready. - await untilRunning(box); + const [running] = await watch.started(); + expect(Math.abs(running.from - readyAt)).toBeLessThan(100); + expect(running.to - running.from).toBeCloseTo(1000, -1); await expect(holdAnimations(page, box)).rejects.toThrow(/runs backward/); await box.evaluate(el => el.getAnimations().forEach(a => a.finish())); @@ -677,7 +680,7 @@ test.describe('holding', () => { ); }); const box = page.locator('#box'); - await untilRunning(box, 50); + await waitForAnimation(box, {minProgressMs: 50}); const [held] = await holdAnimations(page, box); // Two frames later, the held animation is still where the hold read it. const after = await box.evaluate( @@ -720,7 +723,7 @@ test.describe('holding', () => { await releaseAnimations(page); // The first hold was released too: the move runs to its end, and rest is // its end, not the frame it was held on. - await untilIdle(box); + await waitForRest(box); expect(await box.evaluate(el => getComputedStyle(el).transform)).toBe( 'matrix(1, 0, 0, 1, 300, 0)', ); @@ -759,7 +762,7 @@ test.describe('holding', () => { }), ); const box = page.locator('#box'); - await untilRunning(box); + await waitForAnimation(box); // The later-started element is held first. const [first] = await holdAnimations(page, box); const [later] = await holdAnimations(page, page.locator('#early')); @@ -783,7 +786,7 @@ test.describe('holding', () => { el.getBoundingClientRect(); el.style.transform = 'translateX(300px)'; }); - await untilRunning(box); + await waitForAnimation(box); expect(await holdAnimations(page, box)).toHaveLength(1); // The page sends the box back: the held transition is cancelled, and a // new one runs from where the box is. @@ -806,7 +809,7 @@ test.describe('holding', () => { await seekAnimations(page, 150); expect(await state()).toEqual({cancelled: 'idle', held: 0}); await releaseAnimations(page); - await untilIdle(box); + await waitForRest(box); expect(await state()).toEqual({cancelled: 'idle', held: 0}); expect(await box.evaluate(el => getComputedStyle(el).transform)).toBe( 'none', @@ -841,7 +844,259 @@ test.describe('holding', () => { }); }); -test.describe('held ends', () => { +test.describe('animation state', () => { + test('waitForAnimation reports a running transition and waitForRest resolves after it', async ({ + page, + }) => { + await fixture(page); + const box = page.locator('#box'); + await box.evaluate(el => { + el.style.transition = 'transform 300ms linear'; + void (el as HTMLElement).offsetWidth; + el.style.transform = 'translateY(100px)'; + }); + const [running] = await waitForAnimation(box); + expect(running.kind).toBe('CSSTransition'); + expect(running.name).toBe('transform'); + expect(running.durationMs).toBe(300); + expect(running.to - running.from).toBeCloseTo(300, 0); + await waitForRest(box); + const settledAt = await page.evaluate( + () => performance.timeOrigin + performance.now(), + ); + expect(settledAt).toBeGreaterThanOrEqual(running.to); + const transform = await box.evaluate(el => getComputedStyle(el).transform); + expect(parseTransform(transform).translateY).toBeCloseTo(100, 1); + }); + + test("waitForAnimation's progress starts after a transition's delay", async ({ + page, + }) => { + await fixture(page); + const box = page.locator('#box'); + await box.evaluate(el => { + el.style.transition = 'transform 300ms linear 200ms'; + void (el as HTMLElement).offsetWidth; + el.style.transform = 'translateY(100px)'; + }); + await waitForAnimation(box, {minProgressMs: 50}); + // 50 ms along the curve: past the 200 ms delay, not inside it. + const currentTime = await box.evaluate(el => + Number(el.getAnimations()[0]?.currentTime), + ); + expect(currentTime).toBeGreaterThanOrEqual(250); + await waitForRest(box); + }); + + test('waitForAnimation reports a short animation that ended before it looked', async ({ + page, + }) => { + await fixture(page); + const box = page.locator('#box'); + await waitForRest(box); + await box.evaluate(el => { + el.style.transition = 'transform 60ms linear'; + void (el as HTMLElement).offsetWidth; + el.style.transform = 'translateY(40px)'; + }); + // A cold page answers late: the transition is over before the first look. + await page.waitForTimeout(250); + const [ended] = await waitForAnimation(box); + expect(ended.kind).toBe('CSSTransition'); + expect(ended.durationMs).toBe(60); + expect(ended.finishedAt).not.toBeNull(); + expect(ended.finishedAt ?? 0).toBeGreaterThanOrEqual(ended.to - 1); + }); + + test("a watch taken before the action catches an entrance on the page's first contact", async ({ + page, + }) => { + await fixture(page); + const box = page.locator('#box'); + const watch = await watchAnimations(box); + await box.evaluate(el => + el.animate([{opacity: 0}, {opacity: 1}], {duration: 50}), + ); + await page.waitForTimeout(250); + const [entrance] = await watch.started(); + expect(entrance.kind).toBe('Animation'); + expect(entrance.durationMs).toBe(50); + expect(entrance.finishedAt).not.toBeNull(); + }); + + test("a watch taken on the page's first contact ignores an animation already running", async ({ + page, + }) => { + await fixture(page); + const box = page.locator('#box'); + // An endless spinner, started before the harness has touched the page. + await box.evaluate(el => + el.animate([{rotate: '0deg'}, {rotate: '360deg'}], { + duration: 1000, + iterations: Infinity, + }), + ); + const watch = await watchAnimations(box); + await expect(watch.started({timeout: 300})).rejects.toThrow( + /no animation started/, + ); + await box.evaluate(el => + el.animate([{opacity: 1}, {opacity: 0.5}], {duration: 300}), + ); + const started = await watch.started(); + expect(started.map(a => a.durationMs)).toEqual([300]); + }); + + test('a watch reports only what started after it', async ({page}) => { + await fixture(page); + const box = page.locator('#box'); + await box.evaluate(el => + el.animate([{opacity: 0}, {opacity: 1}], {duration: 40}), + ); + await page.waitForTimeout(150); + const watch = await watchAnimations(box); + await expect(watch.started({timeout: 300})).rejects.toThrow( + /no animation started/, + ); + await box.evaluate(el => + el.animate([{opacity: 1}, {opacity: 0.5}], {duration: 70}), + ); + const started = await watch.started(); + expect(started.map(a => a.durationMs)).toEqual([70]); + }); + + test('an entrance the page cancels when it ends keeps its start', async ({ + page, + }) => { + await fixture(page); + const box = page.locator('#box'); + await page.addStyleTag({ + content: + '@keyframes enter { from { opacity: 0 } to { opacity: 1 } }' + + ' .entering { animation: enter 60ms linear }', + }); + const watch = await watchAnimations(box); + // The page's time origin, where a lost start lands, is well before the + // trigger. + await page.waitForTimeout(200); + // Removing the class on animationend cancels the finished CSS animation, + // which nulls its startTime. + const triggeredAt = await box.evaluate(el => { + el.addEventListener( + 'animationend', + () => el.classList.remove('entering'), + {once: true}, + ); + el.classList.add('entering'); + return performance.timeOrigin + performance.now(); + }); + await expect + .poll(() => box.evaluate(el => el.classList.contains('entering'))) + .toBe(false); + const [entrance] = await watch.started(); + expect(entrance.kind).toBe('CSSAnimation'); + expect(entrance.finishedAt).not.toBeNull(); + // It started on a frame after the trigger, not at the page's time origin. + expect(entrance.from).toBeGreaterThan(triggeredAt - 17); + expect(entrance.from).toBeLessThan(triggeredAt + 500); + expect(entrance.to - entrance.from).toBeCloseTo(60, 3); + expect(entrance.finishedAt ?? 0).toBeGreaterThanOrEqual(entrance.to - 1); + }); + + test('a run going backward is dated from when it began: with a delay, ended before the look, or cancelled when it ends', async ({ + page, + }) => { + await fixture(page); + const box = page.locator('#box'); + const frames = [ + {translate: '0px 0px', offset: 0}, + {translate: '100px 0px', offset: 1}, + ]; + + // A 300 ms delay plays last when going backward: the curve starts at once. + let watch = await watchAnimations(box); + let readyAt = await box.evaluate(async (el, keyframes) => { + const animation = el.animate(keyframes, {duration: 400, delay: 300}); + animation.reverse(); + await animation.ready; + return performance.timeOrigin + performance.now(); + }, frames); + const [delayed] = await watch.started(); + expect(Math.abs(delayed.from - readyAt)).toBeLessThan(50); + expect(delayed.to - delayed.from).toBeCloseTo(400, 0); + await box.evaluate(el => el.getAnimations().forEach(a => a.finish())); + + // A short one that ended before the look keeps its curve inside its life. + watch = await watchAnimations(box); + readyAt = await box.evaluate(async (el, keyframes) => { + const animation = el.animate(keyframes, {duration: 60}); + animation.reverse(); + await animation.ready; + return performance.timeOrigin + performance.now(); + }, frames); + await page.waitForTimeout(200); + const [ended] = await watch.started(); + expect(ended.finishedAt).not.toBeNull(); + expect(Math.abs(ended.from - readyAt)).toBeLessThan(50); + expect(ended.to).toBeLessThanOrEqual((ended.finishedAt ?? 0) + 1); + + // One the page cancels when it ends keeps its start. + watch = await watchAnimations(box); + readyAt = await box.evaluate(async (el, keyframes) => { + const animation = el.animate(keyframes, {duration: 60}); + animation.addEventListener('finish', () => animation.cancel(), { + once: true, + }); + animation.reverse(); + await animation.ready; + return performance.timeOrigin + performance.now(); + }, frames); + await page.waitForTimeout(200); + const [cancelled] = await watch.started(); + expect(Math.abs(cancelled.from - readyAt)).toBeLessThan(50); + expect(cancelled.to - cancelled.from).toBeCloseTo(60, 0); + + // Read right after finish(), before the finish event is delivered: the + // scheduled end goes through the rate (backward, it is the startTime). + for (const rate of [-1, 2]) { + watch = await watchAnimations(box); + await box.evaluate( + async (el, [keyframes, playbackRate]) => { + const animation = el.animate(keyframes as Keyframe[], { + duration: 1000, + }); + animation.playbackRate = playbackRate as number; + await animation.ready; + animation.finish(); + }, + [frames, rate] as const, + ); + const [read] = await watch.started(); + expect(read.finishedAt).not.toBeNull(); + expect(Math.abs((read.finishedAt ?? 0) - read.to)).toBeLessThan(50); + } + + // Going backward, an end delay presents nothing: a run still inside it + // has not progressed. + await box.evaluate((el, keyframes) => { + const animation = el.animate(keyframes, {duration: 400, endDelay: 500}); + animation.reverse(); + }, frames); + await expect( + waitForAnimation(box, {minProgressMs: 100, timeout: 300}), + ).rejects.toThrow(); + await box.evaluate(el => el.getAnimations().forEach(a => a.finish())); + + // Nor is it rest: waitForRest waits out the end delay and the curve. + const restFrom = await box.evaluate((el, keyframes) => { + el.animate(keyframes, {duration: 600, endDelay: 500}).reverse(); + return performance.now(); + }, frames); + await waitForRest(box, {timeout: 5_000}); + const restAt = await box.evaluate(() => performance.now()); + expect(restAt - restFrom).toBeGreaterThan(1_000); + }); + test('an end value held by the keyframes is not part of the curve, and individual transform properties are sampled', async ({ page, }) => { @@ -859,6 +1114,20 @@ test.describe('held ends', () => { {duration: 5300, fill: 'forwards'}, ); }); + const startedAt = await page.evaluate(() => performance.now()); + const [active] = await waitForAnimation(box); + expect(active.to - active.from).toBeCloseTo(300, 0); // the curve, not the hold + await waitForRest(box, {timeout: 3_000}); + const after = await box.evaluate(el => ({ + at: performance.now(), + running: el.getAnimations().some(a => a.playState === 'running'), + translate: getComputedStyle(el).translate, + })); + expect(after.running).toBe(true); // still holding its end value + expect(after.at - startedAt).toBeGreaterThanOrEqual(300); + expect(after.at - startedAt).toBeLessThan(1_500); + expect(after.translate).toBe('0px 120px'); + const held = await holdAnimations(page, box); expect(held[0].curveEndMs).toBeCloseTo(300, 0); const [mid, end] = await sampleCurve(box, [150, 300]); From 2bcd5b2328e2e3e7c5e3cdafc7566cd70e4f8022 Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Mon, 26 Oct 2026 02:50:42 -0700 Subject: [PATCH 65/93] docs(interaction): the waits read a run reversed before it starts; a mid-run reverse or replay reads as its first run Co-authored-by: Christopher Chedeau --- internal/interaction-harness/README.md | 10 ++++++---- internal/interaction-harness/src/curves.ts | 4 ++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index a55cc41603655..22ab218dbe48e 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -61,10 +61,12 @@ than `playState`, does not wait the hold out. That holds for an animation that plays forward under a linear or cubic-bézier effect easing (the time the run starts is solved from the curve); under any other effect easing (steps, `linear()` stops, an overshoot) or another direction the curve is read to its -full end, and `waitForRest` waits the hold out. An animation running backward (`reverse()`, a -negative playback rate) is reported by `waitForAnimation` from when it -started, but `holdAnimations` refuses it: seeking and sampling run a curve -forward. +full end, and `waitForRest` waits the hold out. An animation reversed before it starts (`reverse()` +on a new animation, a negative playback rate from the start) is reported by +`waitForAnimation` from when it started, but `holdAnimations` refuses it: +seeking and sampling run a curve forward. The recorder keeps one entry per +`Animation` object, so one reversed or replayed mid-run is reported as its +first run; take a fresh animation for each run a proof reads. **Waiting for an animation.** The page records every animation from the first call into `curves.ts` on it (and on every later document): a frame loop on the diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index 90522f5d7a42c..4f3bcd82a00d9 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -811,6 +811,10 @@ export interface ActiveAnimation { * finished is reported too, with `finishedAt` set. For an action that may be * the page's first contact with this module, take `watchAnimations(locator)` * before the action instead. + * + * A run going backward is read when it was reversed before it started; the + * recorder keeps one entry per `Animation` object, so an animation reversed + * or replayed mid-run is reported as its first run. */ export async function waitForAnimation( locator: Locator, From 571f08898eef33c5e1a3a4675d61330081c53acd Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Mon, 26 Oct 2026 02:25:39 -0700 Subject: [PATCH 66/93] fix(interaction): openStory turns Storybook's highlight module off MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Storybook 10's highlight module observes #storybook-root and, on every insertion, reads the computed style of every element on the page inside the inserting task: one exit copy in a Button story carried 117 getComputedStyle calls and two forced style recalculations, which a proof then counted as the component's. openStory sets the module's own __STORYBOOK_HIGHLIGHT_INITIALIZED before the preview runs, so the module returns at once, and checks that its root element is absent once the story renders (a Storybook that stops reading the flag fails loudly). openStory(…, {storybookHighlight: true}) keeps it on, in a page that has not turned it off. A self-test on a built story counts the reads and style recalculations an insertion carries: none by default, every element's with the module kept on. --- internal/interaction-harness/README.md | 8 ++ internal/interaction-harness/src/story.ts | 47 ++++++++- .../src/storybook.interaction.browser.spec.ts | 97 +++++++++++++++++++ 3 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 internal/interaction-harness/src/storybook.interaction.browser.spec.ts diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index fe932bb8941b4..3a15f9f5e9438 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -69,6 +69,14 @@ dispatch latency, and the events go through the compositor's gesture detection the limits below. Read positions a frame after a move: Chromium delivers `pointermove` aligned to animation frames. +**Stories, not Storybook.** `openStory` turns Storybook's highlight module off +on every page it opens, and checks that its root element is absent. Left on, +the module observes `#storybook-root` and, on every insertion, reads the +computed style of every element on the page inside the inserting task (117 +reads and two forced style recalculations for one exit copy in a Button story), +so a proof would measure Storybook as well as the component. +`openStory(page, origin, id, args, {storybookHighlight: true})` keeps it on. + **Curve sampling.** Pause every running animation (CSS transitions and animations are Web Animations too), seek to a time, read `getComputedStyle`: the presented value at that exact time, independent of frame timing. Finished diff --git a/internal/interaction-harness/src/story.ts b/internal/interaction-harness/src/story.ts index 598f75d852cfe..11411a9d6211f 100644 --- a/internal/interaction-harness/src/story.ts +++ b/internal/interaction-harness/src/story.ts @@ -6,7 +6,8 @@ * static server over a built Storybook * @output `phone` (the touch-phone browser context every interaction proof * runs in), `withStorybook` (serve the built Storybook for a spec file), and - * `openStory` (load one story and wait until it has rendered). + * `openStory` (load one story, with Storybook's highlight module off, and + * wait until it has rendered). * @position Stories are the reproduction path, as for every other browser * check in this repository: a scenario used for verification is checked in, * not built in a one-off page. @@ -59,13 +60,47 @@ export function withStorybook(test: TestType): () => string { }; } -/** Loads a story in the page and waits for its first render, its fonts and two frames. */ +/** Pages whose Storybook highlight module `openStory` turned off. */ +const highlightOff = new WeakSet(); + +export interface OpenStoryOptions { + /** + * Keeps Storybook's highlight module on. It is off by default: when on, it + * observes `#storybook-root` and, on every insertion, reads the computed + * style of every element on the page inside the inserting task, so a proof + * would measure Storybook as well as the component. + */ + readonly storybookHighlight?: boolean; +} + +/** + * Loads a story in the page and waits for its first render, its fonts and two + * frames. Storybook's highlight module is off (see `OpenStoryOptions`), and + * that is checked: its root element must be absent once the story renders. + */ export async function openStory( page: Page, origin: string, storyId: string, args?: Readonly>, + {storybookHighlight = false}: OpenStoryOptions = {}, ): Promise { + if (storybookHighlight) { + if (highlightOff.has(page)) { + throw new Error( + "openStory: this page already turned Storybook's highlight module off; open the story in a new page to keep it on", + ); + } + } else if (!highlightOff.has(page)) { + highlightOff.add(page); + // Storybook's highlight module returns at once when this is set before + // its preview runs. + await page.addInitScript(() => { + ( + globalThis as unknown as {__STORYBOOK_HIGHLIGHT_INITIALIZED?: boolean} + ).__STORYBOOK_HIGHLIGHT_INITIALIZED = true; + }); + } const params = new URLSearchParams({id: storyId, viewMode: 'story'}); if (args != null) { params.set( @@ -86,6 +121,14 @@ export async function openStory( requestAnimationFrame(() => requestAnimationFrame(() => resolve())), ); }); + if ( + !storybookHighlight && + (await page.locator('#storybook-highlights-root').count()) > 0 + ) { + throw new Error( + "openStory: Storybook's highlight module is on (#storybook-highlights-root exists), so every insertion would also read the style of every element on the page; this Storybook may no longer read __STORYBOOK_HIGHLIGHT_INITIALIZED", + ); + } } /** Resolves after `count` animation frames have been presented in the page. */ diff --git a/internal/interaction-harness/src/storybook.interaction.browser.spec.ts b/internal/interaction-harness/src/storybook.interaction.browser.spec.ts new file mode 100644 index 0000000000000..3219a83d659cd --- /dev/null +++ b/internal/interaction-harness/src/storybook.interaction.browser.spec.ts @@ -0,0 +1,97 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file storybook.interaction.browser.spec.ts + * @input The built Storybook (`withStorybook`) and one of its stories + * @output `openStory` turns Storybook's highlight module off, so an insertion + * in a story carries no `getComputedStyle` scan of the page; with the module + * on (the control), every insertion carries one + * @position Self-test for story.ts; it needs the built Storybook, as every + * component proof does + */ + +import {expect, test, type Page} from '@playwright/test'; +import {openStory, withStorybook} from './story'; +import {countBetween, traceRendering} from './trace'; + +const storybook = withStorybook(test); + +/** Counts the page's `getComputedStyle` calls, from before Storybook loads. */ +function countStyleReads(): void { + const w = window as unknown as {__styleReads: number}; + w.__styleReads = 0; + const read = window.getComputedStyle.bind(window); + window.getComputedStyle = ((element: Element, pseudo?: string | null) => { + w.__styleReads += 1; + return read(element, pseudo); + }) as typeof window.getComputedStyle; +} + +/** + * Inserts an exit copy of the story's first element into `#storybook-root`, + * as an exit animation's copy is, lets the insertion's mutation microtasks run, + * and reports what ran between its marks. + */ +async function insertCopy(page: Page) { + return page.evaluate(async () => { + const w = window as unknown as {__styleReads: number}; + const root = document.getElementById('storybook-root') as HTMLElement; + const reads = w.__styleReads; + const from = + performance.timeOrigin + performance.mark('copy:start').startTime; + const copy = (root.firstElementChild as HTMLElement).cloneNode( + true, + ) as HTMLElement; + copy.setAttribute('aria-hidden', 'true'); + root.appendChild(copy); + // A MutationObserver's callback is a microtask the insertion queued: + // these run after it. + await Promise.resolve(); + await Promise.resolve(); + const to = performance.timeOrigin + performance.mark('copy:end').startTime; + const scanned = w.__styleReads - reads; + const elements = document.body.querySelectorAll('*').length; + copy.remove(); + return {from, to, scanned, elements}; + }); +} + +test("openStory turns Storybook's highlight module off: an insertion in a story carries no getComputedStyle scan of the page", async ({ + page, + browserName, +}) => { + test.skip( + browserName !== 'chromium', + 'rendering traces are Chromium-only (README: engine limits)', + ); + // The harness default. + await page.addInitScript(countStyleReads); + await openStory(page, storybook(), 'core-button--primary'); + expect(await page.locator('#storybook-highlights-root').count()).toBe(0); + let off = {from: 0, to: 0, scanned: 0, elements: 0}; + const offTrace = await traceRendering(page, async () => { + off = await insertCopy(page); + }); + expect(off.scanned).toBe(0); + expect(countBetween(offTrace, off.from, off.to).UpdateLayoutTree ?? 0).toBe( + 0, + ); + + // Control: Storybook's module kept on reads the style of every element on + // the page inside the inserting task. + const control = await page.context().newPage(); + await control.addInitScript(countStyleReads); + await openStory(control, storybook(), 'core-button--primary', undefined, { + storybookHighlight: true, + }); + expect(await control.locator('#storybook-highlights-root').count()).toBe(1); + let on = {from: 0, to: 0, scanned: 0, elements: 0}; + const onTrace = await traceRendering(control, async () => { + on = await insertCopy(control); + }); + expect(on.scanned).toBeGreaterThanOrEqual(on.elements); + expect( + countBetween(onTrace, on.from, on.to).UpdateLayoutTree ?? 0, + ).toBeGreaterThan(0); + await control.close(); +}); From 4c41132317b85f08f1848d38ffceb28ea07da131 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Mon, 26 Oct 2026 03:06:02 -0700 Subject: [PATCH 67/93] fix(interaction): a trace sets apart only the harness's own reads, never a test's evaluate traceRendering filed every DevTools-run task that set off no page script in devtoolsTasks. A test's own page.evaluate that calls page code directly (a story's API, a hook) leaves no listener or timer in the trace, so its work was filed there too, and a "no long task" proof passed over it. The harness now marks its own page-side reads (console.timeStamp with HARNESS_READ) in waitForAnimation and in blockMainThread without during. devtoolsTasks holds only tasks that carry that mark and ran no page script; every other long task, a test's evaluate included, is the page's. The self-test adds a 60 ms page function a test's evaluate calls directly: it is a long task, and without the mark in waitForAnimation its read is one too. --- internal/interaction-harness/README.md | 9 +++--- internal/interaction-harness/src/busy.ts | 2 ++ internal/interaction-harness/src/curves.ts | 2 ++ .../src/harness.interaction.browser.spec.ts | 24 ++++++++++++++- internal/interaction-harness/src/trace.ts | 30 ++++++++++++++----- 5 files changed, 55 insertions(+), 12 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 3a15f9f5e9438..d11572fb329c6 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -181,10 +181,11 @@ compositor's frames in the judged part. WebKit has no such trace: interaction; `countBetween` counts the steady middle of an animation (its first and last frames may commit). A transform/opacity animation does no Layout or Paint there; a layout-property animation does one per frame. Its -`longTasks` are the page's own: a task that ran a DevTools command (a harness -read such as `waitForAnimation` describing keyframes, or `blockMainThread` -without `during`) is in `devtoolsTasks` instead, unless page script ran -inside it (a listener or timer the evaluate set off). +`longTasks` are the page's own: only a task the harness marked as its own read +(`waitForAnimation` describing keyframes, `blockMainThread` without `during`) +is in `devtoolsTasks` instead, and only when no page script ran inside it. A +test's own `page.evaluate` stays a long task, whether it calls page code +directly or sets it off. **Storyboards.** Deterministic: frames come from seeking held animations or from stepping scripted input, never from wall-clock capture. The JSON trace diff --git a/internal/interaction-harness/src/busy.ts b/internal/interaction-harness/src/busy.ts index 161edd91e209f..52de62fb5676a 100644 --- a/internal/interaction-harness/src/busy.ts +++ b/internal/interaction-harness/src/busy.ts @@ -67,6 +67,8 @@ export async function blockMainThread( const minProgress = Math.max(minProgressMs, ONE_FRAME_MS); if (during == null) { return page.evaluate(duration => { + // The harness's own work: a trace sets it apart (trace.ts HARNESS_READ). + console.timeStamp('harness-read'); const from = performance.timeOrigin + performance.now(); const end = performance.now() + duration; let spins = 0; diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index a51443c7fa578..c81fbd75cd731 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -843,6 +843,8 @@ export async function waitForAnimation( for (;;) { const found = await locator.evaluate( (element, {minProgress, since: watchedFrom}) => { + // The harness's own read: a trace sets it apart (trace.ts HARNESS_READ). + console.timeStamp('harness-read'); const w = window as unknown as { __harnessCurveEnd: (a: Animation) => number; __harnessAnimations: { diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index b96f7b11a1671..6573168e7f5c9 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -672,7 +672,7 @@ test.describe('frame evidence', () => { expect(await readLongTasks(page, block)).toEqual([]); }); - test("a harness read inside a trace is not the page's long task, while page script an evaluate sets off is", async ({ + test("a harness read inside a trace is not the page's long task, while a test's own evaluate is, whether it calls the page directly or sets it off", async ({ page, browserName, }) => { @@ -698,11 +698,23 @@ test.describe('frame evidence', () => { } }), ); + // Page code a test can call directly, as through a story's own API. + await page.evaluate(() => { + (window as unknown as {pageWork: (ms: number) => void}).pageWork = ( + ms: number, + ) => { + const end = performance.now() + ms; + while (performance.now() < end) { + // spin + } + }; + }); await frames(page, 3); const now = () => page.evaluate(() => performance.timeOrigin + performance.now()); const read = {from: 0, to: 0}; const click = {from: 0, to: 0}; + const direct = {from: 0, to: 0}; let timer = {from: 0, to: 0}; const trace = await traceRendering(page, async () => { read.from = await now(); @@ -726,6 +738,12 @@ test.describe('frame evidence', () => { click.from = await now(); await slow.evaluate(el => (el as HTMLElement).click()); click.to = await now(); + // Page code a test's evaluate calls directly: 60 ms of the page's work. + direct.from = await now(); + await page.evaluate(() => + (window as unknown as {pageWork: (ms: number) => void}).pageWork(60), + ); + direct.to = await now(); }); const within = (w: {from: number; to: number}) => (t: {start: number}) => t.start >= w.from - 20 && t.start < w.to; @@ -737,6 +755,10 @@ test.describe('frame evidence', () => { expect( trace.longTasks.some(t => within(click)(t) && t.durationMs >= 55), ).toBe(true); + expect( + trace.longTasks.some(t => within(direct)(t) && t.durationMs >= 55), + ).toBe(true); + expect(trace.devtoolsTasks.filter(within(direct))).toEqual([]); }); }); diff --git a/internal/interaction-harness/src/trace.ts b/internal/interaction-harness/src/trace.ts index 63dd620ee8f2c..41cd401b30c25 100644 --- a/internal/interaction-harness/src/trace.ts +++ b/internal/interaction-harness/src/trace.ts @@ -22,6 +22,14 @@ import type {CDPSession, Page} from '@playwright/test'; +/** + * The mark a harness function puts on its own page-side work + * (`console.timeStamp('harness-read')` at the start of the function it + * evaluates in the page), so a trace can tell the harness's reads from page + * code a test's own evaluate runs. + */ +export const HARNESS_READ = 'harness-read'; + /** Trace events that run page script: a listener, a timer, a frame callback. */ const PAGE_SCRIPT_EVENTS = new Set([ 'EventDispatch', @@ -55,9 +63,9 @@ export interface RenderingTrace { readonly durationMs: number; /** * The page's own main-thread tasks longer than 50 ms inside the window (long - * tasks), epoch ms. A task that ran a DevTools command (a `page.evaluate`) - * is one only if page script ran inside it (an event listener, a timer, a - * frame callback it set off); otherwise it is in `devtoolsTasks`. + * tasks), epoch ms. A task that ran a test's own DevTools command (a + * `page.evaluate`) is one, whether it calls page code directly or sets it + * off; only the harness's own marked reads are in `devtoolsTasks`. */ readonly longTasks: readonly { readonly start: number; @@ -65,9 +73,9 @@ export interface RenderingTrace { }[]; /** * Main-thread tasks longer than 50 ms inside the window that ran a DevTools - * command and no page script, epoch ms: a harness read inside the body - * (`waitForAnimation` describing every animation's keyframes) or - * `blockMainThread(page, ms)` without `during`. + * command the harness marked as its own read (`HARNESS_READ`) and no page + * script, epoch ms: `waitForAnimation` describing every animation's + * keyframes, or `blockMainThread(page, ms)` without `during`. */ readonly devtoolsTasks: readonly { readonly start: number; @@ -192,8 +200,16 @@ export async function traceRendering( {mojo_interface_tag?: unknown} | undefined )?.mojo_interface_tag === 'blink.mojom.DevToolsSession', ); + // Only the harness's own reads are set apart: a test's evaluate may call + // page code directly, which leaves no listener or timer in the trace. + const harnessRead = inside.some( + e => + e.name === 'TimeStamp' && + (e.args?.data as {message?: unknown} | undefined)?.message === + HARNESS_READ, + ); const pageScript = inside.some(e => PAGE_SCRIPT_EVENTS.has(e.name)); - (devtools && !pageScript ? devtoolsTasks : longTasks).push({ + (devtools && harnessRead && !pageScript ? devtoolsTasks : longTasks).push({ start: toEpochMs(task.ts), durationMs: (task.dur ?? 0) / 1000, }); From 9320644717a5870b9349424caaca109b83839b91 Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Mon, 26 Oct 2026 03:33:37 -0700 Subject: [PATCH 68/93] fix(interaction): a wait keeps waiting while the action's animation is pending waitForAnimation and watch.started() left pending animations (no start time yet) out of the running set, so an earlier animation that had finished answered for the one the action started. A pending animation in scope now keeps the wait going. Self-test: a short animation ends, then a longer one starts; the wait returns the longer one (it returned the 40 ms one before). The README and JSDoc say how a mid-run reverse or replay is misread. Co-authored-by: Christopher Chedeau --- internal/interaction-harness/README.md | 5 +++-- internal/interaction-harness/src/curves.ts | 17 +++++++++----- .../src/harness.interaction.browser.spec.ts | 22 +++++++++++++++++++ 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 22ab218dbe48e..1b57ad1ddc4bd 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -65,8 +65,9 @@ full end, and `waitForRest` waits the hold out. An animation reversed before it on a new animation, a negative playback rate from the start) is reported by `waitForAnimation` from when it started, but `holdAnimations` refuses it: seeking and sampling run a curve forward. The recorder keeps one entry per -`Animation` object, so one reversed or replayed mid-run is reported as its -first run; take a fresh animation for each run a proof reads. +`Animation` object, so one reversed or replayed mid-run is misread: a watch +taken before the reversal rejects, and an unwatched wait returns at once with +a curve dated before it. Take a fresh animation for each run a proof reads. **Waiting for an animation.** The page records every animation from the first call into `curves.ts` on it (and on every later document): a frame loop on the diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index 4f3bcd82a00d9..eb4a3f6a9f559 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -812,9 +812,10 @@ export interface ActiveAnimation { * the page's first contact with this module, take `watchAnimations(locator)` * before the action instead. * - * A run going backward is read when it was reversed before it started; the + * A run going backward is read when it was reversed before it started. The * recorder keeps one entry per `Animation` object, so an animation reversed - * or replayed mid-run is reported as its first run. + * or replayed mid-run is misread: a watch taken before the reversal rejects, + * and an unwatched wait returns at once with a curve dated before it. */ export async function waitForAnimation( locator: Locator, @@ -912,12 +913,16 @@ export async function waitForAnimation( const recorded = recorder.log.filter( e => e.seenAt >= after && inside(e.animation), ); - // Running: since the watch, or (without one) any on the element. - const running = ( + // Running: since the watch, or (without one) any on the element. A + // pending one (no start time yet) counts too: the action's animation + // may still be pending at the first look, and an earlier one that + // finished must not answer for it. + const live = ( watchedFrom == null ? element.getAnimations({subtree: true}) : recorded.filter(e => e.finishedAt == null).map(e => e.animation) - ).filter(a => a.playState === 'running' && a.startTime != null); + ).filter(a => a.playState === 'running'); + const running = live.filter(a => a.startTime != null); // Progress along the curve, on the timeline: a delay presents nothing // (going backward, the end delay plays first), a run going backward // (`reverse()`) progresses from its end, and a faster rate presents @@ -936,7 +941,7 @@ export async function waitForAnimation( if (progressed.length > 0) { return progressed.map(a => describe(a, null, null)); } - if (running.length > 0) { + if (live.length > 0) { return []; } // Finished: by its finish event, or by its state when the event has not diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 6e806d693488d..b0ac741e0f093 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -947,6 +947,28 @@ test.describe('animation state', () => { expect(started.map(a => a.durationMs)).toEqual([300]); }); + test('a wait answers with the animation the action started, not one that finished before it', async ({ + page, + }) => { + await fixture(page); + const box = page.locator('#box'); + const frames = [{opacity: 0.2}, {opacity: 1}]; + // The recorder runs and is armed from here. + await waitForRest(box); + // A short one runs and ends; then the action starts a longer one, still + // pending when the wait first looks. + await box.evaluate( + (el, keyframes) => el.animate(keyframes, {duration: 40}).finished, + frames, + ); + await box.evaluate((el, keyframes) => { + el.animate(keyframes, {duration: 1000}); + }, frames); + const [found] = await waitForAnimation(box); + expect(found.durationMs).toBe(1000); + await box.evaluate(el => el.getAnimations().forEach(a => a.finish())); + }); + test('a watch reports only what started after it', async ({page}) => { await fixture(page); const box = page.locator('#box'); From 156e4b35342207379f2b9033161fe7bc5afb93e9 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Mon, 26 Oct 2026 03:46:00 -0700 Subject: [PATCH 69/93] test(interaction): openStory's highlight self-test runs in both engines Storybook's highlight module scans the page on every insertion in WebKit as in Chromium, so the self-test now counts the insertion's getComputedStyle calls in both engines; only Chromium's trace counts the style recalculations they force. It no longer serves Storybook for an engine it skips. --- .../src/storybook.interaction.browser.spec.ts | 49 +++++++++++-------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/internal/interaction-harness/src/storybook.interaction.browser.spec.ts b/internal/interaction-harness/src/storybook.interaction.browser.spec.ts index 3219a83d659cd..a711feb62cb84 100644 --- a/internal/interaction-harness/src/storybook.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/storybook.interaction.browser.spec.ts @@ -5,7 +5,8 @@ * @input The built Storybook (`withStorybook`) and one of its stories * @output `openStory` turns Storybook's highlight module off, so an insertion * in a story carries no `getComputedStyle` scan of the page; with the module - * on (the control), every insertion carries one + * on (the control), every insertion carries one. Both engines; Chromium's + * trace also counts the style recalculations the scan forces * @position Self-test for story.ts; it needs the built Storybook, as every * component proof does */ @@ -60,22 +61,31 @@ test("openStory turns Storybook's highlight module off: an insertion in a story page, browserName, }) => { - test.skip( - browserName !== 'chromium', - 'rendering traces are Chromium-only (README: engine limits)', - ); + // The insertion's reads in every engine, and in Chromium (whose trace the + // harness reads) the style recalculations between its marks. + const measure = async (p: Page) => { + if (browserName !== 'chromium') { + return {copy: await insertCopy(p), styleRecalc: null}; + } + let copy = {from: 0, to: 0, scanned: 0, elements: 0}; + const trace = await traceRendering(p, async () => { + copy = await insertCopy(p); + }); + return { + copy, + styleRecalc: + countBetween(trace, copy.from, copy.to).UpdateLayoutTree ?? 0, + }; + }; // The harness default. await page.addInitScript(countStyleReads); await openStory(page, storybook(), 'core-button--primary'); expect(await page.locator('#storybook-highlights-root').count()).toBe(0); - let off = {from: 0, to: 0, scanned: 0, elements: 0}; - const offTrace = await traceRendering(page, async () => { - off = await insertCopy(page); - }); - expect(off.scanned).toBe(0); - expect(countBetween(offTrace, off.from, off.to).UpdateLayoutTree ?? 0).toBe( - 0, - ); + const off = await measure(page); + expect(off.copy.scanned).toBe(0); + if (off.styleRecalc != null) { + expect(off.styleRecalc).toBe(0); + } // Control: Storybook's module kept on reads the style of every element on // the page inside the inserting task. @@ -85,13 +95,10 @@ test("openStory turns Storybook's highlight module off: an insertion in a story storybookHighlight: true, }); expect(await control.locator('#storybook-highlights-root').count()).toBe(1); - let on = {from: 0, to: 0, scanned: 0, elements: 0}; - const onTrace = await traceRendering(control, async () => { - on = await insertCopy(control); - }); - expect(on.scanned).toBeGreaterThanOrEqual(on.elements); - expect( - countBetween(onTrace, on.from, on.to).UpdateLayoutTree ?? 0, - ).toBeGreaterThan(0); + const on = await measure(control); + expect(on.copy.scanned).toBeGreaterThanOrEqual(on.copy.elements); + if (on.styleRecalc != null) { + expect(on.styleRecalc).toBeGreaterThan(0); + } await control.close(); }); From 9d5c20a8fec4ea887d0e02efdbc39b93b5ec213c Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Mon, 26 Oct 2026 03:58:02 -0700 Subject: [PATCH 70/93] fix(interaction): a traced waitForRest skips scroll- and view-timeline animations Inside a trace, waitForRest waits on timers for each animation's scheduled end. A scroll- or view-timeline animation (DateInput's wheel rows) has none: its time is a percentage, the wait came out NaN, and the timer re-armed with no delay until the deadline, where the untraced wait rests. It now skips animations that are not on the document timeline, as holdAnimations does. Self-test: a scroll-timeline row inside a trace; it rejected at the deadline before (Chromium, 3x repeat pass). --- internal/interaction-harness/src/curves.ts | 6 ++++ .../src/harness.interaction.browser.spec.ts | 28 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index 519a63257c9d0..1dfc80d5f70a7 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -1167,6 +1167,12 @@ export async function waitForRest( const scheduled = () => { let wait = 0; for (const a of element.getAnimations({subtree: true})) { + // A scroll- or view-timeline animation moves only when something + // scrolls: it has no scheduled end to wait for (holdAnimations + // skips it the same way). + if (!(a.timeline instanceof DocumentTimeline)) { + continue; + } if ( a.pending || (a.playState === 'running' && a.startTime == null) diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 983e622da8e7c..2206445551062 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -865,6 +865,34 @@ test.describe('frame evidence', () => { expect(outcome).toBe('rested'); }); + test('inside a trace, waitForRest rests beside a scroll-timeline animation', async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'rendering traces are Chromium-only (README: engine limits)', + ); + await fixture(page); + const scroller = page.locator('#scroller'); + await waitForRest(scroller); + let outcome = ''; + await traceRendering(page, async () => { + // A row driven by its scroller's timeline: it moves only on scroll. + await scroller.evaluate(el => { + const row = el.firstElementChild as HTMLElement; + row.animate([{opacity: 0.5}, {opacity: 1}], { + timeline: new ScrollTimeline({source: el}), + }); + }); + outcome = await waitForRest(scroller, {timeout: 2000}).then( + () => 'rested', + (error: Error) => error.message, + ); + }); + expect(outcome).toBe('rested'); + }); + test('a CSS transition and animation that start and end inside a trace are reported, and lay out nothing in between', async ({ page, browserName, From cb907b7ff0b5514392508f434827395bbe0f8361 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Mon, 26 Oct 2026 04:04:40 -0700 Subject: [PATCH 71/93] fix(interaction): the busy verdict judges a sparse cast from its first moved frame, and the harness's waits run on the real timers With a compositor record, the moved criterion counted every compositor frame in the judged part, but only a gap between two moved frames can be credited to the compositor, so the frames before the first and after the last moved frame read as misses: at one screencast frame in six, a composited animation failed in 5 of 7 runs, where the same frames passed without the record. The criterion now counts the compositor's frames from the first to the last moved frame; the gaps before and after are the stall check's, which already measures the window's edges. blockMainThread(..., {during}) and sampleRects waited on the page's own requestAnimationFrame and setTimeout, which freezeTimers holds, so neither settled under a frozen timer gate, though the harness's own waits are documented as not held. They now run on the gate's real frame and timeout (the gate exposes its real setTimeout too). A unit case at one frame in six and a self-test under a frozen gate fail on the previous code. --- internal/interaction-harness/README.md | 6 ++- internal/interaction-harness/src/busy.test.ts | 10 +++++ internal/interaction-harness/src/busy.ts | 44 ++++++++++++++----- internal/interaction-harness/src/frames.ts | 14 +++++- .../src/harness.interaction.browser.spec.ts | 28 ++++++++++++ internal/interaction-harness/src/story.ts | 2 +- 6 files changed, 88 insertions(+), 16 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 7fb3ece733bb0..aae90534f5465 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -175,7 +175,8 @@ a stall as before. A gap that opens or closes the judged part, or holds a delivered frame that did not move, is judged from the screencast, so a frozen element fails whatever else the compositor draws (a rAF animation's block presents no compositor frame at all). With the record, the 90% counts the -compositor's frames in the judged part. WebKit has no such trace: +compositor's frames from the first to the last moved frame; the gaps before and +after are the stall check's. WebKit has no such trace: `stop()` resolves to `[]` and the screencast is judged alone. **No Layout / Paint.** `traceRendering` records a Chromium trace around the @@ -195,7 +196,8 @@ evidence. A component that finishes an animation from a timer (a transitionend backstop, a cleanup render) would do so behind a held storyboard's back: `installTimerGate` before the animation starts and `freezeTimers` while photographing. The page can still cancel a held callback, a held frame callback -gets the time it actually runs, and the harness's own frame waits are not held. +gets the time it actually runs, and the harness's own waits (its frame waits, +`blockMainThread` with `during`, `sampleRects`) are not held. An animation that repeats forever does not set a storyboard's default length; with nothing else to film, pass `times`. A touch filmstrip keeps the scripted event `timeStamp`s (Chromium), but a handler that reads `performance.now()` or diff --git a/internal/interaction-harness/src/busy.test.ts b/internal/interaction-harness/src/busy.test.ts index 6d25a3c30e328..0b433ef6c7c3f 100644 --- a/internal/interaction-harness/src/busy.test.ts +++ b/internal/interaction-harness/src/busy.test.ts @@ -360,6 +360,16 @@ const skipFrames = (): TrackedFrame[] => ); describe('busyWindowProof with the compositor record', () => { + it('with a compositor record, judges a sparse cast from its first to its last moved frame', () => { + // One screencast frame in six, as on a loaded machine, with the compositor + // presenting every frame: the frames before the first and after the last + // moved one are the stall check's, not misses. + const sparse = moving().filter((_, i) => i % 6 === 0); + expect( + busyWindowProof(sparse, BLOCK, {presented: drawn()}).verdict.failures, + ).toEqual([]); + }); + it('reads frames the screencast skipped while the compositor presented them as presented: a recorded run that fails without the record', () => { const frames = skipFrames(); const alone = busyWindowProof(frames, skip.block, {settledWithin: 1}); diff --git a/internal/interaction-harness/src/busy.ts b/internal/interaction-harness/src/busy.ts index 52de62fb5676a..5374a7634b247 100644 --- a/internal/interaction-harness/src/busy.ts +++ b/internal/interaction-harness/src/busy.ts @@ -81,6 +81,18 @@ export async function blockMainThread( return during.evaluate( (element, {duration, minProgress, wait}) => new Promise((resolve, reject) => { + // The harness's own waits run on the real timers: a frozen timer gate + // (story.ts) holds only the page's. + const gate = ( + window as unknown as { + __harnessTimerGate?: { + frame?: typeof requestAnimationFrame; + timeout?: typeof setTimeout; + }; + } + ).__harnessTimerGate; + const frame = gate?.frame ?? window.requestAnimationFrame.bind(window); + const later = gate?.timeout ?? window.setTimeout.bind(window); const deadline = performance.now() + wait; const tick = () => { const running = element @@ -95,7 +107,7 @@ export async function blockMainThread( // The same loop as above, from a task queued in the frame that saw // the animation: it runs once that frame's rendering update has // handed the animation to the compositor. - setTimeout(() => { + later(() => { const from = performance.timeOrigin + performance.now(); const end = performance.now() + duration; while (performance.now() < end) { @@ -106,10 +118,10 @@ export async function blockMainThread( } else if (performance.now() > deadline) { reject(new Error('blockMainThread: no animation started')); } else { - requestAnimationFrame(tick); + frame(tick); } }; - requestAnimationFrame(tick); + frame(tick); }), {duration: ms, minProgress, wait: timeout}, ); @@ -189,7 +201,8 @@ export interface BusyProof extends FrameStats { /** * `moved` (and `recorderSkipped`) over the frames judged: the frames the * screencast delivered in the judged part, or with a compositor record the - * compositor's frames there; `expectedFrames` when nothing was delivered. + * compositor's frames from the first to the last moved frame; + * `expectedFrames` when nothing moved or was delivered. */ readonly movedFraction: number; /** The judgment against the limits (see `busyWindowProof`). */ @@ -216,7 +229,8 @@ export interface BusyProof extends FrameStats { * (a short window may still miss one): the frames the screencast delivered * in the judged part, so an irregular cast is judged on what it delivered * and a frame it did not deliver is a gap for the stall check, not a miss; - * with a compositor record, the compositor's frames there; + * with a compositor record, the compositor's frames from the first to the + * last moved frame (the gaps before and after are the stall check's); * - the judged part spans at least `minJudgedFrames` frames. Fewer means the * animation under test was not moving during the block (it settled before * the block, or the block caught another animation), so there is nothing to @@ -436,9 +450,11 @@ export function busyWindowProof( ); const moved = changes.length; // The frames judged for motion: the screencast's own frames in the judged - // part (each `changes` entry is one of them), or the compositor's frames - // when recorded. A part that delivered nothing is judged against its span, - // so a frozen animation misses here too. + // part (each `changes` entry is one of them), or, with a compositor record, + // the compositor's frames from the first to the last moved frame (the gaps + // before and after them are the stall check's). A part that moved nothing, + // or delivered nothing, is judged against its span, so a frozen animation + // misses here too. const deliveredFrames = tracked.filter( f => f.box != null && @@ -446,9 +462,15 @@ export function busyWindowProof( f.t <= judged.to, ).length; const judgedFrames = - compositorFrameMs != null || deliveredFrames === 0 - ? expectedFrames - : deliveredFrames; + compositorFrameMs != null + ? changes.length > 0 + ? Math.round( + (changes[changes.length - 1] - changes[0]) / compositorFrameMs, + ) + 1 + : expectedFrames + : deliveredFrames === 0 + ? expectedFrames + : deliveredFrames; const movedFraction = (moved + recorderSkipped) / Math.max(1, judgedFrames); // The cast's position step, and the motion per interval it has to resolve. const positions = tracked.flatMap(f => (f.box == null ? [] : [f.box[axis]])); diff --git a/internal/interaction-harness/src/frames.ts b/internal/interaction-harness/src/frames.ts index 466e8d37cfcbb..83841d259c17f 100644 --- a/internal/interaction-harness/src/frames.ts +++ b/internal/interaction-harness/src/frames.ts @@ -427,6 +427,16 @@ export async function sampleRects( opacity: number; }[] >(resolve => { + // The harness's own waits run on the real timers: a frozen timer gate + // (story.ts) holds only the page's. + const gate = ( + window as unknown as { + __harnessTimerGate?: { + frame?: typeof requestAnimationFrame; + }; + } + ).__harnessTimerGate; + const frame = gate?.frame ?? window.requestAnimationFrame.bind(window); const samples: { t: number; box: {x: number; y: number; width: number; height: number} | null; @@ -444,12 +454,12 @@ export async function sampleRects( opacity: Number(style.opacity), }); if (now < end) { - requestAnimationFrame(tick); + frame(tick); } else { resolve(samples); } }; - requestAnimationFrame(tick); + frame(tick); }), durationMs, ); diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index a8fcd5c46be1b..854548ee6bb2e 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -1856,6 +1856,34 @@ test.describe('storyboards', {tag: '@storyboard'}, () => { } }); + test("the harness's own waits run on the real timers: a frozen timer gate holds neither blockMainThread's block nor sampleRects", async ({ + page, + }) => { + await fixture(page); + await installTimerGate(page); + const box = page.locator('#box'); + await box.evaluate(el => { + el.animate( + [{transform: 'translateX(0px)'}, {transform: 'translateX(200px)'}], + {duration: 1_500, easing: 'linear'}, + ); + }); + await freezeTimers(page); + // Settled, or still held after 5 s. + const within = (work: Promise) => + Promise.race([ + work.then(() => 'settled'), + new Promise(resolve => + setTimeout(() => resolve('held'), 5_000), + ), + ]); + expect( + await within(blockMainThread(page, 100, {during: box, timeout: 2_000})), + ).toBe('settled'); + expect(await within(sampleRects(box, 200))).toBe('settled'); + await thawTimers(page); + }); + test('the timer gate holds a component timer until the filmstrip is done', async ({ page, }, testInfo) => { diff --git a/internal/interaction-harness/src/story.ts b/internal/interaction-harness/src/story.ts index 11411a9d6211f..627969a6637f3 100644 --- a/internal/interaction-harness/src/story.ts +++ b/internal/interaction-harness/src/story.ts @@ -162,7 +162,7 @@ const TIMER_GATE = `(() => { const realInterval = window.setInterval.bind(window); const realFrame = window.requestAnimationFrame.bind(window); const realCancelFrame = window.cancelAnimationFrame.bind(window); - const gate = {frozen: false, queue: [], frame: realFrame}; + const gate = {frozen: false, queue: [], frame: realFrame, timeout: realTimeout}; window.__harnessTimerGate = gate; const drop = key => { gate.queue = gate.queue.filter(entry => entry.key !== key); }; window.setTimeout = (fn, ms, ...args) => { From 462d82caaf50831156607916a18bdd392f6a2369 Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Mon, 26 Oct 2026 04:07:15 -0700 Subject: [PATCH 72/93] fix(interaction): a scroll- or view-timeline animation does not hold up a wait waitForAnimation and watch.started() counted a scroll- or view-timeline animation (DateInput's wheel rows) as running; it never progresses on its own, so the wait never answered with the animation that finished and rejected at its timeout. Only document-timeline animations count now, as for holdAnimations. Self-test: a scroll-timeline row beside a short animation that ends; it rejected before (Chromium, 3x repeat pass). The installCurveEnd doc comment sits above installCurveEnd again. Co-authored-by: Christopher Chedeau --- internal/interaction-harness/src/curves.ts | 25 +++++++++++-------- .../src/harness.interaction.browser.spec.ts | 21 ++++++++++++++++ 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index eb4a3f6a9f559..d988382178a32 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -24,14 +24,6 @@ import {parseTransform, type TimedValue, type Transform2D} from './math'; /** How far inside its end a seek past an animation's end lands, ms. */ export const END_INSET_MS = 0.5; -/** - * Installs `window.__harnessCurveEnd(animation)`: the local time (ms, delay - * included) where the animation's effect stops changing. An animation whose - * keyframes end in a run of identical values (an end value held so that a - * later interruption can read it) ends where that run starts, when it plays - * forward under a linear or cubic-bézier effect easing (the time the run - * starts is solved from the curve); any other ends at its end time. - */ /** * Page side: records every animation the document runs, from its first frame * or its creation (a frame loop on the page's real requestAnimationFrame, and @@ -163,6 +155,14 @@ async function installRecorder(page: Page): Promise { await page.evaluate(animationRecorder); } +/** + * Installs `window.__harnessCurveEnd(animation)`: the local time (ms, delay + * included) where the animation's effect stops changing. An animation whose + * keyframes end in a run of identical values (an end value held so that a + * later interruption can read it) ends where that run starts, when it plays + * forward under a linear or cubic-bézier effect easing (the time the run + * starts is solved from the curve); any other ends at its end time. + */ async function installCurveEnd(page: Page): Promise { await installRecorder(page); await page.evaluate(() => { @@ -916,12 +916,17 @@ export async function waitForAnimation( // Running: since the watch, or (without one) any on the element. A // pending one (no start time yet) counts too: the action's animation // may still be pending at the first look, and an earlier one that - // finished must not answer for it. + // finished must not answer for it. Only document-timeline animations + // count, as for holdAnimations: a scroll- or view-timeline one moves + // only when something scrolls and never progresses on its own. const live = ( watchedFrom == null ? element.getAnimations({subtree: true}) : recorded.filter(e => e.finishedAt == null).map(e => e.animation) - ).filter(a => a.playState === 'running'); + ).filter( + a => + a.playState === 'running' && a.timeline instanceof DocumentTimeline, + ); const running = live.filter(a => a.startTime != null); // Progress along the curve, on the timeline: a delay presents nothing // (going backward, the end delay plays first), a run going backward diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index b0ac741e0f093..1f293b8034c38 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -969,6 +969,27 @@ test.describe('animation state', () => { await box.evaluate(el => el.getAnimations().forEach(a => a.finish())); }); + test('a scroll-timeline animation in scope does not hold up a wait', async ({ + page, + }) => { + await fixture(page); + const scroller = page.locator('#scroller'); + await waitForRest(scroller); + // A row driven by its scroller's timeline (it moves only on scroll), and + // a short document animation that ends. + await scroller.evaluate(el => { + const [row, other] = [...el.children] as HTMLElement[]; + row.animate([{opacity: 0.5}, {opacity: 1}], { + timeline: new ScrollTimeline({source: el}), + }); + return other.animate([{opacity: 0.2}, {opacity: 1}], {duration: 40}) + .finished; + }); + const [found] = await waitForAnimation(scroller, {timeout: 2000}); + expect(found.durationMs).toBe(40); + expect(found.finishedAt).not.toBeNull(); + }); + test('a watch reports only what started after it', async ({page}) => { await fixture(page); const box = page.locator('#box'); From 803e6d67cc56bbdd43f7e710be8f9f0b084128ac Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Mon, 26 Oct 2026 04:43:53 -0700 Subject: [PATCH 73/93] test(interaction): the long-task self-test judges the harness read, not the page's frames around it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "a harness read inside a trace is not the page's long task…" also asserted that no long task started anywhere in the read's window. On CI's runner each frame of the fixture's 1,200 running rows took 51-54 ms, the page's own long tasks, and failed it; at 4x CPU locally they take 126-220 ms. The read itself was set apart correctly. The test now asserts the read's own classification, and removes the rows before the timer, click and direct-call windows, so each holds only its own work. It passes at 4x CPU, where the previous assertion failed. --- .../src/harness.interaction.browser.spec.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 854548ee6bb2e..06b674c60d698 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -720,6 +720,10 @@ test.describe('frame evidence', () => { read.from = await now(); await waitForAnimation(page.locator('#list')); read.to = await now(); + // On a slow machine each frame of 1,200 running rows is a long task of + // the page's own; the windows below hold only their own work. + await page.evaluate(() => document.getElementById('list')?.remove()); + await frames(page, 2); // The page's own task: a timer that spins 60 ms. timer = await page.evaluate( () => @@ -747,8 +751,9 @@ test.describe('frame evidence', () => { }); const within = (w: {from: number; to: number}) => (t: {start: number}) => t.start >= w.from - 20 && t.start < w.to; + // The read itself is set apart. The rows' frames around it may be long + // tasks too, the page's own: they stay in longTasks. expect(trace.devtoolsTasks.some(within(read))).toBe(true); - expect(trace.longTasks.filter(within(read))).toEqual([]); expect( trace.longTasks.some(t => within(timer)(t) && t.durationMs >= 55), ).toBe(true); From a9be409b35c185cb0d6b12ff12f21175fcb20488 Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Mon, 26 Oct 2026 04:52:57 -0700 Subject: [PATCH 74/93] fix(interaction): a scroll-driven animation at its end is not a finished curve waitForAnimation's finished branch read every recorded animation, so a scroll- or view-timeline animation that reached its end (a scroller scrolled to its end) came back as finished with a NaN duration and interval. The recorded set now keeps document-timeline animations only. The scroll-timeline self-test scrolls its scroller to the end first; it returned the NaN entry before (Chromium, 3x repeat pass). The README, JSDoc and file header name watchAnimations where they list what moves the wait window and what the file exports. Co-authored-by: Christopher Chedeau --- internal/interaction-harness/README.md | 2 +- internal/interaction-harness/src/curves.ts | 12 +++++++++--- .../src/harness.interaction.browser.spec.ts | 3 +++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 1b57ad1ddc4bd..95931b9147759 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -74,7 +74,7 @@ call into `curves.ts` on it (and on every later document): a frame loop on the page's own `requestAnimationFrame`, the CSS animation and transition events, and `Element.animate` as it is called, pending animations included. So `waitForAnimation` also reports one that started after the last -`waitForRest` or `waitForAnimation` and has already finished (`finishedAt` +`waitForRest`, `waitForAnimation` or `watchAnimations` and has already finished (`finishedAt` set): a short entrance no longer races the first look on a cold page. When the action may be the page's first contact with the harness, take `watchAnimations(locator)` before it and read `watch.started()` after. diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index d988382178a32..ba7a3bc75bdf3 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -11,7 +11,8 @@ * transform/opacity at each t, with the individual `translate`, `rotate` and * `scale` properties composed in), `releaseAnimations`, `curveOf`, which * picks one channel out of the samples for ./math's `compareCurve`, and the - * animation-state waits `waitForAnimation` and `waitForRest`. + * animation-state waits `waitForAnimation`, `watchAnimations` and + * `waitForRest`. * @position Curve sampling: the exact presented value of an animation at a * fixed time, independent of frame timing, so a curve can be compared with * the UIKit reference to a fraction of a pixel. A paused animation's effect @@ -807,7 +808,7 @@ export interface ActiveAnimation { * It does not race a short animation: the page records every animation from * its first frame (from the first call into this module on the page: this, * `waitForRest`, `watchAnimations`, sampling or holding), so one that started - * after the last `waitForRest` or `waitForAnimation` and has already + * after the last `waitForRest`, `waitForAnimation` or `watchAnimations` and has already * finished is reported too, with `finishedAt` set. For an action that may be * the page's first contact with this module, take `watchAnimations(locator)` * before the action instead. @@ -910,8 +911,13 @@ export async function waitForAnimation( ); }; const after = watchedFrom ?? recorder.armedAt(); + // Document-timeline animations only, as for holdAnimations: a scroll- + // or view-timeline one that reached its end is not a finished curve. const recorded = recorder.log.filter( - e => e.seenAt >= after && inside(e.animation), + e => + e.seenAt >= after && + inside(e.animation) && + e.animation.timeline instanceof DocumentTimeline, ); // Running: since the watch, or (without one) any on the element. A // pending one (no start time yet) counts too: the action's animation diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 1f293b8034c38..9e706e438a9dd 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -982,6 +982,9 @@ test.describe('animation state', () => { row.animate([{opacity: 0.5}, {opacity: 1}], { timeline: new ScrollTimeline({source: el}), }); + // Scrolled to its end, the scroll-driven one has finished too: it is + // not a finished curve either. + el.scrollTop = el.scrollHeight; return other.animate([{opacity: 0.2}, {opacity: 1}], {duration: 40}) .finished; }); From 02ff48fafe68a5a99e13af7e3c42e2482d032eae Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Mon, 26 Oct 2026 05:27:11 -0700 Subject: [PATCH 75/93] fix(interaction): a watch stamps after the newest entry, and an exit removed at its end is still reported - Arming the recorder now records first and stamps strictly after the newest entry: a watch taken right after a step that changed style no longer reports that step's transition (created by the arm's own style read) as started after it. - Each entry keeps its target's ancestors as recorded, so an exit whose element the page removes at its end is still reported for the container it left, instead of a wait rejecting with nothing recorded. A self-test for each; both failed before (Chromium, 3x repeat pass). Co-authored-by: Christopher Chedeau --- internal/interaction-harness/src/curves.ts | 24 +++++++++- .../src/harness.interaction.browser.spec.ts | 46 +++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index ba7a3bc75bdf3..00dc51d7e122c 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -74,11 +74,22 @@ function animationRecorder(): void { return; } seen.add(animation); + // The target's ancestors as recorded: an exit whose element the page + // removes at its end is still reported for the container it left. + const ancestors: Element[] = []; + for ( + let node = (animation.effect as KeyframeEffect | null)?.target ?? null; + node != null; + node = node.parentElement + ) { + ancestors.push(node); + } const entry = { animation, seenAt: performance.now(), finishedAt: null as number | null, startedAt: playStart(animation), + ancestors, }; if (entry.startedAt == null) { unstarted.push(entry); @@ -140,7 +151,15 @@ function animationRecorder(): void { w.__harnessAnimations = { log, record, - arm: () => (armedAt = performance.now()), + // Record first, then stamp strictly after the newest entry: a step that + // changed style just before has its transition created by that read, + // and it started before the arm. + arm: () => { + record(); + const newest = log.length === 0 ? 0 : log[log.length - 1].seenAt; + armedAt = Math.max(performance.now(), newest + 0.001); + return armedAt; + }, armedAt: () => armedAt, }; } @@ -844,6 +863,7 @@ export async function waitForAnimation( seenAt: number; finishedAt: number | null; startedAt: number | null; + ancestors: Element[]; }[]; record: () => void; armedAt: () => number; @@ -916,7 +936,7 @@ export async function waitForAnimation( const recorded = recorder.log.filter( e => e.seenAt >= after && - inside(e.animation) && + (inside(e.animation) || e.ancestors.includes(element)) && e.animation.timeline instanceof DocumentTimeline, ); // Running: since the watch, or (without one) any on the element. A diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 9e706e438a9dd..d778abe993f4b 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -993,6 +993,52 @@ test.describe('animation state', () => { expect(found.finishedAt).not.toBeNull(); }); + test('a watch taken right after a style change does not report that change', async ({ + page, + }) => { + await fixture(page); + const box = page.locator('#box'); + await waitForRest(box); + // The step: a transition the next style read creates. + await box.evaluate(el => { + el.style.transition = 'opacity 300ms linear'; + el.style.opacity = '0.5'; + }); + const watch = await watchAnimations(box); + await expect(watch.started({timeout: 200})).rejects.toThrow( + /waitForAnimation/, + ); + await box.evaluate(el => el.getAnimations().forEach(a => a.finish())); + }); + + test('an exit whose element the page removes at its end is still reported', async ({ + page, + }) => { + await fixture(page); + await page.evaluate(() => { + const wrap = document.createElement('div'); + wrap.id = 'wrap'; + const leaf = document.createElement('div'); + leaf.id = 'leaf'; + leaf.style.cssText = 'width:20px;height:20px;background:#c33'; + wrap.append(leaf); + document.body.append(wrap); + }); + const wrap = page.locator('#wrap'); + const watch = await watchAnimations(wrap); + await page.evaluate(() => { + const leaf = document.getElementById('leaf') as HTMLElement; + const exit = leaf.animate([{opacity: 1}, {opacity: 0}], {duration: 60}); + exit.finished.then(() => leaf.remove()); + }); + await expect + .poll(() => page.evaluate(() => document.getElementById('leaf') == null)) + .toBe(true); + const [exit] = await watch.started(); + expect(exit.durationMs).toBe(60); + expect(exit.finishedAt).not.toBeNull(); + }); + test('a watch reports only what started after it', async ({page}) => { await fixture(page); const box = page.locator('#box'); From 528dd2da76f52b26e404751653603155b5ee972d Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Mon, 26 Oct 2026 05:32:21 -0700 Subject: [PATCH 76/93] fix(interaction): after an on-screen start, the busy verdict judges the frames it credits as skipped With a compositor record and an on-screen start, the moved criterion counted from the first moved frame after the on-screen frame, while the gap from the on-screen frame to it was credited as frames the screencast skipped. When the cast skipped the frames right after the on-screen start, those credits had no place in the denominator: four unmoved frames that fail on a full cast ("31 of 35 expected frames moved") passed as "26/30 moved and 5 skipped by the screencast (103%)", and a clean run read 117%. The criterion now counts the compositor's frames from the on-screen frame, so both sides cover the same frames: the skipped cast fails like the full one, and a clean run reads at most 100%. A busy.test.ts case pins all three. --- internal/interaction-harness/src/busy.test.ts | 30 +++++++++++++++++++ internal/interaction-harness/src/busy.ts | 13 ++++---- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/internal/interaction-harness/src/busy.test.ts b/internal/interaction-harness/src/busy.test.ts index 0b433ef6c7c3f..64b981a442c9e 100644 --- a/internal/interaction-harness/src/busy.test.ts +++ b/internal/interaction-harness/src/busy.test.ts @@ -370,6 +370,36 @@ describe('busyWindowProof with the compositor record', () => { ).toEqual([]); }); + it('with a compositor record and an on-screen start, counts the frames it credits as skipped in what it judges', () => { + // On screen 87 ms into the block, then four delivered frames that show + // no new position: that fails on a full cast. + const misses = [15, 20, 25, 30]; + const missing = startingAt(1087).map((f, i, all) => + misses.includes(i) + ? ({...f, box: {...all[i - 1].box!}} as TrackedFrame) + : f, + ); + const full = busyWindowProof(missing, BLOCK, {presented: drawn()}); + expect(full.verdict.failures.join('\n')).toMatch(/expected frames moved/); + // The screencast skipping the five frames after the on-screen frame + // changes nothing: the compositor presented them, and they are judged. + const skip = (frames: TrackedFrame[]) => + frames.filter((_, i) => i < 2 || i > 6); + const skipped = busyWindowProof(skip(missing), BLOCK, { + presented: drawn(), + }); + expect(skipped.recorderSkipped).toBe(5); + expect(skipped.verdict.failures.join('\n')).toMatch( + /expected frames moved/, + ); + // And a run with no misses never reads as more than every frame moved. + const clean = busyWindowProof(skip(startingAt(1087)), BLOCK, { + presented: drawn(), + }); + expect(clean.verdict.failures).toEqual([]); + expect(clean.movedFraction).toBeLessThanOrEqual(1); + }); + it('reads frames the screencast skipped while the compositor presented them as presented: a recorded run that fails without the record', () => { const frames = skipFrames(); const alone = busyWindowProof(frames, skip.block, {settledWithin: 1}); diff --git a/internal/interaction-harness/src/busy.ts b/internal/interaction-harness/src/busy.ts index 5374a7634b247..208500c41e1c3 100644 --- a/internal/interaction-harness/src/busy.ts +++ b/internal/interaction-harness/src/busy.ts @@ -452,9 +452,11 @@ export function busyWindowProof( // The frames judged for motion: the screencast's own frames in the judged // part (each `changes` entry is one of them), or, with a compositor record, // the compositor's frames from the first to the last moved frame (the gaps - // before and after them are the stall check's). A part that moved nothing, - // or delivered nothing, is judged against its span, so a frozen animation - // misses here too. + // before and after them are the stall check's). After an on-screen start + // they count from the on-screen frame, which `changes` leaves out, so the + // frames credited as skipped after it are judged too. A part that moved + // nothing, or delivered nothing, is judged against its span, so a frozen + // animation misses here too. const deliveredFrames = tracked.filter( f => f.box != null && @@ -465,8 +467,9 @@ export function busyWindowProof( compositorFrameMs != null ? changes.length > 0 ? Math.round( - (changes[changes.length - 1] - changes[0]) / compositorFrameMs, - ) + 1 + (changes[changes.length - 1] - (onScreenFrom ?? changes[0])) / + compositorFrameMs, + ) + (onScreenFrom == null ? 1 : 0) : expectedFrames : deliveredFrames === 0 ? expectedFrames From 51688cdf3c3a8bc02eeb76e004e90261506ff623 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Mon, 26 Oct 2026 06:47:17 -0700 Subject: [PATCH 77/93] fix(interaction): waitForAnimation and a watch's started() stand down while a trace runs Polled inside a traced body, each look added a Layout of an anchored element in every frame (24 in the anchored fixture's steady middle). Four reads did it, each alone: Playwright's locator.evaluate (each call makes a main-thread frame), the recorder's document-wide read, an unwatched wait's element.getAnimations(), and effect.getComputedTiming(), which updates a running animation on demand and restyles its target. While a trace runs, record() from outside the recorder and arm()'s read stand down; the wait takes the element's animations from the recorder's log, judges a started one by its own times (playState only before it has a start time), reads timing through getTiming(), and reaches the element through a handle to what the locator matches at each look. The same window now lays out 0 times. --- internal/interaction-harness/README.md | 6 +- internal/interaction-harness/src/curves.ts | 167 ++++++++++++++++-- .../src/harness.interaction.browser.spec.ts | 44 +++++ internal/interaction-harness/src/trace.ts | 5 +- 4 files changed, 200 insertions(+), 22 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index b9f1cb4b320b3..01b9b07ee2270 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -192,7 +192,11 @@ down too: it waits on timers for the scheduled ends of the element's curves (looking `frames` times a frame apart before settling on none, so an animation the action starts a frame or two later is waited for) and reads no style until the trace ends (in the anchored fixture's steady middle, -24 Layouts before, 0 now). The trace's `longTasks` are the page's own: only a +24 Layouts before, 0 now). So do `waitForAnimation` and a watch's +`started()`: they take the element's animations from the recorder's log, +judge a started one by its own times, and reach the element through a handle, +since each `locator.evaluate` makes a main-thread frame (there too, 24 +Layouts before, 0 now). The trace's `longTasks` are the page's own: only a task the harness marked as its own read (`waitForAnimation` describing keyframes, `blockMainThread` without `during`) is in `devtoolsTasks` instead, and only when no page script ran inside it. A test's own `page.evaluate` stays a diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index 487b6a10d9839..b10c0361179c6 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -203,7 +203,15 @@ function animationRecorder(): void { }; w.__harnessAnimations = { log, - record, + // A read from outside the recorder (a wait's poll) stands down while a + // trace runs, as the frame loop does: document.getAnimations() flushes + // style. CSS start events and the animate() wrapper still log what + // starts, and resume() catches up. + record: () => { + if (!tracing()) { + record(); + } + }, // Catches up on what started during a trace, then runs its frames again. resume: () => { record(); @@ -211,9 +219,12 @@ function animationRecorder(): void { }, // Record first, then stamp strictly after the newest entry: a step that // changed style just before has its transition created by that read, - // and it started before the arm. + // and it started before the arm. While a trace runs the read stands + // down, as record() does. arm: () => { - record(); + if (!tracing()) { + record(); + } const newest = log.length === 0 ? 0 : log[log.length - 1].seenAt; armedAt = Math.max(performance.now(), newest + 0.001); return armedAt; @@ -246,10 +257,47 @@ async function installCurveEnd(page: Page): Promise { await page.evaluate(() => { const w = window as unknown as { __harnessCurveEnd?: (animation: Animation) => number; + __harnessTiming?: ( + effect: AnimationEffect | null | undefined, + ) => ComputedEffectTiming | null; + }; + // An effect's timing as getComputedTiming() reports it. Inside a rendering + // trace it is worked out from getTiming() instead: getComputedTiming() + // updates a running animation on demand, which restyles its target in the + // next frame (and lays out an anchored one), exactly what a trace counts. + w.__harnessTiming ??= effect => { + if (effect == null) { + return null; + } + if ( + (window as unknown as {__harnessRenderingTrace?: boolean}) + .__harnessRenderingTrace !== true + ) { + return effect.getComputedTiming(); + } + const timing = effect.getTiming(); + const duration = + typeof timing.duration === 'number' ? timing.duration : 0; + const iterations = Number(timing.iterations ?? 1); + const delay = Number(timing.delay ?? 0); + const endDelay = Number(timing.endDelay ?? 0); + const activeDuration = + duration === 0 || iterations === 0 ? 0 : duration * iterations; + const derived: ComputedEffectTiming = { + ...timing, + duration, + iterations, + delay, + endDelay, + activeDuration, + endTime: Math.max(delay + activeDuration + endDelay, 0), + }; + return derived; }; if (w.__harnessCurveEnd) { return; } + const timingOf = w.__harnessTiming; const timingKeys = new Set([ 'offset', 'computedOffset', @@ -265,7 +313,7 @@ async function installCurveEnd(page: Page): Promise { ); w.__harnessCurveEnd = animation => { const effect = animation.effect as KeyframeEffect | null; - const timing = effect?.getComputedTiming(); + const timing = timingOf(effect); const delay = Number(timing?.delay ?? 0); const active = Number(timing?.activeDuration ?? 0); if ( @@ -875,6 +923,37 @@ export interface ActiveAnimation { readonly finishedAt: number | null; } +/** + * Runs `fn` on `locator`'s element: through the locator, or, `quietly`, + * through a handle to the element it matches now (none yet: `[]`). Each + * locator.evaluate makes a main-thread frame, which a trace counts; + * `elementHandles()` and a handle's evaluate make none. + */ +async function readElement( + locator: Locator, + quietly: boolean, + fn: (element: Element, arg: A) => R[], + arg: A, +): Promise { + const run = fn as unknown as (element: Element, arg: unknown) => R[]; + if (!quietly) { + return locator.evaluate(run, arg as unknown); + } + const handles = await locator.elementHandles(); + try { + if (handles.length > 1) { + throw new Error( + `waitForAnimation: ${String(locator)} matches ${handles.length} elements`, + ); + } + return handles.length === 0 + ? [] + : await handles[0].evaluate(run, arg as unknown); + } finally { + await Promise.all(handles.map(handle => handle.dispose())); + } +} + /** * Waits until an animation is running on `locator`'s element (or inside it) * and has presented `minProgressMs` of its curve (after any delay), and @@ -894,6 +973,12 @@ export interface ActiveAnimation { * recorder keeps one entry per `Animation` object, so an animation reversed * or replayed mid-run is misread: a watch taken before the reversal rejects, * and an unwatched wait returns at once with a curve dated before it. + * + * Started inside a `traceRendering` body, it stands down as the recorder + * does, so it adds no frame the trace counts: it takes the element's + * animations from the recorder's log, judges a started one by its own times, + * and reaches the element through a handle to what the locator matches at + * each look. */ export async function waitForAnimation( locator: Locator, @@ -909,14 +994,29 @@ export async function waitForAnimation( } = {}, ): Promise { const deadline = Date.now() + timeout; - await installCurveEnd(locator.page()); + const page = locator.page(); + await installCurveEnd(page); + // Inside a rendering trace the wait stands down (see the page function), + // and it reads the element through handles: each locator.evaluate makes a + // main-thread frame (a Layout of an anchored element), where a handle's + // evaluate adds none. + const traced = await page.evaluate( + () => + (window as unknown as {__harnessRenderingTrace?: boolean}) + .__harnessRenderingTrace === true, + ); for (;;) { - const found = await locator.evaluate( + const found = await readElement( + locator, + traced, (element, {minProgress, since: watchedFrom}) => { // The harness's own read: a trace sets it apart (trace.ts HARNESS_READ). console.timeStamp('harness-read'); const w = window as unknown as { __harnessCurveEnd: (a: Animation) => number; + __harnessTiming: ( + effect: AnimationEffect | null | undefined, + ) => ComputedEffectTiming | null; __harnessAnimations: { log: { animation: Animation; @@ -931,6 +1031,13 @@ export async function waitForAnimation( }; const curveEnd = w.__harnessCurveEnd; const recorder = w.__harnessAnimations; + // Inside a rendering trace the wait stands down as the recorder does: + // record() reads nothing, the animations come from the recorder's log + // rather than getAnimations(), and a started one runs by its own + // times, since playState flushes a CSS animation's style. + const traced = + (window as unknown as {__harnessRenderingTrace?: boolean}) + .__harnessRenderingTrace === true; recorder.record(); const describe = ( a: Animation, @@ -938,7 +1045,7 @@ export async function waitForAnimation( recordedStart: number | null, ) => { const effect = a.effect as KeyframeEffect | null; - const timing = effect?.getComputedTiming(); + const timing = w.__harnessTiming(effect); // The recorded start stands in once the page has cancelled it. A // run going backward began at its end: its startTime is where its // current time would reach 0, a whole run later. @@ -1005,21 +1112,36 @@ export async function waitForAnimation( // finished must not answer for it. Only document-timeline animations // count, as for holdAnimations: a scroll- or view-timeline one moves // only when something scrolls and never progresses on its own. + const runs = (a: Animation) => { + if (!(a.timeline instanceof DocumentTimeline)) { + return false; + } + if (!traced || a.startTime == null) { + return a.playState === 'running'; + } + const time = Number(a.currentTime ?? 0); + return a.playbackRate < 0 + ? time > 0 + : time < Number(w.__harnessTiming(a.effect)?.endTime ?? 0); + }; + const onElement = () => + recorder.log.filter( + e => inside(e.animation) || e.ancestors.includes(element), + ); const live = ( - watchedFrom == null + watchedFrom == null && !traced ? element.getAnimations({subtree: true}) - : recorded.filter(e => e.finishedAt == null).map(e => e.animation) - ).filter( - a => - a.playState === 'running' && a.timeline instanceof DocumentTimeline, - ); + : (watchedFrom == null ? onElement() : recorded) + .filter(e => e.finishedAt == null) + .map(e => e.animation) + ).filter(runs); const running = live.filter(a => a.startTime != null); // Progress along the curve, on the timeline: a delay presents nothing // (going backward, the end delay plays first), a run going backward // (`reverse()`) progresses from its end, and a faster rate presents // more curve per ms. const progressed = running.filter(a => { - const timing = a.effect?.getComputedTiming(); + const timing = w.__harnessTiming(a.effect); const time = Number(a.currentTime ?? 0); const local = a.playbackRate < 0 @@ -1044,13 +1166,20 @@ export async function waitForAnimation( return e.finishedAt; } const a = e.animation; - if (a.playState !== 'finished' || a.startTime == null) { + if (a.startTime == null) { + return null; + } + const end = Number(w.__harnessTiming(a.effect)?.endTime ?? 0); + // Traced, by its own times; otherwise by its state. + const time = Number(a.currentTime ?? 0); + const done = traced + ? a.playbackRate < 0 + ? time <= 0 + : time >= end + : a.playState === 'finished'; + if (!done) { return null; } - const end = Number( - (a.effect as KeyframeEffect | null)?.getComputedTiming().endTime ?? - 0, - ); return a.playbackRate < 0 ? Number(a.startTime) : Number(a.startTime) + end / Math.abs(a.playbackRate || 1); diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 72435eb14f9f1..9447f4ebb1024 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -735,6 +735,50 @@ test.describe('frame evidence', () => { expect(trace.window.to - started).toBeGreaterThanOrEqual(590); }); + test('a waitForAnimation or watch.started() awaited inside a traced body stands down too, so the trace counts no Layout from it', async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'rendering traces are Chromium-only (README: engine limits)', + ); + await page.route('http://anchored.test/', route => + route.fulfill({contentType: 'text/html', body: ANCHORED}), + ); + await page.goto('http://anchored.test/'); + const pop = page.locator('#pop'); + await waitForRest(pop); + const watch = await watchAnimations(pop); + let started = 0; + let waited: {durationMs: number}[] = []; + let watched: {durationMs: number}[] = []; + const trace = await traceRendering(page, async () => { + started = await pop.evaluate(el => { + el.animate( + [ + {transform: 'translateY(0px)', opacity: 1}, + {transform: 'translateY(100px)', opacity: 0.5}, + ], + {duration: 600}, + ); + return performance.timeOrigin + performance.now(); + }); + // Both poll until 450 ms of the curve has played, across the steady + // middle; a read of the page's animations each time would lay it out. + waited = await waitForAnimation(pop, {minProgressMs: 450}); + watched = await watch.started({minProgressMs: 450}); + await page.waitForTimeout(200); + }); + expect(countBetween(trace, started + 100, started + 500).Layout ?? 0).toBe( + 0, + ); + // Each still answered with the animation once 450 ms of it had played. + expect(waited.map(a => a.durationMs)).toEqual([600]); + expect(watched.map(a => a.durationMs)).toEqual([600]); + expect(trace.window.to - started).toBeGreaterThanOrEqual(590); + }); + test('a waitForRest awaited inside a traced body waits for an animation the action starts two frames later', async ({ page, browserName, diff --git a/internal/interaction-harness/src/trace.ts b/internal/interaction-harness/src/trace.ts index cd5217d7c3bea..5d09183ec149b 100644 --- a/internal/interaction-harness/src/trace.ts +++ b/internal/interaction-harness/src/trace.ts @@ -116,8 +116,9 @@ interface TraceEvent { /** * Traces the page while `body` runs and counts the rendering work its main * thread did between a start mark (placed just before `body`) and an end mark - * (just after). The harness's animation recorder and `waitForRest` stand down - * meanwhile, so a body may wait with either and add no frame of its own. + * (just after). The harness's animation recorder, `waitForRest`, + * `waitForAnimation` and a watch's `started()` stand down meanwhile, so a body + * may wait with any of them and add no frame of its own. */ export async function traceRendering( page: Page, From 5ad117013e9149fc93f2a2f400b9209dfaa7f757 Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Mon, 26 Oct 2026 07:48:54 -0700 Subject: [PATCH 78/93] fix(interaction): the WebKit container runs as the invoking user under docker Docker ran the Playwright image as root over the bind-mounted checkout, so the test-results/ it wrote were root's and the next host run failed with EACCES clearing them. Under docker the container now runs with the host uid and gid (HOME=/tmp); rootless podman already maps the user. The README says so. --- internal/interaction-harness/README.md | 4 +++- .../scripts/webkit-container.mjs | 17 ++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 21570f608a721..63a94e388b33b 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -23,7 +23,9 @@ pnpm storyboards # only the tests tagged @storyboard On Linux distributions whose system libraries are older than Playwright's WebKit build needs, run the WebKit project inside the official Playwright -image (podman or docker; the repository is mounted at the same path): +image (podman or docker; the repository is mounted at the same path, and +under docker the container runs as your user so its `test-results/` stay +yours): ```sh node internal/interaction-harness/scripts/webkit-container.mjs [playwright test args…] diff --git a/internal/interaction-harness/scripts/webkit-container.mjs b/internal/interaction-harness/scripts/webkit-container.mjs index 32cc0452396df..af48a1c7cf1f5 100644 --- a/internal/interaction-harness/scripts/webkit-container.mjs +++ b/internal/interaction-harness/scripts/webkit-container.mjs @@ -12,7 +12,10 @@ * Storybook and the evidence directory are the host's own. The image tag * matches the installed playwright, so its browsers match the client. * Uses podman when present, else docker. `--network=host` lets the specs' - * local Storybook server and the browser reach each other. + * local Storybook server and the browser reach each other. Under docker the + * container runs as the invoking user (rootless podman already maps it), so + * the `test-results/` it writes into the mounted checkout stay the host + * user's and the next host run can clear them. */ import {spawnSync} from 'node:child_process'; import fs from 'node:fs'; @@ -47,9 +50,21 @@ const env = [ ] .filter(name => process.env[name] != null) .flatMap(name => ['-e', `${name}=${process.env[name]}`]); +// Docker runs the image as root unless told otherwise; the files it writes +// into the bind mount would then be root's. +const user = + engine === 'docker' && typeof process.getuid === 'function' + ? [ + '--user', + `${process.getuid()}:${process.getgid?.() ?? process.getuid()}`, + '-e', + 'HOME=/tmp', + ] + : []; const command = [ 'run', '--rm', + ...user, '--network=host', '--ipc=host', '-v', From 49460a660126c89a0dddd131be1ea1c17c51cb6b Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Mon, 26 Oct 2026 07:50:04 -0700 Subject: [PATCH 79/93] docs(interaction): a traced waitForAnimation on a CSS @keyframes animation still flushes its style The traceRendering JSDoc and the README said waitForAnimation and a watch's started() stand down inside a trace. That holds for script animations and CSS transitions; on a CSS @keyframes animation (and, unwatched, a cancelled CSS transition) they read the animation's timing, which flushes its style. Both now say so and point to waitForRest or a wait after the trace. --- internal/interaction-harness/README.md | 6 +++++- internal/interaction-harness/src/trace.ts | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 01b9b07ee2270..a6fe3093c2345 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -196,7 +196,11 @@ style until the trace ends (in the anchored fixture's steady middle, `started()`: they take the element's animations from the recorder's log, judge a started one by its own times, and reach the element through a handle, since each `locator.evaluate` makes a main-thread frame (there too, 24 -Layouts before, 0 now). The trace's `longTasks` are the page's own: only a +Layouts before, 0 now). That holds for script animations and CSS +transitions. On a CSS `@keyframes` animation (and, unwatched, a cancelled CSS +transition), `waitForAnimation` and `started()` read the animation's timing, +which flushes its style and adds Layouts to the trace: wait on one with +`waitForRest`, or after the trace. The trace's `longTasks` are the page's own: only a task the harness marked as its own read (`waitForAnimation` describing keyframes, `blockMainThread` without `during`) is in `devtoolsTasks` instead, and only when no page script ran inside it. A test's own `page.evaluate` stays a diff --git a/internal/interaction-harness/src/trace.ts b/internal/interaction-harness/src/trace.ts index 5d09183ec149b..43095115bec91 100644 --- a/internal/interaction-harness/src/trace.ts +++ b/internal/interaction-harness/src/trace.ts @@ -118,7 +118,11 @@ interface TraceEvent { * thread did between a start mark (placed just before `body`) and an end mark * (just after). The harness's animation recorder, `waitForRest`, * `waitForAnimation` and a watch's `started()` stand down meanwhile, so a body - * may wait with any of them and add no frame of its own. + * may wait with any of them and add no frame of its own, with one gap: on a + * CSS `@keyframes` animation (and, for an unwatched `waitForAnimation`, a + * cancelled CSS transition) the wait reads its timing, which flushes that + * animation's style, so wait on such an animation with `waitForRest`, or + * after the trace. */ export async function traceRendering( page: Page, From 38ea7d9c32526177a0e05cb8cb8e7252dfd4c655 Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Mon, 26 Oct 2026 08:22:52 -0700 Subject: [PATCH 80/93] fix(interaction): rootless docker runs the WebKit container without --user Rootless Docker maps the container's root to the invoking user, and any other container uid to a subordinate host uid that cannot write the mounted checkout, so --user made the WebKit run fail with EACCES there. The script passes --user only to a rootful daemon (docker info reports no name=rootless); the README and header say so. --- internal/interaction-harness/README.md | 4 ++-- .../scripts/webkit-container.mjs | 24 +++++++++++++------ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 63a94e388b33b..31c5149c2d407 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -24,8 +24,8 @@ pnpm storyboards # only the tests tagged @storyboard On Linux distributions whose system libraries are older than Playwright's WebKit build needs, run the WebKit project inside the official Playwright image (podman or docker; the repository is mounted at the same path, and -under docker the container runs as your user so its `test-results/` stay -yours): +under a rootful docker daemon the container runs as your user so its +`test-results/` stay yours): ```sh node internal/interaction-harness/scripts/webkit-container.mjs [playwright test args…] diff --git a/internal/interaction-harness/scripts/webkit-container.mjs b/internal/interaction-harness/scripts/webkit-container.mjs index af48a1c7cf1f5..cb20fd44aa725 100644 --- a/internal/interaction-harness/scripts/webkit-container.mjs +++ b/internal/interaction-harness/scripts/webkit-container.mjs @@ -12,10 +12,10 @@ * Storybook and the evidence directory are the host's own. The image tag * matches the installed playwright, so its browsers match the client. * Uses podman when present, else docker. `--network=host` lets the specs' - * local Storybook server and the browser reach each other. Under docker the - * container runs as the invoking user (rootless podman already maps it), so - * the `test-results/` it writes into the mounted checkout stay the host - * user's and the next host run can clear them. + * local Storybook server and the browser reach each other. Under a rootful + * docker daemon the container runs as the invoking user (rootless podman and + * rootless docker already map it), so the `test-results/` it writes into the + * mounted checkout stay the host user's and the next host run can clear them. */ import {spawnSync} from 'node:child_process'; import fs from 'node:fs'; @@ -50,10 +50,20 @@ const env = [ ] .filter(name => process.env[name] != null) .flatMap(name => ['-e', `${name}=${process.env[name]}`]); -// Docker runs the image as root unless told otherwise; the files it writes -// into the bind mount would then be root's. +// A rootful Docker daemon runs the image as root unless told otherwise; the +// files it writes into the bind mount would then be root's. Rootless Docker +// maps the container's root to the invoking user (as podman does), and any +// other container uid to a subordinate host uid that cannot write the +// checkout, so it gets no `--user`. +const rootlessDocker = + engine === 'docker' && + /name=rootless/.test( + spawnSync('docker', ['info', '--format', '{{json .SecurityOptions}}'], { + encoding: 'utf8', + }).stdout ?? '', + ); const user = - engine === 'docker' && typeof process.getuid === 'function' + engine === 'docker' && !rootlessDocker && typeof process.getuid === 'function' ? [ '--user', `${process.getuid()}:${process.getgid?.() ?? process.getuid()}`, From 7f0f5c49c4494a2210948f6ef132f050e154480c Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Mon, 26 Oct 2026 09:00:05 -0700 Subject: [PATCH 81/93] docs(interaction): waitForAnimation's JSDoc names the CSS @keyframes gap, as the README does A traced waitForAnimation on a CSS @keyframes animation (and, unwatched, next to a cancelled CSS transition) reads the animation's timing, which flushes its style. The README's No Layout / Paint paragraph and traceRendering's JSDoc say so; waitForAnimation's own JSDoc still said it adds no frame. --- internal/interaction-harness/src/curves.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index 25743cb647f0a..0318feb433925 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -1010,7 +1010,10 @@ async function withinTimeout( * does, so it adds no frame the trace counts: it takes the element's * animations from the recorder's log, judges a started one by its own times, * and reaches the element through a handle to what the locator matches at - * each look. + * each look. That holds for script animations and CSS transitions. On a CSS + * `@keyframes` animation (and, unwatched, a cancelled CSS transition) it reads + * the animation's timing, which flushes its style and adds Layouts to the + * trace: wait on one with `waitForRest`, or after the trace. */ export async function waitForAnimation( locator: Locator, From 4045092bf289c5059ba8da7752827a44c13c7f1c Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Mon, 26 Oct 2026 09:05:15 -0700 Subject: [PATCH 82/93] chore(interaction): move four instruments out of the frame evidence Scope: castStoryboard, sampleRects, openStory's Storybook highlight default and the WebKit container's shader cache move to a follow-up pull request, unchanged, with their self-tests and README text. The frame evidence keeps the screencast, marker tracking, frame stats and the busy-main-thread proof. --- internal/interaction-harness/README.md | 25 +-- .../scripts/webkit-container.mjs | 26 ---- internal/interaction-harness/src/frames.ts | 70 +-------- .../src/harness.interaction.browser.spec.ts | 142 +----------------- internal/interaction-harness/src/story.ts | 47 +----- .../interaction-harness/src/storyboard.ts | 64 +------- .../src/storybook.interaction.browser.spec.ts | 104 ------------- 7 files changed, 12 insertions(+), 466 deletions(-) delete mode 100644 internal/interaction-harness/src/storybook.interaction.browser.spec.ts diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 8247bb710163c..7daf770400e6e 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -33,12 +33,6 @@ under a rootful docker daemon the container runs as your user so its node internal/interaction-harness/scripts/webkit-container.mjs [playwright test args…] ``` -WebKit composites through Mesa's software GL there, which compiles each shader -the first time a page draws it (450–700 ms for the first frame of a blurred or -newly animated layer). The script keeps Mesa's shader cache on the host -(`~/.cache/astryx-webkit-shaders/`, or `ASTRYX_WEBKIT_SHADER_CACHE`), so -only the first run on a machine pays for them. - Evidence (storyboards, curve traces, busy-main-thread frames) is written to `test-results/interaction-evidence/` (override with `ASTRYX_INTERACTION_EVIDENCE`) and attached to the test report. It is never @@ -50,12 +44,12 @@ committed: a pull request links or attaches it. | --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact final-move speed, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. | | `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | -| `frames.ts` | `startScreencast` (timestamped composited frames), `markElement` (a marker chip or outline), `trackMarker` (marker box per frame), `sampleRects` (per-rAF rects), `frameStats`. | +| `frames.ts` | `startScreencast` (timestamped composited frames), `markElement` (a marker chip or outline), `trackMarker` (marker box per frame), `frameStats`. | | `busy.ts` | `blockMainThread(page, ms)` and `busyWindowProof(tracked, window)`: frames presented, frames that moved, longest stall, a verdict; `presented` adds the compositor's own frames. | | `trace.ts` | `traceRendering(page, body)`: main-thread Layout / Paint / UpdateLayoutTree / PrePaint counts, `countBetween`, long tasks; `installLongTaskLog` / `readLongTasks`; `startCompositorTrace`: when the compositor presents. | | `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t), `waitForAnimation`, `watchAnimations`, `waitForRest`, `releaseAnimations`. | | `math.ts` | Oracles independent of the code under test: `cubicBezier`, `timingFunction`, `springStep` (SwiftUI response / dampingFraction, initial velocity), `projectDecay`, `rubberBand`, `parseTransform`, `compareCurve`, `timeline`. | -| `storyboard.ts` | `animationStoryboard` (seek to t = 0, 16.7, 33.3 … ms), `touchStoryboard` (a frame per scripted event), `castStoryboard` (real screencast frames): PNG contact sheet + JSON trace. | +| `storyboard.ts` | `animationStoryboard` (seek to t = 0, 16.7, 33.3 … ms), `touchStoryboard` (a frame per scripted event): PNG contact sheet + JSON trace. | | `output.ts` | `evidenceDir`, `writeEvidence`: where evidence goes and how it is attached to the report. | | `story.ts` | `phone`, `withStorybook`, `openStory`, `frames`, and the timer gate (`installTimerGate`, `freezeTimers`, `thawTimers`). | @@ -71,14 +65,6 @@ dispatch latency, and the events go through the compositor's gesture detection the limits below. Read positions a frame after a move: Chromium delivers `pointermove` aligned to animation frames. -**Stories, not Storybook.** `openStory` turns Storybook's highlight module off -on every page it opens, and checks that its root element is absent. Left on, -the module observes `#storybook-root` and, on every insertion, reads the -computed style of every element on the page inside the inserting task (117 -reads and two forced style recalculations for one exit copy in a Button story), -so a proof would measure Storybook as well as the component. -`openStory(page, origin, id, args, {storybookHighlight: true})` keeps it on. - **Curve sampling.** Pause every running animation (CSS transitions and animations are Web Animations too), seek to a time, read `getComputedStyle`: the presented value at that exact time, independent of frame timing. Finished @@ -149,8 +135,7 @@ allowed; a frame it did not deliver is a gap, for the stall check). Pass `maxMedianFrameMs` (20 at 60 Hz) only in a proof that claims display rate. `verdict.failures` names each miss with its numbers, and `summary` carries the raw numbers for the report. Frames are judged in presentation order: a loaded -machine can deliver them out of order. `castStoryboard` turns the frames of the -block into a filmstrip. +machine can deliver them out of order. A just-started animation can reach the screen after the block began (50–100 ms later in WebKit on a loaded machine). When the cast saw the element at rest @@ -199,14 +184,14 @@ backstop, a cleanup render) would do so behind a held storyboard's back: `installTimerGate` before the animation starts and `freezeTimers` while photographing. The page can still cancel a held callback, a held frame callback gets the time it actually runs, and the harness's own waits (its frame waits, -`blockMainThread` with `during`, `sampleRects`) are not held. +`blockMainThread` with `during`) are not held. An animation that repeats forever does not set a storyboard's default length; with nothing else to film, pass `times`. A touch filmstrip keeps the scripted event `timeStamp`s (Chromium), but a handler that reads `performance.now()` or `Date.now()` sees one event per photograph, about 100 ms apart, so a clock-timed release flings slower when filmed than when played: a storyboard cannot show such a gesture; record it in real time instead, with -`startScreencast` and `castStoryboard`. Chromium can return an empty copy of +`startScreencast`. Chromium can return an empty copy of the page's surface right after it changes ("Unable to capture screenshot"); a storyboard takes that photograph again two frames later, up to three attempts in all, and lists each retake under `recaptures` in the trace. diff --git a/internal/interaction-harness/scripts/webkit-container.mjs b/internal/interaction-harness/scripts/webkit-container.mjs index 11ef11bdfda80..cb20fd44aa725 100644 --- a/internal/interaction-harness/scripts/webkit-container.mjs +++ b/internal/interaction-harness/scripts/webkit-container.mjs @@ -16,19 +16,10 @@ * docker daemon the container runs as the invoking user (rootless podman and * rootless docker already map it), so the `test-results/` it writes into the * mounted checkout stay the host user's and the next host run can clear them. - * - * WebKit composites through Mesa's software GL in the image, which compiles - * each shader variant the first time a page draws it: 450–700 ms for the - * first frame of a blurred or newly animated layer, long enough for a short - * animation to end before it is ever presented. Mesa caches what it compiles, - * and the cache is kept on the host (`~/.cache/astryx-webkit-shaders/`, - * or `ASTRYX_WEBKIT_SHADER_CACHE`), so a run starts with every shader an - * earlier run compiled instead of paying for them in its first tests. */ import {spawnSync} from 'node:child_process'; import fs from 'node:fs'; import {createRequire} from 'node:module'; -import os from 'node:os'; import path from 'node:path'; import {fileURLToPath} from 'node:url'; @@ -51,19 +42,6 @@ if (engine == null) { process.exit(1); } const args = process.argv.slice(2); -// Where the container sees the host's shader cache (see the header). -const containerShaderCache = '/astryx-shader-cache'; -const shaderCache = - process.env.ASTRYX_WEBKIT_SHADER_CACHE ?? - path.join( - os.homedir(), - '.cache', - 'astryx-webkit-shaders', - image.replace(/[^\w.-]+/g, '_'), - ); -fs.mkdirSync(shaderCache, {recursive: true}); -// The harness self-test reads this marker inside the container. -fs.writeFileSync(path.join(shaderCache, 'ASTRYX-SHADER-CACHE'), `${image}\n`); const env = [ 'ASTRYX_STORYBOOK_DIR', 'ASTRYX_INTERACTION_EVIDENCE', @@ -101,10 +79,6 @@ const command = [ '--ipc=host', '-v', `${root}:${root}`, - '-v', - `${shaderCache}:${containerShaderCache}`, - '-e', - `MESA_SHADER_CACHE_DIR=${containerShaderCache}`, '-w', root, ...env, diff --git a/internal/interaction-harness/src/frames.ts b/internal/interaction-harness/src/frames.ts index 83841d259c17f..c61ea7e171c6f 100644 --- a/internal/interaction-harness/src/frames.ts +++ b/internal/interaction-harness/src/frames.ts @@ -7,12 +7,11 @@ * `Screencast` domain) * @output `startScreencast` (timestamped composited frames), `markElement` * (a marker outline the frames can find), `trackMarker` (per-frame marker - * box), `sampleRects` (per-rAF rect sampling), `frameStats`. + * box), `frameStats`. * @position Frame evidence for motion proofs. The screencast is captured off * the page's main thread, from what the compositor presents, so it keeps * delivering frames while a script blocks the main thread: that is what makes - * the busy-main-thread proof possible. Rect sampling is the cheap path when - * the main thread is free. + * the busy-main-thread proof possible. * * Marker tracking needs no image library: frames are decoded and scanned in a * scratch page of the same browser (createImageBitmap + canvas), so the @@ -400,71 +399,6 @@ export async function trackMarker( } } -export interface RectSample { - /** Epoch ms of the rAF callback. */ - readonly t: number; - readonly box: Box | null; - readonly transform: string; - readonly opacity: number; -} - -/** - * Samples the element's rect, transform and opacity on every animation frame - * for `durationMs`. Main-thread sampling: it stops while the main thread is - * blocked, which is the point of the screencast path above. - */ -export async function sampleRects( - locator: Locator, - durationMs: number, -): Promise { - return locator.evaluate( - (element, duration) => - new Promise< - { - t: number; - box: {x: number; y: number; width: number; height: number} | null; - transform: string; - opacity: number; - }[] - >(resolve => { - // The harness's own waits run on the real timers: a frozen timer gate - // (story.ts) holds only the page's. - const gate = ( - window as unknown as { - __harnessTimerGate?: { - frame?: typeof requestAnimationFrame; - }; - } - ).__harnessTimerGate; - const frame = gate?.frame ?? window.requestAnimationFrame.bind(window); - const samples: { - t: number; - box: {x: number; y: number; width: number; height: number} | null; - transform: string; - opacity: number; - }[] = []; - const end = performance.now() + duration; - const tick = (now: number) => { - const rect = element.getBoundingClientRect(); - const style = getComputedStyle(element); - samples.push({ - t: performance.timeOrigin + now, - box: {x: rect.x, y: rect.y, width: rect.width, height: rect.height}, - transform: style.transform, - opacity: Number(style.opacity), - }); - if (now < end) { - frame(tick); - } else { - resolve(samples); - } - }; - frame(tick); - }), - durationMs, - ); -} - export interface FrameStats { readonly frames: number; /** Frames whose tracked value differs from the frame before. */ diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index a25c9f5ee36af..e1d1a422b177f 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -25,19 +25,14 @@ * two frames later; a touch filmstrip keeps scripted event times while a * handler clock sees its pacing; the timer gate holds a component timer * until the filmstrip is done, drops callbacks the page cancels, stamps held - * frames with the time they run, and leaves the harness its own frames; a - * cast filmstrip keeps exactly the presented frames of its window, labelled - * from its zero, as PNG. + * frames with the time they run, and leaves the harness its own frames. * @position The harness's own test. A component spec trusts these * instruments; this file is why it may. */ -import fs from 'node:fs'; -import path from 'node:path'; import {expect, test, type Page} from '@playwright/test'; import { animationStoryboard, - castStoryboard, blockMainThread, busyWindowProof, compareCurve, @@ -61,7 +56,6 @@ import { readLongTasks, releaseAnimations, sampleCurve, - sampleRects, seekAnimations, springStep, startCompositorTrace, @@ -514,83 +508,6 @@ test.describe('frame evidence', () => { expect(proof.monotonic).toBe(true); }); - test('in the WebKit container, compiled shaders outlive the run', async ({ - browser, - }, testInfo) => { - // The official Playwright image keeps its browsers here. - test.skip( - testInfo.project.name !== 'webkit' || - process.env.PLAYWRIGHT_BROWSERS_PATH !== '/ms-playwright', - 'runs inside webkit-container.mjs', - ); - // A cache inside the container dies with it: every run then compiled each - // shader again, 450–700 ms in the first frame of a blurred layer. - const dir = process.env.MESA_SHADER_CACHE_DIR; - expect(dir, 'MESA_SHADER_CACHE_DIR names the host-kept cache').toBeTruthy(); - // Written on the host by the script before the container starts. - expect( - fs.readFileSync(path.join(dir ?? '', 'ASTRYX-SHADER-CACHE'), 'utf8'), - ).toContain('playwright'); - // Mesa writes what it compiles there once a page draws a blurred layer. - // Earlier runs have filled the cache already, so this run's browser gets - // an empty directory beside them, in the same host-kept mount, and the - // test watches its own writes arrive. - const fresh = fs.mkdtempSync(path.join(dir ?? '', 'self-test-')); - try { - const own = await browser.browserType().launch({ - env: {...process.env, MESA_SHADER_CACHE_DIR: fresh}, - }); - try { - const page = await own.newPage(); - await page.setContent( - '
', - ); - await page.evaluate( - () => - document - .getElementById('glass') - ?.animate([{scale: '1'}, {scale: '0.9'}], {duration: 200}) - .finished, - ); - } finally { - await own.close(); - } - expect(fs.readdirSync(fresh, {recursive: true})).not.toEqual([]); - } finally { - fs.rmSync(fresh, {recursive: true, force: true}); - } - }); - - test('rect sampling reads one rect per animation frame while the main thread is free', async ({ - page, - browserName, - }) => { - await fixture(page); - const box = page.locator('#box'); - await box.evaluate(el => { - el.animate( - [{transform: 'translateX(0px)'}, {transform: 'translateX(200px)'}], - {duration: 600, easing: 'linear'}, - ); - }); - const samples = await sampleRects(box, 400); - const xs = samples.map(s => s.box?.x ?? NaN); - const gaps = samples - .slice(1) - .map((s, i) => s.t - samples[i].t) - .sort((a, b) => a - b); - const cadence = gaps[Math.floor(gaps.length / 2)]; - expect(cadence).toBeLessThan(34); - expect(samples.length).toBeGreaterThanOrEqual(10); - if (browserName === 'chromium') { - // One sample per presented frame at 60 Hz. (Headless WebKit's animation - // frames are irregular, so only the cadence is checked there.) - expect(samples.length).toBeGreaterThanOrEqual(20); - } - expect(xs.every((x, i) => i === 0 || x >= xs[i - 1])).toBe(true); - expect(xs.at(-1) ?? 0).toBeGreaterThan(xs[0] + 80); - }); - test('the trace counts no Layout for a transform animation and one per frame for `top`', async ({ page, browserName, @@ -1954,7 +1871,7 @@ test.describe('storyboards', {tag: '@storyboard'}, () => { } }); - test("the harness's own waits run on the real timers: a frozen timer gate holds neither blockMainThread's block nor sampleRects", async ({ + test("the harness's own waits run on the real timers: a frozen timer gate does not hold blockMainThread's block", async ({ page, }) => { await fixture(page); @@ -1978,7 +1895,6 @@ test.describe('storyboards', {tag: '@storyboard'}, () => { expect( await within(blockMainThread(page, 100, {during: box, timeout: 2_000})), ).toBe('settled'); - expect(await within(sampleRects(box, 200))).toBe('settled'); await thawTimers(page); }); @@ -2167,58 +2083,4 @@ test.describe('storyboards', {tag: '@storyboard'}, () => { expect(state).toMatchObject({timer: false, frame: false}); expect(state.heldFor).toBeGreaterThanOrEqual(140); }); - - test('a cast filmstrip keeps exactly the presented frames of its window, labelled from its zero', async ({ - page, - }, testInfo) => { - await fixture(page); - const box = page.locator('#box'); - // JPEG frames (WebKit's are always JPEG) take the conversion path. - const cast = await startScreencast(page, {format: 'jpeg'}); - await box.evaluate(el => - el.animate( - [{transform: 'translateX(0px)'}, {transform: 'translateX(200px)'}], - {duration: 600}, - ), - ); - await page.waitForTimeout(500); - const presented = await cast.stop(); - expect(presented.length).toBeGreaterThan(8); - // A window of the middle frames, zeroed on one of them. - const zero = presented[2].t; - const from = presented[3].t; - const to = presented[presented.length - 3].t; - const inside = presented.filter(f => f.t >= from && f.t <= to); - const seen: number[] = []; - const film = await castStoryboard(page, presented, { - name: 'harness-self-test-cast', - zero, - from, - to, - data: t => { - seen.push(t); - return {t}; - }, - }); - expect(film.frames.map(f => f.t)).toEqual(inside.map(f => f.t - zero)); - expect(film.frames.map(f => f.label)).toEqual( - inside.map(f => `t=${(f.t - zero).toFixed(1)} ms`), - ); - expect(seen).toEqual(inside.map(f => f.t)); - // Every cell is a PNG, whatever the screencast delivered. - for (const frame of film.frames) { - expect([...frame.png.subarray(0, 4)]).toEqual([0x89, 0x50, 0x4e, 0x47]); - } - const files = await film.write(testInfo); - expect(files['sheet.png']).toMatch(/sheet\.png$/); - // A window with no presented frame in it films nothing. - const empty = await castStoryboard(page, presented, { - name: 'harness-self-test-cast-empty', - zero, - from: to + 1, - to: to + 2, - }); - expect(empty.frames).toEqual([]); - await expect(empty.write(testInfo)).rejects.toThrow(/no frames/); - }); }); diff --git a/internal/interaction-harness/src/story.ts b/internal/interaction-harness/src/story.ts index 627969a6637f3..5a1f2237c6725 100644 --- a/internal/interaction-harness/src/story.ts +++ b/internal/interaction-harness/src/story.ts @@ -6,8 +6,7 @@ * static server over a built Storybook * @output `phone` (the touch-phone browser context every interaction proof * runs in), `withStorybook` (serve the built Storybook for a spec file), and - * `openStory` (load one story, with Storybook's highlight module off, and - * wait until it has rendered). + * `openStory` (load one story and wait until it has rendered). * @position Stories are the reproduction path, as for every other browser * check in this repository: a scenario used for verification is checked in, * not built in a one-off page. @@ -60,47 +59,13 @@ export function withStorybook(test: TestType): () => string { }; } -/** Pages whose Storybook highlight module `openStory` turned off. */ -const highlightOff = new WeakSet(); - -export interface OpenStoryOptions { - /** - * Keeps Storybook's highlight module on. It is off by default: when on, it - * observes `#storybook-root` and, on every insertion, reads the computed - * style of every element on the page inside the inserting task, so a proof - * would measure Storybook as well as the component. - */ - readonly storybookHighlight?: boolean; -} - -/** - * Loads a story in the page and waits for its first render, its fonts and two - * frames. Storybook's highlight module is off (see `OpenStoryOptions`), and - * that is checked: its root element must be absent once the story renders. - */ +/** Loads a story in the page and waits for its first render, its fonts and two frames. */ export async function openStory( page: Page, origin: string, storyId: string, args?: Readonly>, - {storybookHighlight = false}: OpenStoryOptions = {}, ): Promise { - if (storybookHighlight) { - if (highlightOff.has(page)) { - throw new Error( - "openStory: this page already turned Storybook's highlight module off; open the story in a new page to keep it on", - ); - } - } else if (!highlightOff.has(page)) { - highlightOff.add(page); - // Storybook's highlight module returns at once when this is set before - // its preview runs. - await page.addInitScript(() => { - ( - globalThis as unknown as {__STORYBOOK_HIGHLIGHT_INITIALIZED?: boolean} - ).__STORYBOOK_HIGHLIGHT_INITIALIZED = true; - }); - } const params = new URLSearchParams({id: storyId, viewMode: 'story'}); if (args != null) { params.set( @@ -121,14 +86,6 @@ export async function openStory( requestAnimationFrame(() => requestAnimationFrame(() => resolve())), ); }); - if ( - !storybookHighlight && - (await page.locator('#storybook-highlights-root').count()) > 0 - ) { - throw new Error( - "openStory: Storybook's highlight module is on (#storybook-highlights-root exists), so every insertion would also read the style of every element on the page; this Storybook may no longer read __STORYBOOK_HIGHLIGHT_INITIALIZED", - ); - } } /** Resolves after `count` animation frames have been presented in the page. */ diff --git a/internal/interaction-harness/src/storyboard.ts b/internal/interaction-harness/src/storyboard.ts index 4a9cbec360bb4..2dc22fdc8cf04 100644 --- a/internal/interaction-harness/src/storyboard.ts +++ b/internal/interaction-harness/src/storyboard.ts @@ -336,8 +336,7 @@ export interface TouchStoryboardOptions extends StoryboardOptions { * photograph per event, about 100 ms between moves instead of the script's * 16.7. Its release velocity is that much lower, so a flick that dismisses * when played can settle back when filmed: a storyboard cannot show such a - * gesture; record it in real time instead (`startScreencast`, - * `castStoryboard`). + * gesture; record it in real time instead. */ export async function touchStoryboard( page: Page, @@ -365,64 +364,3 @@ export async function touchStoryboard( }); return storyboard; } - -/** - * A filmstrip of screencast frames (real presented frames, e.g. the frames of - * a busy-main-thread window), labelled with their time relative to `zero`. - * Evidence for what the compositor presented, not a deterministic storyboard. - */ -export async function castStoryboard( - page: Page, - frames: readonly { - readonly t: number; - readonly data: string; - readonly format: 'png' | 'jpeg'; - }[], - options: StoryboardOptions & { - readonly zero: number; - readonly from?: number; - readonly to?: number; - readonly data?: (t: number) => Readonly>; - }, -): Promise { - const storyboard = new Storyboard(page, options); - for (const frame of frames) { - if (options.from != null && frame.t < options.from) { - continue; - } - if (options.to != null && frame.t > options.to) { - continue; - } - const png = - frame.format === 'png' - ? Buffer.from(frame.data, 'base64') - : await toPng(page, frame.data); - const t = frame.t - options.zero; - storyboard.frames.push({ - label: `t=${t.toFixed(1)} ms`, - t, - data: options.data?.(frame.t) ?? {}, - png, - }); - } - return storyboard; -} - -async function toPng(page: Page, jpegBase64: string): Promise { - const scratch = await page.context().newPage(); - try { - const url = await scratch.evaluate(async data => { - const bitmap = await createImageBitmap( - await (await fetch(`data:image/jpeg;base64,${data}`)).blob(), - ); - const canvas = document.createElement('canvas'); - canvas.width = bitmap.width; - canvas.height = bitmap.height; - canvas.getContext('2d')?.drawImage(bitmap, 0, 0); - return canvas.toDataURL('image/png'); - }, jpegBase64); - return Buffer.from(url.slice(url.indexOf(',') + 1), 'base64'); - } finally { - await scratch.close(); - } -} diff --git a/internal/interaction-harness/src/storybook.interaction.browser.spec.ts b/internal/interaction-harness/src/storybook.interaction.browser.spec.ts deleted file mode 100644 index a711feb62cb84..0000000000000 --- a/internal/interaction-harness/src/storybook.interaction.browser.spec.ts +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. - -/** - * @file storybook.interaction.browser.spec.ts - * @input The built Storybook (`withStorybook`) and one of its stories - * @output `openStory` turns Storybook's highlight module off, so an insertion - * in a story carries no `getComputedStyle` scan of the page; with the module - * on (the control), every insertion carries one. Both engines; Chromium's - * trace also counts the style recalculations the scan forces - * @position Self-test for story.ts; it needs the built Storybook, as every - * component proof does - */ - -import {expect, test, type Page} from '@playwright/test'; -import {openStory, withStorybook} from './story'; -import {countBetween, traceRendering} from './trace'; - -const storybook = withStorybook(test); - -/** Counts the page's `getComputedStyle` calls, from before Storybook loads. */ -function countStyleReads(): void { - const w = window as unknown as {__styleReads: number}; - w.__styleReads = 0; - const read = window.getComputedStyle.bind(window); - window.getComputedStyle = ((element: Element, pseudo?: string | null) => { - w.__styleReads += 1; - return read(element, pseudo); - }) as typeof window.getComputedStyle; -} - -/** - * Inserts an exit copy of the story's first element into `#storybook-root`, - * as an exit animation's copy is, lets the insertion's mutation microtasks run, - * and reports what ran between its marks. - */ -async function insertCopy(page: Page) { - return page.evaluate(async () => { - const w = window as unknown as {__styleReads: number}; - const root = document.getElementById('storybook-root') as HTMLElement; - const reads = w.__styleReads; - const from = - performance.timeOrigin + performance.mark('copy:start').startTime; - const copy = (root.firstElementChild as HTMLElement).cloneNode( - true, - ) as HTMLElement; - copy.setAttribute('aria-hidden', 'true'); - root.appendChild(copy); - // A MutationObserver's callback is a microtask the insertion queued: - // these run after it. - await Promise.resolve(); - await Promise.resolve(); - const to = performance.timeOrigin + performance.mark('copy:end').startTime; - const scanned = w.__styleReads - reads; - const elements = document.body.querySelectorAll('*').length; - copy.remove(); - return {from, to, scanned, elements}; - }); -} - -test("openStory turns Storybook's highlight module off: an insertion in a story carries no getComputedStyle scan of the page", async ({ - page, - browserName, -}) => { - // The insertion's reads in every engine, and in Chromium (whose trace the - // harness reads) the style recalculations between its marks. - const measure = async (p: Page) => { - if (browserName !== 'chromium') { - return {copy: await insertCopy(p), styleRecalc: null}; - } - let copy = {from: 0, to: 0, scanned: 0, elements: 0}; - const trace = await traceRendering(p, async () => { - copy = await insertCopy(p); - }); - return { - copy, - styleRecalc: - countBetween(trace, copy.from, copy.to).UpdateLayoutTree ?? 0, - }; - }; - // The harness default. - await page.addInitScript(countStyleReads); - await openStory(page, storybook(), 'core-button--primary'); - expect(await page.locator('#storybook-highlights-root').count()).toBe(0); - const off = await measure(page); - expect(off.copy.scanned).toBe(0); - if (off.styleRecalc != null) { - expect(off.styleRecalc).toBe(0); - } - - // Control: Storybook's module kept on reads the style of every element on - // the page inside the inserting task. - const control = await page.context().newPage(); - await control.addInitScript(countStyleReads); - await openStory(control, storybook(), 'core-button--primary', undefined, { - storybookHighlight: true, - }); - expect(await control.locator('#storybook-highlights-root').count()).toBe(1); - const on = await measure(control); - expect(on.copy.scanned).toBeGreaterThanOrEqual(on.copy.elements); - if (on.styleRecalc != null) { - expect(on.styleRecalc).toBeGreaterThan(0); - } - await control.close(); -}); From 6762cc544b735b6ba65bf70c2fc4bb58665afb73 Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Mon, 26 Oct 2026 09:05:55 -0700 Subject: [PATCH 83/93] chore(interaction): move rendering traces and the long-task log out of the frame evidence Scope: traceRendering, countBetween and the long-task log (with the harness-read marks the trace uses, their self-tests and README text) move to a follow-up pull request, unchanged. trace.ts keeps startCompositorTrace, which the busy-main-thread proof reads. --- internal/interaction-harness/README.md | 15 +- internal/interaction-harness/src/busy.ts | 2 - internal/interaction-harness/src/curves.ts | 2 - .../src/harness.interaction.browser.spec.ts | 183 +----------- internal/interaction-harness/src/trace.ts | 268 +----------------- 5 files changed, 11 insertions(+), 459 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 7daf770400e6e..468ae2c695e0b 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -11,7 +11,7 @@ pixels per CSS px, touch, mobile viewport rules), in Chromium and WebKit. The harness proves its own instruments in [`src/harness.interaction.browser.spec.ts`](src/harness.interaction.browser.spec.ts): every positive check has a negative control beside it (a rAF animation that -freezes, a `top` animation that lays out every frame). +freezes under a blocked main thread). ## Running @@ -46,7 +46,7 @@ committed: a pull request links or attaches it. | `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | | `frames.ts` | `startScreencast` (timestamped composited frames), `markElement` (a marker chip or outline), `trackMarker` (marker box per frame), `frameStats`. | | `busy.ts` | `blockMainThread(page, ms)` and `busyWindowProof(tracked, window)`: frames presented, frames that moved, longest stall, a verdict; `presented` adds the compositor's own frames. | -| `trace.ts` | `traceRendering(page, body)`: main-thread Layout / Paint / UpdateLayoutTree / PrePaint counts, `countBetween`, long tasks; `installLongTaskLog` / `readLongTasks`; `startCompositorTrace`: when the compositor presents. | +| `trace.ts` | `startCompositorTrace`: when the compositor presents each frame (for the busy-main-thread proof). | | `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t), `waitForAnimation`, `watchAnimations`, `waitForRest`, `releaseAnimations`. | | `math.ts` | Oracles independent of the code under test: `cubicBezier`, `timingFunction`, `springStep` (SwiftUI response / dampingFraction, initial velocity), `projectDecay`, `rubberBand`, `parseTransform`, `compareCurve`, `timeline`. | | `storyboard.ts` | `animationStoryboard` (seek to t = 0, 16.7, 33.3 … ms), `touchStoryboard` (a frame per scripted event): PNG contact sheet + JSON trace. | @@ -166,16 +166,6 @@ compositor's frames from the first to the last moved frame; the gaps before and after are the stall check's. WebKit has no such trace: `stop()` resolves to `[]` and the screencast is judged alone. -**No Layout / Paint.** `traceRendering` records a Chromium trace around the -interaction; `countBetween` counts the steady middle of an animation (its -first and last frames may commit). A transform/opacity animation does no -Layout or Paint there; a layout-property animation does one per frame. Its -`longTasks` are the page's own: only a task the harness marked as its own read -(`waitForAnimation` describing keyframes, `blockMainThread` without `during`) -is in `devtoolsTasks` instead, and only when no page script ran inside it. A -test's own `page.evaluate` stays a long task, whether it calls page code -directly or sets it off. - **Storyboards.** Deterministic: frames come from seeking held animations or from stepping scripted input, never from wall-clock capture. The JSON trace carries each frame's measurements and the spec asserts on it; the sheet is @@ -208,7 +198,6 @@ in all, and lists each retake under `recaptures` in the trace. | Click synthesised from a touch | yes | only from `driver.tap` (WebKit's tap gesture), not from a raw touch sequence | | Curve sampling, storyboards, input log | yes | yes | | Screencast frame evidence (busy main thread) | yes | yes (WebKit's own screencast; frames keep coming while the main thread is blocked) | -| Rendering trace, long tasks | yes | no (Playwright exposes neither) | | Compositor presentation record | yes | no: `stop()` resolves to `[]`; the busy verdict judges the screencast alone | What WebKit cannot show here — native scroll hand-off, flings, a finger diff --git a/internal/interaction-harness/src/busy.ts b/internal/interaction-harness/src/busy.ts index 208500c41e1c3..cad49ad187c82 100644 --- a/internal/interaction-harness/src/busy.ts +++ b/internal/interaction-harness/src/busy.ts @@ -67,8 +67,6 @@ export async function blockMainThread( const minProgress = Math.max(minProgressMs, ONE_FRAME_MS); if (during == null) { return page.evaluate(duration => { - // The harness's own work: a trace sets it apart (trace.ts HARNESS_READ). - console.timeStamp('harness-read'); const from = performance.timeOrigin + performance.now(); const end = performance.now() + duration; let spins = 0; diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index 52d187ea0a6a3..69b9908066f62 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -900,8 +900,6 @@ export async function waitForAnimation( evaluateTimeout => locator.evaluate( (element, {minProgress, since: watchedFrom}) => { - // The harness's own read: a trace sets it apart (trace.ts HARNESS_READ). - console.timeStamp('harness-read'); const w = window as unknown as { __harnessCurveEnd: (a: Animation) => number; __harnessAnimations: { diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index e1d1a422b177f..31842fd92dbfb 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -8,9 +8,7 @@ * page; a flick flings natively (Chromium); a second finger and a pinch * arrive as distinct pointers; the screencast shows a compositor animation * advancing through a blocked main thread while a rAF animation freezes; - * the trace counts zero Layout for a transform animation and one per frame - * for a `top` animation; the long-task log records the page's own long - * task but not a DevTools evaluate, which the trace counts; sampled curves + * sampled curves * match the oracles, individual transform properties included; a sample at * an end reports the time it was read at; holding takes only running * document-timeline animations, each at its own offset, holds each where it @@ -36,7 +34,6 @@ import { blockMainThread, busyWindowProof, compareCurve, - countBetween, cubicBezier, curveOf, drag, @@ -45,7 +42,6 @@ import { freezeTimers, holdAnimations, installInputLog, - installLongTaskLog, installTimerGate, markElement, merge, @@ -53,7 +49,6 @@ import { PARTIAL_LIFT_SKIP, pinch, readInputLog, - readLongTasks, releaseAnimations, sampleCurve, seekAnimations, @@ -66,7 +61,6 @@ import { timeline, touchDriver, touchStoryboard, - traceRendering, trackMarker, waitForAnimation, waitForRest, @@ -507,181 +501,6 @@ test.describe('frame evidence', () => { expect(proof.verdict.failures, proof.summary).toEqual([]); expect(proof.monotonic).toBe(true); }); - - test('the trace counts no Layout for a transform animation and one per frame for `top`', async ({ - page, - browserName, - }) => { - test.skip( - browserName !== 'chromium', - 'rendering traces are Chromium-only (README: engine limits)', - ); - await fixture(page); - const box = page.locator('#box'); - // Count the animation's steady middle: starting and ending it may lay out once each. - const animate = async (property: 'transform' | 'top') => { - let started = 0; - const trace = await traceRendering(page, async () => { - started = await box.evaluate((el, prop) => { - const keyframes = - prop === 'transform' - ? [ - {transform: 'translateY(0px)'}, - {transform: 'translateY(200px)'}, - ] - : [{top: '360px'}, {top: '560px'}]; - el.animate(keyframes, {duration: 600}); - return performance.timeOrigin + performance.now(); - }, property); - await page.waitForTimeout(700); - }); - return countBetween(trace, started + 100, started + 500); - }; - const transform = await animate('transform'); - const top = await animate('top'); - expect(transform.Layout ?? 0).toBe(0); - expect(transform.Paint ?? 0).toBe(0); - // ~24 frames in 400 ms: one layout per frame. - expect(top.Layout ?? 0).toBeGreaterThan(18); - }); - - test("the long-task log records the page's own long task but not a DevTools evaluate, which the trace reports apart", async ({ - page, - browserName, - }) => { - test.skip( - browserName !== 'chromium', - 'the Long Tasks API and rendering traces are Chromium-only', - ); - await installLongTaskLog(page); - await fixture(page); - // The page's own task: a timer that spins 120 ms. - const own = await page.evaluate( - () => - new Promise<{from: number; to: number}>(resolve => { - setTimeout(() => { - const from = performance.timeOrigin + performance.now(); - const end = performance.now() + 120; - while (performance.now() < end) { - // spin - } - resolve({from, to: performance.timeOrigin + performance.now()}); - }, 0); - }), - ); - await frames(page, 2); - const recorded = await readLongTasks(page, own); - expect(recorded).toHaveLength(1); - expect(recorded[0].durationMs).toBeGreaterThanOrEqual(110); - // A DevTools evaluate that spins 300 ms: the trace reports it apart from - // the page's long tasks, and the log never sees it. - let block = {from: 0, to: 0}; - const trace = await traceRendering(page, async () => { - block = await blockMainThread(page, 300); - }); - await frames(page, 2); - const inBlock = (t: {start: number}) => - t.start < block.to && t.start >= block.from - 20; - expect( - trace.devtoolsTasks.some(t => t.durationMs >= 290 && inBlock(t)), - ).toBe(true); - expect(trace.longTasks.filter(inBlock)).toEqual([]); - expect(await readLongTasks(page, block)).toEqual([]); - }); - - test("a harness read inside a trace is not the page's long task, while a test's own evaluate is, whether it calls the page directly or sets it off", async ({ - page, - browserName, - }) => { - test.skip( - browserName !== 'chromium', - 'rendering traces are Chromium-only (README: engine limits)', - ); - // Enough animated rows that describing their keyframes takes a DevTools - // task over 50 ms. - const rows = Array.from( - {length: 1200}, - (_, i) => `
row ${i}
`, - ).join(''); - await page.setContent( - `
${rows}
`, - ); - const slow = page.locator('#slow'); - await slow.evaluate(el => - el.addEventListener('click', () => { - const end = performance.now() + 60; - while (performance.now() < end) { - // spin - } - }), - ); - // Page code a test can call directly, as through a story's own API. - await page.evaluate(() => { - (window as unknown as {pageWork: (ms: number) => void}).pageWork = ( - ms: number, - ) => { - const end = performance.now() + ms; - while (performance.now() < end) { - // spin - } - }; - }); - await frames(page, 3); - const now = () => - page.evaluate(() => performance.timeOrigin + performance.now()); - const read = {from: 0, to: 0}; - const click = {from: 0, to: 0}; - const direct = {from: 0, to: 0}; - let timer = {from: 0, to: 0}; - const trace = await traceRendering(page, async () => { - read.from = await now(); - await waitForAnimation(page.locator('#list')); - read.to = await now(); - // On a slow machine each frame of 1,200 running rows is a long task of - // the page's own; the windows below hold only their own work. - await page.evaluate(() => document.getElementById('list')?.remove()); - await frames(page, 2); - // The page's own task: a timer that spins 60 ms. - timer = await page.evaluate( - () => - new Promise<{from: number; to: number}>(resolve => { - setTimeout(() => { - const from = performance.timeOrigin + performance.now(); - const end = performance.now() + 60; - while (performance.now() < end) { - // spin - } - resolve({from, to: performance.timeOrigin + performance.now()}); - }, 0); - }), - ); - // Page script run inside a DevTools evaluate: a 60 ms click handler. - click.from = await now(); - await slow.evaluate(el => (el as HTMLElement).click()); - click.to = await now(); - // Page code a test's evaluate calls directly: 60 ms of the page's work. - direct.from = await now(); - await page.evaluate(() => - (window as unknown as {pageWork: (ms: number) => void}).pageWork(60), - ); - direct.to = await now(); - }); - const within = (w: {from: number; to: number}) => (t: {start: number}) => - t.start >= w.from - 20 && t.start < w.to; - // The read itself is set apart. The rows' frames around it may be long - // tasks too, the page's own: they stay in longTasks. - expect(trace.devtoolsTasks.some(within(read))).toBe(true); - expect( - trace.longTasks.some(t => within(timer)(t) && t.durationMs >= 55), - ).toBe(true); - expect( - trace.longTasks.some(t => within(click)(t) && t.durationMs >= 55), - ).toBe(true); - expect( - trace.longTasks.some(t => within(direct)(t) && t.durationMs >= 55), - ).toBe(true); - expect(trace.devtoolsTasks.filter(within(direct))).toEqual([]); - }); }); test.describe('curve sampling', () => { diff --git a/internal/interaction-harness/src/trace.ts b/internal/interaction-harness/src/trace.ts index 41cd401b30c25..04ab7b35560d1 100644 --- a/internal/interaction-harness/src/trace.ts +++ b/internal/interaction-harness/src/trace.ts @@ -2,106 +2,17 @@ /** * @file trace.ts - * @input Uses a Playwright `Page` (Chromium: CDP `Tracing`) and the page's - * PerformanceObserver for long tasks - * @output `traceRendering` — counts of the main thread's rendering work - * (Layout, Paint, UpdateLayoutTree, PrePaint, …) between two marks — and - * `installLongTaskLog` / `readLongTasks`. - * @position The "no Layout/Paint during a transform/opacity animation" proof. - * A compositor animation costs the main thread nothing per frame; one that - * animates a layout property, or writes styles from JavaScript each frame, - * shows a Layout (and usually a Paint) per frame. Chromium only: WebKit - * exposes no comparable trace to Playwright. - * - * Counting is scoped to the renderer main thread that ran the page's own - * `performance.mark` calls, and to the time between the start and end marks, - * so other pages, the browser process and setup work are excluded. - * - * SYNC: Keep RENDERING_EVENTS equal to the README's list. + * @input Uses a Playwright `Page` (Chromium: CDP `Tracing`) + * @output `startCompositorTrace`: when the compositor presented each frame, + * for the busy-main-thread proof's `presented`. + * @position Frame evidence beside the screencast: on a loaded machine + * Chromium's screencast can skip frames the compositor still presents, and + * this record tells such a skip from a stall. Chromium only: WebKit exposes + * no comparable trace to Playwright. */ import type {CDPSession, Page} from '@playwright/test'; -/** - * The mark a harness function puts on its own page-side work - * (`console.timeStamp('harness-read')` at the start of the function it - * evaluates in the page), so a trace can tell the harness's reads from page - * code a test's own evaluate runs. - */ -export const HARNESS_READ = 'harness-read'; - -/** Trace events that run page script: a listener, a timer, a frame callback. */ -const PAGE_SCRIPT_EVENTS = new Set([ - 'EventDispatch', - 'FunctionCall', - 'TimerFire', - 'FireAnimationFrame', - 'FireIdleCallback', -]); - -/** Main-thread trace events that are rendering work, by their trace names. */ -export const RENDERING_EVENTS = [ - 'UpdateLayoutTree', // style recalculation - 'Layout', - 'PrePaint', - 'Paint', - 'Layerize', - 'UpdateLayer', - 'ScrollLayer', - 'HitTest', -] as const; - -export interface RenderingTrace { - /** Count per trace event name inside the window, on the page's main thread. */ - readonly counts: Readonly>; - /** Shorthand for the four that matter most. */ - readonly layout: number; - readonly paint: number; - readonly styleRecalc: number; - readonly prePaint: number; - /** Window length, ms. */ - readonly durationMs: number; - /** - * The page's own main-thread tasks longer than 50 ms inside the window (long - * tasks), epoch ms. A task that ran a test's own DevTools command (a - * `page.evaluate`) is one, whether it calls page code directly or sets it - * off; only the harness's own marked reads are in `devtoolsTasks`. - */ - readonly longTasks: readonly { - readonly start: number; - readonly durationMs: number; - }[]; - /** - * Main-thread tasks longer than 50 ms inside the window that ran a DevTools - * command the harness marked as its own read (`HARNESS_READ`) and no page - * script, epoch ms: `waitForAnimation` describing every animation's - * keyframes, or `blockMainThread(page, ms)` without `during`. - */ - readonly devtoolsTasks: readonly { - readonly start: number; - readonly durationMs: number; - }[]; - /** Every counted rendering event with its epoch-ms time, for sub-window counts. */ - readonly events: readonly {readonly name: string; readonly t: number}[]; - /** The window, epoch ms on the page's clock. */ - readonly window: {readonly from: number; readonly to: number}; -} - -/** Counts rendering events by name inside `[from, to]` (epoch ms). */ -export function countBetween( - trace: RenderingTrace, - from: number, - to: number, -): Record { - const counts: Record = {}; - for (const e of trace.events) { - if (e.t >= from && e.t <= to) { - counts[e.name] = (counts[e.name] ?? 0) + 1; - } - } - return counts; -} - interface TraceEvent { name: string; cat: string; @@ -113,128 +24,12 @@ interface TraceEvent { args?: Record; } -/** - * Traces the page while `body` runs and counts the rendering work its main - * thread did between a start mark (placed just before `body`) and an end mark - * (just after). - */ -export async function traceRendering( - page: Page, - body: () => Promise, -): Promise { - const cdp: CDPSession = await page.context().newCDPSession(page); - const events: TraceEvent[] = []; - cdp.on('Tracing.dataCollected', ({value}) => { - events.push(...(value as unknown as TraceEvent[])); - }); - const complete = new Promise(resolve => - cdp.once('Tracing.tracingComplete', () => resolve()), - ); - await cdp.send('Tracing.start', { - transferMode: 'ReportEvents', - traceConfig: { - includedCategories: [ - 'devtools.timeline', - 'disabled-by-default-devtools.timeline', - 'blink.user_timing', - 'toplevel', - ], - excludedCategories: ['*'], - }, - }); - const tag = `harness-${Math.random().toString(36).slice(2, 8)}`; - const mark = (name: string) => - page.evaluate( - n => performance.timeOrigin + performance.mark(n).startTime, - name, - ); - const fromEpoch = await mark(`${tag}:start`); - await body(); - const toEpoch = await mark(`${tag}:end`); - await cdp.send('Tracing.end'); - await complete; - await cdp.detach().catch(() => {}); - - const start = events.find(e => e.name === `${tag}:start`); - const end = events.find(e => e.name === `${tag}:end`); - if (start == null || end == null) { - throw new Error('traceRendering: the window marks were not traced'); - } - const onMain = (e: TraceEvent) => - e.pid === start.pid && - e.tid === start.tid && - e.ts >= start.ts && - e.ts <= end.ts; - // Trace time (µs, monotonic) -> epoch ms, anchored on the start mark. - const toEpochMs = (ts: number) => fromEpoch + (ts - start.ts) / 1000; - const counts: Record = {}; - const counted: {name: string; t: number}[] = []; - for (const e of events) { - if (!onMain(e) || e.ph === 'E') { - continue; - } - if ((RENDERING_EVENTS as readonly string[]).includes(e.name)) { - counts[e.name] = (counts[e.name] ?? 0) + 1; - counted.push({name: e.name, t: toEpochMs(e.ts)}); - } - } - // A task that ran a DevTools command received it over the DevTools - // session's pipe; page script it set off shows as its listener, timer or - // frame callback. - const main = events.filter(e => onMain(e) && e.ph !== 'E'); - const longTasks: {start: number; durationMs: number}[] = []; - const devtoolsTasks: {start: number; durationMs: number}[] = []; - for (const task of main) { - if (task.name !== 'RunTask' || (task.dur ?? 0) <= 50_000) { - continue; - } - const end = task.ts + (task.dur ?? 0); - const inside = main.filter( - e => e !== task && e.ts >= task.ts && e.ts <= end, - ); - const devtools = inside.some( - e => - e.name === 'Receive mojo message' && - ( - e.args?.chrome_mojo_event_info as - {mojo_interface_tag?: unknown} | undefined - )?.mojo_interface_tag === 'blink.mojom.DevToolsSession', - ); - // Only the harness's own reads are set apart: a test's evaluate may call - // page code directly, which leaves no listener or timer in the trace. - const harnessRead = inside.some( - e => - e.name === 'TimeStamp' && - (e.args?.data as {message?: unknown} | undefined)?.message === - HARNESS_READ, - ); - const pageScript = inside.some(e => PAGE_SCRIPT_EVENTS.has(e.name)); - (devtools && harnessRead && !pageScript ? devtoolsTasks : longTasks).push({ - start: toEpochMs(task.ts), - durationMs: (task.dur ?? 0) / 1000, - }); - } - return { - counts, - layout: counts.Layout ?? 0, - paint: counts.Paint ?? 0, - styleRecalc: counts.UpdateLayoutTree ?? 0, - prePaint: counts.PrePaint ?? 0, - durationMs: (end.ts - start.ts) / 1000, - longTasks, - devtoolsTasks, - events: counted, - window: {from: fromEpoch, to: toEpoch}, - }; -} - /** * Records when the compositor presents each frame, for `busyWindowProof`'s * `presented`: a Chromium trace of `Display::FrameDisplayed`, on the page * clock. Start it with the screencast, before the block, and stop it after. * Engines without a CDP trace (WebKit) resolve `stop()` to [], and the - * verdict then judges the screencast alone. It runs its own trace, so not - * inside `traceRendering`. + * verdict then judges the screencast alone. */ export async function startCompositorTrace( page: Page, @@ -284,50 +79,3 @@ export async function startCompositorTrace( }, }; } - -const LONG_TASK_SCRIPT = `(() => { - if (window.__harnessLongTasks || typeof PerformanceObserver === 'undefined') return; - const tasks = []; - window.__harnessLongTasks = tasks; - try { - new PerformanceObserver(list => { - for (const entry of list.getEntries()) tasks.push({start: performance.timeOrigin + entry.startTime, durationMs: entry.duration, name: entry.name}); - }).observe({type: 'longtask', buffered: true}); - } catch {} -})()`; - -/** - * Records every long task (> 50 ms) the page runs (install before navigation; - * Chromium), through the Long Tasks API. That sees the tasks of the page's own - * event loop: a timer, an event handler, `blockMainThread(page, ms, {during})`. - * A DevTools `page.evaluate`, such as `blockMainThread(page, ms)` without - * `during`, is not one of them and is never recorded here; `traceRendering` - * reports it in `devtoolsTasks`, apart from the page's `longTasks`. - */ -export async function installLongTaskLog(page: Page): Promise { - await page.addInitScript(LONG_TASK_SCRIPT); -} - -export interface LongTask { - /** Epoch ms. */ - readonly start: number; - readonly durationMs: number; - readonly name: string; -} - -/** Long tasks recorded since navigation, optionally only those overlapping a window. */ -export async function readLongTasks( - page: Page, - range?: {from: number; to: number}, -): Promise { - const tasks = await page.evaluate( - () => - (window as unknown as {__harnessLongTasks?: LongTask[]}) - .__harnessLongTasks ?? [], - ); - return range == null - ? tasks - : tasks.filter( - t => t.start < range.to && t.start + t.durationMs > range.from, - ); -} From 03058f60f10fe57690487edf7c92ba2f9b066ca7 Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Mon, 26 Oct 2026 09:07:07 -0700 Subject: [PATCH 84/93] chore(interaction): move the compositor presentation record out of the frame evidence Scope: startCompositorTrace (trace.ts), its Chromium self-test and the recorded skip fixture with its unit test move to the follow-up pull request with the rendering traces, unchanged. busyWindowProof keeps its presented option and its synthetic compositor-record tests; the README and JSDoc say where a record comes from without naming the moved helper. --- internal/interaction-harness/README.md | 11 +- .../src/busy.skip.fixture.ts | 125 ------------------ internal/interaction-harness/src/busy.test.ts | 33 ----- internal/interaction-harness/src/busy.ts | 8 +- .../src/harness.interaction.browser.spec.ts | 80 ----------- internal/interaction-harness/src/index.ts | 1 - internal/interaction-harness/src/trace.ts | 81 ------------ 7 files changed, 8 insertions(+), 331 deletions(-) delete mode 100644 internal/interaction-harness/src/busy.skip.fixture.ts delete mode 100644 internal/interaction-harness/src/trace.ts diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 468ae2c695e0b..d509d31e5b1e6 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -46,7 +46,6 @@ committed: a pull request links or attaches it. | `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | | `frames.ts` | `startScreencast` (timestamped composited frames), `markElement` (a marker chip or outline), `trackMarker` (marker box per frame), `frameStats`. | | `busy.ts` | `blockMainThread(page, ms)` and `busyWindowProof(tracked, window)`: frames presented, frames that moved, longest stall, a verdict; `presented` adds the compositor's own frames. | -| `trace.ts` | `startCompositorTrace`: when the compositor presents each frame (for the busy-main-thread proof). | | `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t), `waitForAnimation`, `watchAnimations`, `waitForRest`, `releaseAnimations`. | | `math.ts` | Oracles independent of the code under test: `cubicBezier`, `timingFunction`, `springStep` (SwiftUI response / dampingFraction, initial velocity), `projectDecay`, `rubberBand`, `parseTransform`, `compareCurve`, `timeline`. | | `storyboard.ts` | `animationStoryboard` (seek to t = 0, 16.7, 33.3 … ms), `touchStoryboard` (a frame per scripted event): PNG contact sheet + JSON trace. | @@ -152,9 +151,8 @@ two steps per frame, which never fails it. Move a proof's element at least two steps a frame: the self-tests move 560 px in 1,600 ms, about 4.8 px a frame. On a loaded machine Chromium's screencast can skip frames the compositor -still presents. `startCompositorTrace(page)`, started with the screencast and -stopped after the block, records when the compositor presented each frame -(`Display::FrameDisplayed`); pass it as `presented`. A gap between two moved +still presents. Pass when the compositor presented each frame (epoch ms on the +page clock, from a trace of `Display::FrameDisplayed`) as `presented`. A gap between two moved frames with nothing delivered inside it is then measured between the compositor's own frames: the frames it presented there count as presented (`recorderSkipped`, in the summary), and a gap it did not present through is @@ -163,8 +161,8 @@ delivered frame that did not move, is judged from the screencast, so a frozen element fails whatever else the compositor draws (a rAF animation's block presents no compositor frame at all). With the record, the 90% counts the compositor's frames from the first to the last moved frame; the gaps before and -after are the stall check's. WebKit has no such trace: -`stop()` resolves to `[]` and the screencast is judged alone. +after are the stall check's. Without a record (WebKit has no such trace), the +screencast is judged alone. **Storyboards.** Deterministic: frames come from seeking held animations or from stepping scripted input, never from wall-clock capture. The JSON trace @@ -198,7 +196,6 @@ in all, and lists each retake under `recaptures` in the trace. | Click synthesised from a touch | yes | only from `driver.tap` (WebKit's tap gesture), not from a raw touch sequence | | Curve sampling, storyboards, input log | yes | yes | | Screencast frame evidence (busy main thread) | yes | yes (WebKit's own screencast; frames keep coming while the main thread is blocked) | -| Compositor presentation record | yes | no: `stop()` resolves to `[]`; the busy verdict judges the screencast alone | What WebKit cannot show here — native scroll hand-off, flings, a finger lifting while another stays down, finger tracking under a busy main thread, and diff --git a/internal/interaction-harness/src/busy.skip.fixture.ts b/internal/interaction-harness/src/busy.skip.fixture.ts deleted file mode 100644 index 409886f12e6f8..0000000000000 --- a/internal/interaction-harness/src/busy.skip.fixture.ts +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. - -/** - * @file busy.skip.fixture.ts - * @input One recorded busy-main-thread run in headless Chromium under - * parallel load: a compositor coast through a 566 ms block - * @output `frames`: the screencast frames the marker was tracked in, as - * [ms after the block started, y in CSS px, or null where the marker was not - * found]; `presented`: when the compositor presented each frame - * (`Display::FrameDisplayed`), ms after the block started; `block`: the - * block's span on the same clock. The screencast skipped frames at about - * +366 and +466 ms while the compositor kept presenting every 16.7 ms. - * @position Fixture for busy.test.ts - */ - -export const block = {from: 0, to: 566.1}; - -export const frames: ReadonlyArray = [ - [-1145.8, null], - [-1114, null], - [-1063.9, null], - [-1031.1, null], - [-996.6, null], - [-964.3, null], - [-930.7, null], - [-896.7, null], - [-864, null], - [-830.6, null], - [-797.8, null], - [-762.7, null], - [-731.3, null], - [-686.5, null], - [-680.8, null], - [-651, null], - [-634.1, null], - [-620.7, null], - [-615.8, null], - [-514.4, null], - [-495, null], - [-472.4, null], - [-465.3, null], - [-448.5, null], - [-430.3, null], - [-414.8, null], - [-398.3, null], - [-379.5, null], - [-364.2, null], - [-348.3, null], - [-330.6, null], - [-315, null], - [-297.6, null], - [-281.9, null], - [-265.1, null], - [-247.6, null], - [-230.8, null], - [-215.3, null], - [-198.6, null], - [-181, null], - [-165.3, null], - [-147.7, null], - [-132, null], - [-115.4, null], - [-98.5, null], - [-81.2, 504], - [-65.2, 500], - [-48.5, 496], - [-30.9, 491], - [-15.2, 487], - [1.3, 484], - [32.8, 480], - [49.8, 476], - [66.3, 473], - [82.8, 469], - [99.7, 466], - [116.1, 463], - [132.9, 460], - [149.5, 457], - [166.3, 454], - [182.9, 451], - [199.4, 448], - [216.2, 446], - [232.8, 443], - [249.4, 441], - [266.1, 438], - [282.7, 436], - [299.5, 434], - [316, 432], - [332.7, 430], - [349.5, 428], - [366.1, 426], - [433.1, 418], - [449.7, 417], - [466.2, 415], - [516.2, 411], - [532.8, 409], - [549.5, 408], - [566.2, 407], - [650.6, 401], - [666.2, 400], - [682.9, 399], - [699.6, 398], - [716.3, 397], - [816.1, 391], - [832.8, 391], - [849.5, 390], - [866.2, 389], - [882.9, 388], - [899.5, 388], - [916.1, 387], - [1033.8, 383], - [1050.1, 383], - [1066.7, 382], - [1083.3, 382], - [1099.5, 381], - [1117.3, 381], - [1134.2, 380], - [1149.5, 380], -]; - -export const presented: readonly number[] = [ - -98.7, -81.3, -65.3, -48.7, -31.1, -15.3, 1.3, 32.5, 49.6, 66.1, 82.7, 99.6, - 115.9, 132.6, 149.3, 166.1, 182.6, 199.3, 216, 232.6, 249.3, 266, 282.6, - 299.2, 315.8, 332.6, 349.3, 366, 381.5, 398.3, 415.1, 432.9, 449.6, 466.1, - 481.5, 498.3, 516, 532.6, 549.4, 566.1, 581.7, 598.4, 615, 631.6, 650.3, 666, -]; diff --git a/internal/interaction-harness/src/busy.test.ts b/internal/interaction-harness/src/busy.test.ts index 64b981a442c9e..00071a28643ef 100644 --- a/internal/interaction-harness/src/busy.test.ts +++ b/internal/interaction-harness/src/busy.test.ts @@ -14,7 +14,6 @@ import {describe, expect, it} from 'vitest'; import {busyWindowProof} from './busy'; -import * as skip from './busy.skip.fixture'; import {inPresentationOrder, type TrackedFrame} from './frames'; const FRAME = 1000 / 60; @@ -350,15 +349,6 @@ function drawn( return draws; } -const skipFrames = (): TrackedFrame[] => - skip.frames.map( - ([t, y]) => - ({ - t, - box: y == null ? null : {x: 0, y, width: 10, height: 10}, - }) as TrackedFrame, - ); - describe('busyWindowProof with the compositor record', () => { it('with a compositor record, judges a sparse cast from its first to its last moved frame', () => { // One screencast frame in six, as on a loaded machine, with the compositor @@ -400,29 +390,6 @@ describe('busyWindowProof with the compositor record', () => { expect(clean.movedFraction).toBeLessThanOrEqual(1); }); - it('reads frames the screencast skipped while the compositor presented them as presented: a recorded run that fails without the record', () => { - const frames = skipFrames(); - const alone = busyWindowProof(frames, skip.block, {settledWithin: 1}); - expect(alone.verdict.failures.join('\n')).toMatch( - /longest stall 6\d\.\d ms = 4 intervals > 3/, - ); - // Every frame the screencast delivered moved: only the stall remains, - // which the record resolves. - expect(alone.verdict.failures.join('\n')).not.toMatch( - /expected frames moved/, - ); - const proof = busyWindowProof(frames, skip.block, { - settledWithin: 1, - presented: skip.presented, - }); - expect(proof.verdict.failures).toEqual([]); - expect(proof.recorderSkipped).toBe(5); - expect(proof.longestStallFrames).toBeLessThanOrEqual(2); - expect(proof.summary).toMatch( - /the screencast skipped 5 frames the compositor presented \(at \+3\d\d ms, \+4\d\d ms\)/, - ); - }); - it('counts a gap the compositor drew through as skipped, and one it did not draw through as a stall', () => { const k = at(1300); const gap: [number, number] = [ diff --git a/internal/interaction-harness/src/busy.ts b/internal/interaction-harness/src/busy.ts index cad49ad187c82..96a0b642456be 100644 --- a/internal/interaction-harness/src/busy.ts +++ b/internal/interaction-harness/src/busy.ts @@ -249,8 +249,8 @@ export interface BusyProof extends FrameStats { * no step and read as not moved, which the proof notes (never a failure). * * A loaded machine's screencast can skip frames the compositor still presents - * (Chromium). Given the compositor's record (`presented`, from - * `startCompositorTrace`), a gap between two moved frames with no frame + * (Chromium). Given the compositor's record (`presented`: when it presented + * each frame), a gap between two moved frames with no frame * delivered inside it is measured between the compositor's own frames: it is * a stall only where the compositor presented nothing, and the frames it did * present there are neither stalls nor misses (`recorderSkipped`). A gap @@ -307,8 +307,8 @@ export function busyWindowProof( settledAt?: number; /** * When the compositor presented each frame, epoch ms on the page clock - * (`startCompositorTrace`; Chromium). Empty or absent, the screencast is - * judged alone. + * (Chromium's `Display::FrameDisplayed`). Empty or absent, the screencast + * is judged alone. */ presented?: readonly number[]; } = {}, diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 31842fd92dbfb..7464538cdce79 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -53,7 +53,6 @@ import { sampleCurve, seekAnimations, springStep, - startCompositorTrace, startScreencast, Storyboard, tap, @@ -342,85 +341,6 @@ test.describe('frame evidence', () => { expect(slowRaf.verdict.failures.join('\n')).toMatch(/longest stall/); }); - test('frames the screencast skips while the compositor presents them are not a stall; a frozen animation still fails with the compositor record', async ({ - page, - browserName, - }, testInfo) => { - test.skip( - browserName !== 'chromium', - 'compositor traces are Chromium-only (README: engine limits)', - ); - await busyFixture(page); - const box = page.locator('#box'); - await markElement(box, {mode: 'outline'}); - const run = async (driver: 'compositor' | 'raf') => { - await box.evaluate((el, how) => { - el.getAnimations().forEach(a => a.cancel()); - el.style.transform = 'none'; - if (how === 'compositor') { - el.animate( - [{transform: 'translateY(0px)'}, {transform: 'translateY(560px)'}], - {duration: 1600, easing: 'linear'}, - ); - } else { - const start = performance.now(); - const step = (now: number) => { - const p = Math.min(1, (now - start) / 1600); - el.style.transform = `translateY(${560 * p}px)`; - if (p < 1) { - requestAnimationFrame(step); - } - }; - requestAnimationFrame(step); - } - }, driver); - const cast = await startScreencast(page); - const compositor = await startCompositorTrace(page); - await page.waitForTimeout(200); - const window = await blockMainThread(page, 700); - await page.waitForTimeout(200); - const presented = await compositor.stop(); - const tracked = await trackMarker(page, await cast.stop()); - return {window, tracked, presented}; - }; - - const moving = await run('compositor'); - expect(moving.presented.length).toBeGreaterThan(0); - // The screencast as a loaded machine delivers it: four frames in a row - // skipped mid-block, which the compositor still presented. - const mid = moving.tracked.findIndex(f => f.t > moving.window.from + 350); - const skipped = moving.tracked.filter((_, i) => i < mid || i >= mid + 4); - const alone = busyWindowProof(skipped, moving.window, {axis: 'y'}); - expect(alone.verdict.failures.join('\n'), alone.summary).toMatch( - /longest stall/, - ); - const proof = busyWindowProof(skipped, moving.window, { - axis: 'y', - presented: moving.presented, - }); - testInfo.annotations.push({ - type: 'busy-main-thread compositor record', - description: proof.summary, - }); - expect(proof.verdict.failures, proof.summary).toEqual([]); - expect(proof.recorderSkipped).toBeGreaterThanOrEqual(4); - - // A rAF animation presents no change for the whole block: the record - // excuses nothing, whatever the compositor drew meanwhile. - const frozen = await run('raf'); - const still = busyWindowProof(frozen.tracked, frozen.window, { - axis: 'y', - presented: frozen.presented, - }); - testInfo.annotations.push({ - type: 'busy-main-thread rAF control, compositor record', - description: `${still.summary}; compositor frames inside the block: ${frozen.presented.filter(t => t > frozen.window.from && t < frozen.window.to).length}`, - }); - expect(still.verdict.pass).toBe(false); - expect(still.recorderSkipped).toBe(0); - expect(still.verdict.failures.join('\n')).toMatch(/longest stall/); - }); - test('a block that starts during a just-started transition still lets the compositor advance it', async ({ page, }, testInfo) => { diff --git a/internal/interaction-harness/src/index.ts b/internal/interaction-harness/src/index.ts index 7a45ff45dfe41..2bcb127f36d88 100644 --- a/internal/interaction-harness/src/index.ts +++ b/internal/interaction-harness/src/index.ts @@ -21,4 +21,3 @@ export * from './scripts'; export * from './story'; export * from './storyboard'; export * from './touch'; -export * from './trace'; diff --git a/internal/interaction-harness/src/trace.ts b/internal/interaction-harness/src/trace.ts deleted file mode 100644 index 04ab7b35560d1..0000000000000 --- a/internal/interaction-harness/src/trace.ts +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. - -/** - * @file trace.ts - * @input Uses a Playwright `Page` (Chromium: CDP `Tracing`) - * @output `startCompositorTrace`: when the compositor presented each frame, - * for the busy-main-thread proof's `presented`. - * @position Frame evidence beside the screencast: on a loaded machine - * Chromium's screencast can skip frames the compositor still presents, and - * this record tells such a skip from a stall. Chromium only: WebKit exposes - * no comparable trace to Playwright. - */ - -import type {CDPSession, Page} from '@playwright/test'; - -interface TraceEvent { - name: string; - cat: string; - ph: string; - ts: number; - dur?: number; - pid: number; - tid: number; - args?: Record; -} - -/** - * Records when the compositor presents each frame, for `busyWindowProof`'s - * `presented`: a Chromium trace of `Display::FrameDisplayed`, on the page - * clock. Start it with the screencast, before the block, and stop it after. - * Engines without a CDP trace (WebKit) resolve `stop()` to [], and the - * verdict then judges the screencast alone. - */ -export async function startCompositorTrace( - page: Page, -): Promise<{stop(): Promise}> { - let cdp: CDPSession; - try { - cdp = await page.context().newCDPSession(page); - } catch { - return {stop: async () => []}; - } - const events: TraceEvent[] = []; - cdp.on('Tracing.dataCollected', ({value}) => { - events.push(...(value as unknown as TraceEvent[])); - }); - const complete = new Promise(resolve => - cdp.once('Tracing.tracingComplete', () => resolve()), - ); - await cdp.send('Tracing.start', { - transferMode: 'ReportEvents', - traceConfig: { - includedCategories: [ - 'disabled-by-default-display.framedisplayed', - 'blink.user_timing', - ], - excludedCategories: ['*'], - }, - }); - const tag = `harness-compositor-${Math.random().toString(36).slice(2, 8)}`; - const epoch = await page.evaluate( - n => performance.timeOrigin + performance.mark(n).startTime, - tag, - ); - return { - async stop() { - await cdp.send('Tracing.end'); - await complete; - await cdp.detach().catch(() => {}); - const mark = events.find(e => e.name === tag); - if (mark == null) { - throw new Error('startCompositorTrace: the clock mark was not traced'); - } - // Trace time (µs, monotonic) -> epoch ms, anchored on the mark. - return events - .filter(e => e.name === 'Display::FrameDisplayed') - .map(e => epoch + (e.ts - mark.ts) / 1000) - .sort((a, b) => a - b); - }, - }; -} From bf5950fa9c0406231c7387b676faf49b05a958ea Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Mon, 26 Oct 2026 09:07:08 -0700 Subject: [PATCH 85/93] feat(interaction): Chromium rendering traces, the compositor presentation record and the long-task log traceRendering counts the main thread's rendering work (Layout, Paint, UpdateLayoutTree, PrePaint, ...) between two marks, for the no Layout / Paint proof of a compositor animation; countBetween reads its steady middle. startCompositorTrace records when the compositor presented each frame, for busyWindowProof's presented. installLongTaskLog / readLongTasks record the page's own long tasks; a harness read is marked (console.timeStamp 'harness-read') so a trace reports it in devtoolsTasks, apart from the page's longTasks. Self-tests in Chromium, the recorded skip fixture, README. Split out of the frame evidence pull request for scope; the code is unchanged. Co-authored-by: Christopher Chedeau --- internal/interaction-harness/README.md | 24 +- .../src/busy.skip.fixture.ts | 125 +++++++ internal/interaction-harness/src/busy.test.ts | 33 ++ internal/interaction-harness/src/busy.ts | 10 +- internal/interaction-harness/src/curves.ts | 2 + .../src/harness.interaction.browser.spec.ts | 263 +++++++++++++- internal/interaction-harness/src/index.ts | 1 + internal/interaction-harness/src/trace.ts | 333 ++++++++++++++++++ 8 files changed, 781 insertions(+), 10 deletions(-) create mode 100644 internal/interaction-harness/src/busy.skip.fixture.ts create mode 100644 internal/interaction-harness/src/trace.ts diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index d509d31e5b1e6..7daf770400e6e 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -11,7 +11,7 @@ pixels per CSS px, touch, mobile viewport rules), in Chromium and WebKit. The harness proves its own instruments in [`src/harness.interaction.browser.spec.ts`](src/harness.interaction.browser.spec.ts): every positive check has a negative control beside it (a rAF animation that -freezes under a blocked main thread). +freezes, a `top` animation that lays out every frame). ## Running @@ -46,6 +46,7 @@ committed: a pull request links or attaches it. | `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | | `frames.ts` | `startScreencast` (timestamped composited frames), `markElement` (a marker chip or outline), `trackMarker` (marker box per frame), `frameStats`. | | `busy.ts` | `blockMainThread(page, ms)` and `busyWindowProof(tracked, window)`: frames presented, frames that moved, longest stall, a verdict; `presented` adds the compositor's own frames. | +| `trace.ts` | `traceRendering(page, body)`: main-thread Layout / Paint / UpdateLayoutTree / PrePaint counts, `countBetween`, long tasks; `installLongTaskLog` / `readLongTasks`; `startCompositorTrace`: when the compositor presents. | | `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t), `waitForAnimation`, `watchAnimations`, `waitForRest`, `releaseAnimations`. | | `math.ts` | Oracles independent of the code under test: `cubicBezier`, `timingFunction`, `springStep` (SwiftUI response / dampingFraction, initial velocity), `projectDecay`, `rubberBand`, `parseTransform`, `compareCurve`, `timeline`. | | `storyboard.ts` | `animationStoryboard` (seek to t = 0, 16.7, 33.3 … ms), `touchStoryboard` (a frame per scripted event): PNG contact sheet + JSON trace. | @@ -151,8 +152,9 @@ two steps per frame, which never fails it. Move a proof's element at least two steps a frame: the self-tests move 560 px in 1,600 ms, about 4.8 px a frame. On a loaded machine Chromium's screencast can skip frames the compositor -still presents. Pass when the compositor presented each frame (epoch ms on the -page clock, from a trace of `Display::FrameDisplayed`) as `presented`. A gap between two moved +still presents. `startCompositorTrace(page)`, started with the screencast and +stopped after the block, records when the compositor presented each frame +(`Display::FrameDisplayed`); pass it as `presented`. A gap between two moved frames with nothing delivered inside it is then measured between the compositor's own frames: the frames it presented there count as presented (`recorderSkipped`, in the summary), and a gap it did not present through is @@ -161,8 +163,18 @@ delivered frame that did not move, is judged from the screencast, so a frozen element fails whatever else the compositor draws (a rAF animation's block presents no compositor frame at all). With the record, the 90% counts the compositor's frames from the first to the last moved frame; the gaps before and -after are the stall check's. Without a record (WebKit has no such trace), the -screencast is judged alone. +after are the stall check's. WebKit has no such trace: +`stop()` resolves to `[]` and the screencast is judged alone. + +**No Layout / Paint.** `traceRendering` records a Chromium trace around the +interaction; `countBetween` counts the steady middle of an animation (its +first and last frames may commit). A transform/opacity animation does no +Layout or Paint there; a layout-property animation does one per frame. Its +`longTasks` are the page's own: only a task the harness marked as its own read +(`waitForAnimation` describing keyframes, `blockMainThread` without `during`) +is in `devtoolsTasks` instead, and only when no page script ran inside it. A +test's own `page.evaluate` stays a long task, whether it calls page code +directly or sets it off. **Storyboards.** Deterministic: frames come from seeking held animations or from stepping scripted input, never from wall-clock capture. The JSON trace @@ -196,6 +208,8 @@ in all, and lists each retake under `recaptures` in the trace. | Click synthesised from a touch | yes | only from `driver.tap` (WebKit's tap gesture), not from a raw touch sequence | | Curve sampling, storyboards, input log | yes | yes | | Screencast frame evidence (busy main thread) | yes | yes (WebKit's own screencast; frames keep coming while the main thread is blocked) | +| Rendering trace, long tasks | yes | no (Playwright exposes neither) | +| Compositor presentation record | yes | no: `stop()` resolves to `[]`; the busy verdict judges the screencast alone | What WebKit cannot show here — native scroll hand-off, flings, a finger lifting while another stays down, finger tracking under a busy main thread, and diff --git a/internal/interaction-harness/src/busy.skip.fixture.ts b/internal/interaction-harness/src/busy.skip.fixture.ts new file mode 100644 index 0000000000000..409886f12e6f8 --- /dev/null +++ b/internal/interaction-harness/src/busy.skip.fixture.ts @@ -0,0 +1,125 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file busy.skip.fixture.ts + * @input One recorded busy-main-thread run in headless Chromium under + * parallel load: a compositor coast through a 566 ms block + * @output `frames`: the screencast frames the marker was tracked in, as + * [ms after the block started, y in CSS px, or null where the marker was not + * found]; `presented`: when the compositor presented each frame + * (`Display::FrameDisplayed`), ms after the block started; `block`: the + * block's span on the same clock. The screencast skipped frames at about + * +366 and +466 ms while the compositor kept presenting every 16.7 ms. + * @position Fixture for busy.test.ts + */ + +export const block = {from: 0, to: 566.1}; + +export const frames: ReadonlyArray = [ + [-1145.8, null], + [-1114, null], + [-1063.9, null], + [-1031.1, null], + [-996.6, null], + [-964.3, null], + [-930.7, null], + [-896.7, null], + [-864, null], + [-830.6, null], + [-797.8, null], + [-762.7, null], + [-731.3, null], + [-686.5, null], + [-680.8, null], + [-651, null], + [-634.1, null], + [-620.7, null], + [-615.8, null], + [-514.4, null], + [-495, null], + [-472.4, null], + [-465.3, null], + [-448.5, null], + [-430.3, null], + [-414.8, null], + [-398.3, null], + [-379.5, null], + [-364.2, null], + [-348.3, null], + [-330.6, null], + [-315, null], + [-297.6, null], + [-281.9, null], + [-265.1, null], + [-247.6, null], + [-230.8, null], + [-215.3, null], + [-198.6, null], + [-181, null], + [-165.3, null], + [-147.7, null], + [-132, null], + [-115.4, null], + [-98.5, null], + [-81.2, 504], + [-65.2, 500], + [-48.5, 496], + [-30.9, 491], + [-15.2, 487], + [1.3, 484], + [32.8, 480], + [49.8, 476], + [66.3, 473], + [82.8, 469], + [99.7, 466], + [116.1, 463], + [132.9, 460], + [149.5, 457], + [166.3, 454], + [182.9, 451], + [199.4, 448], + [216.2, 446], + [232.8, 443], + [249.4, 441], + [266.1, 438], + [282.7, 436], + [299.5, 434], + [316, 432], + [332.7, 430], + [349.5, 428], + [366.1, 426], + [433.1, 418], + [449.7, 417], + [466.2, 415], + [516.2, 411], + [532.8, 409], + [549.5, 408], + [566.2, 407], + [650.6, 401], + [666.2, 400], + [682.9, 399], + [699.6, 398], + [716.3, 397], + [816.1, 391], + [832.8, 391], + [849.5, 390], + [866.2, 389], + [882.9, 388], + [899.5, 388], + [916.1, 387], + [1033.8, 383], + [1050.1, 383], + [1066.7, 382], + [1083.3, 382], + [1099.5, 381], + [1117.3, 381], + [1134.2, 380], + [1149.5, 380], +]; + +export const presented: readonly number[] = [ + -98.7, -81.3, -65.3, -48.7, -31.1, -15.3, 1.3, 32.5, 49.6, 66.1, 82.7, 99.6, + 115.9, 132.6, 149.3, 166.1, 182.6, 199.3, 216, 232.6, 249.3, 266, 282.6, + 299.2, 315.8, 332.6, 349.3, 366, 381.5, 398.3, 415.1, 432.9, 449.6, 466.1, + 481.5, 498.3, 516, 532.6, 549.4, 566.1, 581.7, 598.4, 615, 631.6, 650.3, 666, +]; diff --git a/internal/interaction-harness/src/busy.test.ts b/internal/interaction-harness/src/busy.test.ts index 00071a28643ef..64b981a442c9e 100644 --- a/internal/interaction-harness/src/busy.test.ts +++ b/internal/interaction-harness/src/busy.test.ts @@ -14,6 +14,7 @@ import {describe, expect, it} from 'vitest'; import {busyWindowProof} from './busy'; +import * as skip from './busy.skip.fixture'; import {inPresentationOrder, type TrackedFrame} from './frames'; const FRAME = 1000 / 60; @@ -349,6 +350,15 @@ function drawn( return draws; } +const skipFrames = (): TrackedFrame[] => + skip.frames.map( + ([t, y]) => + ({ + t, + box: y == null ? null : {x: 0, y, width: 10, height: 10}, + }) as TrackedFrame, + ); + describe('busyWindowProof with the compositor record', () => { it('with a compositor record, judges a sparse cast from its first to its last moved frame', () => { // One screencast frame in six, as on a loaded machine, with the compositor @@ -390,6 +400,29 @@ describe('busyWindowProof with the compositor record', () => { expect(clean.movedFraction).toBeLessThanOrEqual(1); }); + it('reads frames the screencast skipped while the compositor presented them as presented: a recorded run that fails without the record', () => { + const frames = skipFrames(); + const alone = busyWindowProof(frames, skip.block, {settledWithin: 1}); + expect(alone.verdict.failures.join('\n')).toMatch( + /longest stall 6\d\.\d ms = 4 intervals > 3/, + ); + // Every frame the screencast delivered moved: only the stall remains, + // which the record resolves. + expect(alone.verdict.failures.join('\n')).not.toMatch( + /expected frames moved/, + ); + const proof = busyWindowProof(frames, skip.block, { + settledWithin: 1, + presented: skip.presented, + }); + expect(proof.verdict.failures).toEqual([]); + expect(proof.recorderSkipped).toBe(5); + expect(proof.longestStallFrames).toBeLessThanOrEqual(2); + expect(proof.summary).toMatch( + /the screencast skipped 5 frames the compositor presented \(at \+3\d\d ms, \+4\d\d ms\)/, + ); + }); + it('counts a gap the compositor drew through as skipped, and one it did not draw through as a stall', () => { const k = at(1300); const gap: [number, number] = [ diff --git a/internal/interaction-harness/src/busy.ts b/internal/interaction-harness/src/busy.ts index 96a0b642456be..208500c41e1c3 100644 --- a/internal/interaction-harness/src/busy.ts +++ b/internal/interaction-harness/src/busy.ts @@ -67,6 +67,8 @@ export async function blockMainThread( const minProgress = Math.max(minProgressMs, ONE_FRAME_MS); if (during == null) { return page.evaluate(duration => { + // The harness's own work: a trace sets it apart (trace.ts HARNESS_READ). + console.timeStamp('harness-read'); const from = performance.timeOrigin + performance.now(); const end = performance.now() + duration; let spins = 0; @@ -249,8 +251,8 @@ export interface BusyProof extends FrameStats { * no step and read as not moved, which the proof notes (never a failure). * * A loaded machine's screencast can skip frames the compositor still presents - * (Chromium). Given the compositor's record (`presented`: when it presented - * each frame), a gap between two moved frames with no frame + * (Chromium). Given the compositor's record (`presented`, from + * `startCompositorTrace`), a gap between two moved frames with no frame * delivered inside it is measured between the compositor's own frames: it is * a stall only where the compositor presented nothing, and the frames it did * present there are neither stalls nor misses (`recorderSkipped`). A gap @@ -307,8 +309,8 @@ export function busyWindowProof( settledAt?: number; /** * When the compositor presented each frame, epoch ms on the page clock - * (Chromium's `Display::FrameDisplayed`). Empty or absent, the screencast - * is judged alone. + * (`startCompositorTrace`; Chromium). Empty or absent, the screencast is + * judged alone. */ presented?: readonly number[]; } = {}, diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index 69b9908066f62..52d187ea0a6a3 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -900,6 +900,8 @@ export async function waitForAnimation( evaluateTimeout => locator.evaluate( (element, {minProgress, since: watchedFrom}) => { + // The harness's own read: a trace sets it apart (trace.ts HARNESS_READ). + console.timeStamp('harness-read'); const w = window as unknown as { __harnessCurveEnd: (a: Animation) => number; __harnessAnimations: { diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 7464538cdce79..e1d1a422b177f 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -8,7 +8,9 @@ * page; a flick flings natively (Chromium); a second finger and a pinch * arrive as distinct pointers; the screencast shows a compositor animation * advancing through a blocked main thread while a rAF animation freezes; - * sampled curves + * the trace counts zero Layout for a transform animation and one per frame + * for a `top` animation; the long-task log records the page's own long + * task but not a DevTools evaluate, which the trace counts; sampled curves * match the oracles, individual transform properties included; a sample at * an end reports the time it was read at; holding takes only running * document-timeline animations, each at its own offset, holds each where it @@ -34,6 +36,7 @@ import { blockMainThread, busyWindowProof, compareCurve, + countBetween, cubicBezier, curveOf, drag, @@ -42,6 +45,7 @@ import { freezeTimers, holdAnimations, installInputLog, + installLongTaskLog, installTimerGate, markElement, merge, @@ -49,10 +53,12 @@ import { PARTIAL_LIFT_SKIP, pinch, readInputLog, + readLongTasks, releaseAnimations, sampleCurve, seekAnimations, springStep, + startCompositorTrace, startScreencast, Storyboard, tap, @@ -60,6 +66,7 @@ import { timeline, touchDriver, touchStoryboard, + traceRendering, trackMarker, waitForAnimation, waitForRest, @@ -341,6 +348,85 @@ test.describe('frame evidence', () => { expect(slowRaf.verdict.failures.join('\n')).toMatch(/longest stall/); }); + test('frames the screencast skips while the compositor presents them are not a stall; a frozen animation still fails with the compositor record', async ({ + page, + browserName, + }, testInfo) => { + test.skip( + browserName !== 'chromium', + 'compositor traces are Chromium-only (README: engine limits)', + ); + await busyFixture(page); + const box = page.locator('#box'); + await markElement(box, {mode: 'outline'}); + const run = async (driver: 'compositor' | 'raf') => { + await box.evaluate((el, how) => { + el.getAnimations().forEach(a => a.cancel()); + el.style.transform = 'none'; + if (how === 'compositor') { + el.animate( + [{transform: 'translateY(0px)'}, {transform: 'translateY(560px)'}], + {duration: 1600, easing: 'linear'}, + ); + } else { + const start = performance.now(); + const step = (now: number) => { + const p = Math.min(1, (now - start) / 1600); + el.style.transform = `translateY(${560 * p}px)`; + if (p < 1) { + requestAnimationFrame(step); + } + }; + requestAnimationFrame(step); + } + }, driver); + const cast = await startScreencast(page); + const compositor = await startCompositorTrace(page); + await page.waitForTimeout(200); + const window = await blockMainThread(page, 700); + await page.waitForTimeout(200); + const presented = await compositor.stop(); + const tracked = await trackMarker(page, await cast.stop()); + return {window, tracked, presented}; + }; + + const moving = await run('compositor'); + expect(moving.presented.length).toBeGreaterThan(0); + // The screencast as a loaded machine delivers it: four frames in a row + // skipped mid-block, which the compositor still presented. + const mid = moving.tracked.findIndex(f => f.t > moving.window.from + 350); + const skipped = moving.tracked.filter((_, i) => i < mid || i >= mid + 4); + const alone = busyWindowProof(skipped, moving.window, {axis: 'y'}); + expect(alone.verdict.failures.join('\n'), alone.summary).toMatch( + /longest stall/, + ); + const proof = busyWindowProof(skipped, moving.window, { + axis: 'y', + presented: moving.presented, + }); + testInfo.annotations.push({ + type: 'busy-main-thread compositor record', + description: proof.summary, + }); + expect(proof.verdict.failures, proof.summary).toEqual([]); + expect(proof.recorderSkipped).toBeGreaterThanOrEqual(4); + + // A rAF animation presents no change for the whole block: the record + // excuses nothing, whatever the compositor drew meanwhile. + const frozen = await run('raf'); + const still = busyWindowProof(frozen.tracked, frozen.window, { + axis: 'y', + presented: frozen.presented, + }); + testInfo.annotations.push({ + type: 'busy-main-thread rAF control, compositor record', + description: `${still.summary}; compositor frames inside the block: ${frozen.presented.filter(t => t > frozen.window.from && t < frozen.window.to).length}`, + }); + expect(still.verdict.pass).toBe(false); + expect(still.recorderSkipped).toBe(0); + expect(still.verdict.failures.join('\n')).toMatch(/longest stall/); + }); + test('a block that starts during a just-started transition still lets the compositor advance it', async ({ page, }, testInfo) => { @@ -421,6 +507,181 @@ test.describe('frame evidence', () => { expect(proof.verdict.failures, proof.summary).toEqual([]); expect(proof.monotonic).toBe(true); }); + + test('the trace counts no Layout for a transform animation and one per frame for `top`', async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'rendering traces are Chromium-only (README: engine limits)', + ); + await fixture(page); + const box = page.locator('#box'); + // Count the animation's steady middle: starting and ending it may lay out once each. + const animate = async (property: 'transform' | 'top') => { + let started = 0; + const trace = await traceRendering(page, async () => { + started = await box.evaluate((el, prop) => { + const keyframes = + prop === 'transform' + ? [ + {transform: 'translateY(0px)'}, + {transform: 'translateY(200px)'}, + ] + : [{top: '360px'}, {top: '560px'}]; + el.animate(keyframes, {duration: 600}); + return performance.timeOrigin + performance.now(); + }, property); + await page.waitForTimeout(700); + }); + return countBetween(trace, started + 100, started + 500); + }; + const transform = await animate('transform'); + const top = await animate('top'); + expect(transform.Layout ?? 0).toBe(0); + expect(transform.Paint ?? 0).toBe(0); + // ~24 frames in 400 ms: one layout per frame. + expect(top.Layout ?? 0).toBeGreaterThan(18); + }); + + test("the long-task log records the page's own long task but not a DevTools evaluate, which the trace reports apart", async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'the Long Tasks API and rendering traces are Chromium-only', + ); + await installLongTaskLog(page); + await fixture(page); + // The page's own task: a timer that spins 120 ms. + const own = await page.evaluate( + () => + new Promise<{from: number; to: number}>(resolve => { + setTimeout(() => { + const from = performance.timeOrigin + performance.now(); + const end = performance.now() + 120; + while (performance.now() < end) { + // spin + } + resolve({from, to: performance.timeOrigin + performance.now()}); + }, 0); + }), + ); + await frames(page, 2); + const recorded = await readLongTasks(page, own); + expect(recorded).toHaveLength(1); + expect(recorded[0].durationMs).toBeGreaterThanOrEqual(110); + // A DevTools evaluate that spins 300 ms: the trace reports it apart from + // the page's long tasks, and the log never sees it. + let block = {from: 0, to: 0}; + const trace = await traceRendering(page, async () => { + block = await blockMainThread(page, 300); + }); + await frames(page, 2); + const inBlock = (t: {start: number}) => + t.start < block.to && t.start >= block.from - 20; + expect( + trace.devtoolsTasks.some(t => t.durationMs >= 290 && inBlock(t)), + ).toBe(true); + expect(trace.longTasks.filter(inBlock)).toEqual([]); + expect(await readLongTasks(page, block)).toEqual([]); + }); + + test("a harness read inside a trace is not the page's long task, while a test's own evaluate is, whether it calls the page directly or sets it off", async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'rendering traces are Chromium-only (README: engine limits)', + ); + // Enough animated rows that describing their keyframes takes a DevTools + // task over 50 ms. + const rows = Array.from( + {length: 1200}, + (_, i) => `
row ${i}
`, + ).join(''); + await page.setContent( + `
${rows}
`, + ); + const slow = page.locator('#slow'); + await slow.evaluate(el => + el.addEventListener('click', () => { + const end = performance.now() + 60; + while (performance.now() < end) { + // spin + } + }), + ); + // Page code a test can call directly, as through a story's own API. + await page.evaluate(() => { + (window as unknown as {pageWork: (ms: number) => void}).pageWork = ( + ms: number, + ) => { + const end = performance.now() + ms; + while (performance.now() < end) { + // spin + } + }; + }); + await frames(page, 3); + const now = () => + page.evaluate(() => performance.timeOrigin + performance.now()); + const read = {from: 0, to: 0}; + const click = {from: 0, to: 0}; + const direct = {from: 0, to: 0}; + let timer = {from: 0, to: 0}; + const trace = await traceRendering(page, async () => { + read.from = await now(); + await waitForAnimation(page.locator('#list')); + read.to = await now(); + // On a slow machine each frame of 1,200 running rows is a long task of + // the page's own; the windows below hold only their own work. + await page.evaluate(() => document.getElementById('list')?.remove()); + await frames(page, 2); + // The page's own task: a timer that spins 60 ms. + timer = await page.evaluate( + () => + new Promise<{from: number; to: number}>(resolve => { + setTimeout(() => { + const from = performance.timeOrigin + performance.now(); + const end = performance.now() + 60; + while (performance.now() < end) { + // spin + } + resolve({from, to: performance.timeOrigin + performance.now()}); + }, 0); + }), + ); + // Page script run inside a DevTools evaluate: a 60 ms click handler. + click.from = await now(); + await slow.evaluate(el => (el as HTMLElement).click()); + click.to = await now(); + // Page code a test's evaluate calls directly: 60 ms of the page's work. + direct.from = await now(); + await page.evaluate(() => + (window as unknown as {pageWork: (ms: number) => void}).pageWork(60), + ); + direct.to = await now(); + }); + const within = (w: {from: number; to: number}) => (t: {start: number}) => + t.start >= w.from - 20 && t.start < w.to; + // The read itself is set apart. The rows' frames around it may be long + // tasks too, the page's own: they stay in longTasks. + expect(trace.devtoolsTasks.some(within(read))).toBe(true); + expect( + trace.longTasks.some(t => within(timer)(t) && t.durationMs >= 55), + ).toBe(true); + expect( + trace.longTasks.some(t => within(click)(t) && t.durationMs >= 55), + ).toBe(true); + expect( + trace.longTasks.some(t => within(direct)(t) && t.durationMs >= 55), + ).toBe(true); + expect(trace.devtoolsTasks.filter(within(direct))).toEqual([]); + }); }); test.describe('curve sampling', () => { diff --git a/internal/interaction-harness/src/index.ts b/internal/interaction-harness/src/index.ts index 2bcb127f36d88..7a45ff45dfe41 100644 --- a/internal/interaction-harness/src/index.ts +++ b/internal/interaction-harness/src/index.ts @@ -21,3 +21,4 @@ export * from './scripts'; export * from './story'; export * from './storyboard'; export * from './touch'; +export * from './trace'; diff --git a/internal/interaction-harness/src/trace.ts b/internal/interaction-harness/src/trace.ts new file mode 100644 index 0000000000000..41cd401b30c25 --- /dev/null +++ b/internal/interaction-harness/src/trace.ts @@ -0,0 +1,333 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file trace.ts + * @input Uses a Playwright `Page` (Chromium: CDP `Tracing`) and the page's + * PerformanceObserver for long tasks + * @output `traceRendering` — counts of the main thread's rendering work + * (Layout, Paint, UpdateLayoutTree, PrePaint, …) between two marks — and + * `installLongTaskLog` / `readLongTasks`. + * @position The "no Layout/Paint during a transform/opacity animation" proof. + * A compositor animation costs the main thread nothing per frame; one that + * animates a layout property, or writes styles from JavaScript each frame, + * shows a Layout (and usually a Paint) per frame. Chromium only: WebKit + * exposes no comparable trace to Playwright. + * + * Counting is scoped to the renderer main thread that ran the page's own + * `performance.mark` calls, and to the time between the start and end marks, + * so other pages, the browser process and setup work are excluded. + * + * SYNC: Keep RENDERING_EVENTS equal to the README's list. + */ + +import type {CDPSession, Page} from '@playwright/test'; + +/** + * The mark a harness function puts on its own page-side work + * (`console.timeStamp('harness-read')` at the start of the function it + * evaluates in the page), so a trace can tell the harness's reads from page + * code a test's own evaluate runs. + */ +export const HARNESS_READ = 'harness-read'; + +/** Trace events that run page script: a listener, a timer, a frame callback. */ +const PAGE_SCRIPT_EVENTS = new Set([ + 'EventDispatch', + 'FunctionCall', + 'TimerFire', + 'FireAnimationFrame', + 'FireIdleCallback', +]); + +/** Main-thread trace events that are rendering work, by their trace names. */ +export const RENDERING_EVENTS = [ + 'UpdateLayoutTree', // style recalculation + 'Layout', + 'PrePaint', + 'Paint', + 'Layerize', + 'UpdateLayer', + 'ScrollLayer', + 'HitTest', +] as const; + +export interface RenderingTrace { + /** Count per trace event name inside the window, on the page's main thread. */ + readonly counts: Readonly>; + /** Shorthand for the four that matter most. */ + readonly layout: number; + readonly paint: number; + readonly styleRecalc: number; + readonly prePaint: number; + /** Window length, ms. */ + readonly durationMs: number; + /** + * The page's own main-thread tasks longer than 50 ms inside the window (long + * tasks), epoch ms. A task that ran a test's own DevTools command (a + * `page.evaluate`) is one, whether it calls page code directly or sets it + * off; only the harness's own marked reads are in `devtoolsTasks`. + */ + readonly longTasks: readonly { + readonly start: number; + readonly durationMs: number; + }[]; + /** + * Main-thread tasks longer than 50 ms inside the window that ran a DevTools + * command the harness marked as its own read (`HARNESS_READ`) and no page + * script, epoch ms: `waitForAnimation` describing every animation's + * keyframes, or `blockMainThread(page, ms)` without `during`. + */ + readonly devtoolsTasks: readonly { + readonly start: number; + readonly durationMs: number; + }[]; + /** Every counted rendering event with its epoch-ms time, for sub-window counts. */ + readonly events: readonly {readonly name: string; readonly t: number}[]; + /** The window, epoch ms on the page's clock. */ + readonly window: {readonly from: number; readonly to: number}; +} + +/** Counts rendering events by name inside `[from, to]` (epoch ms). */ +export function countBetween( + trace: RenderingTrace, + from: number, + to: number, +): Record { + const counts: Record = {}; + for (const e of trace.events) { + if (e.t >= from && e.t <= to) { + counts[e.name] = (counts[e.name] ?? 0) + 1; + } + } + return counts; +} + +interface TraceEvent { + name: string; + cat: string; + ph: string; + ts: number; + dur?: number; + pid: number; + tid: number; + args?: Record; +} + +/** + * Traces the page while `body` runs and counts the rendering work its main + * thread did between a start mark (placed just before `body`) and an end mark + * (just after). + */ +export async function traceRendering( + page: Page, + body: () => Promise, +): Promise { + const cdp: CDPSession = await page.context().newCDPSession(page); + const events: TraceEvent[] = []; + cdp.on('Tracing.dataCollected', ({value}) => { + events.push(...(value as unknown as TraceEvent[])); + }); + const complete = new Promise(resolve => + cdp.once('Tracing.tracingComplete', () => resolve()), + ); + await cdp.send('Tracing.start', { + transferMode: 'ReportEvents', + traceConfig: { + includedCategories: [ + 'devtools.timeline', + 'disabled-by-default-devtools.timeline', + 'blink.user_timing', + 'toplevel', + ], + excludedCategories: ['*'], + }, + }); + const tag = `harness-${Math.random().toString(36).slice(2, 8)}`; + const mark = (name: string) => + page.evaluate( + n => performance.timeOrigin + performance.mark(n).startTime, + name, + ); + const fromEpoch = await mark(`${tag}:start`); + await body(); + const toEpoch = await mark(`${tag}:end`); + await cdp.send('Tracing.end'); + await complete; + await cdp.detach().catch(() => {}); + + const start = events.find(e => e.name === `${tag}:start`); + const end = events.find(e => e.name === `${tag}:end`); + if (start == null || end == null) { + throw new Error('traceRendering: the window marks were not traced'); + } + const onMain = (e: TraceEvent) => + e.pid === start.pid && + e.tid === start.tid && + e.ts >= start.ts && + e.ts <= end.ts; + // Trace time (µs, monotonic) -> epoch ms, anchored on the start mark. + const toEpochMs = (ts: number) => fromEpoch + (ts - start.ts) / 1000; + const counts: Record = {}; + const counted: {name: string; t: number}[] = []; + for (const e of events) { + if (!onMain(e) || e.ph === 'E') { + continue; + } + if ((RENDERING_EVENTS as readonly string[]).includes(e.name)) { + counts[e.name] = (counts[e.name] ?? 0) + 1; + counted.push({name: e.name, t: toEpochMs(e.ts)}); + } + } + // A task that ran a DevTools command received it over the DevTools + // session's pipe; page script it set off shows as its listener, timer or + // frame callback. + const main = events.filter(e => onMain(e) && e.ph !== 'E'); + const longTasks: {start: number; durationMs: number}[] = []; + const devtoolsTasks: {start: number; durationMs: number}[] = []; + for (const task of main) { + if (task.name !== 'RunTask' || (task.dur ?? 0) <= 50_000) { + continue; + } + const end = task.ts + (task.dur ?? 0); + const inside = main.filter( + e => e !== task && e.ts >= task.ts && e.ts <= end, + ); + const devtools = inside.some( + e => + e.name === 'Receive mojo message' && + ( + e.args?.chrome_mojo_event_info as + {mojo_interface_tag?: unknown} | undefined + )?.mojo_interface_tag === 'blink.mojom.DevToolsSession', + ); + // Only the harness's own reads are set apart: a test's evaluate may call + // page code directly, which leaves no listener or timer in the trace. + const harnessRead = inside.some( + e => + e.name === 'TimeStamp' && + (e.args?.data as {message?: unknown} | undefined)?.message === + HARNESS_READ, + ); + const pageScript = inside.some(e => PAGE_SCRIPT_EVENTS.has(e.name)); + (devtools && harnessRead && !pageScript ? devtoolsTasks : longTasks).push({ + start: toEpochMs(task.ts), + durationMs: (task.dur ?? 0) / 1000, + }); + } + return { + counts, + layout: counts.Layout ?? 0, + paint: counts.Paint ?? 0, + styleRecalc: counts.UpdateLayoutTree ?? 0, + prePaint: counts.PrePaint ?? 0, + durationMs: (end.ts - start.ts) / 1000, + longTasks, + devtoolsTasks, + events: counted, + window: {from: fromEpoch, to: toEpoch}, + }; +} + +/** + * Records when the compositor presents each frame, for `busyWindowProof`'s + * `presented`: a Chromium trace of `Display::FrameDisplayed`, on the page + * clock. Start it with the screencast, before the block, and stop it after. + * Engines without a CDP trace (WebKit) resolve `stop()` to [], and the + * verdict then judges the screencast alone. It runs its own trace, so not + * inside `traceRendering`. + */ +export async function startCompositorTrace( + page: Page, +): Promise<{stop(): Promise}> { + let cdp: CDPSession; + try { + cdp = await page.context().newCDPSession(page); + } catch { + return {stop: async () => []}; + } + const events: TraceEvent[] = []; + cdp.on('Tracing.dataCollected', ({value}) => { + events.push(...(value as unknown as TraceEvent[])); + }); + const complete = new Promise(resolve => + cdp.once('Tracing.tracingComplete', () => resolve()), + ); + await cdp.send('Tracing.start', { + transferMode: 'ReportEvents', + traceConfig: { + includedCategories: [ + 'disabled-by-default-display.framedisplayed', + 'blink.user_timing', + ], + excludedCategories: ['*'], + }, + }); + const tag = `harness-compositor-${Math.random().toString(36).slice(2, 8)}`; + const epoch = await page.evaluate( + n => performance.timeOrigin + performance.mark(n).startTime, + tag, + ); + return { + async stop() { + await cdp.send('Tracing.end'); + await complete; + await cdp.detach().catch(() => {}); + const mark = events.find(e => e.name === tag); + if (mark == null) { + throw new Error('startCompositorTrace: the clock mark was not traced'); + } + // Trace time (µs, monotonic) -> epoch ms, anchored on the mark. + return events + .filter(e => e.name === 'Display::FrameDisplayed') + .map(e => epoch + (e.ts - mark.ts) / 1000) + .sort((a, b) => a - b); + }, + }; +} + +const LONG_TASK_SCRIPT = `(() => { + if (window.__harnessLongTasks || typeof PerformanceObserver === 'undefined') return; + const tasks = []; + window.__harnessLongTasks = tasks; + try { + new PerformanceObserver(list => { + for (const entry of list.getEntries()) tasks.push({start: performance.timeOrigin + entry.startTime, durationMs: entry.duration, name: entry.name}); + }).observe({type: 'longtask', buffered: true}); + } catch {} +})()`; + +/** + * Records every long task (> 50 ms) the page runs (install before navigation; + * Chromium), through the Long Tasks API. That sees the tasks of the page's own + * event loop: a timer, an event handler, `blockMainThread(page, ms, {during})`. + * A DevTools `page.evaluate`, such as `blockMainThread(page, ms)` without + * `during`, is not one of them and is never recorded here; `traceRendering` + * reports it in `devtoolsTasks`, apart from the page's `longTasks`. + */ +export async function installLongTaskLog(page: Page): Promise { + await page.addInitScript(LONG_TASK_SCRIPT); +} + +export interface LongTask { + /** Epoch ms. */ + readonly start: number; + readonly durationMs: number; + readonly name: string; +} + +/** Long tasks recorded since navigation, optionally only those overlapping a window. */ +export async function readLongTasks( + page: Page, + range?: {from: number; to: number}, +): Promise { + const tasks = await page.evaluate( + () => + (window as unknown as {__harnessLongTasks?: LongTask[]}) + .__harnessLongTasks ?? [], + ); + return range == null + ? tasks + : tasks.filter( + t => t.start < range.to && t.start + t.durationMs > range.from, + ); +} From d12ed28093036cb80015e36f7989f7fc48bec2b2 Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Mon, 26 Oct 2026 09:07:09 -0700 Subject: [PATCH 86/93] feat(interaction): cast filmstrips, rect sampling, a Storybook highlight default and the WebKit shader cache - castStoryboard: a filmstrip of real screencast frames (e.g. a busy window), labelled from a zero. - sampleRects: one element rect per animation frame, the cheap path when the main thread is free. - openStory turns Storybook's highlight module off (it reads every element's computed style on each insertion, inside the inserting task) and checks it is off; storybookHighlight: true keeps it. - The WebKit container keeps Mesa's shader cache on the host, so only the first run on a machine pays 450-700 ms per new shader. Self-tests for each, README. Split out of the frame evidence pull request for scope; the code is unchanged. Co-authored-by: Christopher Chedeau --- internal/interaction-harness/README.md | 25 ++- .../scripts/webkit-container.mjs | 26 ++++ internal/interaction-harness/src/frames.ts | 70 ++++++++- .../src/harness.interaction.browser.spec.ts | 142 +++++++++++++++++- internal/interaction-harness/src/story.ts | 47 +++++- .../interaction-harness/src/storyboard.ts | 64 +++++++- .../src/storybook.interaction.browser.spec.ts | 104 +++++++++++++ 7 files changed, 466 insertions(+), 12 deletions(-) create mode 100644 internal/interaction-harness/src/storybook.interaction.browser.spec.ts diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 7daf770400e6e..8247bb710163c 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -33,6 +33,12 @@ under a rootful docker daemon the container runs as your user so its node internal/interaction-harness/scripts/webkit-container.mjs [playwright test args…] ``` +WebKit composites through Mesa's software GL there, which compiles each shader +the first time a page draws it (450–700 ms for the first frame of a blurred or +newly animated layer). The script keeps Mesa's shader cache on the host +(`~/.cache/astryx-webkit-shaders/`, or `ASTRYX_WEBKIT_SHADER_CACHE`), so +only the first run on a machine pays for them. + Evidence (storyboards, curve traces, busy-main-thread frames) is written to `test-results/interaction-evidence/` (override with `ASTRYX_INTERACTION_EVIDENCE`) and attached to the test report. It is never @@ -44,12 +50,12 @@ committed: a pull request links or attaches it. | --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact final-move speed, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. | | `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | -| `frames.ts` | `startScreencast` (timestamped composited frames), `markElement` (a marker chip or outline), `trackMarker` (marker box per frame), `frameStats`. | +| `frames.ts` | `startScreencast` (timestamped composited frames), `markElement` (a marker chip or outline), `trackMarker` (marker box per frame), `sampleRects` (per-rAF rects), `frameStats`. | | `busy.ts` | `blockMainThread(page, ms)` and `busyWindowProof(tracked, window)`: frames presented, frames that moved, longest stall, a verdict; `presented` adds the compositor's own frames. | | `trace.ts` | `traceRendering(page, body)`: main-thread Layout / Paint / UpdateLayoutTree / PrePaint counts, `countBetween`, long tasks; `installLongTaskLog` / `readLongTasks`; `startCompositorTrace`: when the compositor presents. | | `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t), `waitForAnimation`, `watchAnimations`, `waitForRest`, `releaseAnimations`. | | `math.ts` | Oracles independent of the code under test: `cubicBezier`, `timingFunction`, `springStep` (SwiftUI response / dampingFraction, initial velocity), `projectDecay`, `rubberBand`, `parseTransform`, `compareCurve`, `timeline`. | -| `storyboard.ts` | `animationStoryboard` (seek to t = 0, 16.7, 33.3 … ms), `touchStoryboard` (a frame per scripted event): PNG contact sheet + JSON trace. | +| `storyboard.ts` | `animationStoryboard` (seek to t = 0, 16.7, 33.3 … ms), `touchStoryboard` (a frame per scripted event), `castStoryboard` (real screencast frames): PNG contact sheet + JSON trace. | | `output.ts` | `evidenceDir`, `writeEvidence`: where evidence goes and how it is attached to the report. | | `story.ts` | `phone`, `withStorybook`, `openStory`, `frames`, and the timer gate (`installTimerGate`, `freezeTimers`, `thawTimers`). | @@ -65,6 +71,14 @@ dispatch latency, and the events go through the compositor's gesture detection the limits below. Read positions a frame after a move: Chromium delivers `pointermove` aligned to animation frames. +**Stories, not Storybook.** `openStory` turns Storybook's highlight module off +on every page it opens, and checks that its root element is absent. Left on, +the module observes `#storybook-root` and, on every insertion, reads the +computed style of every element on the page inside the inserting task (117 +reads and two forced style recalculations for one exit copy in a Button story), +so a proof would measure Storybook as well as the component. +`openStory(page, origin, id, args, {storybookHighlight: true})` keeps it on. + **Curve sampling.** Pause every running animation (CSS transitions and animations are Web Animations too), seek to a time, read `getComputedStyle`: the presented value at that exact time, independent of frame timing. Finished @@ -135,7 +149,8 @@ allowed; a frame it did not deliver is a gap, for the stall check). Pass `maxMedianFrameMs` (20 at 60 Hz) only in a proof that claims display rate. `verdict.failures` names each miss with its numbers, and `summary` carries the raw numbers for the report. Frames are judged in presentation order: a loaded -machine can deliver them out of order. +machine can deliver them out of order. `castStoryboard` turns the frames of the +block into a filmstrip. A just-started animation can reach the screen after the block began (50–100 ms later in WebKit on a loaded machine). When the cast saw the element at rest @@ -184,14 +199,14 @@ backstop, a cleanup render) would do so behind a held storyboard's back: `installTimerGate` before the animation starts and `freezeTimers` while photographing. The page can still cancel a held callback, a held frame callback gets the time it actually runs, and the harness's own waits (its frame waits, -`blockMainThread` with `during`) are not held. +`blockMainThread` with `during`, `sampleRects`) are not held. An animation that repeats forever does not set a storyboard's default length; with nothing else to film, pass `times`. A touch filmstrip keeps the scripted event `timeStamp`s (Chromium), but a handler that reads `performance.now()` or `Date.now()` sees one event per photograph, about 100 ms apart, so a clock-timed release flings slower when filmed than when played: a storyboard cannot show such a gesture; record it in real time instead, with -`startScreencast`. Chromium can return an empty copy of +`startScreencast` and `castStoryboard`. Chromium can return an empty copy of the page's surface right after it changes ("Unable to capture screenshot"); a storyboard takes that photograph again two frames later, up to three attempts in all, and lists each retake under `recaptures` in the trace. diff --git a/internal/interaction-harness/scripts/webkit-container.mjs b/internal/interaction-harness/scripts/webkit-container.mjs index cb20fd44aa725..11ef11bdfda80 100644 --- a/internal/interaction-harness/scripts/webkit-container.mjs +++ b/internal/interaction-harness/scripts/webkit-container.mjs @@ -16,10 +16,19 @@ * docker daemon the container runs as the invoking user (rootless podman and * rootless docker already map it), so the `test-results/` it writes into the * mounted checkout stay the host user's and the next host run can clear them. + * + * WebKit composites through Mesa's software GL in the image, which compiles + * each shader variant the first time a page draws it: 450–700 ms for the + * first frame of a blurred or newly animated layer, long enough for a short + * animation to end before it is ever presented. Mesa caches what it compiles, + * and the cache is kept on the host (`~/.cache/astryx-webkit-shaders/`, + * or `ASTRYX_WEBKIT_SHADER_CACHE`), so a run starts with every shader an + * earlier run compiled instead of paying for them in its first tests. */ import {spawnSync} from 'node:child_process'; import fs from 'node:fs'; import {createRequire} from 'node:module'; +import os from 'node:os'; import path from 'node:path'; import {fileURLToPath} from 'node:url'; @@ -42,6 +51,19 @@ if (engine == null) { process.exit(1); } const args = process.argv.slice(2); +// Where the container sees the host's shader cache (see the header). +const containerShaderCache = '/astryx-shader-cache'; +const shaderCache = + process.env.ASTRYX_WEBKIT_SHADER_CACHE ?? + path.join( + os.homedir(), + '.cache', + 'astryx-webkit-shaders', + image.replace(/[^\w.-]+/g, '_'), + ); +fs.mkdirSync(shaderCache, {recursive: true}); +// The harness self-test reads this marker inside the container. +fs.writeFileSync(path.join(shaderCache, 'ASTRYX-SHADER-CACHE'), `${image}\n`); const env = [ 'ASTRYX_STORYBOOK_DIR', 'ASTRYX_INTERACTION_EVIDENCE', @@ -79,6 +101,10 @@ const command = [ '--ipc=host', '-v', `${root}:${root}`, + '-v', + `${shaderCache}:${containerShaderCache}`, + '-e', + `MESA_SHADER_CACHE_DIR=${containerShaderCache}`, '-w', root, ...env, diff --git a/internal/interaction-harness/src/frames.ts b/internal/interaction-harness/src/frames.ts index c61ea7e171c6f..83841d259c17f 100644 --- a/internal/interaction-harness/src/frames.ts +++ b/internal/interaction-harness/src/frames.ts @@ -7,11 +7,12 @@ * `Screencast` domain) * @output `startScreencast` (timestamped composited frames), `markElement` * (a marker outline the frames can find), `trackMarker` (per-frame marker - * box), `frameStats`. + * box), `sampleRects` (per-rAF rect sampling), `frameStats`. * @position Frame evidence for motion proofs. The screencast is captured off * the page's main thread, from what the compositor presents, so it keeps * delivering frames while a script blocks the main thread: that is what makes - * the busy-main-thread proof possible. + * the busy-main-thread proof possible. Rect sampling is the cheap path when + * the main thread is free. * * Marker tracking needs no image library: frames are decoded and scanned in a * scratch page of the same browser (createImageBitmap + canvas), so the @@ -399,6 +400,71 @@ export async function trackMarker( } } +export interface RectSample { + /** Epoch ms of the rAF callback. */ + readonly t: number; + readonly box: Box | null; + readonly transform: string; + readonly opacity: number; +} + +/** + * Samples the element's rect, transform and opacity on every animation frame + * for `durationMs`. Main-thread sampling: it stops while the main thread is + * blocked, which is the point of the screencast path above. + */ +export async function sampleRects( + locator: Locator, + durationMs: number, +): Promise { + return locator.evaluate( + (element, duration) => + new Promise< + { + t: number; + box: {x: number; y: number; width: number; height: number} | null; + transform: string; + opacity: number; + }[] + >(resolve => { + // The harness's own waits run on the real timers: a frozen timer gate + // (story.ts) holds only the page's. + const gate = ( + window as unknown as { + __harnessTimerGate?: { + frame?: typeof requestAnimationFrame; + }; + } + ).__harnessTimerGate; + const frame = gate?.frame ?? window.requestAnimationFrame.bind(window); + const samples: { + t: number; + box: {x: number; y: number; width: number; height: number} | null; + transform: string; + opacity: number; + }[] = []; + const end = performance.now() + duration; + const tick = (now: number) => { + const rect = element.getBoundingClientRect(); + const style = getComputedStyle(element); + samples.push({ + t: performance.timeOrigin + now, + box: {x: rect.x, y: rect.y, width: rect.width, height: rect.height}, + transform: style.transform, + opacity: Number(style.opacity), + }); + if (now < end) { + frame(tick); + } else { + resolve(samples); + } + }; + frame(tick); + }), + durationMs, + ); +} + export interface FrameStats { readonly frames: number; /** Frames whose tracked value differs from the frame before. */ diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index e1d1a422b177f..a25c9f5ee36af 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -25,14 +25,19 @@ * two frames later; a touch filmstrip keeps scripted event times while a * handler clock sees its pacing; the timer gate holds a component timer * until the filmstrip is done, drops callbacks the page cancels, stamps held - * frames with the time they run, and leaves the harness its own frames. + * frames with the time they run, and leaves the harness its own frames; a + * cast filmstrip keeps exactly the presented frames of its window, labelled + * from its zero, as PNG. * @position The harness's own test. A component spec trusts these * instruments; this file is why it may. */ +import fs from 'node:fs'; +import path from 'node:path'; import {expect, test, type Page} from '@playwright/test'; import { animationStoryboard, + castStoryboard, blockMainThread, busyWindowProof, compareCurve, @@ -56,6 +61,7 @@ import { readLongTasks, releaseAnimations, sampleCurve, + sampleRects, seekAnimations, springStep, startCompositorTrace, @@ -508,6 +514,83 @@ test.describe('frame evidence', () => { expect(proof.monotonic).toBe(true); }); + test('in the WebKit container, compiled shaders outlive the run', async ({ + browser, + }, testInfo) => { + // The official Playwright image keeps its browsers here. + test.skip( + testInfo.project.name !== 'webkit' || + process.env.PLAYWRIGHT_BROWSERS_PATH !== '/ms-playwright', + 'runs inside webkit-container.mjs', + ); + // A cache inside the container dies with it: every run then compiled each + // shader again, 450–700 ms in the first frame of a blurred layer. + const dir = process.env.MESA_SHADER_CACHE_DIR; + expect(dir, 'MESA_SHADER_CACHE_DIR names the host-kept cache').toBeTruthy(); + // Written on the host by the script before the container starts. + expect( + fs.readFileSync(path.join(dir ?? '', 'ASTRYX-SHADER-CACHE'), 'utf8'), + ).toContain('playwright'); + // Mesa writes what it compiles there once a page draws a blurred layer. + // Earlier runs have filled the cache already, so this run's browser gets + // an empty directory beside them, in the same host-kept mount, and the + // test watches its own writes arrive. + const fresh = fs.mkdtempSync(path.join(dir ?? '', 'self-test-')); + try { + const own = await browser.browserType().launch({ + env: {...process.env, MESA_SHADER_CACHE_DIR: fresh}, + }); + try { + const page = await own.newPage(); + await page.setContent( + '
', + ); + await page.evaluate( + () => + document + .getElementById('glass') + ?.animate([{scale: '1'}, {scale: '0.9'}], {duration: 200}) + .finished, + ); + } finally { + await own.close(); + } + expect(fs.readdirSync(fresh, {recursive: true})).not.toEqual([]); + } finally { + fs.rmSync(fresh, {recursive: true, force: true}); + } + }); + + test('rect sampling reads one rect per animation frame while the main thread is free', async ({ + page, + browserName, + }) => { + await fixture(page); + const box = page.locator('#box'); + await box.evaluate(el => { + el.animate( + [{transform: 'translateX(0px)'}, {transform: 'translateX(200px)'}], + {duration: 600, easing: 'linear'}, + ); + }); + const samples = await sampleRects(box, 400); + const xs = samples.map(s => s.box?.x ?? NaN); + const gaps = samples + .slice(1) + .map((s, i) => s.t - samples[i].t) + .sort((a, b) => a - b); + const cadence = gaps[Math.floor(gaps.length / 2)]; + expect(cadence).toBeLessThan(34); + expect(samples.length).toBeGreaterThanOrEqual(10); + if (browserName === 'chromium') { + // One sample per presented frame at 60 Hz. (Headless WebKit's animation + // frames are irregular, so only the cadence is checked there.) + expect(samples.length).toBeGreaterThanOrEqual(20); + } + expect(xs.every((x, i) => i === 0 || x >= xs[i - 1])).toBe(true); + expect(xs.at(-1) ?? 0).toBeGreaterThan(xs[0] + 80); + }); + test('the trace counts no Layout for a transform animation and one per frame for `top`', async ({ page, browserName, @@ -1871,7 +1954,7 @@ test.describe('storyboards', {tag: '@storyboard'}, () => { } }); - test("the harness's own waits run on the real timers: a frozen timer gate does not hold blockMainThread's block", async ({ + test("the harness's own waits run on the real timers: a frozen timer gate holds neither blockMainThread's block nor sampleRects", async ({ page, }) => { await fixture(page); @@ -1895,6 +1978,7 @@ test.describe('storyboards', {tag: '@storyboard'}, () => { expect( await within(blockMainThread(page, 100, {during: box, timeout: 2_000})), ).toBe('settled'); + expect(await within(sampleRects(box, 200))).toBe('settled'); await thawTimers(page); }); @@ -2083,4 +2167,58 @@ test.describe('storyboards', {tag: '@storyboard'}, () => { expect(state).toMatchObject({timer: false, frame: false}); expect(state.heldFor).toBeGreaterThanOrEqual(140); }); + + test('a cast filmstrip keeps exactly the presented frames of its window, labelled from its zero', async ({ + page, + }, testInfo) => { + await fixture(page); + const box = page.locator('#box'); + // JPEG frames (WebKit's are always JPEG) take the conversion path. + const cast = await startScreencast(page, {format: 'jpeg'}); + await box.evaluate(el => + el.animate( + [{transform: 'translateX(0px)'}, {transform: 'translateX(200px)'}], + {duration: 600}, + ), + ); + await page.waitForTimeout(500); + const presented = await cast.stop(); + expect(presented.length).toBeGreaterThan(8); + // A window of the middle frames, zeroed on one of them. + const zero = presented[2].t; + const from = presented[3].t; + const to = presented[presented.length - 3].t; + const inside = presented.filter(f => f.t >= from && f.t <= to); + const seen: number[] = []; + const film = await castStoryboard(page, presented, { + name: 'harness-self-test-cast', + zero, + from, + to, + data: t => { + seen.push(t); + return {t}; + }, + }); + expect(film.frames.map(f => f.t)).toEqual(inside.map(f => f.t - zero)); + expect(film.frames.map(f => f.label)).toEqual( + inside.map(f => `t=${(f.t - zero).toFixed(1)} ms`), + ); + expect(seen).toEqual(inside.map(f => f.t)); + // Every cell is a PNG, whatever the screencast delivered. + for (const frame of film.frames) { + expect([...frame.png.subarray(0, 4)]).toEqual([0x89, 0x50, 0x4e, 0x47]); + } + const files = await film.write(testInfo); + expect(files['sheet.png']).toMatch(/sheet\.png$/); + // A window with no presented frame in it films nothing. + const empty = await castStoryboard(page, presented, { + name: 'harness-self-test-cast-empty', + zero, + from: to + 1, + to: to + 2, + }); + expect(empty.frames).toEqual([]); + await expect(empty.write(testInfo)).rejects.toThrow(/no frames/); + }); }); diff --git a/internal/interaction-harness/src/story.ts b/internal/interaction-harness/src/story.ts index 5a1f2237c6725..627969a6637f3 100644 --- a/internal/interaction-harness/src/story.ts +++ b/internal/interaction-harness/src/story.ts @@ -6,7 +6,8 @@ * static server over a built Storybook * @output `phone` (the touch-phone browser context every interaction proof * runs in), `withStorybook` (serve the built Storybook for a spec file), and - * `openStory` (load one story and wait until it has rendered). + * `openStory` (load one story, with Storybook's highlight module off, and + * wait until it has rendered). * @position Stories are the reproduction path, as for every other browser * check in this repository: a scenario used for verification is checked in, * not built in a one-off page. @@ -59,13 +60,47 @@ export function withStorybook(test: TestType): () => string { }; } -/** Loads a story in the page and waits for its first render, its fonts and two frames. */ +/** Pages whose Storybook highlight module `openStory` turned off. */ +const highlightOff = new WeakSet(); + +export interface OpenStoryOptions { + /** + * Keeps Storybook's highlight module on. It is off by default: when on, it + * observes `#storybook-root` and, on every insertion, reads the computed + * style of every element on the page inside the inserting task, so a proof + * would measure Storybook as well as the component. + */ + readonly storybookHighlight?: boolean; +} + +/** + * Loads a story in the page and waits for its first render, its fonts and two + * frames. Storybook's highlight module is off (see `OpenStoryOptions`), and + * that is checked: its root element must be absent once the story renders. + */ export async function openStory( page: Page, origin: string, storyId: string, args?: Readonly>, + {storybookHighlight = false}: OpenStoryOptions = {}, ): Promise { + if (storybookHighlight) { + if (highlightOff.has(page)) { + throw new Error( + "openStory: this page already turned Storybook's highlight module off; open the story in a new page to keep it on", + ); + } + } else if (!highlightOff.has(page)) { + highlightOff.add(page); + // Storybook's highlight module returns at once when this is set before + // its preview runs. + await page.addInitScript(() => { + ( + globalThis as unknown as {__STORYBOOK_HIGHLIGHT_INITIALIZED?: boolean} + ).__STORYBOOK_HIGHLIGHT_INITIALIZED = true; + }); + } const params = new URLSearchParams({id: storyId, viewMode: 'story'}); if (args != null) { params.set( @@ -86,6 +121,14 @@ export async function openStory( requestAnimationFrame(() => requestAnimationFrame(() => resolve())), ); }); + if ( + !storybookHighlight && + (await page.locator('#storybook-highlights-root').count()) > 0 + ) { + throw new Error( + "openStory: Storybook's highlight module is on (#storybook-highlights-root exists), so every insertion would also read the style of every element on the page; this Storybook may no longer read __STORYBOOK_HIGHLIGHT_INITIALIZED", + ); + } } /** Resolves after `count` animation frames have been presented in the page. */ diff --git a/internal/interaction-harness/src/storyboard.ts b/internal/interaction-harness/src/storyboard.ts index 2dc22fdc8cf04..4a9cbec360bb4 100644 --- a/internal/interaction-harness/src/storyboard.ts +++ b/internal/interaction-harness/src/storyboard.ts @@ -336,7 +336,8 @@ export interface TouchStoryboardOptions extends StoryboardOptions { * photograph per event, about 100 ms between moves instead of the script's * 16.7. Its release velocity is that much lower, so a flick that dismisses * when played can settle back when filmed: a storyboard cannot show such a - * gesture; record it in real time instead. + * gesture; record it in real time instead (`startScreencast`, + * `castStoryboard`). */ export async function touchStoryboard( page: Page, @@ -364,3 +365,64 @@ export async function touchStoryboard( }); return storyboard; } + +/** + * A filmstrip of screencast frames (real presented frames, e.g. the frames of + * a busy-main-thread window), labelled with their time relative to `zero`. + * Evidence for what the compositor presented, not a deterministic storyboard. + */ +export async function castStoryboard( + page: Page, + frames: readonly { + readonly t: number; + readonly data: string; + readonly format: 'png' | 'jpeg'; + }[], + options: StoryboardOptions & { + readonly zero: number; + readonly from?: number; + readonly to?: number; + readonly data?: (t: number) => Readonly>; + }, +): Promise { + const storyboard = new Storyboard(page, options); + for (const frame of frames) { + if (options.from != null && frame.t < options.from) { + continue; + } + if (options.to != null && frame.t > options.to) { + continue; + } + const png = + frame.format === 'png' + ? Buffer.from(frame.data, 'base64') + : await toPng(page, frame.data); + const t = frame.t - options.zero; + storyboard.frames.push({ + label: `t=${t.toFixed(1)} ms`, + t, + data: options.data?.(frame.t) ?? {}, + png, + }); + } + return storyboard; +} + +async function toPng(page: Page, jpegBase64: string): Promise { + const scratch = await page.context().newPage(); + try { + const url = await scratch.evaluate(async data => { + const bitmap = await createImageBitmap( + await (await fetch(`data:image/jpeg;base64,${data}`)).blob(), + ); + const canvas = document.createElement('canvas'); + canvas.width = bitmap.width; + canvas.height = bitmap.height; + canvas.getContext('2d')?.drawImage(bitmap, 0, 0); + return canvas.toDataURL('image/png'); + }, jpegBase64); + return Buffer.from(url.slice(url.indexOf(',') + 1), 'base64'); + } finally { + await scratch.close(); + } +} diff --git a/internal/interaction-harness/src/storybook.interaction.browser.spec.ts b/internal/interaction-harness/src/storybook.interaction.browser.spec.ts new file mode 100644 index 0000000000000..a711feb62cb84 --- /dev/null +++ b/internal/interaction-harness/src/storybook.interaction.browser.spec.ts @@ -0,0 +1,104 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file storybook.interaction.browser.spec.ts + * @input The built Storybook (`withStorybook`) and one of its stories + * @output `openStory` turns Storybook's highlight module off, so an insertion + * in a story carries no `getComputedStyle` scan of the page; with the module + * on (the control), every insertion carries one. Both engines; Chromium's + * trace also counts the style recalculations the scan forces + * @position Self-test for story.ts; it needs the built Storybook, as every + * component proof does + */ + +import {expect, test, type Page} from '@playwright/test'; +import {openStory, withStorybook} from './story'; +import {countBetween, traceRendering} from './trace'; + +const storybook = withStorybook(test); + +/** Counts the page's `getComputedStyle` calls, from before Storybook loads. */ +function countStyleReads(): void { + const w = window as unknown as {__styleReads: number}; + w.__styleReads = 0; + const read = window.getComputedStyle.bind(window); + window.getComputedStyle = ((element: Element, pseudo?: string | null) => { + w.__styleReads += 1; + return read(element, pseudo); + }) as typeof window.getComputedStyle; +} + +/** + * Inserts an exit copy of the story's first element into `#storybook-root`, + * as an exit animation's copy is, lets the insertion's mutation microtasks run, + * and reports what ran between its marks. + */ +async function insertCopy(page: Page) { + return page.evaluate(async () => { + const w = window as unknown as {__styleReads: number}; + const root = document.getElementById('storybook-root') as HTMLElement; + const reads = w.__styleReads; + const from = + performance.timeOrigin + performance.mark('copy:start').startTime; + const copy = (root.firstElementChild as HTMLElement).cloneNode( + true, + ) as HTMLElement; + copy.setAttribute('aria-hidden', 'true'); + root.appendChild(copy); + // A MutationObserver's callback is a microtask the insertion queued: + // these run after it. + await Promise.resolve(); + await Promise.resolve(); + const to = performance.timeOrigin + performance.mark('copy:end').startTime; + const scanned = w.__styleReads - reads; + const elements = document.body.querySelectorAll('*').length; + copy.remove(); + return {from, to, scanned, elements}; + }); +} + +test("openStory turns Storybook's highlight module off: an insertion in a story carries no getComputedStyle scan of the page", async ({ + page, + browserName, +}) => { + // The insertion's reads in every engine, and in Chromium (whose trace the + // harness reads) the style recalculations between its marks. + const measure = async (p: Page) => { + if (browserName !== 'chromium') { + return {copy: await insertCopy(p), styleRecalc: null}; + } + let copy = {from: 0, to: 0, scanned: 0, elements: 0}; + const trace = await traceRendering(p, async () => { + copy = await insertCopy(p); + }); + return { + copy, + styleRecalc: + countBetween(trace, copy.from, copy.to).UpdateLayoutTree ?? 0, + }; + }; + // The harness default. + await page.addInitScript(countStyleReads); + await openStory(page, storybook(), 'core-button--primary'); + expect(await page.locator('#storybook-highlights-root').count()).toBe(0); + const off = await measure(page); + expect(off.copy.scanned).toBe(0); + if (off.styleRecalc != null) { + expect(off.styleRecalc).toBe(0); + } + + // Control: Storybook's module kept on reads the style of every element on + // the page inside the inserting task. + const control = await page.context().newPage(); + await control.addInitScript(countStyleReads); + await openStory(control, storybook(), 'core-button--primary', undefined, { + storybookHighlight: true, + }); + expect(await control.locator('#storybook-highlights-root').count()).toBe(1); + const on = await measure(control); + expect(on.copy.scanned).toBeGreaterThanOrEqual(on.copy.elements); + if (on.styleRecalc != null) { + expect(on.styleRecalc).toBeGreaterThan(0); + } + await control.close(); +}); From 0906c6551514065c39872c62688bd4c90a04ece8 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Mon, 26 Oct 2026 09:33:37 -0700 Subject: [PATCH 87/93] feat(interaction): a rendering trace records why styles recalculated, and countBetween can leave out Chrome's own compositor animation update traceRendering(page, body, {invalidations: true}) also records each style recalculation's reason and element, the animations that ran (one entry per animation, as Chrome reuses an animation's trace id once it is gone) and whether each ran on the compositor, and when script ran. countBetween(trace, from, to, {skipCompositorAnimationUpdates: true}) then leaves out a pass Chrome makes on its own for compositor animations: every recalculation since the previous pass has the reason Animation (or is a pseudo-element's Conditional restyle following its element's), on an element with an animation on the compositor at the time, with no script in between. Unit cases include a recorded trace of a Popover presenting on a loaded machine. --- internal/interaction-harness/README.md | 24 +- .../src/harness.interaction.browser.spec.ts | 54 +++ .../src/trace.popover.fixture.ts | 333 ++++++++++++++++++ .../interaction-harness/src/trace.test.ts | 188 ++++++++++ internal/interaction-harness/src/trace.ts | 254 ++++++++++++- 5 files changed, 840 insertions(+), 13 deletions(-) create mode 100644 internal/interaction-harness/src/trace.popover.fixture.ts create mode 100644 internal/interaction-harness/src/trace.test.ts diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 7daf770400e6e..47f959c5c88bd 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -40,18 +40,18 @@ committed: a pull request links or attaches it. ## Instruments -| Module | What it gives a spec | -| --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact final-move speed, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. | -| `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | -| `frames.ts` | `startScreencast` (timestamped composited frames), `markElement` (a marker chip or outline), `trackMarker` (marker box per frame), `frameStats`. | -| `busy.ts` | `blockMainThread(page, ms)` and `busyWindowProof(tracked, window)`: frames presented, frames that moved, longest stall, a verdict; `presented` adds the compositor's own frames. | -| `trace.ts` | `traceRendering(page, body)`: main-thread Layout / Paint / UpdateLayoutTree / PrePaint counts, `countBetween`, long tasks; `installLongTaskLog` / `readLongTasks`; `startCompositorTrace`: when the compositor presents. | -| `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t), `waitForAnimation`, `watchAnimations`, `waitForRest`, `releaseAnimations`. | -| `math.ts` | Oracles independent of the code under test: `cubicBezier`, `timingFunction`, `springStep` (SwiftUI response / dampingFraction, initial velocity), `projectDecay`, `rubberBand`, `parseTransform`, `compareCurve`, `timeline`. | -| `storyboard.ts` | `animationStoryboard` (seek to t = 0, 16.7, 33.3 … ms), `touchStoryboard` (a frame per scripted event): PNG contact sheet + JSON trace. | -| `output.ts` | `evidenceDir`, `writeEvidence`: where evidence goes and how it is attached to the report. | -| `story.ts` | `phone`, `withStorybook`, `openStory`, `frames`, and the timer gate (`installTimerGate`, `freezeTimers`, `thawTimers`). | +| Module | What it gives a spec | +| --------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact final-move speed, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. | +| `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | +| `frames.ts` | `startScreencast` (timestamped composited frames), `markElement` (a marker chip or outline), `trackMarker` (marker box per frame), `frameStats`. | +| `busy.ts` | `blockMainThread(page, ms)` and `busyWindowProof(tracked, window)`: frames presented, frames that moved, longest stall, a verdict; `presented` adds the compositor's own frames. | +| `trace.ts` | `traceRendering(page, body, {invalidations?})`: main-thread Layout / Paint / UpdateLayoutTree / PrePaint counts, `countBetween` for a sub-window (optionally without Chrome's own compositor animation updates), long tasks; `installLongTaskLog` / `readLongTasks`; `startCompositorTrace`: when the compositor presents. | +| `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t), `waitForAnimation`, `watchAnimations`, `waitForRest`, `releaseAnimations`. | +| `math.ts` | Oracles independent of the code under test: `cubicBezier`, `timingFunction`, `springStep` (SwiftUI response / dampingFraction, initial velocity), `projectDecay`, `rubberBand`, `parseTransform`, `compareCurve`, `timeline`. | +| `storyboard.ts` | `animationStoryboard` (seek to t = 0, 16.7, 33.3 … ms), `touchStoryboard` (a frame per scripted event): PNG contact sheet + JSON trace. | +| `output.ts` | `evidenceDir`, `writeEvidence`: where evidence goes and how it is attached to the report. | +| `story.ts` | `phone`, `withStorybook`, `openStory`, `frames`, and the timer gate (`installTimerGate`, `freezeTimers`, `thawTimers`). | ## The proofs diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index e1d1a422b177f..5507ad621686e 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -545,6 +545,60 @@ test.describe('frame evidence', () => { expect(top.Layout ?? 0).toBeGreaterThan(18); }); + test('with invalidations, a trace tells a compositor animation from one the compositor cannot run', async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'rendering traces are Chromium-only (README: engine limits)', + ); + await fixture(page); + const box = page.locator('#box'); + const animate = async (property: 'transform' | 'top') => { + let started = 0; + const trace = await traceRendering( + page, + async () => { + started = await box.evaluate((el, prop) => { + el.animate( + prop === 'transform' + ? [ + {transform: 'translateY(0px)'}, + {transform: 'translateY(200px)'}, + ] + : [{top: '360px'}, {top: '560px'}], + {duration: 600}, + ); + return performance.timeOrigin + performance.now(); + }, property); + await page.waitForTimeout(700); + }, + {invalidations: true}, + ); + return {trace, started}; + }; + const transform = await animate('transform'); + expect(transform.trace.animations).toEqual([ + expect.objectContaining({composited: true}), + ]); + expect( + transform.trace.styleInvalidations?.some( + inv => inv.reason === 'Animation', + ), + ).toBe(true); + const top = await animate('top'); + expect(top.trace.animations).toEqual([ + expect.objectContaining({composited: false}), + ]); + // Its per-frame layout is the page's: none of it is left out. + expect( + countBetween(top.trace, top.started + 100, top.started + 500, { + skipCompositorAnimationUpdates: true, + }).Layout ?? 0, + ).toBeGreaterThan(18); + }); + test("the long-task log records the page's own long task but not a DevTools evaluate, which the trace reports apart", async ({ page, browserName, diff --git a/internal/interaction-harness/src/trace.popover.fixture.ts b/internal/interaction-harness/src/trace.popover.fixture.ts new file mode 100644 index 0000000000000..7023d8020a1c8 --- /dev/null +++ b/internal/interaction-harness/src/trace.popover.fixture.ts @@ -0,0 +1,333 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file trace.popover.fixture.ts + * @input A rendering trace recorded with `{invalidations: true}` while a + * Popover presented under a touch on a loaded machine (Chromium) + * @output POPOVER_PRESENT_UNDER_LOAD: its Animation events, rendering events, + * style recalculations and script starts, ms from the present's start + * @position Fixture for trace.test.ts + * + * The popover layer is trace node 6; its present runs two compositor + * animations under the ids 0x25 and 0x26, and the dismissal's exit copy + * (node 59) gets the same ids once they end. In the present's window + * [150, 482] Chrome restyles the layer on its own (`Animation`), then its + * `::backdrop` (`Conditional ::backdrop`), with one Layout and Paint and no + * script since the previous pass. + */ + +export const POPOVER_PRESENT_UNDER_LOAD = { + window: [150, 482], + animationEvents: [ + { + name: 'Animation', + ph: 'b', + ts: -86600, + id2: { + local: '0x25', + }, + args: { + data: { + name: '', + nodeId: 6, + state: 'running', + }, + }, + }, + { + name: 'Animation', + ph: 'e', + ts: -72600, + id2: { + local: '0x25', + }, + args: { + data: {}, + }, + }, + { + name: 'Animation', + ph: 'b', + ts: -54100, + id2: { + local: '0x25', + }, + args: { + data: { + name: '', + nodeId: 6, + state: 'running', + }, + }, + }, + { + name: 'Animation', + ph: 'b', + ts: -52800, + id2: { + local: '0x26', + }, + args: { + data: { + name: '', + nodeId: 6, + state: 'running', + }, + }, + }, + { + name: 'Animation', + ph: 'n', + ts: -1900, + id2: { + local: '0x25', + }, + args: { + data: { + compositeFailed: 0, + }, + }, + }, + { + name: 'Animation', + ph: 'n', + ts: -1900, + id2: { + local: '0x26', + }, + args: { + data: { + compositeFailed: 0, + }, + }, + }, + { + name: 'Animation', + ph: 'n', + ts: 17400, + id2: { + local: '0x25', + }, + args: { + data: { + state: 'running', + }, + }, + }, + { + name: 'Animation', + ph: 'n', + ts: 17400, + id2: { + local: '0x26', + }, + args: { + data: { + state: 'running', + }, + }, + }, + { + name: 'Animation', + ph: 'e', + ts: 533600, + id2: { + local: '0x25', + }, + args: { + data: {}, + }, + }, + { + name: 'Animation', + ph: 'e', + ts: 537900, + id2: { + local: '0x26', + }, + args: { + data: {}, + }, + }, + { + name: 'Animation', + ph: 'b', + ts: 649200, + id2: { + local: '0x25', + }, + args: { + data: { + name: '', + nodeId: 59, + state: 'running', + }, + }, + }, + { + name: 'Animation', + ph: 'b', + ts: 650200, + id2: { + local: '0x26', + }, + args: { + data: { + name: '', + nodeId: 59, + state: 'running', + }, + }, + }, + { + name: 'Animation', + ph: 'n', + ts: 660900, + id2: { + local: '0x25', + }, + args: { + data: { + compositeFailed: 0, + }, + }, + }, + { + name: 'Animation', + ph: 'n', + ts: 661000, + id2: { + local: '0x26', + }, + args: { + data: { + compositeFailed: 0, + }, + }, + }, + { + name: 'Animation', + ph: 'n', + ts: 668300, + id2: { + local: '0x25', + }, + args: { + data: { + state: 'running', + }, + }, + }, + { + name: 'Animation', + ph: 'n', + ts: 668300, + id2: { + local: '0x26', + }, + args: { + data: { + state: 'running', + }, + }, + }, + { + name: 'Animation', + ph: 'e', + ts: 1202300, + id2: { + local: '0x25', + }, + args: { + data: {}, + }, + }, + { + name: 'Animation', + ph: 'e', + ts: 1202400, + id2: { + local: '0x26', + }, + args: { + data: {}, + }, + }, + ], + events: [ + { + name: 'UpdateLayoutTree', + t: 118.6, + }, + { + name: 'Layout', + t: 119.6, + }, + { + name: 'UpdateLayoutTree', + t: 120.1, + }, + { + name: 'Layout', + t: 126, + }, + { + name: 'UpdateLayoutTree', + t: 151, + }, + { + name: 'Layout', + t: 151.6, + }, + { + name: 'UpdateLayoutTree', + t: 159.3, + }, + { + name: 'Layout', + t: 159.9, + }, + { + name: 'PrePaint', + t: 160.4, + }, + { + name: 'Paint', + t: 160.4, + }, + { + name: 'Paint', + t: 160.5, + }, + { + name: 'Layerize', + t: 160.5, + }, + ], + styleInvalidations: [ + { + t: 118.6, + reason: 'Animation', + nodeId: 6, + }, + { + t: 120.1, + reason: 'Conditional ::backdrop', + nodeId: 6, + }, + { + t: 150.9, + reason: 'Animation', + nodeId: 6, + }, + { + t: 157.4, + reason: 'Conditional ::backdrop', + nodeId: 6, + }, + ], + scripts: [ + 3.4, 3.5, 3.6, 6.7, 6.7, 13.8, 13.9, 13.9, 13.9, 14.1, 14.2, 14.3, 14.3, 72, + 72, + ], +} as const; diff --git a/internal/interaction-harness/src/trace.test.ts b/internal/interaction-harness/src/trace.test.ts new file mode 100644 index 0000000000000..4b4cc6b4c24b3 --- /dev/null +++ b/internal/interaction-harness/src/trace.test.ts @@ -0,0 +1,188 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file trace.test.ts + * @input countBetween, traceAnimations, hand-built rendering traces and one + * recorded trace (trace.popover.fixture.ts) + * @output Tests which rendering passes countBetween leaves out with + * skipCompositorAnimationUpdates: only Chrome's own update for compositor + * animations, never work the page caused + * @position Unit tests for internal/interaction-harness/src/trace.ts + */ + +import {describe, expect, it} from 'vitest'; +import {POPOVER_PRESENT_UNDER_LOAD} from './trace.popover.fixture'; +import { + countBetween, + traceAnimations, + type RenderingTrace, + type TraceAnimation, +} from './trace'; + +const PASS = ['UpdateLayoutTree', 'Layout', 'PrePaint', 'Paint', 'Paint']; + +function trace( + passes: number[], + extra: Pick, +): RenderingTrace { + const events = passes.flatMap(t => + PASS.map((name, k) => ({name, t: t + k * 0.1})), + ); + return { + counts: {}, + layout: 0, + paint: 0, + styleRecalc: 0, + prePaint: 0, + durationMs: 500, + longTasks: [], + devtoolsTasks: [], + events, + window: {from: 0, to: 500}, + ...extra, + }; +} + +const COMPOSITED = [ + {nodeId: 7, composited: true}, + {nodeId: 10, composited: true}, +]; + +describe('countBetween with skipCompositorAnimationUpdates', () => { + it('leaves out a pass Chrome made on its own for compositor animations', () => { + const t = trace([4, 154], { + animations: COMPOSITED, + styleInvalidations: [ + {t: 1, reason: 'Inline CSS style declaration was mutated', nodeId: 7}, + {t: 153.7, reason: 'Animation', nodeId: 7}, + {t: 153.8, reason: 'Animation', nodeId: 10}, + ], + scripts: [0.5], + }); + expect(countBetween(t, 150, 400)).toMatchObject({Layout: 1, Paint: 2}); + expect( + countBetween(t, 150, 400, {skipCompositorAnimationUpdates: true}), + ).toEqual({}); + // The start, which the page's style writes caused, still counts. + expect( + countBetween(t, 0, 400, {skipCompositorAnimationUpdates: true}), + ).toMatchObject({Layout: 1}); + }); + + it('counts the pass when script ran since the previous one', () => { + const t = trace([4, 154], { + animations: COMPOSITED, + styleInvalidations: [{t: 153.7, reason: 'Animation', nodeId: 7}], + scripts: [150], + }); + expect( + countBetween(t, 150, 400, {skipCompositorAnimationUpdates: true}), + ).toMatchObject({Layout: 1}); + }); + + it('counts the pass when any recalculation has another reason', () => { + const t = trace([4, 154], { + animations: COMPOSITED, + styleInvalidations: [ + {t: 153.7, reason: 'Animation', nodeId: 7}, + { + t: 153.8, + reason: 'Inline CSS style declaration was mutated', + nodeId: 10, + }, + ], + scripts: [], + }); + expect( + countBetween(t, 150, 400, {skipCompositorAnimationUpdates: true}), + ).toMatchObject({Layout: 1}); + }); + + it('counts every pass of an animation the compositor cannot run', () => { + const t = trace([4, 21, 38, 154], { + animations: [{nodeId: 7, composited: false}], + styleInvalidations: [20, 37, 153].map(at => ({ + t: at, + reason: 'Animation', + nodeId: 7, + })), + scripts: [], + }); + expect( + countBetween(t, 15, 400, {skipCompositorAnimationUpdates: true}), + ).toMatchObject({Layout: 3}); + }); + + it('needs a trace recorded with invalidations', () => { + const t = trace([4], {}); + expect(() => + countBetween(t, 0, 400, {skipCompositorAnimationUpdates: true}), + ).toThrow(/invalidations: true/); + }); +}); + +describe('a recorded trace: a popover presenting on a loaded machine', () => { + const recorded = POPOVER_PRESENT_UNDER_LOAD; + const animations = traceAnimations(recorded.animationEvents, ts => ts / 1000); + const withAnimations = (list: readonly TraceAnimation[]): RenderingTrace => ({ + ...trace([], { + animations: list, + styleInvalidations: recorded.styleInvalidations, + scripts: recorded.scripts, + }), + events: recorded.events, + }); + const [from, to] = recorded.window; + + it('keeps one entry per animation when Chrome reuses an id for a later one', () => { + // The layer's present (node 6) and, under the same two ids, the exit copy + // the dismissal plays on (node 59). + expect(animations.filter(a => a.nodeId === 6 && a.composited).length).toBe( + 2, + ); + expect(animations.filter(a => a.nodeId === 59 && a.composited).length).toBe( + 2, + ); + for (const a of animations.filter(a => a.nodeId === 6 && a.composited)) { + expect(a.from).toBeLessThan(from); + expect(a.to).toBeGreaterThan(to); + } + }); + + it("leaves out Chrome's own update of the layer and of its ::backdrop", () => { + const t = withAnimations(animations); + expect(countBetween(t, from, to)).toMatchObject({Layout: 2, Paint: 2}); + expect( + countBetween(t, from, to, {skipCompositorAnimationUpdates: true}), + ).toEqual({}); + }); + + it('counts the passes when script ran before them', () => { + const t = { + ...withAnimations(animations), + scripts: [...recorded.scripts, 140], + }; + expect( + countBetween(t, from, to, {skipCompositorAnimationUpdates: true}), + ).toMatchObject({Layout: 2}); + }); + + it('counts them when the layer has no compositor animation at the time', () => { + const ended = animations.map(a => + a.nodeId === 6 ? {...a, to: Math.min(a.to ?? Infinity, 100)} : a, + ); + expect( + countBetween(withAnimations(ended), from, to, { + skipCompositorAnimationUpdates: true, + }), + ).toMatchObject({Layout: 2}); + const mainThread = animations.map(a => + a.nodeId === 6 ? {...a, composited: false} : a, + ); + expect( + countBetween(withAnimations(mainThread), from, to, { + skipCompositorAnimationUpdates: true, + }), + ).toMatchObject({Layout: 2}); + }); +}); diff --git a/internal/interaction-harness/src/trace.ts b/internal/interaction-harness/src/trace.ts index 41cd401b30c25..a35a97259fab3 100644 --- a/internal/interaction-harness/src/trace.ts +++ b/internal/interaction-harness/src/trace.ts @@ -17,6 +17,20 @@ * `performance.mark` calls, and to the time between the start and end marks, * so other pages, the browser process and setup work are excluded. * + * With `{invalidations: true}` the trace also records why each style + * recalculation happened, which elements animations ran on and whether they + * ran on the compositor, and when script ran. `countBetween` can then leave + * out the one rendering pass Chrome makes on its own for a compositor + * animation (its start time arriving from the compositor, late on a loaded + * machine): a pass whose style recalculations all have the reason + * `Animation`, on elements with an animation on the compositor at the time + * (one entry per animation, as Chrome reuses an animation's trace id once it + * is gone), with no script since the previous pass. A pseudo-element's + * `Conditional ::backdrop` restyle on such an element counts with the + * element's own, in that pass or the one before. Anything the page causes (a + * style write, a layout read, an animation the compositor cannot run) is + * still counted. + * * SYNC: Keep RENDERING_EVENTS equal to the README's list. */ @@ -85,6 +99,129 @@ export interface RenderingTrace { readonly events: readonly {readonly name: string; readonly t: number}[]; /** The window, epoch ms on the page's clock. */ readonly window: {readonly from: number; readonly to: number}; + /** + * With `invalidations`: each style recalculation Chrome scheduled, with its + * reason (`Animation`, `Inline CSS style declaration was mutated`, …) and + * the element's trace node id, epoch ms. + */ + readonly styleInvalidations?: readonly { + readonly t: number; + readonly reason: string; + readonly nodeId?: number; + }[]; + /** + * With `invalidations`: one entry per animation that ran, with the element + * it ran on, whether it ran on the compositor (Chrome reported no reason it + * could not) and when it ran, epoch ms (`to` absent while it still runs). + */ + readonly animations?: readonly TraceAnimation[]; + /** + * With `invalidations`: when script started on the main thread (function + * calls, animation frame callbacks, timers, event dispatch), epoch ms. + */ + readonly scripts?: readonly number[]; +} + +/** An animation as a trace shows it (see `traceAnimations`). */ +export interface TraceAnimation { + readonly nodeId: number; + readonly composited: boolean; + readonly from?: number; + readonly to?: number; +} + +export interface TraceRenderingOptions { + /** + * Also record style recalculation reasons, the animations and whether they + * ran on the compositor, and script, for + * `countBetween(…, {skipCompositorAnimationUpdates: true})`. + */ + invalidations?: boolean; +} + +export interface CountBetweenOptions { + /** + * Leave out a rendering pass Chrome makes on its own for compositor + * animations: every style recalculation since the previous pass has the + * reason `Animation` (or is a pseudo-element's `Conditional ::…` restyle + * following that element's) and is on an element with an animation on the + * compositor at the time, and no script ran in between. Needs a trace + * recorded with `{invalidations: true}`. + */ + skipCompositorAnimationUpdates?: boolean; +} + +// Rendering events closer than this belong to one pass (one frame's style, +// layout and paint); passes are a frame apart. +const PASS_GAP_MS = 4; + +/** + * The times of the rendering events that belong to passes Chrome made on its + * own for compositor animations (see `CountBetweenOptions`). + */ +function compositorAnimationUpdates(trace: RenderingTrace): Set { + const {styleInvalidations, animations, scripts} = trace; + if (styleInvalidations == null || animations == null || scripts == null) { + throw new Error( + 'countBetween: skipCompositorAnimationUpdates needs traceRendering(…, {invalidations: true})', + ); + } + // An element counts as composited at a time when an animation on it runs + // on the compositor then and none on the main thread does. + const alive = (a: TraceAnimation, t: number) => + (a.from == null || a.from <= t) && (a.to == null || t <= a.to); + const compositedAt = (nodeId: number | undefined, t: number) => + nodeId != null && + animations.some(a => a.nodeId === nodeId && a.composited && alive(a, t)) && + !animations.some(a => a.nodeId === nodeId && !a.composited && alive(a, t)); + const events = [...trace.events].sort((a, b) => a.t - b.t); + const skipped = new Set(); + let previousEnd = -Infinity; + // The elements Chrome restyled on its own in the previous pass. + let ownBefore = new Set(); + let i = 0; + while (i < events.length) { + let j = i; + while ( + j + 1 < events.length && + events[j + 1].t - events[j].t <= PASS_GAP_MS + ) { + j += 1; + } + const start = events[i].t; + const end = events[j].t; + const causes = styleInvalidations.filter( + inv => inv.t > previousEnd && inv.t <= end, + ); + // A pseudo-element (`::backdrop`) restyles after the element it belongs + // to: `Conditional ::backdrop` is Chrome's own when that element's + // restyle is, in this pass or the one before. + const own = new Set( + causes + .filter(inv => inv.reason === 'Animation' && inv.nodeId != null) + .map(inv => inv.nodeId as number), + ); + const isAnimationUpdate = + causes.length > 0 && + causes.every( + inv => + (inv.reason === 'Animation' || + (inv.reason.startsWith('Conditional ::') && + inv.nodeId != null && + (own.has(inv.nodeId) || ownBefore.has(inv.nodeId)))) && + compositedAt(inv.nodeId, start), + ) && + !scripts.some(t => t > previousEnd && t <= start); + if (isAnimationUpdate) { + for (let k = i; k <= j; k += 1) { + skipped.add(events[k].t); + } + } + ownBefore = isAnimationUpdate ? own : new Set(); + previousEnd = end; + i = j + 1; + } + return skipped; } /** Counts rendering events by name inside `[from, to]` (epoch ms). */ @@ -92,10 +229,14 @@ export function countBetween( trace: RenderingTrace, from: number, to: number, + {skipCompositorAnimationUpdates = false}: CountBetweenOptions = {}, ): Record { + const skipped = skipCompositorAnimationUpdates + ? compositorAnimationUpdates(trace) + : new Set(); const counts: Record = {}; for (const e of trace.events) { - if (e.t >= from && e.t <= to) { + if (e.t >= from && e.t <= to && !skipped.has(e.t)) { counts[e.name] = (counts[e.name] ?? 0) + 1; } } @@ -113,6 +254,76 @@ interface TraceEvent { args?: Record; } +/** A trace event as `traceAnimations` reads it. */ +export interface AnimationTraceEvent { + readonly name: string; + readonly ph: string; + readonly ts: number; + readonly id?: string; + readonly id2?: {readonly local?: string}; + readonly args?: Record; +} + +/** + * The animations a trace shows, one entry each, from Chrome's `Animation` + * events: the element (`nodeId`), whether it ran on the compositor and when. + * Chrome names an animation's events by an id it reuses once that animation + * is gone (a popover's exit copy gets its present's ids), so a begin event + * opens a new entry under its id and later events for the id add to it. + * `toMs` turns a trace timestamp into the times the trace reports. + */ +export function traceAnimations( + events: readonly AnimationTraceEvent[], + toMs: (ts: number) => number, +): TraceAnimation[] { + const open = new Map< + string, + {nodeId?: number; composited?: boolean; from: number; to?: number} + >(); + const all: { + nodeId?: number; + composited?: boolean; + from: number; + to?: number; + }[] = []; + for (const e of [...events].sort((a, b) => a.ts - b.ts)) { + if (e.name !== 'Animation') { + continue; + } + const id = e.id2?.local ?? e.id ?? ''; + const d = ((e.args as {data?: Record} | undefined)?.data ?? + {}) as Record; + let entry = open.get(id); + if (e.ph === 'b' || entry == null) { + entry = {from: toMs(e.ts)}; + open.set(id, entry); + all.push(entry); + } + if (typeof d.nodeId === 'number') { + entry.nodeId = d.nodeId; + } + if (typeof d.compositeFailed === 'number') { + entry.composited = (entry.composited ?? true) && d.compositeFailed === 0; + } + if (e.ph === 'e') { + entry.to = toMs(e.ts); + open.delete(id); + } + } + return all.flatMap(({nodeId, composited, from, to}) => + nodeId == null + ? [] + : [ + { + nodeId, + composited: composited ?? false, + from, + ...(to == null ? {} : {to}), + }, + ], + ); +} + /** * Traces the page while `body` runs and counts the rendering work its main * thread did between a start mark (placed just before `body`) and an end mark @@ -121,6 +332,7 @@ interface TraceEvent { export async function traceRendering( page: Page, body: () => Promise, + {invalidations = false}: TraceRenderingOptions = {}, ): Promise { const cdp: CDPSession = await page.context().newCDPSession(page); const events: TraceEvent[] = []; @@ -138,6 +350,9 @@ export async function traceRendering( 'disabled-by-default-devtools.timeline', 'blink.user_timing', 'toplevel', + ...(invalidations + ? ['disabled-by-default-devtools.timeline.invalidationTracking'] + : []), ], excludedCategories: ['*'], }, @@ -178,6 +393,42 @@ export async function traceRendering( counted.push({name: e.name, t: toEpochMs(e.ts)}); } } + const extra: { + -readonly [ + K in 'styleInvalidations' | 'animations' | 'scripts' + ]?: RenderingTrace[K]; + } = {}; + if (invalidations) { + const data = (e: TraceEvent) => + ((e.args as {data?: Record} | undefined)?.data ?? + {}) as Record; + extra.animations = traceAnimations( + events.filter(e => e.pid === start.pid), + toEpochMs, + ); + extra.styleInvalidations = events + .filter(e => onMain(e) && e.name === 'StyleRecalcInvalidationTracking') + .map(e => { + const d = data(e); + return { + t: toEpochMs(e.ts), + reason: String(d.reason ?? ''), + nodeId: typeof d.nodeId === 'number' ? d.nodeId : undefined, + }; + }); + extra.scripts = events + .filter( + e => + onMain(e) && + [ + 'FunctionCall', + 'FireAnimationFrame', + 'TimerFire', + 'EventDispatch', + ].includes(e.name), + ) + .map(e => toEpochMs(e.ts)); + } // A task that ran a DevTools command received it over the DevTools // session's pipe; page script it set off shows as its listener, timer or // frame callback. @@ -225,6 +476,7 @@ export async function traceRendering( devtoolsTasks, events: counted, window: {from: fromEpoch, to: toEpoch}, + ...extra, }; } From 66c651fd44b3fc84bf5404fffdd30cba8cbed34a Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Mon, 26 Oct 2026 12:07:46 -0700 Subject: [PATCH 88/93] fix(interaction): a watch taken inside a trace right after a style change does not report that change Inside a trace arm() skipped the page-wide read that creates a pending transition, so a watch taken right after a style change counted that change as started after it. watchAnimations now arms through a handle to its element, and arm() reads that element's own animations while a trace runs (resolving its pending style change, which the next frame would resolve anyway). Traced twin of the existing self-test: 3 of 3 failed before; the harness self-tests pass (61 x 2, Chromium). --- internal/interaction-harness/src/curves.ts | 46 ++++++++++++++----- .../src/harness.interaction.browser.spec.ts | 22 +++++++++ 2 files changed, 57 insertions(+), 11 deletions(-) diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index 0318feb433925..f1e7c1a27abd3 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -219,11 +219,17 @@ function animationRecorder(): void { }, // Record first, then stamp strictly after the newest entry: a step that // changed style just before has its transition created by that read, - // and it started before the arm. While a trace runs the read stands - // down, as record() does. - arm: () => { + // and it started before the arm. While a trace runs the page-wide read + // stands down, as record() does; the watched element's own animations + // are still read, which resolves its pending style change (one the next + // frame would resolve anyway, so the trace gains no rendering step). + arm: (element?: Element) => { if (!tracing()) { record(); + } else if (element != null) { + for (const animation of element.getAnimations({subtree: true})) { + remember(animation); + } } const newest = log.length === 0 ? 0 : log[log.length - 1].seenAt; armedAt = Math.max(performance.now(), newest + 0.001); @@ -1253,13 +1259,31 @@ export async function waitForAnimation( } } -/** Marks now as the point later waits count animations from. */ -async function armRecorder(page: Page): Promise { - return page.evaluate(() => - ( - window as unknown as {__harnessAnimations: {arm: () => number}} - ).__harnessAnimations.arm(), - ); +/** + * Marks now as the point later waits count animations from. With a locator, + * its element's pending style change is resolved first, also inside a trace + * (reached through a handle, which adds no main-thread frame). + */ +async function armRecorder(page: Page, locator?: Locator): Promise { + const handles = locator == null ? [] : await locator.elementHandles(); + try { + if (handles.length === 1) { + return await handles[0].evaluate(element => + ( + window as unknown as { + __harnessAnimations: {arm: (element?: Element) => number}; + } + ).__harnessAnimations.arm(element as Element), + ); + } + return await page.evaluate(() => + ( + window as unknown as {__harnessAnimations: {arm: () => number}} + ).__harnessAnimations.arm(), + ); + } finally { + await Promise.all(handles.map(handle => handle.dispose())); + } } export interface AnimationWatch { @@ -1287,7 +1311,7 @@ export async function watchAnimations( locator: Locator, ): Promise { await installCurveEnd(locator.page()); - const since = await armRecorder(locator.page()); + const since = await armRecorder(locator.page(), locator); return { started: options => waitForAnimation(locator, {...options, since}), }; diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 1748133585812..715486fe816a6 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -1988,6 +1988,28 @@ test.describe('animation state', () => { await box.evaluate(el => el.getAnimations().forEach(a => a.finish())); }); + test('inside a rendering trace too, a watch taken right after a style change does not report that change', async ({ + page, + browserName, + }) => { + test.skip(browserName !== 'chromium', 'rendering traces are Chromium-only'); + await fixture(page); + const box = page.locator('#box'); + await waitForRest(box); + await traceRendering(page, async () => { + // The step: a transition the next style read creates. + await box.evaluate(el => { + el.style.transition = 'opacity 300ms linear'; + el.style.opacity = '0.5'; + }); + const watch = await watchAnimations(box); + await expect(watch.started({timeout: 200})).rejects.toThrow( + /waitForAnimation/, + ); + }); + await box.evaluate(el => el.getAnimations().forEach(a => a.finish())); + }); + test('an exit whose element the page removes at its end is still reported', async ({ page, }) => { From 56135012d31d7ab88efcc14c994f7f743b336825 Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Mon, 26 Oct 2026 13:08:51 -0700 Subject: [PATCH 89/93] docs(interaction): the README's evidence list and busy recipe match what ships Nothing in the frame evidence writes busy-main-thread frames to the evidence directory, so the README no longer lists them there; and the busy recipe's block is spec:AST-094 FR5's 1 s rather than 500-1000 ms. --- internal/interaction-harness/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index d509d31e5b1e6..ae4e6d62b677e 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -33,7 +33,7 @@ under a rootful docker daemon the container runs as your user so its node internal/interaction-harness/scripts/webkit-container.mjs [playwright test args…] ``` -Evidence (storyboards, curve traces, busy-main-thread frames) is written to +Evidence (storyboards, curve traces) is written to `test-results/interaction-evidence/` (override with `ASTRYX_INTERACTION_EVIDENCE`) and attached to the test report. It is never committed: a pull request links or attaches it. @@ -103,7 +103,8 @@ action may be the page's first contact with the harness, take **Busy main thread.** Mark the element, start the screencast, start the animation, wait until it runs (`waitForAnimation`: an animation cannot start -while the thread is held), block the main thread 500–1000 ms, stop. Always +while the thread is held), block the main thread for 1 s (`spec:AST-094` +FR5's verification row), stop. Always start the gesture or animation first and block mid-flight, never before: in headless Chromium a touch that starts while the main thread is busy does not scroll or start a gesture even with only passive listeners (the input From 852d55c2620b6fa0598821eeaa1f85a6685121ea Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Mon, 26 Oct 2026 13:09:55 -0700 Subject: [PATCH 90/93] fix(interaction): readLongTasks fails closed, and the README lists every counted event readLongTasks returned [] when the log never ran on the document (installed after navigating) or the engine has no Long Tasks API (WebKit), so a "no long task" assertion could pass without evidence. The init script now records whether the observer runs, and readLongTasks throws in both cases. Self-test: reading without installing throws. The README's trace.ts row now lists all eight RENDERING_EVENTS its SYNC note mirrors. --- internal/interaction-harness/README.md | 24 ++++++------- .../src/harness.interaction.browser.spec.ts | 11 ++++++ internal/interaction-harness/src/trace.ts | 34 +++++++++++++++---- 3 files changed, 51 insertions(+), 18 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index b847594451fad..7ccf1214a908c 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -40,18 +40,18 @@ committed: a pull request links or attaches it. ## Instruments -| Module | What it gives a spec | -| --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact final-move speed, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. | -| `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | -| `frames.ts` | `startScreencast` (timestamped composited frames), `markElement` (a marker chip or outline), `trackMarker` (marker box per frame), `frameStats`. | -| `busy.ts` | `blockMainThread(page, ms)` and `busyWindowProof(tracked, window)`: frames presented, frames that moved, longest stall, a verdict; `presented` adds the compositor's own frames. | -| `trace.ts` | `traceRendering(page, body)`: main-thread Layout / Paint / UpdateLayoutTree / PrePaint counts, `countBetween`, long tasks; `installLongTaskLog` / `readLongTasks`; `startCompositorTrace`: when the compositor presents. | -| `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t), `waitForAnimation`, `watchAnimations`, `waitForRest`, `releaseAnimations`. | -| `math.ts` | Oracles independent of the code under test: `cubicBezier`, `timingFunction`, `springStep` (SwiftUI response / dampingFraction, initial velocity), `projectDecay`, `rubberBand`, `parseTransform`, `compareCurve`, `timeline`. | -| `storyboard.ts` | `animationStoryboard` (seek to t = 0, 16.7, 33.3 … ms), `touchStoryboard` (a frame per scripted event): PNG contact sheet + JSON trace. | -| `output.ts` | `evidenceDir`, `writeEvidence`: where evidence goes and how it is attached to the report. | -| `story.ts` | `phone`, `withStorybook`, `openStory`, `frames`, and the timer gate (`installTimerGate`, `freezeTimers`, `thawTimers`). | +| Module | What it gives a spec | +| --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact final-move speed, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. | +| `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | +| `frames.ts` | `startScreencast` (timestamped composited frames), `markElement` (a marker chip or outline), `trackMarker` (marker box per frame), `frameStats`. | +| `busy.ts` | `blockMainThread(page, ms)` and `busyWindowProof(tracked, window)`: frames presented, frames that moved, longest stall, a verdict; `presented` adds the compositor's own frames. | +| `trace.ts` | `traceRendering(page, body)`: main-thread counts of UpdateLayoutTree / Layout / PrePaint / Paint / Layerize / UpdateLayer / ScrollLayer / HitTest, `countBetween`, long tasks; `installLongTaskLog` / `readLongTasks`; `startCompositorTrace`: when the compositor presents. | +| `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t), `waitForAnimation`, `watchAnimations`, `waitForRest`, `releaseAnimations`. | +| `math.ts` | Oracles independent of the code under test: `cubicBezier`, `timingFunction`, `springStep` (SwiftUI response / dampingFraction, initial velocity), `projectDecay`, `rubberBand`, `parseTransform`, `compareCurve`, `timeline`. | +| `storyboard.ts` | `animationStoryboard` (seek to t = 0, 16.7, 33.3 … ms), `touchStoryboard` (a frame per scripted event): PNG contact sheet + JSON trace. | +| `output.ts` | `evidenceDir`, `writeEvidence`: where evidence goes and how it is attached to the report. | +| `story.ts` | `phone`, `withStorybook`, `openStory`, `frames`, and the timer gate (`installTimerGate`, `freezeTimers`, `thawTimers`). | ## The proofs diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index e1d1a422b177f..1ac0c6d0e472a 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -545,6 +545,17 @@ test.describe('frame evidence', () => { expect(top.Layout ?? 0).toBeGreaterThan(18); }); + test('the long-task log fails closed when it never ran on the document', async ({ + page, + }) => { + // No installLongTaskLog before navigating: an empty answer would read as + // "no long task", so it throws instead. + await fixture(page); + await expect(readLongTasks(page)).rejects.toThrow( + /not running on this document/, + ); + }); + test("the long-task log records the page's own long task but not a DevTools evaluate, which the trace reports apart", async ({ page, browserName, diff --git a/internal/interaction-harness/src/trace.ts b/internal/interaction-harness/src/trace.ts index 41cd401b30c25..4789b68fdbf79 100644 --- a/internal/interaction-harness/src/trace.ts +++ b/internal/interaction-harness/src/trace.ts @@ -286,13 +286,16 @@ export async function startCompositorTrace( } const LONG_TASK_SCRIPT = `(() => { - if (window.__harnessLongTasks || typeof PerformanceObserver === 'undefined') return; + if (window.__harnessLongTasks) return; const tasks = []; - window.__harnessLongTasks = tasks; + const log = {tasks, supported: false}; + window.__harnessLongTasks = log; + if (typeof PerformanceObserver === 'undefined' || !(PerformanceObserver.supportedEntryTypes || []).includes('longtask')) return; try { new PerformanceObserver(list => { for (const entry of list.getEntries()) tasks.push({start: performance.timeOrigin + entry.startTime, durationMs: entry.duration, name: entry.name}); }).observe({type: 'longtask', buffered: true}); + log.supported = true; } catch {} })()`; @@ -315,16 +318,35 @@ export interface LongTask { readonly name: string; } -/** Long tasks recorded since navigation, optionally only those overlapping a window. */ +/** + * Long tasks recorded since navigation, optionally only those overlapping a + * window. Fails closed: it throws when the log never ran on this document + * (installed after the page navigated) or the engine has no Long Tasks API + * (WebKit), so an empty answer is evidence of no long task, never of no log. + */ export async function readLongTasks( page: Page, range?: {from: number; to: number}, ): Promise { - const tasks = await page.evaluate( + const log = await page.evaluate( () => - (window as unknown as {__harnessLongTasks?: LongTask[]}) - .__harnessLongTasks ?? [], + ( + window as unknown as { + __harnessLongTasks?: {tasks: LongTask[]; supported: boolean}; + } + ).__harnessLongTasks ?? null, ); + if (log == null) { + throw new Error( + 'readLongTasks: the long-task log is not running on this document; call installLongTaskLog(page) before navigating', + ); + } + if (!log.supported) { + throw new Error( + 'readLongTasks: this engine has no Long Tasks API, so the log records nothing', + ); + } + const tasks = log.tasks; return range == null ? tasks : tasks.filter( From 6ccd5b46cdda1309c4d5598e6acbc01fc22b4192 Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Mon, 26 Oct 2026 13:10:35 -0700 Subject: [PATCH 91/93] fix(interaction): castStoryboard takes no clip it would ignore castStoryboard accepted StoryboardOptions' clip through its options type but never applied it to the frames. Its options now leave clip out, so a caller can't pass one that does nothing. --- internal/interaction-harness/src/storyboard.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/interaction-harness/src/storyboard.ts b/internal/interaction-harness/src/storyboard.ts index 4a9cbec360bb4..a49207c4f81c4 100644 --- a/internal/interaction-harness/src/storyboard.ts +++ b/internal/interaction-harness/src/storyboard.ts @@ -378,7 +378,8 @@ export async function castStoryboard( readonly data: string; readonly format: 'png' | 'jpeg'; }[], - options: StoryboardOptions & { + // A cast is already the screen's frames: no clip is applied to them. + options: Omit & { readonly zero: number; readonly from?: number; readonly to?: number; From 87e516b545364a9f0182162423e7c4c0dc29d821 Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Mon, 26 Oct 2026 13:11:39 -0700 Subject: [PATCH 92/93] docs(interaction): a traced watch restyles its element; take it before the counted window The arm() comment said resolving the watched element's pending style inside a trace adds no rendering step; it restyles the element (2 Layouts and 2 style recalculations for the anchored fixture). The comment, the README's No Layout / Paint paragraph and watchAnimations' JSDoc now say so and tell a proof to take its traced watch before the window it counts. --- internal/interaction-harness/README.md | 5 ++++- internal/interaction-harness/src/curves.ts | 10 +++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index a6677873a7e1a..f474ffa987bad 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -203,7 +203,10 @@ Layouts before, 0 now). That holds for script animations and CSS transitions. On a CSS `@keyframes` animation (and, unwatched, a cancelled CSS transition), `waitForAnimation` and `started()` read the animation's timing, which flushes its style and adds Layouts to the trace: wait on one with -`waitForRest`, or after the trace. The trace's `longTasks` are the page's own: only a +`waitForRest`, or after the trace. Taking a watch (`watchAnimations`) inside +a trace reads the watched element's animations, which restyles it (and lays +out an anchored one), so a proof takes its traced watch before the window it +counts. The trace's `longTasks` are the page's own: only a task the harness marked as its own read (`waitForAnimation` describing keyframes, `blockMainThread` without `during`) is in `devtoolsTasks` instead, and only when no page script ran inside it. A test's own `page.evaluate` stays a diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index f1e7c1a27abd3..9eff0abfb9da9 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -221,8 +221,10 @@ function animationRecorder(): void { // changed style just before has its transition created by that read, // and it started before the arm. While a trace runs the page-wide read // stands down, as record() does; the watched element's own animations - // are still read, which resolves its pending style change (one the next - // frame would resolve anyway, so the trace gains no rendering step). + // are still read, which resolves its pending style change. That read + // restyles the element (and lays out an anchored one) in the task that + // takes the watch, so a traced proof takes its watch before the window + // it counts (README: No Layout / Paint). arm: (element?: Element) => { if (!tracing()) { record(); @@ -1301,7 +1303,9 @@ export interface AnimationWatch { /** * Starts watching `locator`'s element before an action, so the animation the * action starts is caught even when it ends before the test looks (a short - * entrance on a cold page): + * entrance on a cold page). Taking the watch reads the element's animations, + * which restyles it, inside a rendering trace too: take a traced watch before + * the window the proof counts: * * const watch = await watchAnimations(sheet); * await driver.play(tap(button)); From 707e7fe93fa66cf79c712bfd8596faa171e78e90 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Thu, 29 Oct 2026 13:40:56 -0700 Subject: [PATCH 93/93] docs(interaction): the module table keeps its base row; a paragraph documents the invalidations trace The trace.ts row's longer text re-padded the whole module table, so it clashed with every branch that adds a row there (the wheel and pen drivers' webInput.ts row). The table goes back to its base, and a paragraph after "No Layout / Paint" documents traceRendering's {invalidations: true} and countBetween's skipCompositorAnimationUpdates. --- internal/interaction-harness/README.md | 32 ++++++++++++++++---------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index a15db841b5fe8..277fe72b9aff6 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -46,18 +46,18 @@ committed: a pull request links or attaches it. ## Instruments -| Module | What it gives a spec | -| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact final-move speed, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. | -| `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | -| `frames.ts` | `startScreencast` (timestamped composited frames), `markElement` (a marker chip or outline), `trackMarker` (marker box per frame), `sampleRects` (per-rAF rects), `frameStats`. | -| `busy.ts` | `blockMainThread(page, ms)` and `busyWindowProof(tracked, window)`: frames presented, frames that moved, longest stall, a verdict; `presented` adds the compositor's own frames. | -| `trace.ts` | `traceRendering(page, body, {invalidations?})`: main-thread counts of UpdateLayoutTree / Layout / PrePaint / Paint / Layerize / UpdateLayer / ScrollLayer / HitTest, `countBetween` for a sub-window (optionally without Chrome's own compositor animation updates), long tasks; `installLongTaskLog` / `readLongTasks`; `startCompositorTrace`: when the compositor presents. | -| `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t), `waitForAnimation`, `watchAnimations`, `waitForRest`, `releaseAnimations`. | -| `math.ts` | Oracles independent of the code under test: `cubicBezier`, `timingFunction`, `springStep` (SwiftUI response / dampingFraction, initial velocity), `projectDecay`, `rubberBand`, `parseTransform`, `compareCurve`, `timeline`. | -| `storyboard.ts` | `animationStoryboard` (seek to t = 0, 16.7, 33.3 … ms), `touchStoryboard` (a frame per scripted event), `castStoryboard` (real screencast frames): PNG contact sheet + JSON trace. | -| `output.ts` | `evidenceDir`, `writeEvidence`: where evidence goes and how it is attached to the report. | -| `story.ts` | `phone`, `withStorybook`, `openStory`, `frames`, and the timer gate (`installTimerGate`, `freezeTimers`, `thawTimers`). | +| Module | What it gives a spec | +| --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact final-move speed, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. | +| `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | +| `frames.ts` | `startScreencast` (timestamped composited frames), `markElement` (a marker chip or outline), `trackMarker` (marker box per frame), `sampleRects` (per-rAF rects), `frameStats`. | +| `busy.ts` | `blockMainThread(page, ms)` and `busyWindowProof(tracked, window)`: frames presented, frames that moved, longest stall, a verdict; `presented` adds the compositor's own frames. | +| `trace.ts` | `traceRendering(page, body)`: main-thread counts of UpdateLayoutTree / Layout / PrePaint / Paint / Layerize / UpdateLayer / ScrollLayer / HitTest, `countBetween`, long tasks; `installLongTaskLog` / `readLongTasks`; `startCompositorTrace`: when the compositor presents. | +| `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t), `waitForAnimation`, `watchAnimations`, `waitForRest`, `releaseAnimations`. | +| `math.ts` | Oracles independent of the code under test: `cubicBezier`, `timingFunction`, `springStep` (SwiftUI response / dampingFraction, initial velocity), `projectDecay`, `rubberBand`, `parseTransform`, `compareCurve`, `timeline`. | +| `storyboard.ts` | `animationStoryboard` (seek to t = 0, 16.7, 33.3 … ms), `touchStoryboard` (a frame per scripted event), `castStoryboard` (real screencast frames): PNG contact sheet + JSON trace. | +| `output.ts` | `evidenceDir`, `writeEvidence`: where evidence goes and how it is attached to the report. | +| `story.ts` | `phone`, `withStorybook`, `openStory`, `frames`, and the timer gate (`installTimerGate`, `freezeTimers`, `thawTimers`). | ## The proofs @@ -212,6 +212,14 @@ keyframes, `blockMainThread` without `during`) is in `devtoolsTasks` instead, and only when no page script ran inside it. A test's own `page.evaluate` stays a long task, whether it calls page code directly or sets it off. +With `{invalidations: true}`, `traceRendering` also records why each style +recalculation ran and on which element, every animation that ran and whether +it ran on the compositor, and when script ran. `countBetween(trace, from, to, +{skipCompositorAnimationUpdates: true})` then leaves out a pass Chrome makes on +its own for compositor animations: every recalculation since the previous pass +has the reason Animation, on an element with an animation on the compositor at +the time, with no script in between. + **Storyboards.** Deterministic: frames come from seeking held animations or from stepping scripted input, never from wall-clock capture. The JSON trace carries each frame's measurements and the spec asserts on it; the sheet is