diff --git a/.changeset/build-stylex-cross-file-consts.md b/.changeset/build-stylex-cross-file-consts.md new file mode 100644 index 0000000000000..736b20da4327d --- /dev/null +++ b/.changeset/build-stylex-cross-file-consts.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/build': patch +--- + +[fix] `astryxStylex` resolves `defineConsts` values read from another file. With the `{stylexOptions}` form (the one Storybook uses), a file that read only constants from a `.stylex.ts` module lost that import once StyleX compiled it, so the constants never reached the CSS: a constant used as a media-query key failed the build ("Invalid empty selector") and one used as a value shipped as a custom property nothing declares. In either form, a constant defined on the other side of the layer split, or in an Astryx package (compiled with the library class-name prefix) and read from an app file (compiled with its own), stayed unresolved. + +@vjeux diff --git a/.changeset/click-guard.md b/.changeset/click-guard.md new file mode 100644 index 0000000000000..7376b26fef796 --- /dev/null +++ b/.changeset/click-guard.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: the gesture recognizers and the menu press model share one guard for the browser's click after a gesture that already acted, swallowing it for 400ms or until the next press. Keyboard and assistive-technology clicks (`detail` 0) always pass. + +@vjeux diff --git a/.changeset/gesture-arena-moving-cancel.md b/.changeset/gesture-arena-moving-cancel.md new file mode 100644 index 0000000000000..5b7e52cad131a --- /dev/null +++ b/.changeset/gesture-arena-moving-cancel.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: a gesture that cancels itself on a finger's move (a long press dragged past its tolerance) keeps that finger claimed until it lifts, so the page never takes the drag over midway; a resting finger is still given back when its gesture is cancelled. Not exported from the package entry point. + +@vjeux diff --git a/.changeset/gesture-browser-negotiation.md b/.changeset/gesture-browser-negotiation.md new file mode 100644 index 0000000000000..1930e34496d7e --- /dev/null +++ b/.changeset/gesture-browser-negotiation.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: touch-action derivation, the native scroll probe and haptics for the gesture system. Not exported from the package entry point. + +@vjeux diff --git a/.changeset/gesture-hook-fixes.md b/.changeset/gesture-hook-fixes.md new file mode 100644 index 0000000000000..87f08a0948e31 --- /dev/null +++ b/.changeset/gesture-hook-fixes.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: the gesture hooks call nothing back after unmount; `usePressTracker` brakes with the host by default and keeps one tracker, so an option change or a new ref never drops a press in progress; the recognizer's arena plumbing is `protected`. Not exported from the package entry point. + +@vjeux diff --git a/.changeset/gesture-host-fixes.md b/.changeset/gesture-host-fixes.md new file mode 100644 index 0000000000000..2fc0915c974a6 --- /dev/null +++ b/.changeset/gesture-host-fixes.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: gesture host fixes from review. A page scroll under a still finger keeps the press and its click; watchers and touches nothing can claim get passive listeners only; a claimed pen holds only its own stylus touches; an element's own `touch-action` is never widened; slotted content scrolls with its shadow-tree scroller. Not exported from the package entry point. + +@vjeux diff --git a/.changeset/gesture-host-owner.md b/.changeset/gesture-host-owner.md new file mode 100644 index 0000000000000..ea6ee44bda14a --- /dev/null +++ b/.changeset/gesture-host-owner.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: the gesture host becomes the one document-level touch owner: the finger count, touch gates, and a kept press for continuous-tracking controls, whose element gets `touch-action: manipulation` where the pointer is coarse. A scroll lock's page hold and the finger-scroll watch keep their own listeners and ask the host only about the touches it owns. A second finger fails a one-finger pan that has not claimed, so the page can pinch. A scroll-edge pan holds a lone finger from the first move that pulls away from an edge every scroller under it is at, so those moves are cancelled before it begins; with a second finger anywhere on the screen the page keeps its pinch. The host follows the primary pointer when it changes; what it attaches under a fine pointer is in the every-pointer changeset (`gestures-every-pointer.md`). A press tracker whose hold acts (a stepper's hold-repeat) can bring a selection lock for its presses, as a recognizer can. Internal, for the touch components that adopt them; not exported from the package entry point. + +@vjeux diff --git a/.changeset/gesture-host.md b/.changeset/gesture-host.md new file mode 100644 index 0000000000000..eb9f2fec42db2 --- /dev/null +++ b/.changeset/gesture-host.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: the gesture system's DOM host (claims, click suppression, the browser's takeover, the pointer behind a contact). Not exported from the package entry point. + +@vjeux diff --git a/.changeset/gesture-pan.md b/.changeset/gesture-pan.md index 12eb02fc42391..908da1b367575 100644 --- a/.changeset/gesture-pan.md +++ b/.changeset/gesture-pan.md @@ -2,6 +2,6 @@ '@astryxdesign/core': patch --- -[chore] Internal: the pan gesture recognizer, with UIKit's measured hysteresis and velocity filter. Not exported from the package entry point; no component uses it yet. +[chore] Internal: the pan gesture recognizer, with UIKit's measured hysteresis and velocity filter. Not exported from the package entry point. @vjeux diff --git a/.changeset/gesture-press-tracking.md b/.changeset/gesture-press-tracking.md new file mode 100644 index 0000000000000..1c66836c74243 --- /dev/null +++ b/.changeset/gesture-press-tracking.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: UIControl press tracking with control and cell profiles and the gesture hooks, with every press painter writing `data-astryx-press` through one internal writer in `utils/pressFeedback.ts`. Not exported from the package entry point. + +@vjeux diff --git a/.changeset/gesture-readme.md b/.changeset/gesture-readme.md new file mode 100644 index 0000000000000..9a6293b811b68 --- /dev/null +++ b/.changeset/gesture-readme.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: the gesture system's README for component authors, with its browser proof in Chromium and WebKit. + +@vjeux diff --git a/.changeset/gesture-recognizer-core.md b/.changeset/gesture-recognizer-core.md index 0b0bfc389235f..28caef0d5579c 100644 --- a/.changeset/gesture-recognizer-core.md +++ b/.changeset/gesture-recognizer-core.md @@ -2,6 +2,6 @@ '@astryxdesign/core': patch --- -[chore] Internal: the gesture recognizer core for touch presentations: UIKit's recognizer state machine and the arena that settles which recognizer owns a touch. Not exported from the package entry point; no component uses it yet. +[chore] Internal: the gesture recognizer core for touch presentations: UIKit's recognizer state machine and the arena that settles which recognizer owns a touch. Not exported from the package entry point. @vjeux diff --git a/.changeset/gesture-recognizer-fixes.md b/.changeset/gesture-recognizer-fixes.md new file mode 100644 index 0000000000000..9e6754a78d733 --- /dev/null +++ b/.changeset/gesture-recognizer-fixes.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: gesture recognizer and arena fixes from review; a pan claims nothing before it begins at 10 pt; one velocity tracker for every gesture (`spec:AST-100`: UIKit's measured blend, a scrolling surface's release through the pan's `scrollVelocity()`, and FR2's release rules). Not exported from the package entry point. + +@vjeux diff --git a/.changeset/gesture-swipe-pinch-rotation-edge-hover.md b/.changeset/gesture-swipe-pinch-rotation-edge-hover.md new file mode 100644 index 0000000000000..ffeadde679ef5 --- /dev/null +++ b/.changeset/gesture-swipe-pinch-rotation-edge-hover.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: swipe, pinch, rotation, screen-edge pan and hover gesture recognizers. Not exported from the package entry point. + +@vjeux diff --git a/.changeset/gesture-tap-long-press.md b/.changeset/gesture-tap-long-press.md index 9ef786da65842..f1efef8d29014 100644 --- a/.changeset/gesture-tap-long-press.md +++ b/.changeset/gesture-tap-long-press.md @@ -2,6 +2,6 @@ '@astryxdesign/core': patch --- -[chore] Internal: tap and long-press gesture recognizers with UIKit's measured thresholds. Not exported from the package entry point; no component uses them yet. +[chore] Internal: tap and long-press gesture recognizers with UIKit's measured thresholds. Not exported from the package entry point. @vjeux diff --git a/.changeset/gesture-watcher.md b/.changeset/gesture-watcher.md new file mode 100644 index 0000000000000..bec7cd286e49d --- /dev/null +++ b/.changeset/gesture-watcher.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: a recognizer created with `claimsTouches: false` is a watcher: passive listeners only, simultaneous with every recognizer, and it follows a touch the browser takes for a scroll to the lift. Not exported from the package entry point. + +@vjeux diff --git a/.changeset/gestures-every-pointer.md b/.changeset/gestures-every-pointer.md new file mode 100644 index 0000000000000..30ec76582c1ae --- /dev/null +++ b/.changeset/gestures-every-pointer.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: gestures attach for every pointer, a mouse, a pen and a finger alike (AST-094 FR43). Under a fine primary pointer an element that uses a gesture keeps one passive `pointerdown` listener (a recognizer that claims a moving finger also keeps its blocking `touchmove`), and the page's listeners last only a press. A right click or a ctrl-click on macOS starts no press. A finger's or a pen's hold that does something keeps text unselected under any primary pointer, a touchscreen laptop's included, while a mouse keeps its released behavior (FR41). A mouse or pen drag that a recognizer claims takes no text selection from then on, anywhere on the page, and a finger is read once where the browser sends Touch Events without exposing `ontouchstart`. A press tracker takes a mouse press as a press, painting at once with no scroll brake. The screen-edge pan is a finger's only: it stays off under a mouse, a trackpad and a pen, and takes no `allowedPointerTypes`, and `perEventPointer` is accepted and does nothing. Not exported from the package entry point. + +@vjeux diff --git a/.changeset/gestures-wheel.md b/.changeset/gestures-wheel.md new file mode 100644 index 0000000000000..cc8ec4e57c5d6 --- /dev/null +++ b/.changeset/gestures-wheel.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: a pan takes wheel and trackpad scrolling by default, a mouse wheel's notches included (`allowedScrollTypes`, UIKit's `allowedScrollTypesMask`; `[]` opts out), along its own axis and after the scrollers under the pointer; the screen-edge pan takes none; and a pinch takes a ctrl-wheel, as a trackpad pinches (AST-094 FR43). Not exported from the package entry point. + +@vjeux diff --git a/.changeset/input-stream.md b/.changeset/input-stream.md new file mode 100644 index 0000000000000..61c8b2d52ed22 --- /dev/null +++ b/.changeset/input-stream.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[fix] The touch press feedback and the gesture recognizers now listen on the window's capture phase through one shared input stream: one passive listener per event type for both. An app's own capture-phase `stopPropagation` at the document no longer starves them, and a page of pressables registers no listener that can hold a scroll. + +@vjeux diff --git a/.changeset/layer-exit-copy.md b/.changeset/layer-exit-copy.md new file mode 100644 index 0000000000000..441788643f6d2 --- /dev/null +++ b/.changeset/layer-exit-copy.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: the layer runtime gains one exit copy for touch presentations that animate a popover out: a static, inert copy of the closing popover, shown only in the top layer where the popover was drawn, for the close to play on. + +@vjeux diff --git a/.changeset/layer-input-hold.md b/.changeset/layer-input-hold.md new file mode 100644 index 0000000000000..3acd4e377646f --- /dev/null +++ b/.changeset/layer-input-hold.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: one input hold for every layer (`Layer/inputHold`). While a layer presents, a touch, a press or a click anywhere reaches nothing (keys and assistive technology still act, and a pinch still zooms the page); while it dismisses, the same, and the layer's content is inert. The gesture host holds the layer's touches through its touch gates, and `isInputShield` lets a hit test see past a hold's shield. Not exported from the package entry point; no component uses it yet. + +@vjeux diff --git a/.changeset/lift-module.md b/.changeset/lift-module.md new file mode 100644 index 0000000000000..6d6b6ea44e3c2 --- /dev/null +++ b/.changeset/lift-module.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: one opt-in lift-from-trigger presentation for touch surfaces. The trigger itself lifts into the top layer above a tinted backdrop; a surface grows out of the press beside it on the shared rect morph and shrinks back into it on the layer runtime's exit copy; the trigger returns to its place. Translate, scale and opacity only; nothing mounted at rest, four elements while lifted. A zoom from a thumbnail lifts a copy of the thumbnail's picture into the viewer instead, while the thumbnail hides in place. Not exported from the package entry point; no component uses it yet. + +@vjeux diff --git a/.changeset/lightbox-chrome-toggle.md b/.changeset/lightbox-chrome-toggle.md new file mode 100644 index 0000000000000..579b14aed814f --- /dev/null +++ b/.changeset/lightbox-chrome-toggle.md @@ -0,0 +1,16 @@ +--- +'@astryxdesign/core': patch +--- + +[feat] On a touch screen, tapping a zoomable Lightbox photo now hides the controls and caption, and the background turns black, as in the iOS Photos viewer. The next tap on the photo, or beside it, brings them back. + +Zooming in by touch hides the controls the same way: a double tap as it zooms, or a pinch as soon as it passes the fitted size. Returning to the fitted size shows them. While the photo is zoomed, the background stays black and a tap toggles only the controls. The fades follow Photos' timing: the controls fade over 0.2 s, and the background turns black 0.1 s after them. While the controls are hidden: + +- a tap beside the photo shows them instead of closing the viewer; +- a caption paged into view arrives hidden; +- a cancelled drag down brings them back; +- a keyboard focus on a control shows them. + +A mouse click leaves the controls in place. + +@vjeux diff --git a/.changeset/lightbox-drag-to-dismiss.md b/.changeset/lightbox-drag-to-dismiss.md new file mode 100644 index 0000000000000..a3be598c3bf89 --- /dev/null +++ b/.changeset/lightbox-drag-to-dismiss.md @@ -0,0 +1,9 @@ +--- +'@astryxdesign/core': patch +--- + +[feat] On a touch screen, drag a Lightbox photo down to close the viewer, as in the iOS Photos viewer. The photo follows your finger and shrinks a little as you pull, while the backdrop fades and the controls fade faster. The backdrop fades without its blur, which comes back once it is at rest. Let go moving down, however short the pull, and the photo carries on down as the viewer closes. Flick it back up, or bring it back to where you started, and it springs back into place. A zoomed photo still pans when you drag it; the drag to close starts from the fitted size. Videos and the mouse are unchanged. + +The viewer's controls now sit in their own layers, which fade together during the drag. + +@vjeux diff --git a/.changeset/lightbox-every-pointer.md b/.changeset/lightbox-every-pointer.md new file mode 100644 index 0000000000000..735cc887f63fb --- /dev/null +++ b/.changeset/lightbox-every-pointer.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[feat] Lightbox: a mouse, a pen and a trackpad now use the same gestures as a finger. Drag a photo down to close the viewer; drag sideways, or scroll a trackpad sideways, to move through a gallery; ctrl-wheel to zoom, and out from the fitted size to close; drag or scroll to pan a zoomed photo, which coasts on when released moving. A double-click zooms in at the click, as a double tap does, and a zoom by a pointer hides the controls as a finger's does; the keyboard keeps its 2× zoom about the centre. + +@vjeux diff --git a/.changeset/lightbox-gallery-paging.md b/.changeset/lightbox-gallery-paging.md new file mode 100644 index 0000000000000..15612535de4fb --- /dev/null +++ b/.changeset/lightbox-gallery-paging.md @@ -0,0 +1,9 @@ +--- +'@astryxdesign/core': patch +--- + +[feat] A Lightbox gallery now pages the way the iOS Photos viewer does. On a touch screen, drag or flick between items: the row follows your finger, and the release lands on the next or previous item, one at most. A gap of 40 px separates the items while they move. A zoomed photo pans with your finger until it reaches its edge; a new drag from that edge moves to the next item. A photo you leave zoomed returns to its fitted size once the gallery rests. Under a finger, the previous and next buttons and the arrow keys slide to the item instead of swapping it in place. + +Each item's caption now moves with its item, and the gallery fills the viewer on every screen, so the viewer keeps its size from item to item. `onIndexChange` also reports a swipe. A mouse still moves between items with the buttons and arrow keys. + +@vjeux diff --git a/.changeset/lightbox-hide-click.md b/.changeset/lightbox-hide-click.md new file mode 100644 index 0000000000000..3c736d4e15dd1 --- /dev/null +++ b/.changeset/lightbox-hide-click.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[feat] Lightbox: with a mouse or a pen, a click on a zoomable photo now hides and shows the controls, as a tap does on a touch screen. A double-click still zooms without them. + +@vjeux diff --git a/.changeset/lightbox-pinch-close.md b/.changeset/lightbox-pinch-close.md new file mode 100644 index 0000000000000..9d35eaa8c3282 --- /dev/null +++ b/.changeset/lightbox-pinch-close.md @@ -0,0 +1,11 @@ +--- +'@astryxdesign/core': patch +--- + +[feat] Under a finger, pinching in on a zoomable Lightbox photo at its fitted size now closes the viewer into the photo's thumbnail, as in the iOS Photos viewer, on phones and tablets (a coarse primary pointer). + +While the fingers pinch in, the photo shrinks with them about the point between them and follows it. The backdrop fades on Photos' measured curve and the controls fade faster. A pinch that goes below about 93.5 % closes at the lift unless it is opened again by 5 % first: the photo flies into its thumbnail, or shrinks and fades in place when it has none. Any other pinch springs back to the fitted size. + +A pinch that starts zoomed in still gives way below the fitted size and returns, without closing. + +@vjeux diff --git a/.changeset/lightbox-source-for.md b/.changeset/lightbox-source-for.md new file mode 100644 index 0000000000000..3c2cd171fd494 --- /dev/null +++ b/.changeset/lightbox-source-for.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[experimental] Lightbox: `experimental.sourceFor(index)` returns the element each item zooms out of when it opens and back into when it closes on a touch screen, usually its thumbnail, so every photo of a gallery zooms from its own thumbnail rather than from the element that opened the viewer. It may change or be removed in a patch release. + +@vjeux diff --git a/.changeset/lightbox-touch-zoom.md b/.changeset/lightbox-touch-zoom.md new file mode 100644 index 0000000000000..d335253d11c1a --- /dev/null +++ b/.changeset/lightbox-touch-zoom.md @@ -0,0 +1,9 @@ +--- +'@astryxdesign/core': patch +--- + +[feat] Lightbox's zoom now works under a finger the way the iOS Photos viewer does, on phones and tablets (a coarse primary pointer). Pinch the photo to zoom around your fingers, and move them to pan while you zoom. Past the zoom limits the photo gives way, then eases back when you let go. Double-tap zooms until the photo fills the screen, bringing the tapped point to the centre; double-tap again to go back. A zoomed photo pans with momentum, stretches past its edges and settles back. A mouse, a pen where the primary pointer is fine, and the keyboard keep the zoom they had: 2× about the centre, over the same 200 ms ease-out. A mouse drag and keyboard panning now stop at the photo's edges instead of carrying it out of view. + +On a phone the photo's stage fills the viewer. A pinch on the photo no longer zooms the page, and a pinch on the controls or caption still does. With `hasZoom` on, a tap on the dark area beside the photo closes the viewer after the double-tap wait instead of at once. The zoom animations run on the compositor, so they stay smooth while the page is busy. With `hasZoom` off, nothing changes. + +@vjeux diff --git a/.changeset/lightbox-zoom-transition.md b/.changeset/lightbox-zoom-transition.md new file mode 100644 index 0000000000000..81cb10316d7a8 --- /dev/null +++ b/.changeset/lightbox-zoom-transition.md @@ -0,0 +1,11 @@ +--- +'@astryxdesign/core': patch +--- + +[feat] On a touch screen, a Lightbox photo can now zoom out of its thumbnail when it opens and back into it when it closes, like the iOS Photos viewer (UIKit's zoom transition). A viewer opened from an element that shows a picture (an image, or a button or link around one) zooms from that element for the photo it opened on; a text button opens and closes as before. The photo grows from the thumbnail to its fitted size while the thumbnail's picture fades out, and the backdrop and controls fade in with it. A tap or a drag that lands while it opens is held, so it can't close the viewer, zoom the photo or reach the page; keys and screen readers stay live. + +Closing works the same way: the Close button, Escape, a tap beside the photo, or a swipe to another photo and then a close each send the photo back into the thumbnail of the photo showing, and the viewer stays open until it lands. A photo dragged down shrinks into its thumbnail. Under Reduce Motion the photo cross-fades in place. + +With a mouse the viewer opens and closes at once, as before. + +@vjeux diff --git a/.changeset/long-press-one-source.md b/.changeset/long-press-one-source.md new file mode 100644 index 0000000000000..a73cf7b961b6d --- /dev/null +++ b/.changeset/long-press-one-source.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: `useLongPress` (ContextMenu's long press, and the held finger that opens a menu from its trigger) takes its default duration and tolerance, 500ms and 10px on either axis, from the same source as the long-press gesture recognizer. No behavior changes. + +@vjeux diff --git a/.changeset/menu-trigger-drift.md b/.changeset/menu-trigger-drift.md new file mode 100644 index 0000000000000..adbf976c7b1d0 --- /dev/null +++ b/.changeset/menu-trigger-drift.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[fix] A finger held on a DropdownMenu trigger may drift up to 13px on each axis and still open the menu where the primary pointer is coarse (phones and tablets), as AST-094 FR29 allows; it cancelled past 10px. Where the primary pointer is fine (a touchscreen laptop, a pen tablet), the trigger keeps its 10px. The trigger reads the pointer when its press starts, so no menu or row observes the pointer query. + +@vjeux diff --git a/.changeset/motion-progress.md b/.changeset/motion-progress.md new file mode 100644 index 0000000000000..d7d9d896ba1ff --- /dev/null +++ b/.changeset/motion-progress.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: one progress module for the motion foundation. `progressCurve` gives a spring's progress from 0 to 1 as a closed-form curve, `mapProgress` maps any curve from another (a companion's dim following a sheet, a clamped range), `mapRange` is the clamped linear map they share, and `presentedProgressVelocity` reads the velocity an interrupted motion continues from. One velocity unit throughout: progress per second. Nothing public changes; no component imports it yet. + +@vjeux diff --git a/.changeset/motion-rect-morph.md b/.changeset/motion-rect-morph.md new file mode 100644 index 0000000000000..e06f7961ecfa9 --- /dev/null +++ b/.changeset/motion-rect-morph.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: the rect morph, one way for a touch presentation to grow out of a source rect and shrink back into it. `lerpPath` (a straight line between two rects) and `edgeGrowPath` (UIKit's popover, growing out of a point on its edge) name the rect an element shows at each progress; `morphCurves` turns a path into compositor curves (`translate`, `scale` and a companion `opacity`); `createRectMorph` presents an element along a path and dismisses it back on a copy or on itself, each continuing the other from the rect on screen with its velocity, with Reduced Motion left to the animator. It runs on new progress helpers (`progressCurve`, `mapProgress`, `mapRange`, `presentedProgressVelocity`). Not exported from the package entry point; no component uses it yet. + +@vjeux diff --git a/.changeset/presented-hit-testing.md b/.changeset/presented-hit-testing.md new file mode 100644 index 0000000000000..e27c857277aad --- /dev/null +++ b/.changeset/presented-hit-testing.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] On iOS, a touch on an element the compositor animator is moving reaches what is drawn: the animator reports each moving element's offset from its layout box, and the gesture host sends the touch, press bounds and the tap's click there instead of to the layout position iOS WebKit hit-tests. + +@vjeux diff --git a/.changeset/reduced-motion-record.md b/.changeset/reduced-motion-record.md new file mode 100644 index 0000000000000..2ec44f4c1f9c8 --- /dev/null +++ b/.changeset/reduced-motion-record.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: Reduce Motion for the touch presentations follows design:reduced-motion beyond the animator too. `reducedMotionFade()` gives code beside the animator the animator's own fade (an opacity-only 200 ms cross-fade on a coarse pointer or after a touch release, at once on any other pointer), `animator.reducedMotion()` reports which form a change takes for feedback that plays no fade of its own, and `reducedMotion.stylex.ts` holds the fade's constants for StyleX transitions. The fade's length is not a theme motion role. Not exported from the package entry point. + +@vjeux diff --git a/.changeset/touch-selection-lock-gestures.md b/.changeset/touch-selection-lock-gestures.md new file mode 100644 index 0000000000000..d1f3ab7c2b7fc --- /dev/null +++ b/.changeset/touch-selection-lock-gestures.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: the gesture host applies a recognizer's own `selectionLock` to that recognizer's contact and imports no lock itself; a long press brings one only when it has a hold (a `minimumPressDuration` above 0), and `selectionLock: null` opts out. Not exported from the package entry point. + +@vjeux diff --git a/.changeset/touch-selection-lock.md b/.changeset/touch-selection-lock.md new file mode 100644 index 0000000000000..e5285afc1f8ad --- /dev/null +++ b/.changeset/touch-selection-lock.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[fix] Where the primary pointer is fine (a mouse, a trackpad, a touchscreen laptop, a desktop pen tablet), the press feedback controller no longer registers a touch listener: it reads the primary pointer at the first finger or pen press and keeps its `touchcancel` listener only while that pointer is coarse. + +@vjeux diff --git a/.changeset/uikit-constants-table.md b/.changeset/uikit-constants-table.md index 01fd41a182b0b..2c063cdb50bc7 100644 --- a/.changeset/uikit-constants-table.md +++ b/.changeset/uikit-constants-table.md @@ -2,6 +2,6 @@ '@astryxdesign/core': patch --- -[chore] Internal: the UIKit constants table for touch presentations (springs, scroll physics, timing curves and navigation values), each value with its source and a checked list of the simulator measurements still owed. Not exported from the package entry point; no component reads it yet. +[chore] Internal: the UIKit constants table for touch presentations (springs, scroll physics, timing curves and navigation values), each value with its source and a checked list of the simulator measurements still owed. Not exported from the package entry point. @vjeux diff --git a/.changeset/uikit-motion-animator-pseudo-element.md b/.changeset/uikit-motion-animator-pseudo-element.md new file mode 100644 index 0000000000000..7ccb10bab35da --- /dev/null +++ b/.changeset/uikit-motion-animator-pseudo-element.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: the touch presentations' animator drives a pseudo-element such as a dialog's ::backdrop: a static filling animation holds its value after a curve ends and after `set`, a custom property on its host promotes it, and a Reduce Motion dip borrows and hands back the pseudo-element's own opacity. Not exported from the package entry point. + +@vjeux diff --git a/.changeset/uikit-motion-animator.md b/.changeset/uikit-motion-animator.md index d489e3022e6db..6b23ce1985186 100644 --- a/.changeset/uikit-motion-animator.md +++ b/.changeset/uikit-motion-animator.md @@ -2,6 +2,6 @@ '@astryxdesign/core': patch --- -[chore] Internal: a compositor animator for touch presentations. It plays UIKit's springs, and scroll coasts (built by a separate coast module, so a surface that never coasts does not carry them; `decay` plays momentum, and sends a spring back from past an edge through the Reduce Motion policy: a cross-fade on a coarse pointer or after a touch release, at once elsewhere) as Web Animations on translate, rotate, scale and opacity, so they keep running while JavaScript is busy, and reads values and velocities back from the curves for interruption, retargeting and Reduce Motion cross-fades. Not exported from the package entry point; no component uses it yet. +[chore] Internal: a compositor animator for touch presentations. It plays UIKit's springs, and scroll coasts (built by a separate coast module, so a surface that never coasts does not carry them; `decay` plays momentum, and sends a spring back from past an edge through the Reduce Motion policy: a cross-fade on a coarse pointer or after a touch release, at once elsewhere) as Web Animations on translate, rotate, scale and opacity, so they keep running while JavaScript is busy, and reads values and velocities back from the curves for interruption, retargeting and Reduce Motion cross-fades. Not exported from the package entry point. @vjeux diff --git a/.changeset/uikit-motion-math.md b/.changeset/uikit-motion-math.md index 7af3d73a92a98..cbb9399568857 100644 --- a/.changeset/uikit-motion-math.md +++ b/.changeset/uikit-motion-math.md @@ -2,6 +2,6 @@ '@astryxdesign/core': patch --- -[chore] Internal: the spring, decay, rubber-band and projection math of UIKit for touch presentations, as pure functions checked against an independent integration and against values read from UIKit. Not exported from the package entry point; no component uses it yet. +[chore] Internal: the spring, decay, rubber-band and projection math of UIKit for touch presentations, as pure functions checked against an independent integration and against values read from UIKit. Not exported from the package entry point. @vjeux diff --git a/.changeset/uikit-motion-use-animator.md b/.changeset/uikit-motion-use-animator.md new file mode 100644 index 0000000000000..b234a4e987752 --- /dev/null +++ b/.changeset/uikit-motion-use-animator.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: useAnimator, a React wrapper around the touch presentations' animator that never re-renders per frame, and browser proofs on the interaction harness that the animator's motion keeps running through a blocked main thread in Chromium and WebKit. Not exported from the package entry point. + +@vjeux diff --git a/.changeset/uikit-pager.md b/.changeset/uikit-pager.md new file mode 100644 index 0000000000000..ddfcff9552939 --- /dev/null +++ b/.changeset/uikit-pager.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: a pager for touch presentations that pages like a UIKit paging scroll view. A horizontal pan moves the pages one to one and rubber-bands at the ends. The release picks the page with UIKit's rule and settles on UIKit's paging curve on the compositor, and a finger can catch it mid-flight. Its motion values are internal constants. Not exported from the package entry point. + +@vjeux diff --git a/.changeset/uikit-paging-values.md b/.changeset/uikit-paging-values.md new file mode 100644 index 0000000000000..8096b94de001e --- /dev/null +++ b/.changeset/uikit-paging-values.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: UIKit's paging values and math for touch presentations. The thresholds that pick the page a release settles on, the settle that carries it there and `setContentOffset(animated:)` are measured on iOS 26.2 and replayed against the recorded frames to under half a pixel. Not exported from the package entry point. + +@vjeux diff --git a/.changeset/uikit-zoom-media.md b/.changeset/uikit-zoom-media.md new file mode 100644 index 0000000000000..8ea4eb1cc19eb --- /dev/null +++ b/.changeset/uikit-zoom-media.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: UIScrollView zooming for a photo viewer in the UIKit constants table (the give past the zoom limits, the settle's eased curve and duration law, Photos' aspect-fill double tap and maximum zoom, measured on iOS 26.2), and the zoom stage geometry Lightbox's touch zoom is built on. Not exported from the package entry point; Lightbox's touch zoom reads it. + +@vjeux diff --git a/.changeset/use-finger-scroll.md b/.changeset/use-finger-scroll.md new file mode 100644 index 0000000000000..e48f0b171df03 --- /dev/null +++ b/.changeset/use-finger-scroll.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[chore] Internal: `useFingerScroll(scroller)` tells a finger's scroll from the program's, and the gestures README gains a guide to migrating a raw-touch drag onto the recognizers. Internal to the touch components: not exported from the package entry point or the gestures barrel. + +@vjeux diff --git a/apps/storybook/stories/Lightbox.stories.tsx b/apps/storybook/stories/Lightbox.stories.tsx index e3a6b88af81f4..45389cbba1918 100644 --- a/apps/storybook/stories/Lightbox.stories.tsx +++ b/apps/storybook/stories/Lightbox.stories.tsx @@ -1,7 +1,8 @@ // Copyright (c) Meta Platforms, Inc. and affiliates. -import {useState} from 'react'; +import {useEffect, useRef, useState} from 'react'; import type {Meta, StoryObj} from '@storybook/react'; +import {Dialog} from '@astryxdesign/core/Dialog'; import {Lightbox, useLightbox} from '@astryxdesign/core/Lightbox'; const meta: Meta = { @@ -87,6 +88,30 @@ export const Gallery: Story = { }, }; +/** + * The parent decides whether the viewer closes: here it asks first. When it + * keeps the viewer open, a photo dragged down comes back. + */ +export const ConfirmClose: Story = { + render: () => { + const [isOpen, setIsOpen] = useState(false); + return ( + <> + + { + if (next || window.confirm('Close the photo?')) { + setIsOpen(next); + } + }} + media={{src: SAMPLE_IMAGE, alt: 'Forest path'}} + /> + + ); + }, +}; + export const WithZoom: Story = { render: () => { const [isOpen, setIsOpen] = useState(false); @@ -104,6 +129,245 @@ export const WithZoom: Story = { }, }; +/** + * A gallery a phone can page through: drag or flick between photos (Photos' + * 40 pt gap), pinch or double-tap one to zoom, and drag a zoomed photo to its + * edge before the next drag pages. + */ +export const GalleryWithZoom: Story = { + render: () => { + const [isOpen, setIsOpen] = useState(false); + const [index, setIndex] = useState(0); + return ( + <> + + + + ); + }, +}; + +/** + * A gallery whose captions differ in length: the photo keeps one box + * whichever pages, and so whichever captions, are rendered. + */ +export const GalleryWithCaptions: Story = { + render: () => { + const [isOpen, setIsOpen] = useState(false); + return ( + <> + + + + ); + }, +}; + +/** + * A controlled gallery whose parent keeps its index (it pages only with its + * own buttons, say while a change is confirmed): a swipe returns to the + * photo the parent shows. + */ +export const GalleryKeepsIndex: Story = { + render: () => { + const [isOpen, setIsOpen] = useState(false); + return ( + <> + + {}} + hasZoom + /> + + ); + }, +}; + +/** + * Photos open from their thumbnails and close back into the one showing: + * `experimental.sourceFor` hands the viewer each item's thumbnail, and the photo zooms + * out of it (UIKit's zoom transition). Page to another photo and close, or + * drag it down: it lands in its own thumbnail. + */ +export const ZoomFromThumbnails: Story = { + render: () => { + const [isOpen, setIsOpen] = useState(false); + const [index, setIndex] = useState(0); + const thumbnails = useRef<(HTMLImageElement | null)[]>([]); + return ( + <> +
+ {GALLERY_MEDIA.map((item, i) => ( + + ))} +
+ thumbnails.current[i] ?? null}} + hasZoom + /> + + ); + }, +}; + +/** + * A viewer the page renders only while it is open: closing it, by any path, + * gives the thumbnail back. + */ +export const MountedWhileOpen: Story = { + render: () => { + const [openIndex, setOpenIndex] = useState(null); + const thumbnails = useRef<(HTMLImageElement | null)[]>([]); + return ( + <> +
+ {GALLERY_MEDIA.map((item, i) => ( + + ))} +
+ {openIndex != null && ( + { + if (!open) { + setOpenIndex(null); + } + }} + media={GALLERY_MEDIA} + index={openIndex} + onIndexChange={setOpenIndex} + experimental={{sourceFor: i => thumbnails.current[i] ?? null}} + hasZoom + /> + )} + + ); + }, +}; + +/** + * A photo opened from a dialog: Escape closes the photo first, and while it + * zooms back into its thumbnail the dialog stays. + */ +export const InsideADialog: Story = { + render: () => { + const [isDialogOpen, setIsDialogOpen] = useState(false); + const [isOpen, setIsOpen] = useState(false); + const thumbnail = useRef(null); + return ( + <> + + +
+ +
+ + thumbnail.current}} + hasZoom + /> + + ); + }, +}; + export const WithCaption: Story = { render: () => { const [isOpen, setIsOpen] = useState(false); @@ -125,6 +389,67 @@ export const WithCaption: Story = { }, }; +/** + * Photos and a video in one gallery: the video page has no zoom, and the + * controls a photo's tap hid come back on it. + */ +export const GalleryWithVideo: Story = { + render: () => { + const [isOpen, setIsOpen] = useState(false); + const [index, setIndex] = useState(0); + return ( + <> + + + + ); + }, +}; + +/** + * Media that arrive after the page mounts the viewer (loaded later): it + * behaves as one given its media from the start. + */ +export const MediaArrivesLater: Story = { + render: () => { + const [isOpen, setIsOpen] = useState(false); + const [media, setMedia] = useState([]); + useEffect(() => { + const timer = setTimeout(() => setMedia(GALLERY_MEDIA), 300); + return () => clearTimeout(timer); + }, []); + return ( + <> + + + + ); + }, +}; + export const Video: Story = { render: () => { const [isOpen, setIsOpen] = useState(false); diff --git a/docs/architecture/layer-runtime.md b/docs/architecture/layer-runtime.md index 0fab0e43de33e..5404257cc43cd 100644 --- a/docs/architecture/layer-runtime.md +++ b/docs/architecture/layer-runtime.md @@ -33,6 +33,8 @@ verified_by: packages/core/src/Layer/anchorName.test.ts, packages/core/src/Layer/useLayerDismissal.test.tsx, packages/core/src/Layer/layerDismissalInvariants.test.tsx, + packages/core/src/Layer/layerExitCopy.test.ts, + packages/core/src/Layer/__tests__/layerExitCopy.interaction.browser.spec.ts, packages/core/src/Popover/Popover.test.tsx, packages/core/src/DropdownMenu/DropdownMenu.test.tsx, packages/core/src/ContextMenu/ContextMenu.test.tsx, @@ -62,6 +64,7 @@ deciding_specs: [spec:AST-038, spec:AST-059] "INV7", "INV11", "INV12", + "INV13", "INV14", "INV15" ], @@ -297,6 +300,25 @@ during a press so native light dismissal does not mistake them for outside interaction. Their click action is cancelled as an invoker toggle so the control can perform its own action without closing the layer. +### Exit copy + +The browser hides a popover the moment it closes, whether by a pick, Escape, or +its own light dismiss. A touch presentation whose close stays on screen plays it +on an exit copy: a static copy of the layer that `placeLayerExitCopy` takes as +the layer closes. The copy is laid out on the layer's own box and carries the +transform and opacity the layer is drawn with, so a copy taken while the layer +is still entering holds where it was drawn. It carries no ids, form names, or +form owner; it is `inert`, hidden from assistive technology, and out of pointer +reach; it keeps the layer's scroll positions. It is shown only in the top layer, +as a manual popover, so a clipping, transformed, or sticky ancestor neither cuts +nor moves it; a fixed box or a larger `z-index` is no substitute +(`spec:AST-027` FR8). Where the browser refuses that show during the closing +popover's own toggle, the copy waits, hidden, and enters the top layer once the +toggle is over: after the script that closed the layer, or in the next frame +before it paints. Where there is no top layer (no popover API), or the browser +still refuses, there is no copy and the close is immediate. The presentation +that placed the copy animates it and removes it when its close settles. + ### Current dismissal plumbing `family:overlay-dismissal` owns the shipped Escape/platform-close membership @@ -480,6 +502,12 @@ layers use Layer rendering without joining Escape/platform dismissal. correct, and the runtime polls nothing: an anchor that a scrolled container hides is followed again when an intersection observer reports it back in view. +- **INV13 — One exit copy per closing layer.** A layer's close plays on at most + one exit copy, made by `placeLayerExitCopy`: drawn where the layer was drawn, + inert, hidden from assistive technology, without ids, form names, or form + owner, shown only in the top layer, and removed by its owner when the close + settles. Placing a second copy for the same layer removes the first; another + layer's copy stays. - **INV14 — The runtime owns every layer's shared mechanism.** Internally, the layer runtime owns the shared mechanism layers use: top-layer mounting, close-request delivery, the presentation hold (keeping a layer's chosen presentation from the open until @@ -533,11 +561,13 @@ Current gaps are observable facts, not current target behavior: migration, verification, and completion criteria for these gaps. This record must be updated only as that work ships. -INV14 and INV15 are not met yet, and `spec:AST-003` does not cover them: no -shared presentation hold, exit copy, runtime backdrop tint, or lift module -exists, and Dialog, Drawer, BottomSheet, Lightbox, MobileNav, ToastViewport, -and Selector each build their own top-layer mounting, exit, or backdrop, and -Popover builds its own presentation hold for its sheet presentation. +INV14 and INV15 are not met yet, and `spec:AST-003` does not cover them. The +runtime owns the shared exit copy (INV13, `placeLayerExitCopy`), and the one +lift-from-trigger module exists (INV15, `packages/core/src/liftFromTrigger/`); +no shared presentation hold or runtime backdrop tint exists, Dialog, Drawer, +BottomSheet, Lightbox, MobileNav, ToastViewport, and Selector each build their +own top-layer mounting, exit, or backdrop, and Popover builds its own +presentation hold for its sheet presentation. BottomSheet's own includes its sheet stack's mounting — the re-show that keeps a document's sheets in open order, a step dialog's modal present, and the non-modal sheet's popover host — alongside its scrim; the ordering rules @@ -568,6 +598,12 @@ above are the stack's, the mounting under them is this gap's. entry/return, and controlled close behavior. - A browser fallback change updates public `useLayer` and Popover docs and tests for both the native path and actual fallback. +- An exit-copy change verifies, where each can be observed: the copy's reach, + accessibility and form exclusion in jsdom; its top-layer show (from script + and by light dismiss), its place mid-entrance, and that a layer keeps one + copy, in jsdom and in Chromium and WebKit; and the refused show's wait in + jsdom and on Android Chromium, the engine that refuses a show during another + popover's toggle. ## Owning code @@ -589,6 +625,8 @@ above are the stack's, the mounting under them is this gap's. transitions of a layer and its content across the re-placement's display reset. - `Layer/gestureCounter.ts` owns physical pointer/key gesture identity. +- `Layer/layerExitCopy.ts` owns the exit copy a touch presentation's close plays + on. - `Layer/layerStack.ts`, `Layer/useLayerDismissal.ts`, and `Layer/LayerDepthContext.tsx` own current registration, presence, ordering, and Escape/platform routing. @@ -638,6 +676,7 @@ this current architecture record. | INV3–INV5 | `useLayer.test.tsx` and `anchorName.test.ts` | A mode leaks geometry, RTL resolves from the wrong context, fallback clips, or a sibling anchor is lost | | INV11 | `layerViewportInset.test.ts`, `useLayer.test.tsx`, and the `Core/Layer` viewport-inset stories under the story play guard at a desktop and a phone viewport | A second gutter definition, a `100%`-of-span cap, a layer narrower than its explicit size, a layer box wider than the viewport, a layer inside the gutter, or a layer that slides toward an off-screen anchor | | INV12 | The `Core/Layer` viewport-inset stories' first-frame comparison, and `useLayer.test.tsx`'s synchronous anchor read | A layer whose first painted rectangle differs from its settled one | +| INV13 | `layerExitCopy.test.ts` and `layerExitCopy.interaction.browser.spec.ts` | A copy with an id, a form name, a form owner, or pointer reach; a copy off the layer's drawn box; a copy outside the top layer; a second copy for one layer | | INV6, INV7 | `useLayer.test.tsx`, `Popover.test.tsx`, `DropdownMenu.test.tsx`, and `useMenuHover.test.tsx` | Duplicate close callback or the same press/re-hover reopens a surface | | INV8 | `useLayerDismissal.test.tsx`, `layerDismissalInvariants.test.tsx`, and `useFocusTrap.test.tsx` | Current top registered layer is skipped, two layers close, or a blocker leaks through | | INV9 | Representative Dialog, ContextMenu, Tooltip/HoverCard, and BottomSheet source/tests | A current local channel silently changes ownership or policy | diff --git a/docs/specs/AST-057-row-actions/spec.md b/docs/specs/AST-057-row-actions/spec.md index 2b3f6c57495e1..4c1ed5a60d229 100644 --- a/docs/specs/AST-057-row-actions/spec.md +++ b/docs/specs/AST-057-row-actions/spec.md @@ -131,10 +131,11 @@ and its panels stay put, so no element is added to any row. - **A render prop for a custom panel.** Deferred, not foreclosed: a `render*`-shaped addition can land when a caller needs something the vocabulary cannot say (DEC-1). -- **Gesture tuning.** Axis-lock distance, commit ratio, fling velocity, slide - and spring durations, resistance past the panel: behavior constants in - source, not public API and not design tokens (the same ruling the touch - press model received for its clocks). +- **Gesture tuning.** Axis-lock distance, commit ratio, fling velocity, + resistance past the panel: behavior constants in source, not public API and + not design tokens (the same ruling the touch press model received for its + clocks). The row's slide and spring motion plays on theme motion roles + (`spec:AST-094` FR4). - **Vertical swipes**, and a swipe whose only meaning is dismissal (Toast already has that in `useToastGesture`). - **Public out-of-flow slots on `Item`.** Deliberately not admitted: the @@ -219,9 +220,10 @@ and its panels stay put, so no element is added to any row. | open state | closed, resting open | The row's own. A pointer landing anywhere outside a resting row closes it, which is also how a drag on a neighbour closes it; no group state exists. | closed | `spec:AST-057` | proposed | | the modality split | internal | A mouse drag never starts the swipe; a finger's or a pen's pointer event or a horizontal wheel gesture does, read from the input event, not a prop or a media query (`architecture:interaction-modality` INV6). | — | `spec:AST-057` | proposed | -Not public: axis-lock distance, commit ratio, fling velocity, durations, -resistance, panel widths, the custom property's name. They are behavior -constants. +Not public: axis-lock distance, commit ratio, fling velocity, resistance, +panel widths, the custom property's name. They are behavior constants. The +row's slide and spring motion plays on theme motion roles (`spec:AST-094` +FR4). ## Requirements diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index 31c5149c2d407..1b726088fc9fc 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -1,23 +1,29 @@ # @astryxdesign/interaction-harness Playwright instruments for touch interaction and animation proofs: real touch -input, curve sampling and storyboards. **Internal and unpublished.** Specs live -next to their component as `*.interaction.browser.spec.ts` and run under +input, frame evidence, curve sampling and storyboards. **Internal and +unpublished.** Specs live next to their component as +`*.interaction.browser.spec.ts` and run under [`playwright.interaction.config.ts`](../../playwright.interaction.config.ts) on a phone-sized touch screen (402 × 714 CSS px page, 402 × 874 screen, 3 device pixels per CSS px, touch, mobile viewport rules), in Chromium and WebKit. +The exemplar to copy is +[`BottomSheetDetent.interaction.browser.spec.ts`](../../packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts). The harness proves its own instruments in -[`src/harness.interaction.browser.spec.ts`](src/harness.interaction.browser.spec.ts), -on fixture pages written inline. +[`src/harness.interaction.browser.spec.ts`](src/harness.interaction.browser.spec.ts). +The busy-main-thread proof and the rendering-trace counts have a negative +control beside their positive checks (a rAF animation that freezes under a +blocked main thread, a `top` animation that lays out every frame); other +self-tests check an instrument against a known answer. ## Running ```sh -pnpm storybook:build # component specs read stories from the built Storybook +pnpm storybook:build # the specs read stories from the built Storybook pnpm exec playwright install chromium webkit pnpm test:interaction --project chromium # everything, Chromium -pnpm test:interaction # one spec, both engines +pnpm test:interaction BottomSheetDetent # one spec, both engines pnpm storyboards # only the tests tagged @storyboard ``` @@ -31,6 +37,12 @@ under a rootful docker daemon the container runs as your user so its node internal/interaction-harness/scripts/webkit-container.mjs [playwright test args…] ``` +WebKit composites through Mesa's software GL there, which compiles each shader +the first time a page draws it (450–700 ms for the first frame of a blurred or +newly animated layer). The script keeps Mesa's shader cache on the host +(`~/.cache/astryx-webkit-shaders/`, or `ASTRYX_WEBKIT_SHADER_CACHE`), so +only the first run on a machine pays for them. + Evidence (storyboards, curve traces) is written to `test-results/interaction-evidence/` (override with `ASTRYX_INTERACTION_EVIDENCE`) and attached to the test report. It is never @@ -38,15 +50,19 @@ committed: a pull request links or attaches it. ## Instruments -| Module | What it gives a spec | -| --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact final-move speed, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. | -| `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | -| `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t), `waitForAnimation`, `watchAnimations`, `waitForRest`, `releaseAnimations`. | -| `math.ts` | Oracles independent of the code under test: `cubicBezier`, `timingFunction`, `springStep` (SwiftUI response / dampingFraction, initial velocity), `projectDecay`, `rubberBand`, `parseTransform`, `compareCurve`, `timeline`. | -| `storyboard.ts` | `animationStoryboard` (seek to t = 0, 16.7, 33.3 … ms), `touchStoryboard` (a frame per scripted event): PNG contact sheet + JSON trace. | -| `output.ts` | `evidenceDir`, `writeEvidence`: where evidence goes and how it is attached to the report. | -| `story.ts` | `phone`, `withStorybook`, `openStory`, `frames`, and the timer gate (`installTimerGate`, `freezeTimers`, `thawTimers`). | +| Module | What it gives a spec | +| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact final-move speed, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. | +| `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | +| `webInput.ts` | `webInputDriver(page)`: `wheel({x, y, dx, dy, ctrl})` and `pen(points)` as real engine input, for proofs across pointers (`spec:AST-100` FR4); `wheel` resolves once the page has the step; read back through the input log. | +| `frames.ts` | `startScreencast` (timestamped composited frames), `markElement` (a marker chip or outline), `trackMarker` (marker box per frame), `sampleRects` (per-rAF rects), `frameStats`. | +| `busy.ts` | `blockMainThread(page, ms)` and `busyWindowProof(tracked, window)`: frames presented, frames that moved, longest stall, a verdict; `presented` adds the compositor's own frames. | +| `trace.ts` | `traceRendering(page, body, {invalidations?})`: main-thread counts of UpdateLayoutTree / Layout / PrePaint / Paint / Layerize / UpdateLayer / ScrollLayer / HitTest, `countBetween` for a sub-window (optionally without Chrome's own compositor animation updates), long tasks; `installLongTaskLog` / `readLongTasks`; `startCompositorTrace`: when the compositor presents. | +| `curves.ts` | `holdAnimations`, `seekAnimations`, `sampleCurve` (presented transform / opacity at any t, of the element or, with `pseudoElement`, its `::before` / `::after`), `waitForAnimation`, `watchAnimations`, `waitForRest`, `releaseAnimations`. | +| `math.ts` | Oracles independent of the code under test: `cubicBezier`, `timingFunction`, `springStep` (SwiftUI response / dampingFraction, initial velocity), `projectDecay`, `rubberBand`, `parseTransform`, `compareCurve`, `timeline`. | +| `storyboard.ts` | `animationStoryboard` (seek to t = 0, 16.7, 33.3 … ms), `touchStoryboard` (a frame per scripted event), `castStoryboard` (real screencast frames): PNG contact sheet + JSON trace. | +| `output.ts` | `evidenceDir`, `writeEvidence`: where evidence goes and how it is attached to the report. | +| `story.ts` | `phone`, `withStorybook`, `openStory`, `frames`, and the timer gate (`installTimerGate`, `freezeTimers`, `thawTimers`). | ## The proofs @@ -60,6 +76,14 @@ dispatch latency, and the events go through the compositor's gesture detection the limits below. Read positions a frame after a move: Chromium delivers `pointermove` aligned to animation frames. +**Stories, not Storybook.** `openStory` turns Storybook's highlight module off +on every page it opens, and checks that its root element is absent. Left on, +the module observes `#storybook-root` and, on every insertion, reads the +computed style of every element on the page inside the inserting task (117 +reads and two forced style recalculations for one exit copy in a Button story), +so a proof would measure Storybook as well as the component. +`openStory(page, origin, id, args, {storybookHighlight: true})` keeps it on. + **Curve sampling.** Pause every running animation (CSS transitions and animations are Web Animations too), seek to a time, read `getComputedStyle`: the presented value at that exact time, independent of frame timing. Finished @@ -97,6 +121,109 @@ set): a short entrance no longer races the first look on a cold page. When the action may be the page's first contact with the harness, take `watchAnimations(locator)` before it and read `watch.started()` after. +**Busy main thread.** Mark the element, start the screencast, start the +animation, wait until it runs (`waitForAnimation`: an animation cannot start +while the thread is held), block the main thread for 1 s (`spec:AST-094` +FR5's verification row), stop. Always +start the gesture or animation first and block mid-flight, never before: +in headless Chromium a touch that starts while the main thread is busy does +not scroll or start a gesture even with only passive listeners (the input +waits for the main thread). Tracking a finger under a busy main thread is +therefore proven on the iOS simulator, not here. The +screencast is captured off the main thread, so a compositor animation shows a +new position every frame of the block, and a main-thread animation shows none +(nothing changed, so no frame is sent at all: `busyWindowProof` counts the +whole window as one stall). `settledWithin` (2 px) stops judging at the easing +tail: the screencast rounds to whole pixels, so a tail that still moves shows +plateaus within a pixel or two of rest. A judged part shorter than four of the +screencast's intervals fails: the animation under test was not moving during +the block. With +`during`, `blockMainThread` starts the loop from a task queued in the frame +that saw the animation, after that frame's rendering update: WebKit hands a +newly started animation to its compositor there, and a loop inside the rAF +callback would freeze even a compositor animation. Call it before the release +that starts a settle and await it after, so the page is already waiting when +the settle begins. An animation counts once it is no longer pending and has +run at least one frame: a `minProgressMs` under 16 is raised to 16, since one +created in the frame the page looks is not on the compositor yet. The `verdict` judges the +frames the screencast delivered, whatever its rate (a loaded machine's can +deliver one frame in five or six), and tolerates nothing a frozen animation +does: at least one frame in the judged part, no stall longer than three of the +screencast's intervals (two dropped frames), and at least 90% of the frames it +delivered in the judged part presenting a new position (one miss is always +allowed; a frame it did not deliver is a gap, for the stall check, unless +it delivered fewer than four frames there: then the judged part's span counts +every frame, so one late frame is not "1 of 1 moved"). Pass +`maxMedianFrameMs` (20 at 60 Hz) only in a proof that claims display rate. +`verdict.failures` names each miss with its numbers, and `summary` carries the +raw numbers for the report. Frames are judged in presentation order: a loaded +machine can deliver them out of order. `castStoryboard` turns the frames of the +block into a filmstrip. + +A just-started animation can reach the screen after the block began (50–100 ms +later in WebKit on a loaded machine). When the cast saw the element at rest +before the block (wait for `cast.frameCount()` to pass 0 before starting the +animation), judging starts at its first presented change: the judged part still +ends with the block and still needs four intervals, and `onScreenFromMs` and the +summary report when it came. An element already moving is judged from the +block's start. WebKit's casts are 2 px quantized: the frames come at the +viewport's size, but the page in them is captured at half resolution, so +positions move in 2 CSS px steps (Chromium's in 1). With under two steps of +motion per frame, a frame delivered a few ms early shows no step and reads as +not moved. The proof reports `stepPx` and `pxPerFrame` and notes motion under +two steps per frame, which never fails it. Move a proof's element at least two +steps a frame: the self-tests move 560 px in 1,600 ms, about 4.8 px a frame. + +On a loaded machine Chromium's screencast can skip frames the compositor +still presents. `startCompositorTrace(page)`, started with the screencast and +stopped after the block, records when the compositor presented each frame +(`Display::FrameDisplayed`); pass it as `presented`. The record is +browser-wide (its frames carry no page identity), so it throws when another +page is open as it starts or stops, or opened while it ran. A gap between two moved +frames with nothing delivered inside it is then measured between the +compositor's own frames: the frames it presented there count as presented +(`recorderSkipped`, in the summary), and a gap it did not present through is +a stall as before. A gap that opens or closes the judged part, or holds a +delivered frame that did not move, is judged from the screencast, so a +frozen element fails whatever else the compositor draws (a rAF animation's +block presents no compositor frame at all). With the record, the 90% counts the +compositor's frames from the first to the last moved frame; the gaps before and +after are the stall check's. WebKit has no such trace: +`stop()` resolves to `[]` and the screencast is judged alone. + +**No Layout / Paint.** `traceRendering` records a Chromium trace around the +interaction; `countBetween` counts the steady middle of an animation (its +first and last frames may commit). A transform/opacity animation does no +Layout or Paint there; a layout-property animation does one per frame. The +harness's own animation recorder stands down while the trace runs, since its +frame loop would make every frame a main-thread one, restyling each running +animation's target and laying out an anchored one: it reads only the element a +CSS animation or transition event names, once, when that animation starts, and +catches up afterwards. A `waitForRest` awaited inside the traced body stands +down too: it waits on timers for the scheduled ends of the element's curves +(looking `frames` times a frame apart before settling on none, so an +animation the action starts a frame or two later is waited for) and polls no +style until the trace ends (in the anchored fixture's steady middle, +24 Layouts before, 0 now). Its one read of the element's animations, as it is +called, restyles the element (2 Layouts and 2 style recalculations in the +anchored fixture when called 200 ms into its animation), so await it right +after the action that starts the motion, before the window a proof counts. So do `waitForAnimation` and a watch's +`started()`: they take the element's animations from the recorder's log, +judge a started one by its own times, and reach the element through a handle, +since each `locator.evaluate` makes a main-thread frame (there too, 24 +Layouts before, 0 now). That holds for script animations and CSS +transitions. On a CSS `@keyframes` animation (and, unwatched, a cancelled CSS +transition), `waitForAnimation` and `started()` read the animation's timing, +which flushes its style and adds Layouts to the trace: wait on one with +`waitForRest`, or after the trace. Taking a watch (`watchAnimations`) inside +a trace reads the watched element's animations, which restyles it (and lays +out an anchored one), so a proof takes its traced watch before the window it +counts. The trace's `longTasks` are the page's own: only a +task the harness marked as its own read (`waitForAnimation` describing +keyframes, `blockMainThread` without `during`) is in `devtoolsTasks` instead, +and only when no page script ran inside it. A test's own `page.evaluate` stays a +long task, whether it calls page code directly or sets it off. + **Storyboards.** Deterministic: frames come from seeking held animations or from stepping scripted input, never from wall-clock capture. The JSON trace carries each frame's measurements and the spec asserts on it; the sheet is @@ -104,33 +231,41 @@ evidence. A component that finishes an animation from a timer (a transitionend backstop, a cleanup render) would do so behind a held storyboard's back: `installTimerGate` before the animation starts and `freezeTimers` while photographing. The page can still cancel a held callback, a held frame callback -gets the time it actually runs, and the harness's own frame waits are not held. +gets the time it actually runs, and the harness's own waits (its frame waits, +`blockMainThread` with `during`, `sampleRects`) are not held. An animation that repeats forever does not set a storyboard's default length; with nothing else to film, pass `times`. A touch filmstrip keeps the scripted event `timeStamp`s (Chromium), but a handler that reads `performance.now()` or `Date.now()` sees one event per photograph, about 100 ms apart, so a clock-timed release flings slower when filmed than when played: a storyboard -cannot show such a gesture; record it in real time instead. Chromium can return -an empty copy of the page's surface right after it changes ("Unable to capture -screenshot"); a storyboard takes that photograph again two frames later, up to -three attempts in all, and lists each retake under `recaptures` in the trace. +cannot show such a gesture; record it in real time instead, with +`startScreencast` and `castStoryboard`. Chromium can return an empty copy of +the page's surface right after it changes ("Unable to capture screenshot"); a +storyboard takes that photograph again two frames later, up to three attempts +in all, and lists each retake under `recaptures` in the trace. ## Engine limits (measured with Playwright 1.61) -| | Chromium 149 | WebKit 26.5 (Linux build) | -| ------------------------------------------------------ | ------------ | ----------------------------------------------------------------------------------------------------- | -| Trusted touch + pointer events, multi-touch | yes | yes (a `touchstart` lists only the fingers that landed in it; pointer events exact) | -| Page sees the scripted timestamps | yes, exactly | no: real dispatch time; real-time pacing keeps velocities within ~1 ms | -| Sub-pixel touch positions | yes | no: integer CSS px | -| Native scroll, fling, pinch-zoom from injected touches | yes | no | -| Lifting one finger while others stay down | yes | pointer events right, but that `touchend` lists no remaining `touches`: skip with `PARTIAL_LIFT_SKIP` | -| Click synthesised from a touch | yes | only from `driver.tap` (WebKit's tap gesture), not from a raw touch sequence | -| Curve sampling, storyboards, input log | yes | yes | +| | Chromium 149 | WebKit 26.5 (Linux build) | +| ------------------------------------------------------ | ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------- | +| Trusted touch + pointer events, multi-touch | yes | yes (a `touchstart` lists only the fingers that landed in it; pointer events exact) | +| Page sees the scripted timestamps | yes, exactly | no: real dispatch time; real-time pacing keeps velocities within ~1 ms | +| Sub-pixel touch positions | yes | no: integer CSS px | +| Native scroll, fling, pinch-zoom from injected touches | yes | no | +| Lifting one finger while others stay down | yes | pointer events right, but that `touchend` lists no remaining `touches`: skip with `PARTIAL_LIFT_SKIP` | +| Click synthesised from a touch | yes | only from `driver.tap` (WebKit's tap gesture), not from a raw touch sequence | +| Curve sampling, storyboards, input log | yes | yes | +| Screencast frame evidence (busy main thread) | yes | yes (WebKit's own screencast; frames keep coming while the main thread is blocked) | +| Rendering trace, long tasks | yes | no (Playwright exposes neither) | +| Compositor presentation record | yes | no: `stop()` resolves to `[]`; the busy verdict judges the screencast alone | +| Wheel steps, exact CSS px deltas (`webInput.ts`) | yes | desktop context only (`isMobile: false`): skip with `WHEEL_SKIP`; the step also moves the mouse there (a `pointermove`) and holds Control as a key | +| Pen strokes, pressure, scripted timestamps | yes | no: skip with `PEN_SKIP` | What WebKit cannot show here — native scroll hand-off, flings, a finger -lifting while another stays down, and anything iOS-only — needs Mobile Safari -itself, on a device or the iOS simulator. Skip such a proof on the driver's -flag, citing the reason: `test.skip(!driver.partialLifts, PARTIAL_LIFT_SKIP)`. +lifting while another stays down, finger tracking under a busy main thread, and +anything iOS-only — needs Mobile Safari itself, on a device or the iOS +simulator. Skip such a proof on the driver's flag, citing the reason: +`test.skip(!driver.partialLifts, PARTIAL_LIFT_SKIP)`. The WebKit input command is reached through Playwright's in-process server object, so it needs a locally launched browser (the default), not `connect()`. diff --git a/internal/interaction-harness/package.json b/internal/interaction-harness/package.json index 40cccb26ca206..942eccb9ed8d5 100644 --- a/internal/interaction-harness/package.json +++ b/internal/interaction-harness/package.json @@ -3,7 +3,7 @@ "version": "0.0.0", "private": true, "type": "module", - "description": "Internal: Playwright helpers for touch interaction and animation proofs (real touch input, curve sampling, storyboards). Never published.", + "description": "Internal: Playwright helpers for touch interaction and animation proofs (real touch input, frame evidence, curve sampling, storyboards). Never published.", "main": "./src/index.ts", "exports": { ".": "./src/index.ts", diff --git a/internal/interaction-harness/scripts/webkit-container.mjs b/internal/interaction-harness/scripts/webkit-container.mjs index cb20fd44aa725..a077eae415ed5 100644 --- a/internal/interaction-harness/scripts/webkit-container.mjs +++ b/internal/interaction-harness/scripts/webkit-container.mjs @@ -16,10 +16,19 @@ * docker daemon the container runs as the invoking user (rootless podman and * rootless docker already map it), so the `test-results/` it writes into the * mounted checkout stay the host user's and the next host run can clear them. + * + * WebKit composites through Mesa's software GL in the image, which compiles + * each shader variant the first time a page draws it: 450–700 ms for the + * first frame of a blurred or newly animated layer, long enough for a short + * animation to end before it is ever presented. Mesa caches what it compiles, + * and the cache is kept on the host (`~/.cache/astryx-webkit-shaders/`, + * or `ASTRYX_WEBKIT_SHADER_CACHE`), so a run starts with every shader an + * earlier run compiled instead of paying for them in its first tests. */ import {spawnSync} from 'node:child_process'; import fs from 'node:fs'; import {createRequire} from 'node:module'; +import os from 'node:os'; import path from 'node:path'; import {fileURLToPath} from 'node:url'; @@ -42,6 +51,21 @@ if (engine == null) { process.exit(1); } const args = process.argv.slice(2); +// Where the container sees the host's shader cache (see the header). +const containerShaderCache = '/astryx-shader-cache'; +// Resolved to an absolute path: a bare relative value would be mounted as a +// named volume rather than a host directory. An empty value means the default. +const shaderCache = process.env.ASTRYX_WEBKIT_SHADER_CACHE + ? path.resolve(process.env.ASTRYX_WEBKIT_SHADER_CACHE) + : path.join( + os.homedir(), + '.cache', + 'astryx-webkit-shaders', + image.replace(/[^\w.-]+/g, '_'), + ); +fs.mkdirSync(shaderCache, {recursive: true}); +// The harness self-test reads this marker inside the container. +fs.writeFileSync(path.join(shaderCache, 'ASTRYX-SHADER-CACHE'), `${image}\n`); const env = [ 'ASTRYX_STORYBOOK_DIR', 'ASTRYX_INTERACTION_EVIDENCE', @@ -79,6 +103,10 @@ const command = [ '--ipc=host', '-v', `${root}:${root}`, + '-v', + `${shaderCache}:${containerShaderCache}`, + '-e', + `MESA_SHADER_CACHE_DIR=${containerShaderCache}`, '-w', root, ...env, diff --git a/internal/interaction-harness/src/busy.skip.fixture.ts b/internal/interaction-harness/src/busy.skip.fixture.ts new file mode 100644 index 0000000000000..409886f12e6f8 --- /dev/null +++ b/internal/interaction-harness/src/busy.skip.fixture.ts @@ -0,0 +1,125 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file busy.skip.fixture.ts + * @input One recorded busy-main-thread run in headless Chromium under + * parallel load: a compositor coast through a 566 ms block + * @output `frames`: the screencast frames the marker was tracked in, as + * [ms after the block started, y in CSS px, or null where the marker was not + * found]; `presented`: when the compositor presented each frame + * (`Display::FrameDisplayed`), ms after the block started; `block`: the + * block's span on the same clock. The screencast skipped frames at about + * +366 and +466 ms while the compositor kept presenting every 16.7 ms. + * @position Fixture for busy.test.ts + */ + +export const block = {from: 0, to: 566.1}; + +export const frames: ReadonlyArray = [ + [-1145.8, null], + [-1114, null], + [-1063.9, null], + [-1031.1, null], + [-996.6, null], + [-964.3, null], + [-930.7, null], + [-896.7, null], + [-864, null], + [-830.6, null], + [-797.8, null], + [-762.7, null], + [-731.3, null], + [-686.5, null], + [-680.8, null], + [-651, null], + [-634.1, null], + [-620.7, null], + [-615.8, null], + [-514.4, null], + [-495, null], + [-472.4, null], + [-465.3, null], + [-448.5, null], + [-430.3, null], + [-414.8, null], + [-398.3, null], + [-379.5, null], + [-364.2, null], + [-348.3, null], + [-330.6, null], + [-315, null], + [-297.6, null], + [-281.9, null], + [-265.1, null], + [-247.6, null], + [-230.8, null], + [-215.3, null], + [-198.6, null], + [-181, null], + [-165.3, null], + [-147.7, null], + [-132, null], + [-115.4, null], + [-98.5, null], + [-81.2, 504], + [-65.2, 500], + [-48.5, 496], + [-30.9, 491], + [-15.2, 487], + [1.3, 484], + [32.8, 480], + [49.8, 476], + [66.3, 473], + [82.8, 469], + [99.7, 466], + [116.1, 463], + [132.9, 460], + [149.5, 457], + [166.3, 454], + [182.9, 451], + [199.4, 448], + [216.2, 446], + [232.8, 443], + [249.4, 441], + [266.1, 438], + [282.7, 436], + [299.5, 434], + [316, 432], + [332.7, 430], + [349.5, 428], + [366.1, 426], + [433.1, 418], + [449.7, 417], + [466.2, 415], + [516.2, 411], + [532.8, 409], + [549.5, 408], + [566.2, 407], + [650.6, 401], + [666.2, 400], + [682.9, 399], + [699.6, 398], + [716.3, 397], + [816.1, 391], + [832.8, 391], + [849.5, 390], + [866.2, 389], + [882.9, 388], + [899.5, 388], + [916.1, 387], + [1033.8, 383], + [1050.1, 383], + [1066.7, 382], + [1083.3, 382], + [1099.5, 381], + [1117.3, 381], + [1134.2, 380], + [1149.5, 380], +]; + +export const presented: readonly number[] = [ + -98.7, -81.3, -65.3, -48.7, -31.1, -15.3, 1.3, 32.5, 49.6, 66.1, 82.7, 99.6, + 115.9, 132.6, 149.3, 166.1, 182.6, 199.3, 216, 232.6, 249.3, 266, 282.6, + 299.2, 315.8, 332.6, 349.3, 366, 381.5, 398.3, 415.1, 432.9, 449.6, 466.1, + 481.5, 498.3, 516, 532.6, 549.4, 566.1, 581.7, 598.4, 615, 631.6, 650.3, 666, +]; diff --git a/internal/interaction-harness/src/busy.test.ts b/internal/interaction-harness/src/busy.test.ts new file mode 100644 index 0000000000000..44a2446ecca18 --- /dev/null +++ b/internal/interaction-harness/src/busy.test.ts @@ -0,0 +1,536 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file busy.test.ts + * @input Synthetic tracked frames (no browser) + * @output The busy-main-thread verdict's limits: dropped frames a loaded + * machine produces pass, a frozen animation fails every criterion, a window + * with nothing moving to judge fails, and out-of-order frames are judged in + * presentation order. + * @position Unit test of ./busy and ./frames' ordering; the browser proof + * with a real compositor animation and a rAF control is in + * harness.interaction.browser.spec.ts. + */ + +import {describe, expect, it} from 'vitest'; +import {busyWindowProof} from './busy'; +import * as skip from './busy.skip.fixture'; +import {inPresentationOrder, type TrackedFrame} from './frames'; + +const FRAME = 1000 / 60; +const BLOCK = {from: 1000, to: 1700}; + +/** A marker moving 3 px per frame from t = 800 to 1900, minus `dropped` frame indices. */ +function moving(dropped: readonly number[] = []): TrackedFrame[] { + const frames: TrackedFrame[] = []; + for (let i = 0; 800 + i * FRAME <= 1900; i += 1) { + if (dropped.includes(i)) { + continue; + } + frames.push({ + t: 800 + i * FRAME, + box: {x: 0, y: 3 * i, width: 10, height: 10}, + } as TrackedFrame); + } + return frames; +} + +/** Index of the first frame at or after `t`. */ +const at = (t: number) => Math.ceil((t - 800) / FRAME); + +/** `moving()` with each frame in `repeats` presenting the previous frame's position. */ +function repeating(repeats: readonly number[]): TrackedFrame[] { + return moving().map((f, i) => + repeats.includes(i) + ? ({...f, box: {...f.box!, y: 3 * (i - 1)}} as TrackedFrame) + : f, + ); +} + +/** A marker moving 3 px per frame from t = 800 to 1900, frames `intervals` apart in turn. */ +function cadenced(intervals: readonly number[]): TrackedFrame[] { + const frames: TrackedFrame[] = []; + for (let i = 0, t = 800; t <= 1900; i += 1) { + frames.push({ + t, + box: {x: 0, y: 3 * i, width: 10, height: 10}, + } as TrackedFrame); + t += intervals[i % intervals.length]; + } + return frames; +} + +describe('busyWindowProof verdict', () => { + it('passes a compositor animation with every frame', () => { + const proof = busyWindowProof(moving(), BLOCK); + expect(proof.verdict.failures).toEqual([]); + expect(proof.longestStallFrames).toBe(1); + }); + + it('passes one and two consecutive dropped frames (3 intervals)', () => { + const k = at(1300); + expect(busyWindowProof(moving([k]), BLOCK).verdict.failures).toEqual([]); + const two = busyWindowProof(moving([k, k + 1]), BLOCK); + expect(two.longestStallFrames).toBe(3); + expect(two.verdict.failures).toEqual([]); + }); + + it('fails three consecutive dropped frames', () => { + const k = at(1300); + const proof = busyWindowProof(moving([k, k + 1, k + 2]), BLOCK); + expect(proof.longestStallFrames).toBe(4); + expect(proof.verdict.failures.join('\n')).toMatch(/longest stall/); + }); + + it('fails when more than 10% of the delivered frames do not move', () => { + const k = at(1050); + // 39 frames delivered: up to 3 may show no new position (10% = 3.9). + const scattered = [0, 12, 24].map(d => k + d); + expect( + busyWindowProof(repeating(scattered), BLOCK).verdict.failures, + ).toEqual([]); + const many = [0, 6, 12, 18, 24].map(d => k + d); + expect( + busyWindowProof(repeating(many), BLOCK).verdict.failures.join('\n'), + ).toMatch(/expected frames moved/); + }); + + it('judges a frame the screencast did not deliver as a gap for the stall check, not a miss', () => { + // The same five frames, not delivered: two-interval gaps, and every + // delivered frame moved. + const k = at(1050); + const many = [0, 6, 12, 18, 24].map(d => k + d); + const proof = busyWindowProof(moving(many), BLOCK); + expect(proof.verdict.failures).toEqual([]); + expect(proof.deliveredFrames).toBe(proof.moved); + }); + + it('judges an irregular cast on the frames it delivered', () => { + // A cast cycling 11.7, 11.7 and 17 ms: its median interval would expect + // 56 frames where it delivered 49, every one moved. + const proof = busyWindowProof(cadenced([11.7, 11.7, 17]), BLOCK); + expect(proof.verdict.failures).toEqual([]); + expect(proof.deliveredFrames).toBe(49); + expect(proof.moved).toBe(49); + // Still a stall where it delivers nothing for four frames, and still a + // miss where five delivered frames show no new position. + const gap = cadenced([11.7, 11.7, 17]).filter( + f => f.t < 1300 || f.t > 1360, + ); + expect(busyWindowProof(gap, BLOCK).verdict.failures.join('\n')).toMatch( + /longest stall/, + ); + const unmoved = cadenced([11.7, 11.7, 17]).map((f, i, all) => + [20, 26, 32, 38, 44].includes(i) + ? ({...f, box: {...all[i - 1].box!}} as TrackedFrame) + : f, + ); + expect(busyWindowProof(unmoved, BLOCK).verdict.failures.join('\n')).toMatch( + /expected frames moved/, + ); + }); + + it('judges a window that delivered fewer than four frames against its span', () => { + // A rAF animation frozen by the block, in WebKit: one frame from before + // the block arrives 45 ms into it, then nothing until after it. + const late = moving().filter(f => f.t <= 1000 || f.t >= 1750); + late.push({ + t: 1045, + box: {x: 0, y: 3 * at(1001), width: 10, height: 10}, + } as TrackedFrame); + const proof = busyWindowProof(inPresentationOrder(late), BLOCK); + expect(proof.verdict.failures.join('\n')).toMatch(/expected frames moved/); + }); + + it('lets a short window miss a frame, not three', () => { + // 150 ms judged: nine frames delivered, one miss allowed (10% would be 0.9). + const short = {from: 1000, to: 1190}; + const k = at(1040); + expect(busyWindowProof(repeating([k]), short).verdict.failures).toEqual([]); + expect( + busyWindowProof( + repeating([k, k + 3, k + 6]), + short, + ).verdict.failures.join('\n'), + ).toMatch(/expected frames moved/); + }); + + it('fails a frozen animation on every criterion', () => { + // Frames before and after the block only: nothing changed during it. + const frozen = moving().filter(f => f.t < BLOCK.from || f.t > BLOCK.to); + const proof = busyWindowProof(frozen, BLOCK); + expect(proof.changed).toBe(0); + expect(proof.verdict.pass).toBe(false); + expect(proof.verdict.failures.join('\n')).toMatch(/no frame presented/); + expect(proof.verdict.failures.join('\n')).toMatch(/longest stall/); + expect(proof.verdict.failures.join('\n')).toMatch(/expected frames moved/); + }); + + it('fails a window in which the animation under test was not moving', () => { + // Settled before the block: the judgment ends where it starts. + const settled = moving().map(f => + f.t < 950 + ? f + : ({...f, box: {...f.box!, y: 3 * at(950)}} as TrackedFrame), + ); + const before = busyWindowProof(settled, BLOCK, {settledWithin: 2}); + expect(before.verdict.pass).toBe(false); + expect(before.verdict.failures.join('\n')).toMatch( + /judged = -?\d+ frames < 4: the animation under test was not moving/, + ); + // Settled 40 ms into the block: two frames, too few to judge. + const early = moving().map(f => + f.t < 1040 + ? f + : ({...f, box: {...f.box!, y: 3 * at(1040)}} as TrackedFrame), + ); + expect( + busyWindowProof(early, BLOCK, {settledWithin: 2}).verdict.failures.join( + '\n', + ), + ).toMatch(/frames < 4: the animation under test was not moving/); + }); + + it('judges the frames a slower screencast delivers by default, and display rate only when asked', () => { + // One frame in six, as a loaded machine's screencast delivers: every + // delivered frame moved, so the default verdict passes. + const sixth = moving().filter((_, i) => i % 6 === 0); + expect(busyWindowProof(sixth, BLOCK).verdict.failures).toEqual([]); + // Half rate fails only a proof that claims display rate. + const odd = moving().filter((_, i) => i % 2 === 1); + expect(busyWindowProof(odd, BLOCK).verdict.pass).toBe(true); + expect( + busyWindowProof(odd, BLOCK, {maxMedianFrameMs: 20}).verdict.failures.join( + '\n', + ), + ).toMatch(/median frame interval/); + }); +}); + +describe('busyWindowProof settledWithin', () => { + it('ends the judgment at a rounded easing tail, which would read as a stall', () => { + // An ease-out to y = 24 as the screencast reports it: whole pixels, so the + // tail repeats values while it still moves. + const ys = [ + 120, 100, 82, 67, 55, 46, 39, 34, 30, 28, 27, 26, 26, 26, 25, 25, 24, 24, + 24, 24, + ]; + const frames = ys.map( + (y, i) => + ({ + t: 1000 + i * FRAME, + box: {x: 0, y, width: 10, height: 10}, + }) as TrackedFrame, + ); + const window = {from: 1000, to: 1000 + ys.length * FRAME}; + const loose = busyWindowProof(frames, window, { + settledWithin: 2, + marginMs: 0, + }); + expect(loose.verdict.failures).toEqual([]); + const strict = busyWindowProof(frames, window, { + settledWithin: 1, + marginMs: 0, + }); + expect(strict.longestStallFrames).toBeGreaterThanOrEqual(3); + }); + + it('takes the end value from settledAt over a stale last frame', () => { + // The tail reaches y = 24, then the last frame shows a stale 100 (WebKit + // presents the last committed position once an accelerated animation + // ends while the main thread is still blocked). + const ys = [ + 120, 100, 82, 67, 55, 46, 39, 34, 30, 28, 27, 26, 25, 24, 24, 24, 100, + ]; + const frames = ys.map( + (y, i) => + ({ + t: 1000 + i * FRAME, + box: {x: 0, y, width: 10, height: 10}, + }) as TrackedFrame, + ); + const window = {from: 1000, to: 1000 + ys.length * FRAME}; + const stale = busyWindowProof(frames, window, { + settledWithin: 2, + marginMs: 0, + }); + expect(stale.verdict.pass).toBe(false); + const fromPage = busyWindowProof(frames, window, { + settledWithin: 2, + settledAt: 24, + marginMs: 0, + }); + expect(fromPage.verdict.failures).toEqual([]); + expect(fromPage.judged.to).toBe(frames[11].t); + }); +}); + +/** Seen at rest at t = 900, then on screen from `start`, moving 3 px a frame. */ +function startingAt(start: number): TrackedFrame[] { + const frames = [ + {t: 900, box: {x: 0, y: 0, width: 10, height: 10}} as TrackedFrame, + ]; + for (let i = 1; start + (i - 1) * FRAME <= 1900; i += 1) { + frames.push({ + t: start + (i - 1) * FRAME, + box: {x: 0, y: 3 * i, width: 10, height: 10}, + } as TrackedFrame); + } + return frames; +} + +describe('busyWindowProof on-screen start', () => { + it('judges a just-started animation from its first presented change when it was seen at rest before the block, and reports when', () => { + // On screen 87 ms into the block: four intervals after the judged part's start. + const proof = busyWindowProof(startingAt(1087), BLOCK); + expect(proof.verdict.failures).toEqual([]); + expect(proof.onScreenFromMs).toBeCloseTo(87, 5); + expect(proof.judged.from).toBeCloseTo(1087, 5); + expect(proof.summary).toMatch( + /on screen from \+87 ms \(nothing moved before the block\)/, + ); + }); + + it('still fails an animation that was already moving and stalls at the block start', () => { + const k = at(1020); + const proof = busyWindowProof(moving([k, k + 1, k + 2, k + 3]), BLOCK); + expect(proof.onScreenFromMs).toBeNull(); + expect(proof.verdict.failures.join('\n')).toMatch(/longest stall/); + }); + + it('still fails a stall at the start when no frame before the block shows it at rest', () => { + const unseen = startingAt(1087).filter(f => f.t >= BLOCK.from); + const proof = busyWindowProof(unseen, BLOCK); + expect(proof.onScreenFromMs).toBeNull(); + expect(proof.verdict.failures.join('\n')).toMatch(/longest stall/); + }); + + it('still fails a start the block itself delays until it ends', () => { + const proof = busyWindowProof(startingAt(1720), BLOCK); + expect(proof.onScreenFromMs).toBeNull(); + expect(proof.verdict.failures.join('\n')).toMatch(/no frame presented/); + expect(proof.verdict.failures.join('\n')).toMatch(/longest stall/); + }); + + it('still fails a start that leaves fewer than minJudgedFrames inside the block', () => { + const proof = busyWindowProof(startingAt(1640), BLOCK); + expect(proof.onScreenFromMs).toBeCloseTo(640, 5); + expect(proof.verdict.failures.join('\n')).toMatch( + /frames < 4: the animation under test was not moving/, + ); + }); +}); + +describe('busyWindowProof position step', () => { + it("reports the cast's position step and motion per frame, and notes motion under two steps without failing", () => { + // 2.4 px a frame on a 2 px grid, as WebKit's casts report positions. + const grid = moving().map( + (f, i) => + ({ + ...f, + box: {...f.box!, y: 2 * Math.round((2.4 * i) / 2)}, + }) as TrackedFrame, + ); + const coarse = busyWindowProof(grid, BLOCK); + expect(coarse.stepPx).toBe(2); + expect(coarse.pxPerFrame).toBeCloseTo(2.4, 1); + expect(coarse.notes.join('\n')).toMatch( + /under two of the cast's 2 px position steps/, + ); + expect(coarse.summary).toMatch(/2\.4 px\/frame in 2 px steps; note: /); + expect(coarse.verdict.failures).toEqual([]); + // 3 px a frame in whole pixels: resolvable, nothing to note. + const fine = busyWindowProof(moving(), BLOCK); + expect(fine.stepPx).toBe(1); + expect(fine.pxPerFrame).toBeCloseTo(3, 1); + expect(fine.notes).toEqual([]); + }); +}); + +/** The compositor presenting every frame from `from` to `to`, minus `gaps` ([from, to] spans). */ +function drawn( + gaps: ReadonlyArray = [], + from = 700, + to = 2000, +): number[] { + const draws: number[] = []; + for (let t = from; t <= to; t += FRAME) { + if (!gaps.some(([a, b]) => t > a && t < b)) { + draws.push(t); + } + } + return draws; +} + +const skipFrames = (): TrackedFrame[] => + skip.frames.map( + ([t, y]) => + ({ + t, + box: y == null ? null : {x: 0, y, width: 10, height: 10}, + }) as TrackedFrame, + ); + +describe('busyWindowProof with the compositor record', () => { + it('with a compositor record, judges a sparse cast from its first to its last moved frame', () => { + // One screencast frame in six, as on a loaded machine, with the compositor + // presenting every frame: the frames before the first and after the last + // moved one are the stall check's, not misses. + const sparse = moving().filter((_, i) => i % 6 === 0); + expect( + busyWindowProof(sparse, BLOCK, {presented: drawn()}).verdict.failures, + ).toEqual([]); + }); + + it('with a compositor record and an on-screen start, counts the frames it credits as skipped in what it judges', () => { + // On screen 87 ms into the block, then four delivered frames that show + // no new position: that fails on a full cast. + const misses = [15, 20, 25, 30]; + const missing = startingAt(1087).map((f, i, all) => + misses.includes(i) + ? ({...f, box: {...all[i - 1].box!}} as TrackedFrame) + : f, + ); + const full = busyWindowProof(missing, BLOCK, {presented: drawn()}); + expect(full.verdict.failures.join('\n')).toMatch(/expected frames moved/); + // The screencast skipping the five frames after the on-screen frame + // changes nothing: the compositor presented them, and they are judged. + const skip = (frames: TrackedFrame[]) => + frames.filter((_, i) => i < 2 || i > 6); + const skipped = busyWindowProof(skip(missing), BLOCK, { + presented: drawn(), + }); + expect(skipped.recorderSkipped).toBe(5); + expect(skipped.verdict.failures.join('\n')).toMatch( + /expected frames moved/, + ); + // And a run with no misses never reads as more than every frame moved. + const clean = busyWindowProof(skip(startingAt(1087)), BLOCK, { + presented: drawn(), + }); + expect(clean.verdict.failures).toEqual([]); + expect(clean.movedFraction).toBeLessThanOrEqual(1); + }); + + it('reads frames the screencast skipped while the compositor presented them as presented: a recorded run that fails without the record', () => { + const frames = skipFrames(); + const alone = busyWindowProof(frames, skip.block, {settledWithin: 1}); + expect(alone.verdict.failures.join('\n')).toMatch( + /longest stall 6\d\.\d ms = 4 intervals > 3/, + ); + // Every frame the screencast delivered moved: only the stall remains, + // which the record resolves. + expect(alone.verdict.failures.join('\n')).not.toMatch( + /expected frames moved/, + ); + const proof = busyWindowProof(frames, skip.block, { + settledWithin: 1, + presented: skip.presented, + }); + expect(proof.verdict.failures).toEqual([]); + expect(proof.recorderSkipped).toBe(5); + expect(proof.longestStallFrames).toBeLessThanOrEqual(2); + expect(proof.summary).toMatch( + /the screencast skipped 5 frames the compositor presented \(at \+3\d\d ms, \+4\d\d ms\)/, + ); + }); + + it('counts a gap the compositor drew through as skipped, and one it did not draw through as a stall', () => { + const k = at(1300); + const gap: [number, number] = [ + 800 + (k - 1) * FRAME, + 800 + (k + 4) * FRAME, + ]; + const frames = moving([k, k + 1, k + 2, k + 3]); + expect(busyWindowProof(frames, BLOCK).verdict.pass).toBe(false); + const skipped = busyWindowProof(frames, BLOCK, {presented: drawn()}); + expect(skipped.verdict.failures).toEqual([]); + expect(skipped.recorderSkipped).toBe(4); + const stalled = busyWindowProof(frames, BLOCK, {presented: drawn([gap])}); + expect(stalled.recorderSkipped).toBe(0); + expect(stalled.verdict.failures.join('\n')).toMatch(/longest stall/); + }); + + it('still fails an element that does not move while the compositor presents for something else', () => { + const frozen = moving().map(f => + f.t >= BLOCK.from && f.t <= BLOCK.to + ? ({...f, box: {...f.box!, y: 3 * at(BLOCK.from)}} as TrackedFrame) + : f, + ); + const proof = busyWindowProof(frozen, BLOCK, {presented: drawn()}); + expect(proof.verdict.failures.join('\n')).toMatch(/longest stall/); + expect(proof.verdict.failures.join('\n')).toMatch(/expected frames moved/); + }); + + it('still fails a frozen animation, whether or not the compositor presents during the block', () => { + const frozen = moving().filter(f => f.t < BLOCK.from || f.t > BLOCK.to); + for (const presented of [drawn([[BLOCK.from, BLOCK.to]]), drawn()]) { + const proof = busyWindowProof(frozen, BLOCK, {presented}); + expect(proof.verdict.failures.join('\n')).toMatch(/no frame presented/); + expect(proof.verdict.failures.join('\n')).toMatch(/longest stall/); + } + }); + + it('keeps a gap a stall when the screencast delivered a frame inside it that did not move', () => { + const k = at(1300); + const frames = moving([k, k + 1, k + 3]).map(f => + f.t > 800 + (k + 1.5) * FRAME && f.t < 800 + (k + 2.5) * FRAME + ? ({...f, box: {...f.box!, y: 3 * (k - 1)}} as TrackedFrame) + : f, + ); + const proof = busyWindowProof(frames, BLOCK, {presented: drawn()}); + expect(proof.recorderSkipped).toBe(0); + expect(proof.verdict.failures.join('\n')).toMatch(/longest stall/); + }); + + it('still counts single dropped frames the compositor did not present as missed', () => { + const k = at(1050); + const drops = [0, 6, 12, 18, 24, 30].map(d => k + d); + const frames = moving(drops); + const gaps = drops.map( + i => + [800 + (i - 0.5) * FRAME, 800 + (i + 0.5) * FRAME] as [number, number], + ); + expect( + busyWindowProof(frames, BLOCK, { + presented: drawn(gaps), + }).verdict.failures.join('\n'), + ).toMatch(/expected frames moved/); + expect( + busyWindowProof(frames, BLOCK, {presented: drawn()}).verdict.failures, + ).toEqual([]); + }); + + it('judges a gap at the edge of the judged part from the screencast alone', () => { + // Nothing delivered for the block's first 90 ms: a stall at its start, + // whatever the compositor drew, since no moved frame opens the gap. + const frames = moving().filter( + f => f.t < BLOCK.from || f.t > BLOCK.from + 90, + ); + const proof = busyWindowProof(frames, BLOCK, {presented: drawn()}); + expect(proof.recorderSkipped).toBe(0); + expect(proof.verdict.failures.join('\n')).toMatch(/longest stall/); + }); + + it('treats an empty record as no record, and says so', () => { + const frames = moving([at(1300), at(1300) + 1, at(1300) + 2, at(1300) + 3]); + const proof = busyWindowProof(frames, BLOCK, {presented: []}); + expect(proof.verdict.failures).toEqual( + busyWindowProof(frames, BLOCK).verdict.failures, + ); + expect(proof.summary).toMatch(/no compositor frames recorded/); + }); +}); + +describe('inPresentationOrder', () => { + it('sorts frames delivered out of order, so motion stays monotonic', () => { + const frames = moving(); + const k = at(1040); + const shuffled = [...frames]; + [shuffled[k], shuffled[k + 1]] = [shuffled[k + 1], shuffled[k]]; + expect(busyWindowProof(shuffled, BLOCK).monotonic).toBe(false); + const proof = busyWindowProof(inPresentationOrder(shuffled), BLOCK); + expect(proof.monotonic).toBe(true); + expect(proof.verdict.failures).toEqual([]); + }); +}); diff --git a/internal/interaction-harness/src/busy.ts b/internal/interaction-harness/src/busy.ts new file mode 100644 index 0000000000000..55fd9d670d52a --- /dev/null +++ b/internal/interaction-harness/src/busy.ts @@ -0,0 +1,579 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file busy.ts + * @input Uses a Playwright `Page` + * @output `blockMainThread` — a synchronous busy loop on the page's main thread + * that reports exactly when it ran — and `busyWindowProof`, which reads frame + * evidence for that window. + * @position The busy-main-thread proof: an animation that runs on the + * compositor keeps advancing at display rate while JavaScript holds the main + * thread; one driven from the main thread (rAF, style writes, layout + * properties) freezes for the whole block. Pair it with ./frames + * (`startScreencast` + `trackMarker`), which observes off the main thread. + * + * SYNC: Keep the README's busy-main-thread section equal to this contract. + */ + +import type {Locator, Page} from '@playwright/test'; +import {frameStats, type FrameStats, type TrackedFrame} from './frames'; + +export interface BusyWindow { + /** Epoch ms when the loop started and ended, on the page's own clock. */ + readonly from: number; + readonly to: number; +} + +/** + * Holds the page's main thread for `ms` (a synchronous loop: no task, frame, + * rAF, event or microtask runs meanwhile). Resolves with the window it held. + * + * Block MID-FLIGHT only: start the gesture or animation first. A touch that + * starts while the main thread is busy does not scroll or begin a gesture in + * headless Chromium, even with only passive listeners, and an animation that + * has not started cannot start (see `waitForAnimation`). Finger tracking + * under a busy main thread is a simulator proof, not a Playwright one. + * + * With `during`, the page itself waits until an animation on that element (or + * inside it) has run `minProgressMs`, then blocks from a task it queues in + * that frame. A round trip from the test would start the block later the + * busier the machine is, leaving less of a short animation to judge. The task + * runs after the frame's rendering update, which is where WebKit hands a newly + * started animation to its compositor: a loop inside the rAF callback itself + * holds that hand-off, and even a compositor animation freezes. To catch a + * settle near its start, call it before the release that starts the settle and + * await it after: the page is already waiting when the animation begins. Any + * animation inside `during` counts, so pick an element that runs nothing + * else meanwhile. + * + * An animation counts once it is no longer pending and has run at least one + * frame (`minProgressMs` below 16 ms is raised to 16). One created in the + * frame the page looks is still pending, not yet on the compositor + * (Chromium), and in WebKit its start time resolves before its layer is + * flushed: a block that started on it froze it. + */ +/** One display frame, ms: the least progress `blockMainThread` waits for. */ +const ONE_FRAME_MS = 16; + +export async function blockMainThread( + page: Page, + ms: number, + { + during, + minProgressMs = 16, + timeout = 5_000, + }: {during?: Locator; minProgressMs?: number; timeout?: number} = {}, +): Promise { + const minProgress = Math.max(minProgressMs, ONE_FRAME_MS); + if (during == null) { + return page.evaluate(duration => { + // The harness's own work: a trace sets it apart (trace.ts HARNESS_READ). + console.timeStamp('harness-read'); + const from = performance.timeOrigin + performance.now(); + const end = performance.now() + duration; + let spins = 0; + while (performance.now() < end) { + spins += 1; + } + return {from, to: performance.timeOrigin + performance.now(), spins}; + }, ms); + } + return during.evaluate( + (element, {duration, minProgress, wait}) => + new Promise((resolve, reject) => { + // The harness's own waits run on the real timers: a frozen timer gate + // (story.ts) holds only the page's. + const gate = ( + window as unknown as { + __harnessTimerGate?: { + frame?: typeof requestAnimationFrame; + timeout?: typeof setTimeout; + }; + } + ).__harnessTimerGate; + const frame = gate?.frame ?? window.requestAnimationFrame.bind(window); + const later = gate?.timeout ?? window.setTimeout.bind(window); + const deadline = performance.now() + wait; + const tick = () => { + const running = element + .getAnimations({subtree: true}) + .some( + a => + a.playState === 'running' && + !a.pending && + Number(a.currentTime ?? 0) >= minProgress, + ); + if (running) { + // The same loop as above, from a task queued in the frame that saw + // the animation: it runs once that frame's rendering update has + // handed the animation to the compositor. + later(() => { + const from = performance.timeOrigin + performance.now(); + const end = performance.now() + duration; + while (performance.now() < end) { + // Busy: nothing else runs. + } + resolve({from, to: performance.timeOrigin + performance.now()}); + }, 0); + } else if (performance.now() > deadline) { + reject(new Error('blockMainThread: no animation started')); + } else { + frame(tick); + } + }; + frame(tick); + }), + {duration: ms, minProgress, wait: timeout}, + ); +} + +export interface BusyVerdict { + readonly pass: boolean; + /** Each criterion the window missed, with its measured value and limit. */ + readonly failures: readonly string[]; +} + +export interface BusyProof extends FrameStats { + readonly window: BusyWindow; + /** + * The part of the window judged: inside the margins, up to `settledWithin`, + * and from `onScreenFromMs` when that is set. + */ + readonly judged: {readonly from: number; readonly to: number}; + /** + * When the tracked value was seen at rest before the judged part (a frame + * there, no change in the 250 ms before it: a just-started animation not on + * screen yet), ms after the window's start of its first presented change + * inside the block, where judging starts; null otherwise. + */ + readonly onScreenFromMs: number | null; + /** + * The cast's position step, CSS px: the largest of 0.5, 1 and 2 that every + * tracked position is a multiple of (WebKit's casts are 2: see the README). + */ + readonly stepPx: number; + /** The tracked value's mean motion per screencast interval in the judged part, px. */ + readonly pxPerFrame: number; + /** Facts worth reading beside the verdict, which never fail it. */ + readonly notes: readonly string[]; + /** + * With a compositor record (`presented`): frames the screencast skipped + * between two moved frames while the compositor presented them, which are + * neither stalls nor misses; 0 without one. + */ + readonly recorderSkipped: number; + /** + * The compositor's frame interval around the window, ms, from `presented`; + * null without a usable record (the screencast alone is judged). + */ + readonly compositorFrameMs: number | null; + /** + * The screencast's frame interval, from the frames around the window as well + * as inside it (a short window has few intervals; a frozen one has none). + */ + readonly cadenceMs: number; + /** + * Whole frame intervals in the judged part at the measured cadence (the + * compositor's, when recorded): the `minJudgedFrames` floor reads it. + */ + readonly expectedFrames: number; + /** Frames the screencast delivered in the judged part, with the marker found. */ + readonly deliveredFrames: number; + /** + * Longest time inside the window with no presented change, the window's own + * edges included. A frozen animation presents no new frame at all (nothing + * on screen changed, so the compositor sends none): its stall is the whole + * window. + */ + readonly longestStallMs: number; + /** Where the longest stall starts, ms after the window's start. */ + readonly longestStallAtMs: number; + /** + * `longestStallMs` in screencast intervals, rounded: 1 is no frame missed, + * 2 is one dropped frame. Rounding absorbs timestamp jitter. + */ + readonly longestStallFrames: number; + /** + * Frames in the window that presented a new position (the first compared + * with the last frame before the window). + */ + readonly moved: number; + /** + * `moved` (and `recorderSkipped`) over the frames judged: the frames the + * screencast delivered in the judged part, or with a compositor record the + * compositor's frames from the first to the last moved frame; + * `expectedFrames` when nothing moved, or fewer than `minJudgedFrames` + * frames were delivered. + */ + readonly movedFraction: number; + /** The judgment against the limits (see `busyWindowProof`). */ + readonly verdict: BusyVerdict; + /** One line with the raw numbers, for an annotation or a failure message. */ + readonly summary: string; +} + +/** + * Frame evidence for a busy window: how many frames the screencast presented + * inside it, how many of them moved the tracked axis, and the longest stall. + * The window is trimmed by `marginMs` at both ends so the frame the block + * started on and the one it ended on are not counted. + * + * The verdict judges the frames the screencast delivered, whatever its rate + * (a loaded machine's screencast can deliver one frame in five or six), and + * tolerates nothing a main-thread animation does (it presents no frame for + * the whole block). It passes when: + * - the judged part presented at least one frame, and, only when a proof + * claims display rate and passes `maxMedianFrameMs`, their median interval + * is at most that; + * - no stall spans more than `maxStallFrames` screencast intervals; + * - at least `minMovedFraction` of the frames judged moved the tracked axis + * (a short window may still miss one): the frames the screencast delivered + * in the judged part, so an irregular cast is judged on what it delivered + * and a frame it did not deliver is a gap for the stall check, not a miss + * (unless it delivered fewer than `minJudgedFrames` there: then every frame + * of the judged part's span counts, so one late frame is not 1/1 moved); + * with a compositor record, the compositor's frames from the first to the + * last moved frame (the gaps before and after are the stall check's); + * - the judged part spans at least `minJudgedFrames` frames. Fewer means the + * animation under test was not moving during the block (it settled before + * the block, or the block caught another animation), so there is nothing to + * judge, and that fails rather than passes. + * + * A just-started animation can reach the screen after the block began (its + * hand-off to the compositor lags under load: 50–100 ms in WebKit). When the + * tracked value was seen at rest before the judged part (a frame there, and no + * change in the 250 ms before it), judging starts at its first presented + * change inside the block, which the + * proof reports as `onScreenFromMs`. The judged part still ends with the + * block and the `minJudgedFrames` floor applies to it, so a start the block + * itself delays fails; an animation that was already moving is judged from + * the block's start, and a stall there fails as before. + * + * `stepPx` and `pxPerFrame` say whether a frame's motion is resolvable at + * all: under two position steps per frame, a frame delivered early can show + * no step and read as not moved, which the proof notes (never a failure). + * + * A loaded machine's screencast can skip frames the compositor still presents + * (Chromium). Given the compositor's record (`presented`, from + * `startCompositorTrace`), a gap between two moved frames with no frame + * delivered inside it is measured between the compositor's own frames: it is + * a stall only where the compositor presented nothing, and the frames it did + * present there are neither stalls nor misses (`recorderSkipped`). A gap + * that opens or closes the judged part, or holds a delivered frame that did + * not move, is judged from the screencast as before, so a frozen element + * fails whatever else the compositor draws. + */ +export function busyWindowProof( + tracked: readonly TrackedFrame[], + window: BusyWindow, + { + axis = 'y', + marginMs = 20, + settledWithin, + settledAt, + maxMedianFrameMs = Infinity, + maxStallFrames = 3, + minMovedFraction = 0.9, + minJudgedFrames = 4, + presented, + }: { + axis?: 'x' | 'y' | 'width' | 'height'; + marginMs?: number; + /** + * Opt-in, for a proof that claims display rate: the most the median frame + * interval may be, ms (20 at 60 Hz). Off by default, since it measures the + * screencast's own rate as much as the animation. + */ + maxMedianFrameMs?: number; + /** The most screencast intervals one stall may span (3 = two dropped frames). */ + maxStallFrames?: number; + /** + * The least fraction of the expected frames that must move. One missed + * frame is always allowed, so a window of under ten frames is not judged + * stricter than a long one. + */ + minMovedFraction?: number; + /** The fewest whole frame intervals the judged part of the window may span. */ + minJudgedFrames?: number; + /** + * Stop judging once the value is within this many px of where it ends. + * The screencast reports whole CSS pixels, so an easing tail that still + * moves shows plateaus (26, 26, 26 against a final 24) that would read + * as a stall: 2 is the usual value. + */ + settledWithin?: number; + /** + * Where the value ends, for `settledWithin`: read it from the page once the + * animation has ended (`waitForRest`). Defaults to the last frame's value, + * which can be stale: once an accelerated animation ends while the main + * thread is still blocked, WebKit presents the layer's last committed + * position until the main thread commits the end state. + */ + settledAt?: number; + /** + * When the compositor presented each frame, epoch ms on the page clock + * (`startCompositorTrace`; Chromium). Empty or absent, the screencast is + * judged alone. + */ + presented?: readonly number[]; + } = {}, +): BusyProof { + let end = window.to - marginMs; + if (settledWithin != null) { + const valued = tracked.flatMap(f => + f.box == null ? [] : [{t: f.t, value: f.box[axis]}], + ); + const final = + settledAt ?? (valued.length > 0 ? valued[valued.length - 1].value : null); + const arrived = valued.find( + f => + f.t >= window.from && + final != null && + Math.abs(f.value - final) <= settledWithin, + ); + if (arrived != null) { + end = Math.min(end, arrived.t); + } + } + const inner = {from: window.from + marginMs, to: end}; + const values = tracked + .filter(f => f.t >= inner.from && f.t <= inner.to) + .map(f => ({t: f.t, value: f.box == null ? null : f.box[axis]})); + const before = [...tracked].reverse().find(f => f.t < inner.from); + let previous: number | null = before?.box == null ? null : before.box[axis]; + const inWindow: number[] = []; + for (const v of values) { + if ( + v.value != null && + previous != null && + Math.abs(v.value - previous) > 0.25 + ) { + inWindow.push(v.t); + } + if (v.value != null) { + previous = v.value; + } + } + // Seen at rest before the judged part (a frame there, and no change in the + // 250 ms before it): the first presented change is the animation's on-screen + // start, and judging starts there. With no frame before, nothing is known, + // and the window is judged from its start. + const prior = tracked.filter(f => f.box != null && f.t < inner.from); + let leadFrom = 0; + prior.forEach((f, i) => { + if (f.t < inner.from - 250) { + leadFrom = i; + } + }); + const lead = prior.flatMap(f => (f.box == null ? [] : [f.box[axis]])); + const movedBefore = lead.some( + (value, i) => i > leadFrom && Math.abs(value - lead[i - 1]) > 0.25, + ); + const onScreenFrom = + prior.length > 0 && !movedBefore && inWindow.length > 0 + ? inWindow[0] + : null; + const judged = {from: onScreenFrom ?? inner.from, to: inner.to}; + const changes = inWindow.filter(t => t > judged.from || onScreenFrom == null); + const edges = [judged.from, ...changes, judged.to]; + const around = tracked + .filter(f => f.t >= inner.from - 250 && f.t <= inner.to + 250) + .map(f => f.t); + const intervals = around + .slice(1) + .map((t, i) => t - around[i]) + .sort((a, b) => a - b); + const stats = frameStats( + tracked.map(f => ({t: f.t, value: f.box == null ? null : f.box[axis]})), + judged, + ); + const cadence = + intervals.length >= 4 + ? intervals[Math.floor(intervals.length / 2)] + : stats.medianFrameMs || 1000 / 60; + // The compositor's own frames, when recorded: its interval, from the frames + // around the judged part. + const draws = [...(presented ?? [])] + .filter(t => t >= judged.from - 250 && t <= judged.to + 250) + .sort((a, b) => a - b); + const drawIntervals = draws + .slice(1) + .map((t, i) => t - draws[i]) + .sort((a, b) => a - b); + const compositorFrameMs = + drawIntervals.length >= 4 + ? drawIntervals[Math.floor(drawIntervals.length / 2)] + : null; + const changed = new Set(inWindow); + const delivered = tracked + .filter(f => f.box != null && f.t > judged.from && f.t < judged.to) + .map(f => f.t); + // Each gap between presented changes, in intervals of the channel that saw + // it: the screencast's, or, for a gap between two moved frames with no + // frame delivered inside it, the compositor's own longest interval there. + const gaps = edges.slice(1).map((to, i) => { + const from = edges[i]; + const ms = to - from; + if ( + compositorFrameMs == null || + !changed.has(from) || + !changed.has(to) || + delivered.some(t => t > from && t < to) + ) { + return {from, ms, frames: Math.round(ms / cadence), skipped: 0}; + } + const span = draws.filter( + t => t > from - compositorFrameMs / 2 && t < to + compositorFrameMs / 2, + ); + const longest = + span.length >= 2 + ? Math.min(ms, Math.max(...span.slice(1).map((t, j) => t - span[j]))) + : ms; + const frames = Math.round(longest / compositorFrameMs); + return { + from, + ms: longest, + frames, + skipped: Math.max(0, Math.round(ms / compositorFrameMs) - frames), + }; + }); + const longestGap = gaps.reduce((a, b) => + b.frames > a.frames || (b.frames === a.frames && b.ms > a.ms) ? b : a, + ); + const longestStallMs = longestGap.ms; + // Where the longest stall starts, from the start of the window. + const longestStallAtMs = longestGap.from - window.from; + const longestStallFrames = longestGap.frames; + const skips = gaps.filter(g => g.skipped > 0); + const recorderSkipped = skips.reduce((sum, g) => sum + g.skipped, 0); + // With a compositor record, frames count in its intervals, and the frames + // it presented where the screencast skipped are presented frames. + const expectedFrames = Math.floor( + (judged.to - judged.from) / (compositorFrameMs ?? cadence), + ); + const moved = changes.length; + // The frames judged for motion: the screencast's own frames in the judged + // part (each `changes` entry is one of them), or, with a compositor record, + // the compositor's frames from the first to the last moved frame (the gaps + // before and after them are the stall check's). After an on-screen start + // they count from the on-screen frame, which `changes` leaves out, so the + // frames credited as skipped after it are judged too. A part that moved + // nothing, or delivered fewer than `minJudgedFrames` frames, is judged + // against its span, so a frozen animation misses here too (one late frame + // from before the block is not 1/1 moved). + const deliveredFrames = tracked.filter( + f => + f.box != null && + (onScreenFrom == null ? f.t >= judged.from : f.t > judged.from) && + f.t <= judged.to, + ).length; + const judgedFrames = + compositorFrameMs != null + ? changes.length > 0 + ? Math.round( + (changes[changes.length - 1] - (onScreenFrom ?? changes[0])) / + compositorFrameMs, + ) + (onScreenFrom == null ? 1 : 0) + : expectedFrames + : deliveredFrames < minJudgedFrames + ? expectedFrames + : deliveredFrames; + const movedFraction = (moved + recorderSkipped) / Math.max(1, judgedFrames); + // The cast's position step, and the motion per interval it has to resolve. + const positions = tracked.flatMap(f => (f.box == null ? [] : [f.box[axis]])); + const stepPx = + [2, 1, 0.5].find(step => + positions.every(p => Math.abs(p / step - Math.round(p / step)) < 0.01), + ) ?? 0.5; + const judgedValues = tracked.flatMap(f => + f.box != null && f.t >= judged.from && f.t <= judged.to + ? [{t: f.t, value: f.box[axis]}] + : [], + ); + const first = judgedValues[0]; + const last = judgedValues[judgedValues.length - 1]; + const pxPerFrame = + first == null || last == null || last.t <= first.t + ? 0 + : Math.abs(last.value - first.value) / ((last.t - first.t) / cadence); + const notes: string[] = []; + if (presented != null && compositorFrameMs == null) { + notes.push( + 'no compositor frames recorded around the window: judged from the screencast alone', + ); + } + if (recorderSkipped > 0) { + notes.push( + `the screencast skipped ${recorderSkipped} frames the compositor presented (at ${skips.map(g => `+${(g.from - window.from).toFixed(0)} ms`).join(', ')})`, + ); + } + if (pxPerFrame > 0 && pxPerFrame < 2 * stepPx) { + notes.push( + `${pxPerFrame.toFixed(1)} px/frame is under two of the cast's ${stepPx} px position steps: a frame delivered early can show no step and read as not moved`, + ); + } + const failures: string[] = []; + if (stats.frames === 0) { + failures.push('no frame presented in the judged part'); + } else if (stats.medianFrameMs > maxMedianFrameMs) { + failures.push( + `median frame interval ${stats.medianFrameMs.toFixed(1)} ms > ${maxMedianFrameMs} ms (${stats.frames} frames presented)`, + ); + } + if (longestStallFrames > maxStallFrames) { + failures.push( + `longest stall ${longestStallMs.toFixed(1)} ms = ${longestStallFrames} intervals > ${maxStallFrames}`, + ); + } + if (expectedFrames < minJudgedFrames) { + failures.push( + `${(judged.to - judged.from).toFixed(0)} ms judged = ${expectedFrames} frames < ${minJudgedFrames}: the animation under test was not moving during the block`, + ); + } + const allowedMisses = Math.max( + 1, + Math.floor((1 - minMovedFraction) * judgedFrames + 1e-9), + ); + if (judgedFrames - moved - recorderSkipped > allowedMisses) { + failures.push( + `${moved} of ${judgedFrames} expected frames moved${recorderSkipped > 0 ? ` (and ${recorderSkipped} the screencast skipped)` : ''} (${(movedFraction * 100).toFixed(0)}% < ${(minMovedFraction * 100).toFixed(0)}%)`, + ); + } + const onScreenFromMs = + onScreenFrom == null ? null : onScreenFrom - window.from; + const summary = + `${(judged.to - judged.from).toFixed(0)} ms judged at ${cadence.toFixed(1)} ms/frame: ${stats.frames} frames presented, ` + + `${moved}/${judgedFrames} moved${recorderSkipped > 0 ? ` and ${recorderSkipped} skipped by the screencast` : ''} (${(movedFraction * 100).toFixed(0)}%), ` + + `median ${stats.medianFrameMs.toFixed(1)} ms, ` + + `longest stall ${longestStallMs.toFixed(1)} ms (${longestStallFrames} intervals) at +${longestStallAtMs.toFixed(0)} ms, ` + + `${stats.monotonic ? 'monotonic' : 'not monotonic'}, ` + + `${pxPerFrame.toFixed(1)} px/frame in ${stepPx} px steps` + + (compositorFrameMs == null + ? '' + : `, compositor presenting every ${compositorFrameMs.toFixed(1)} ms`) + + (onScreenFromMs == null + ? '' + : `, on screen from +${onScreenFromMs.toFixed(0)} ms (nothing moved before the block)`) + + notes.map(note => `; note: ${note}`).join(''); + return { + ...stats, + window, + judged, + onScreenFromMs, + stepPx, + pxPerFrame, + notes, + recorderSkipped, + compositorFrameMs, + cadenceMs: cadence, + longestStallMs, + longestStallAtMs, + expectedFrames, + deliveredFrames, + longestStallFrames, + moved, + movedFraction, + verdict: {pass: failures.length === 0, failures}, + summary, + }; +} diff --git a/internal/interaction-harness/src/curves.ts b/internal/interaction-harness/src/curves.ts index 69b9908066f62..3de4befc39f91 100644 --- a/internal/interaction-harness/src/curves.ts +++ b/internal/interaction-harness/src/curves.ts @@ -134,17 +134,61 @@ function animationRecorder(): void { return entry.startedAt == null && entry.animation.playState !== 'idle'; }); }; + // While a rendering trace runs (traceRendering sets + // `__harnessRenderingTrace`), the recorder stands down. Its per-frame + // callback would make every frame a main-thread frame, in which Blink + // restyles each running animation's target, a composited one included (and + // lays out an anchored one), and document.getAnimations() flushes style: + // exactly what the trace counts. So it runs no frame callback, and a CSS + // animation or transition event reads only the animations of the element it + // names, once, at that animation's start (one that ends inside the trace is + // otherwise never seen). The animate() wrapper below still records script + // animations (it reads no style), and traceRendering resumes the recorder + // once the trace ends. + const tracing = () => + (window as unknown as {__harnessRenderingTrace?: boolean}) + .__harnessRenderingTrace === true; + let looping = false; const tick = () => { + if (tracing()) { + looping = false; + return; + } record(); frame(tick); }; + const loop = () => { + if (!looping) { + looping = true; + frame(tick); + } + }; // What already runs when the recorder starts (a spinner on a page the // harness touches for the first time) is seen now, before any watch arms; // first seen a frame later, it would read as started by the watched action. + // (A recorder that starts during a trace reads once here, then waits.) record(); - frame(tick); + loop(); for (const type of ['animationstart', 'transitionrun', 'transitionstart']) { - document.addEventListener(type, record, true); + document.addEventListener( + type, + event => { + if (!tracing()) { + record(); + return; + } + const target = event.target; + if (target instanceof Element) { + // A pseudo-element's animation (`::backdrop`) is in the subtree. + const pseudo = (event as AnimationEvent | TransitionEvent) + .pseudoElement; + for (const animation of target.getAnimations({subtree: !!pseudo})) { + remember(animation); + } + } + }, + true, + ); } // A script animation is recorded as it is made: WebKit can start and finish // a short one between two frames of a fresh page. @@ -159,12 +203,36 @@ function animationRecorder(): void { }; w.__harnessAnimations = { log, - record, + // A read from outside the recorder (a wait's poll) stands down while a + // trace runs, as the frame loop does: document.getAnimations() flushes + // style. CSS start events and the animate() wrapper still log what + // starts, and resume() catches up. + record: () => { + if (!tracing()) { + record(); + } + }, + // Catches up on what started during a trace, then runs its frames again. + resume: () => { + record(); + loop(); + }, // Record first, then stamp strictly after the newest entry: a step that // changed style just before has its transition created by that read, - // and it started before the arm. - arm: () => { - record(); + // and it started before the arm. While a trace runs the page-wide read + // stands down, as record() does; the watched element's own animations + // are still read, which resolves its pending style change. That read + // restyles the element (and lays out an anchored one) in the task that + // takes the watch, so a traced proof takes its watch before the window + // it counts (README: No Layout / Paint). + arm: (element?: Element) => { + if (!tracing()) { + record(); + } else if (element != null) { + for (const animation of element.getAnimations({subtree: true})) { + remember(animation); + } + } const newest = log.length === 0 ? 0 : log[log.length - 1].seenAt; armedAt = Math.max(performance.now(), newest + 0.001); return armedAt; @@ -197,10 +265,47 @@ async function installCurveEnd(page: Page): Promise { await page.evaluate(() => { const w = window as unknown as { __harnessCurveEnd?: (animation: Animation) => number; + __harnessTiming?: ( + effect: AnimationEffect | null | undefined, + ) => ComputedEffectTiming | null; + }; + // An effect's timing as getComputedTiming() reports it. Inside a rendering + // trace it is worked out from getTiming() instead: getComputedTiming() + // updates a running animation on demand, which restyles its target in the + // next frame (and lays out an anchored one), exactly what a trace counts. + w.__harnessTiming ??= effect => { + if (effect == null) { + return null; + } + if ( + (window as unknown as {__harnessRenderingTrace?: boolean}) + .__harnessRenderingTrace !== true + ) { + return effect.getComputedTiming(); + } + const timing = effect.getTiming(); + const duration = + typeof timing.duration === 'number' ? timing.duration : 0; + const iterations = Number(timing.iterations ?? 1); + const delay = Number(timing.delay ?? 0); + const endDelay = Number(timing.endDelay ?? 0); + const activeDuration = + duration === 0 || iterations === 0 ? 0 : duration * iterations; + const derived: ComputedEffectTiming = { + ...timing, + duration, + iterations, + delay, + endDelay, + activeDuration, + endTime: Math.max(delay + activeDuration + endDelay, 0), + }; + return derived; }; if (w.__harnessCurveEnd) { return; } + const timingOf = w.__harnessTiming; const timingKeys = new Set([ 'offset', 'computedOffset', @@ -216,7 +321,7 @@ async function installCurveEnd(page: Page): Promise { ); w.__harnessCurveEnd = animation => { const effect = animation.effect as KeyframeEffect | null; - const timing = effect?.getComputedTiming(); + const timing = timingOf(effect); const delay = Number(timing?.delay ?? 0); const active = Number(timing?.activeDuration ?? 0); if ( @@ -550,7 +655,8 @@ export interface CurveSample { * The time the sampled element presents: `t`, or just inside the end of * its own animations when the sample was taken at or past it (see * `seekAnimations`). Read from the element's own animations only (from - * every held one if none targets it). + * every held one if none targets it); when sampling a pseudo-element, from + * that pseudo-element's. */ readonly presentedAt: number; /** @@ -584,13 +690,19 @@ export interface CurveSample { * held origin, as for `seekAnimations`). Hold the animations first. An animation of the * individual `translate`, `rotate` or `scale` properties is included: they * are composed with `transform` into one matrix. + * + * `pseudoElement` samples the element's `::before` or `::after` instead: + * its presented values, labelled by its own animations (those whose effect + * targets the element with that pseudo-element), as a component that animates + * a pseudo-element (`animatorFor(element, {pseudoElement})`) draws it. */ export async function sampleCurve( locator: Locator, times: readonly number[], + {pseudoElement}: {pseudoElement?: '::before' | '::after'} = {}, ): Promise { const raw = await locator.evaluate( - (element, {ts, inset}) => { + (element, {ts, inset, pseudo}) => { const w = window as unknown as { __harnessHeld?: Animation[]; __harnessHeldStarts?: number[]; @@ -613,12 +725,17 @@ export async function sampleCurve( 'sampleCurve: no held animations (call holdAnimations first)', ); } - // The properties each held animation of this element animates. A - // `::before` or `::after` animation is not the element's own: its - // clock would label the element's samples with another box's time. + // The properties each held animation of the sampled box animates: the + // element's own, or its pseudo-element's when one is sampled. Another + // box's animation (the element's `::before` when the element is + // sampled) is not its own: its clock would label the samples with + // another box's time. const own = held.map(animation => { const effect = animation.effect as KeyframeEffect | null; - if (effect?.target !== element || effect.pseudoElement != null) { + if ( + effect?.target !== element || + (effect.pseudoElement ?? null) !== pseudo + ) { return null; } const properties = new Set(); @@ -667,7 +784,10 @@ export async function sampleCurve( const or = (value: number, fallback: number) => Number.isFinite(value) ? value : fallback; const presentedAt = or(presentedOwn, or(presented, t)); - const style = getComputedStyle(element); + const style = + pseudo == null + ? getComputedStyle(element) + : getComputedStyle(element, pseudo); // translate · rotate · scale · transform, as CSS applies them (all // about the same transform origin, which `transform` excludes too). // A percentage resolves against the border box, unrounded @@ -771,7 +891,7 @@ export async function sampleCurve( }; }); }, - {ts: [...times], inset: END_INSET_MS}, + {ts: [...times], inset: END_INSET_MS, pseudo: pseudoElement ?? null}, ); return raw.map(r => ({ t: r.t, @@ -826,6 +946,47 @@ export interface ActiveAnimation { readonly finishedAt: number | null; } +/** + * Runs `fn` on `locator`'s element: through the locator, or, `quietly`, + * through a handle to the element it matches now (none yet: `[]`). Each + * locator.evaluate makes a main-thread frame, which a trace counts; + * `elementHandles()` and a handle's evaluate make none. + */ +async function readElement( + locator: Locator, + quietly: boolean, + fn: (element: Element, arg: A) => R[], + arg: A, + timeout: number, +): Promise { + const run = fn as unknown as (element: Element, arg: unknown) => R[]; + if (!quietly) { + return locator.evaluate(run, arg as unknown, {timeout}); + } + const handles = await locator.elementHandles(); + try { + if (handles.length > 1) { + throw new Error( + `waitForAnimation: ${String(locator)} matches ${handles.length} elements`, + ); + } + return handles.length === 0 + ? [] + : await handles[0].evaluate(run, arg as unknown); + } finally { + await Promise.all(handles.map(handle => handle.dispose())); + } +} + +/** The rejection for a wait whose element is not in the document. */ +function notInDocument(name: string, cause?: unknown): Error { + return new Error( + `${name}: the element is not in the document. An exit that removes ` + + 'its element is read through its container (watch the parent).', + cause === undefined ? undefined : {cause}, + ); +} + /** * Runs a locator read within the wait's own deadline. A locator waits for * its element with no timeout of its own, so one the page removed (an exit @@ -847,11 +1008,7 @@ async function withinTimeout( if ((await locator.count()) > 0) { return null; } - throw new Error( - `${name}: the element is not in the document. An exit that removes ` + - 'its element is read through its container (watch the parent).', - {cause: error}, - ); + throw notInDocument(name, error); } throw error; } @@ -876,6 +1033,15 @@ async function withinTimeout( * recorder keeps one entry per `Animation` object, so an animation reversed * or replayed mid-run is misread: a watch taken before the reversal rejects, * and an unwatched wait returns at once with a curve dated before it. + * + * Started inside a `traceRendering` body, it stands down as the recorder + * does, so it adds no frame the trace counts: it takes the element's + * animations from the recorder's log, judges a started one by its own times, + * and reaches the element through a handle to what the locator matches at + * each look. That holds for script animations and CSS transitions. On a CSS + * `@keyframes` animation (and, unwatched, a cancelled CSS transition) it reads + * the animation's timing, which flushes its style and adds Layouts to the + * trace: wait on one with `waitForRest`, or after the trace. */ export async function waitForAnimation( locator: Locator, @@ -891,17 +1057,34 @@ export async function waitForAnimation( } = {}, ): Promise { const deadline = Date.now() + timeout; - await installCurveEnd(locator.page()); + const page = locator.page(); + await installCurveEnd(page); + // Inside a rendering trace the wait stands down (see the page function), + // and it reads the element through handles: each locator.evaluate makes a + // main-thread frame (a Layout of an anchored element), where a handle's + // evaluate adds none. + const traced = await page.evaluate( + () => + (window as unknown as {__harnessRenderingTrace?: boolean}) + .__harnessRenderingTrace === true, + ); for (;;) { const read = await withinTimeout( 'waitForAnimation', locator, deadline, evaluateTimeout => - locator.evaluate( + readElement( + locator, + traced, (element, {minProgress, since: watchedFrom}) => { + // The harness's own read: a trace sets it apart (trace.ts HARNESS_READ). + console.timeStamp('harness-read'); const w = window as unknown as { __harnessCurveEnd: (a: Animation) => number; + __harnessTiming: ( + effect: AnimationEffect | null | undefined, + ) => ComputedEffectTiming | null; __harnessAnimations: { log: { animation: Animation; @@ -916,6 +1099,13 @@ export async function waitForAnimation( }; const curveEnd = w.__harnessCurveEnd; const recorder = w.__harnessAnimations; + // Inside a rendering trace the wait stands down as the recorder does: + // record() reads nothing, the animations come from the recorder's log + // rather than getAnimations(), and a started one runs by its own + // times, since playState flushes a CSS animation's style. + const traced = + (window as unknown as {__harnessRenderingTrace?: boolean}) + .__harnessRenderingTrace === true; recorder.record(); const describe = ( a: Animation, @@ -923,7 +1113,7 @@ export async function waitForAnimation( recordedStart: number | null, ) => { const effect = a.effect as KeyframeEffect | null; - const timing = effect?.getComputedTiming(); + const timing = w.__harnessTiming(effect); // The recorded start stands in once the page has cancelled it. A // run going backward began at its end: its startTime is where its // current time would reach 0, a whole run later. @@ -994,24 +1184,36 @@ export async function waitForAnimation( // finished must not answer for it. Only document-timeline animations // count, as for holdAnimations: a scroll- or view-timeline one moves // only when something scrolls and never progresses on its own. + const runs = (a: Animation) => { + if (!(a.timeline instanceof DocumentTimeline)) { + return false; + } + if (!traced || a.startTime == null) { + return a.playState === 'running'; + } + const time = Number(a.currentTime ?? 0); + return a.playbackRate < 0 + ? time > 0 + : time < Number(w.__harnessTiming(a.effect)?.endTime ?? 0); + }; + const onElement = () => + recorder.log.filter( + e => inside(e.animation) || e.ancestors.includes(element), + ); const live = ( - watchedFrom == null + watchedFrom == null && !traced ? element.getAnimations({subtree: true}) - : recorded + : (watchedFrom == null ? onElement() : recorded) .filter(e => e.finishedAt == null) .map(e => e.animation) - ).filter( - a => - a.playState === 'running' && - a.timeline instanceof DocumentTimeline, - ); + ).filter(runs); const running = live.filter(a => a.startTime != null); // Progress along the curve, on the timeline: a delay presents nothing // (going backward, the end delay plays first), a run going backward // (`reverse()`) progresses from its end, and a faster rate presents // more curve per ms. const progressed = running.filter(a => { - const timing = a.effect?.getComputedTiming(); + const timing = w.__harnessTiming(a.effect); const time = Number(a.currentTime ?? 0); const local = a.playbackRate < 0 @@ -1036,13 +1238,20 @@ export async function waitForAnimation( return e.finishedAt; } const a = e.animation; - if (a.playState !== 'finished' || a.startTime == null) { + if (a.startTime == null) { + return null; + } + const end = Number(w.__harnessTiming(a.effect)?.endTime ?? 0); + // Traced, by its own times; otherwise by its state. + const time = Number(a.currentTime ?? 0); + const done = traced + ? a.playbackRate < 0 + ? time <= 0 + : time >= end + : a.playState === 'finished'; + if (!done) { return null; } - const end = Number( - (a.effect as KeyframeEffect | null)?.getComputedTiming() - .endTime ?? 0, - ); return a.playbackRate < 0 ? Number(a.startTime) : Number(a.startTime) + end / Math.abs(a.playbackRate || 1); @@ -1052,7 +1261,7 @@ export async function waitForAnimation( .map(e => describe(e.animation, finishedAt(e), e.startedAt)); }, {minProgress: minProgressMs, since: since ?? null}, - {timeout: evaluateTimeout}, + evaluateTimeout, ), ); const found = read ?? []; @@ -1061,6 +1270,12 @@ export async function waitForAnimation( return found; } if (Date.now() > deadline) { + // A traced read goes through handles, which find no element rather + // than wait for one: a missing element is reported as missing, as the + // untraced read reports it. + if (traced && (await locator.count()) === 0) { + throw notInDocument('waitForAnimation'); + } throw new Error( 'waitForAnimation: no animation started on the element, none running ' + 'and none recorded since the last wait. If the action is the ' + @@ -1072,13 +1287,31 @@ export async function waitForAnimation( } } -/** Marks now as the point later waits count animations from. */ -async function armRecorder(page: Page): Promise { - return page.evaluate(() => - ( - window as unknown as {__harnessAnimations: {arm: () => number}} - ).__harnessAnimations.arm(), - ); +/** + * Marks now as the point later waits count animations from. With a locator, + * its element's pending style change is resolved first, also inside a trace + * (reached through a handle, which adds no main-thread frame). + */ +async function armRecorder(page: Page, locator?: Locator): Promise { + const handles = locator == null ? [] : await locator.elementHandles(); + try { + if (handles.length === 1) { + return await handles[0].evaluate(element => + ( + window as unknown as { + __harnessAnimations: {arm: (element?: Element) => number}; + } + ).__harnessAnimations.arm(element as Element), + ); + } + return await page.evaluate(() => + ( + window as unknown as {__harnessAnimations: {arm: () => number}} + ).__harnessAnimations.arm(), + ); + } finally { + await Promise.all(handles.map(handle => handle.dispose())); + } } export interface AnimationWatch { @@ -1096,7 +1329,9 @@ export interface AnimationWatch { /** * Starts watching `locator`'s element before an action, so the animation the * action starts is caught even when it ends before the test looks (a short - * entrance on a cold page): + * entrance on a cold page). Taking the watch reads the element's animations, + * which restyles it, inside a rendering trace too: take a traced watch before + * the window the proof counts: * * const watch = await watchAnimations(sheet); * await driver.play(tap(button)); @@ -1106,7 +1341,7 @@ export async function watchAnimations( locator: Locator, ): Promise { await installCurveEnd(locator.page()); - const since = await armRecorder(locator.page()); + const since = await armRecorder(locator.page(), locator); return { started: options => waitForAnimation(locator, {...options, since}), }; @@ -1125,6 +1360,19 @@ export async function watchAnimations( * forward run under a linear or cubic-bézier effect easing); under any other * easing or direction, waitForRest waits the hold out, and a hold longer than * `timeout` rejects with "still moving". + * + * While a rendering trace runs (`traceRendering`), it stands down as the + * animation recorder does: a per-frame callback would make every frame a + * main-thread frame, which the trace counts. It then waits on timers for the + * scheduled ends of the curves on the element instead (a run going backward, + * until it finishes), so motion that no animation drives (a script writing + * styles) is not seen until the trace ends. Its one read of the element's + * animations, as it is called, restyles the element (and lays out an anchored + * one) when an animation is already running there: await it right after the + * action that starts the motion, before the window a proof counts, not in its + * middle (README: No Layout / Paint). Finding no animation, it looks again `frames` times a frame + * apart before settling, so one the action starts a few frames later is + * waited for. */ export async function waitForRest( locator: Locator, @@ -1157,15 +1405,94 @@ export async function waitForRest( backdrop, ].join(' | '); }; - // The page's real frames, even while a timer gate holds its callbacks. - const frame = + // The page's real frames and timers, even while a timer gate holds + // its callbacks: a frozen gate never holds the wait itself. + const gate = ( + window as unknown as { + __harnessTimerGate?: { + frame: typeof requestAnimationFrame; + timeout?: typeof setTimeout; + }; + } + ).__harnessTimerGate; + const frame = gate?.frame ?? requestAnimationFrame; + const timeout = gate?.timeout ?? setTimeout; + // Set by traceRendering while it traces (see the recorder above). + const tracing = () => + (window as unknown as {__harnessRenderingTrace?: boolean}) + .__harnessRenderingTrace === true; + const settle = () => { + // Later waits count animations from here. ( - window as unknown as { - __harnessTimerGate?: {frame: typeof requestAnimationFrame}; - } - ).__harnessTimerGate?.frame ?? requestAnimationFrame; - let last = read(); + window as unknown as {__harnessAnimations?: {arm: () => number}} + ).__harnessAnimations?.arm(); + resolve(); + }; + let last = ''; let still = 0; + const frames = () => { + last = read(); + still = 0; + frame(tick); + }; + // While traced: one look at the animations, then a timer to the latest + // scheduled curve end. One not started yet is looked at again a frame + // later; a trace that has ended hands over to the per-frame check. + // Finding nothing settles only after `frames` looks a frame apart, as + // the per-frame check needs `frames` still frames: an animation the + // action starts a frame or two later is still waited for. + let idle = 0; + const scheduled = () => { + let wait = 0; + for (const a of element.getAnimations({subtree: true})) { + // A scroll- or view-timeline animation moves only when something + // scrolls: it has no scheduled end to wait for (holdAnimations + // skips it the same way). + if (!(a.timeline instanceof DocumentTimeline)) { + continue; + } + if ( + a.pending || + (a.playState === 'running' && a.startTime == null) + ) { + wait = Math.max(wait, 16); + } else if (a.playState === 'running') { + // The curve's time left through the playback rate: forward, to + // its (held) end; backward (`reverse()`), to local time 0, + // where it finishes, as the untraced path waits it out. + const rate = a.playbackRate || 1; + const at = Number(a.currentTime ?? 0); + const left = + rate > 0 ? (curveEnd(a) - at) / rate : at / -rate; + wait = Math.max(wait, left); + } + } + if (wait <= 0) { + idle += 1; + if (idle >= need) { + settle(); + return; + } + wait = 16; + } else { + idle = 0; + } + const room = deadline - performance.now(); + timeout( + () => { + if (performance.now() > deadline) { + reject( + new Error('waitForRest: still moving at the deadline'), + ); + } else if (tracing()) { + scheduled(); + } else { + frames(); + } + }, + Math.min(wait, Math.max(0, room)) + 1, + ); + }; const tick = () => { const now = read(); const moving = element.getAnimations({subtree: true}).some( @@ -1182,20 +1509,20 @@ export async function waitForRest( still = !moving && now === last ? still + 1 : 0; last = now; if (still >= need) { - // Later waits count animations from here. - ( - window as unknown as { - __harnessAnimations?: {arm: () => number}; - } - ).__harnessAnimations?.arm(); - resolve(); + settle(); } else if (performance.now() > deadline) { reject(new Error(`waitForRest: still moving (${now})`)); + } else if (tracing()) { + scheduled(); } else { frame(tick); } }; - frame(tick); + if (tracing()) { + scheduled(); + } else { + frames(); + } }), {ms: timeout, need: frames}, {timeout: evaluateTimeout}, diff --git a/internal/interaction-harness/src/frames.ts b/internal/interaction-harness/src/frames.ts new file mode 100644 index 0000000000000..83841d259c17f --- /dev/null +++ b/internal/interaction-harness/src/frames.ts @@ -0,0 +1,531 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file frames.ts + * @input Uses a Playwright `Page`, a `Locator` for the element under test, and + * the engine's screencast (Chromium CDP `Page.startScreencast`; WebKit's + * `Screencast` domain) + * @output `startScreencast` (timestamped composited frames), `markElement` + * (a marker outline the frames can find), `trackMarker` (per-frame marker + * box), `sampleRects` (per-rAF rect sampling), `frameStats`. + * @position Frame evidence for motion proofs. The screencast is captured off + * the page's main thread, from what the compositor presents, so it keeps + * delivering frames while a script blocks the main thread: that is what makes + * the busy-main-thread proof possible. Rect sampling is the cheap path when + * the main thread is free. + * + * Marker tracking needs no image library: frames are decoded and scanned in a + * scratch page of the same browser (createImageBitmap + canvas), so the + * harness adds no dependency. + * + * SYNC: Keep the README's frame-evidence section equal to what this measures. + */ + +import type {CDPSession, Locator, Page} from '@playwright/test'; +import {webkitPageSession} from './touch'; + +export interface CastFrame { + /** Presentation time, epoch ms (the compositor's frame timestamp). */ + readonly t: number; + /** Base64 PNG/JPEG. */ + readonly data: string; + readonly format: 'png' | 'jpeg'; + /** CSS px of the viewport the frame shows. */ + readonly viewportWidth: number; + readonly viewportHeight: number; +} + +export interface Screencast { + /** + * Frames captured so far: wait for the first one to know the cast saw the + * page before an animation starts (`busyWindowProof` then knows its at-rest + * position). + */ + frameCount(): number; + /** Stops and returns every frame captured, in order. */ + stop(): Promise; +} + +export interface ScreencastOptions { + /** Lossless frames by default: marker colours survive PNG, not JPEG. */ + readonly format?: 'png' | 'jpeg'; + readonly quality?: number; + /** + * Keep one presented frame in this many (default 1, every frame): a slower + * evidence channel, like a loaded machine's screencast, for proving that a + * verdict does not depend on the screencast's rate. + */ + readonly everyNthFrame?: number; +} + +/** + * Starts capturing composited frames. Chromium delivers one per presented + * frame (16.7 ms at 60 Hz), timestamped by the compositor; WebKit's own + * screencast delivers one per composited update (about every 14 ms in the + * Linux build), stamped on CLOCK_MONOTONIC and rebased here onto the wall + * clock by that clock's measured offset, so both engines share the page's + * timeline. Both keep delivering while the page's main thread is blocked, and + * neither sends a frame when nothing on screen changed. + */ +export async function startScreencast( + page: Page, + options: ScreencastOptions = {}, +): Promise { + const name = page.context().browser()?.browserType().name(); + const viewport = page.viewportSize() ?? {width: 0, height: 0}; + if (name === 'chromium') { + return chromiumScreencast(page, viewport, options); + } + if (name === 'webkit') { + return webkitScreencast(page, viewport, options); + } + throw new Error( + `startScreencast: unsupported browser ${name ?? '(unknown)'}`, + ); +} + +async function chromiumScreencast( + page: Page, + viewport: {width: number; height: number}, + {format = 'png', quality = 90, everyNthFrame = 1}: ScreencastOptions, +): Promise { + const cdp: CDPSession = await page.context().newCDPSession(page); + const frames: CastFrame[] = []; + const onFrame = (frame: { + data: string; + sessionId: number; + metadata: {timestamp?: number; deviceWidth: number; deviceHeight: number}; + }) => { + frames.push({ + t: (frame.metadata.timestamp ?? Date.now() / 1000) * 1000, + data: frame.data, + format, + viewportWidth: frame.metadata.deviceWidth || viewport.width, + viewportHeight: frame.metadata.deviceHeight || viewport.height, + }); + cdp + .send('Page.screencastFrameAck', {sessionId: frame.sessionId}) + .catch(() => {}); + }; + cdp.on('Page.screencastFrame', onFrame); + // One CSS px per frame px keeps decoding cheap; everyNthFrame 1 = every presented frame. + await cdp.send('Page.startScreencast', { + format, + quality, + maxWidth: viewport.width, + maxHeight: viewport.height, + everyNthFrame, + }); + return { + frameCount: () => frames.length, + async stop() { + await cdp.send('Page.stopScreencast'); + cdp.off('Page.screencastFrame', onFrame); + await cdp.detach().catch(() => {}); + return inPresentationOrder(frames); + }, + }; +} + +async function webkitScreencast( + page: Page, + viewport: {width: number; height: number}, + {quality = 90, everyNthFrame = 1}: ScreencastOptions, +): Promise { + const session = webkitPageSession(page); + const frames: CastFrame[] = []; + let generation = 0; + let delivered = 0; + const onFrame = (payload: unknown) => { + const frame = payload as {data: string; timestamp: number}; + // WebKit's screencast has no everyNthFrame: keep one frame in N here. + delivered += 1; + if ((delivered - 1) % everyNthFrame !== 0) { + session + .send('Screencast.screencastFrameAck', {generation}) + .catch(() => {}); + return; + } + frames.push({ + // WebKit stamps frames in seconds on the monotonic clock; rebased below. + t: frame.timestamp * 1000, + data: frame.data, + format: 'jpeg', + viewportWidth: viewport.width, + viewportHeight: viewport.height, + }); + session.send('Screencast.screencastFrameAck', {generation}).catch(() => {}); + }; + session.on('Screencast.screencastFrame', onFrame); + const started = (await session.send('Screencast.startScreencast', { + width: viewport.width, + height: viewport.height, + toolbarHeight: 0, + quality, + })) as {generation: number}; + generation = started.generation; + // WebKit stamps frames in seconds on CLOCK_MONOTONIC, which this process + // shares: rebase them by the wall clock's offset from that clock, sampled + // together. (Taking the first frame as the start shifted every frame earlier + // by however long that frame took to come, 60–100 ms on a busy page.) + const wallMinusMonotonic = Date.now() - Number(process.hrtime.bigint()) / 1e6; + return { + frameCount: () => frames.length, + async stop() { + await session.send('Screencast.stopScreencast'); + session.off?.('Screencast.screencastFrame', onFrame); + if (frames.length === 0) { + return frames; + } + return inPresentationOrder(frames).map(f => ({ + ...f, + t: f.t + wallMinusMonotonic, + })); + }, + }; +} + +/** + * Frames sorted by their presentation timestamp. A loaded machine can deliver + * screencast frames out of order (a frame stamped 35 ms arriving after one + * stamped 46 ms); judged in arrival order, a steadily moving element would + * appear to step backwards. + */ +export function inPresentationOrder( + frames: readonly T[], +): T[] { + return [...frames].sort((a, b) => a.t - b.t); +} + +/** The marker colour: magenta never occurs in Astryx themes. */ +export const MARKER_RGB: readonly [number, number, number] = [255, 0, 255]; + +export interface MarkerOptions { + readonly rgb?: readonly [number, number, number]; + /** + * `chip` (default): a small square child pinned inside the element's + * top-left corner, above every sibling (z-index max), so nothing the + * component paints can cover it. The element must be its containing block: + * a static element is made `position: relative` while marked. + * `outline`: an inset outline on the element itself (no DOM insertion), for + * elements that cannot take a child; children with their own layers can + * cover parts of it. + */ + readonly mode?: 'chip' | 'outline'; + /** Chip size or outline width, CSS px. */ + readonly size?: number; + /** Chip inset from the element's top-left corner, CSS px. */ + readonly inset?: number; +} + +/** + * Marks the element so screencast frames can find it: the mark moves, scales + * and fades with the element's own layer and never changes its layout. + * Returns an undo. + */ +export async function markElement( + locator: Locator, + options: MarkerOptions = {}, +): Promise<() => Promise> { + const { + rgb = MARKER_RGB, + mode = 'chip', + size = mode === 'chip' ? 16 : 6, + inset = 8, + } = options; + const id = `m${Math.random().toString(36).slice(2, 8)}`; + await locator.evaluate( + (element, {id: markerId, color, size: px, inset: offset, mode: kind}) => { + const el = element as HTMLElement; + el.setAttribute('data-harness-marker', markerId); + if (kind === 'outline') { + const style = document.createElement('style'); + style.setAttribute('data-harness-marker-style', markerId); + style.textContent = `[data-harness-marker="${markerId}"]{outline:${px}px solid rgb(${color.join(',')}) !important;outline-offset:-${px}px !important;}`; + document.head.append(style); + return; + } + if (getComputedStyle(el).position === 'static') { + el.setAttribute('data-harness-marker-position', el.style.position); + el.style.position = 'relative'; + } + const chip = document.createElement('div'); + chip.setAttribute('data-harness-marker-chip', markerId); + chip.setAttribute('aria-hidden', 'true'); + chip.style.cssText = `position:absolute;top:${offset}px;left:${offset}px;width:${px}px;height:${px}px;background:rgb(${color.join(',')});z-index:2147483647;pointer-events:none;`; + el.append(chip); + }, + {id, color: [...rgb], size, inset, mode}, + ); + return async () => { + await locator.page().evaluate(markerId => { + const el = document.querySelector( + `[data-harness-marker="${markerId}"]`, + ); + if (el?.hasAttribute('data-harness-marker-position')) { + el.style.position = + el.getAttribute('data-harness-marker-position') ?? ''; + el.removeAttribute('data-harness-marker-position'); + } + el?.removeAttribute('data-harness-marker'); + document + .querySelector(`[data-harness-marker-chip="${markerId}"]`) + ?.remove(); + document + .querySelector(`[data-harness-marker-style="${markerId}"]`) + ?.remove(); + }, id); + }; +} + +export interface Box { + readonly x: number; + readonly y: number; + readonly width: number; + readonly height: number; +} + +export interface TrackedFrame { + /** Epoch ms. */ + readonly t: number; + /** The marker's bounding box in CSS px, or null when it is not on screen. */ + readonly box: Box | null; + /** Marker pixels found (a fading element finds fewer at the same tolerance). */ + readonly pixels: number; +} + +/** + * Finds the marker in every frame. Decoding runs in a scratch page of the same + * browser, so neither the page under test nor Node needs an image decoder. + */ +export async function trackMarker( + page: Page, + frames: readonly CastFrame[], + { + rgb = MARKER_RGB, + tolerance = 40, + }: {rgb?: readonly [number, number, number]; tolerance?: number} = {}, +): Promise { + const scratch = await page.context().newPage(); + try { + const out: TrackedFrame[] = []; + // Batches keep each evaluate payload moderate. + for (let i = 0; i < frames.length; i += 12) { + const batch = frames.slice(i, i + 12); + const boxes = await scratch.evaluate( + async ({batch: items, color, tol}) => { + const results: { + box: {x: number; y: number; width: number; height: number} | null; + pixels: number; + }[] = []; + for (const item of items) { + const blob = await ( + await fetch(`data:image/${item.format};base64,${item.data}`) + ).blob(); + const bitmap = await createImageBitmap(blob); + const canvas = new OffscreenCanvas(bitmap.width, bitmap.height); + const context = canvas.getContext('2d'); + if (context == null) { + throw new Error('no 2d context'); + } + context.drawImage(bitmap, 0, 0); + const {data, width, height} = context.getImageData( + 0, + 0, + bitmap.width, + bitmap.height, + ); + let minX = Infinity; + let minY = Infinity; + let maxX = -Infinity; + let maxY = -Infinity; + let pixels = 0; + for (let y = 0; y < height; y += 1) { + for (let x = 0; x < width; x += 1) { + const o = (y * width + x) * 4; + if ( + Math.abs(data[o] - color[0]) <= tol && + Math.abs(data[o + 1] - color[1]) <= tol && + Math.abs(data[o + 2] - color[2]) <= tol + ) { + pixels += 1; + if (x < minX) { + minX = x; + } + if (x > maxX) { + maxX = x; + } + if (y < minY) { + minY = y; + } + if (y > maxY) { + maxY = y; + } + } + } + } + const scale = width / item.viewportWidth; + results.push({ + box: + pixels === 0 + ? null + : { + x: minX / scale, + y: minY / scale, + width: (maxX - minX + 1) / scale, + height: (maxY - minY + 1) / scale, + }, + pixels, + }); + } + return results; + }, + { + batch: batch.map(f => ({ + data: f.data, + format: f.format, + viewportWidth: f.viewportWidth, + })), + color: [...rgb], + tol: tolerance, + }, + ); + boxes.forEach((result, k) => + out.push({t: batch[k].t, box: result.box, pixels: result.pixels}), + ); + } + return out; + } finally { + await scratch.close(); + } +} + +export interface RectSample { + /** Epoch ms of the rAF callback. */ + readonly t: number; + readonly box: Box | null; + readonly transform: string; + readonly opacity: number; +} + +/** + * Samples the element's rect, transform and opacity on every animation frame + * for `durationMs`. Main-thread sampling: it stops while the main thread is + * blocked, which is the point of the screencast path above. + */ +export async function sampleRects( + locator: Locator, + durationMs: number, +): Promise { + return locator.evaluate( + (element, duration) => + new Promise< + { + t: number; + box: {x: number; y: number; width: number; height: number} | null; + transform: string; + opacity: number; + }[] + >(resolve => { + // The harness's own waits run on the real timers: a frozen timer gate + // (story.ts) holds only the page's. + const gate = ( + window as unknown as { + __harnessTimerGate?: { + frame?: typeof requestAnimationFrame; + }; + } + ).__harnessTimerGate; + const frame = gate?.frame ?? window.requestAnimationFrame.bind(window); + const samples: { + t: number; + box: {x: number; y: number; width: number; height: number} | null; + transform: string; + opacity: number; + }[] = []; + const end = performance.now() + duration; + const tick = (now: number) => { + const rect = element.getBoundingClientRect(); + const style = getComputedStyle(element); + samples.push({ + t: performance.timeOrigin + now, + box: {x: rect.x, y: rect.y, width: rect.width, height: rect.height}, + transform: style.transform, + opacity: Number(style.opacity), + }); + if (now < end) { + frame(tick); + } else { + resolve(samples); + } + }; + frame(tick); + }), + durationMs, + ); +} + +export interface FrameStats { + readonly frames: number; + /** Frames whose tracked value differs from the frame before. */ + readonly changed: number; + /** Longest stretch without a change inside the window (an open stall at its end included), ms. */ + readonly longestStallMs: number; + /** Median spacing between frames, ms. */ + readonly medianFrameMs: number; + readonly monotonic: boolean; + readonly first: number | null; + readonly last: number | null; +} + +/** Summarises per-frame values (e.g. the marker's y) inside a time window. */ +export function frameStats( + samples: readonly {readonly t: number; readonly value: number | null}[], + window: {readonly from: number; readonly to: number} = { + from: -Infinity, + to: Infinity, + }, + epsilon = 0.25, +): FrameStats { + const inside = samples.filter( + s => s.t >= window.from && s.t <= window.to && s.value != null, + ) as {t: number; value: number}[]; + let changed = 0; + let increasing = 0; + let decreasing = 0; + let stallStart = inside[0]?.t ?? 0; + let longestStallMs = 0; + for (let i = 1; i < inside.length; i += 1) { + const delta = inside[i].value - inside[i - 1].value; + if (Math.abs(delta) > epsilon) { + changed += 1; + if (delta > 0) { + increasing += 1; + } else { + decreasing += 1; + } + longestStallMs = Math.max(longestStallMs, inside[i].t - stallStart); + stallStart = inside[i].t; + } + } + // A stall still open when the window closes counts too: a frozen animation never changes at all. + if (inside.length > 0) { + longestStallMs = Math.max( + longestStallMs, + inside[inside.length - 1].t - stallStart, + ); + } + const gaps = inside + .slice(1) + .map((s, i) => s.t - inside[i].t) + .sort((a, b) => a - b); + return { + frames: inside.length, + changed, + longestStallMs, + medianFrameMs: gaps.length ? gaps[Math.floor(gaps.length / 2)] : 0, + monotonic: increasing === 0 || decreasing === 0, + first: inside[0]?.value ?? null, + last: inside[inside.length - 1]?.value ?? null, + }; +} diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 7e47af95b2c81..08be91b61840e 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -3,32 +3,54 @@ /** * @file harness.interaction.browser.spec.ts * @input Fixture pages written inline (no Storybook), real Chromium/WebKit - * @output Proof that each harness instrument measures what it claims: - * scripted timestamps reach the page; a flick flings natively (Chromium); - * a second finger and a pinch arrive as distinct pointers; sampled curves - * match the oracles; the animation-state waits report a running transition - * and its end, and judge rest by presented values even while an animation - * holds its end, and a watch taken on first contact ignores an animation - * already running and keeps the start of one the page cancels when it ends; individual transform properties are sampled; a sample at + * @output Proof that each harness instrument measures what it claims (the + * busy-main-thread proof and the rendering-trace counts with a negative + * control beside their positive checks): scripted timestamps reach the + * page; a flick flings natively (Chromium); a wheel step arrives as + * exactly its deltas, with Control when held, and a pen stroke as trusted + * pen pointers with its pressure and times (Chromium); a second finger + * and a pinch arrive as distinct pointers; the screencast shows a compositor animation + * advancing through a blocked main thread while a rAF animation freezes; + * the trace counts zero Layout for a transform animation and one per frame + * for a `top` animation, and none from the harness's own animation recorder + * while an anchored element's composited animation runs (a page that asks + * for a frame every frame lays it out in each); the long-task log records + * the page's own long task but not a DevTools evaluate, which the trace + * counts; sampled curves + * match the oracles, individual transform properties included; a sample at * an end reports the time it was read at; holding takes only running * document-timeline animations, each at its own offset, holds each where it * was read, adds a second hold to the held set without moving the first - * hold's times, drops one the page cancels, and releases only them; - * storyboards write a sheet and a trace that follows the curve, an endless - * animation does not set their length, and a photograph Chromium returns - * empty is taken again two frames later; a touch filmstrip keeps scripted - * event times while a handler clock sees its pacing; the timer gate holds a - * component timer until the filmstrip is done, drops callbacks the page - * cancels, stamps held frames with the time they run, and leaves the - * harness its own frames. + * hold's times, drops one the page cancels, and releases only them; the + * animation-state waits report a running transition and its end, and judge + * rest by presented values even while an animation holds its end, and a + * watch taken on first contact ignores an animation already running and + * keeps the start of one the page cancels when it ends, and the recorder + * reads only at CSS starts while a rendering trace runs and catches up + * after it; + * storyboards write a + * sheet and a trace that follows the curve, an endless animation does not + * set their length, and a photograph Chromium returns empty is taken again + * two frames later; a touch filmstrip keeps scripted event times while a + * handler clock sees its pacing; the timer gate holds a component timer + * until the filmstrip is done, drops callbacks the page cancels, stamps held + * frames with the time they run, and leaves the harness its own frames; a + * cast filmstrip keeps exactly the presented frames of its window, labelled + * from its zero, as PNG. * @position The harness's own test. A component spec trusts these * instruments; this file is why it may. */ +import fs from 'node:fs'; +import path from 'node:path'; import {expect, test, type Page} from '@playwright/test'; import { animationStoryboard, + castStoryboard, + blockMainThread, + busyWindowProof, compareCurve, + countBetween, cubicBezier, curveOf, drag, @@ -37,28 +59,49 @@ import { freezeTimers, holdAnimations, installInputLog, + installLongTaskLog, installTimerGate, + markElement, merge, parseTransform, PARTIAL_LIFT_SKIP, pinch, readInputLog, + readLongTasks, releaseAnimations, sampleCurve, + sampleRects, seekAnimations, springStep, + startCompositorTrace, + startScreencast, Storyboard, tap, thawTimers, timeline, touchDriver, touchStoryboard, + traceRendering, + trackMarker, waitForAnimation, waitForRest, watchAnimations, + webInputDriver, + PEN_SKIP, + WHEEL_SKIP, withFinger, } from './index'; +// An element positioned against an anchor: a composited animation of it lays +// out in every main-thread frame, so it shows any per-frame main-thread work. +const ANCHORED = ` + +
`; + const FIXTURE = ` +
`; + +async function busyFixture(page: Page) { + await page.route('http://busy.test/', route => + route.fulfill({contentType: 'text/html', body: BUSY_FIXTURE}), + ); + await page.goto('http://busy.test/'); +} + // A real navigation (not setContent, whose document.open() erases the window // listeners the input log installs). async function fixture(page: Page) { @@ -242,6 +303,1120 @@ test.describe('touch input', () => { }); }); +test.describe('frame evidence', () => { + test('a compositor animation advances every frame through a blocked main thread; a rAF one freezes, whatever the screencast rate', async ({ + page, + }, testInfo) => { + await busyFixture(page); + const box = page.locator('#box'); + await markElement(box, {mode: 'outline'}); + + const run = async (driver: 'compositor' | 'raf', everyNthFrame = 1) => { + await box.evaluate((el, how) => { + el.getAnimations().forEach(a => a.cancel()); + el.style.transform = 'none'; + if (how === 'compositor') { + el.animate( + [{transform: 'translateY(0px)'}, {transform: 'translateY(560px)'}], + {duration: 1600, easing: 'linear'}, + ); + } else { + const start = performance.now(); + const step = (now: number) => { + const p = Math.min(1, (now - start) / 1600); + el.style.transform = `translateY(${560 * p}px)`; + if (p < 1) { + requestAnimationFrame(step); + } + }; + requestAnimationFrame(step); + } + }, driver); + const cast = await startScreencast(page, {everyNthFrame}); + await page.waitForTimeout(200); + const window = await blockMainThread(page, 700); + await page.waitForTimeout(200); + const tracked = await trackMarker(page, await cast.stop()); + return busyWindowProof(tracked, window, {axis: 'y'}); + }; + + const compositor = await run('compositor'); + testInfo.annotations.push({ + type: 'busy-main-thread compositor', + description: compositor.summary, + }); + expect(compositor.verdict.failures, compositor.summary).toEqual([]); + expect(compositor.monotonic).toBe(true); + // Resolvable on the cast's grid: two position steps or more per frame. + expect(compositor.notes, compositor.summary).toEqual([]); + expect(compositor.pxPerFrame).toBeGreaterThanOrEqual(2 * compositor.stepPx); + + const raf = await run('raf'); + testInfo.annotations.push({ + type: 'busy-main-thread rAF control', + description: raf.summary, + }); + // Frozen for the whole block: no presented frame moved (the compositor + // sends none at all when nothing on screen changes), and the verdict that + // passes the compositor animation fails it on every criterion. + expect(raf.changed).toBe(0); + expect(raf.longestStallMs).toBeGreaterThan(600); + expect(raf.verdict.pass).toBe(false); + expect(raf.verdict.failures.join('\n')).toMatch(/longest stall/); + expect(raf.verdict.failures.join('\n')).toMatch(/expected frames moved/); + + // The verdict judges the frames the screencast delivered, not its rate: + // at one frame in six (a loaded machine's ~100 ms), the compositor + // animation still passes and the rAF one still fails. + const slow = await run('compositor', 6); + expect(slow.cadenceMs, slow.summary).toBeGreaterThan(60); + expect(slow.verdict.failures, slow.summary).toEqual([]); + const slowRaf = await run('raf', 6); + expect(slowRaf.verdict.pass, slowRaf.summary).toBe(false); + expect(slowRaf.verdict.failures.join('\n')).toMatch(/longest stall/); + }); + + test('frames the screencast skips while the compositor presents them are not a stall; a frozen animation still fails with the compositor record', async ({ + page, + browserName, + }, testInfo) => { + test.skip( + browserName !== 'chromium', + 'compositor traces are Chromium-only (README: engine limits)', + ); + await busyFixture(page); + const box = page.locator('#box'); + await markElement(box, {mode: 'outline'}); + const run = async (driver: 'compositor' | 'raf') => { + await box.evaluate((el, how) => { + el.getAnimations().forEach(a => a.cancel()); + el.style.transform = 'none'; + if (how === 'compositor') { + el.animate( + [{transform: 'translateY(0px)'}, {transform: 'translateY(560px)'}], + {duration: 1600, easing: 'linear'}, + ); + } else { + const start = performance.now(); + const step = (now: number) => { + const p = Math.min(1, (now - start) / 1600); + el.style.transform = `translateY(${560 * p}px)`; + if (p < 1) { + requestAnimationFrame(step); + } + }; + requestAnimationFrame(step); + } + }, driver); + const cast = await startScreencast(page); + const compositor = await startCompositorTrace(page); + await page.waitForTimeout(200); + const window = await blockMainThread(page, 700); + await page.waitForTimeout(200); + const presented = await compositor.stop(); + const tracked = await trackMarker(page, await cast.stop()); + return {window, tracked, presented}; + }; + + const moving = await run('compositor'); + expect(moving.presented.length).toBeGreaterThan(0); + // The screencast as a loaded machine delivers it: four frames in a row + // skipped mid-block, which the compositor still presented. + const mid = moving.tracked.findIndex(f => f.t > moving.window.from + 350); + const skipped = moving.tracked.filter((_, i) => i < mid || i >= mid + 4); + const alone = busyWindowProof(skipped, moving.window, {axis: 'y'}); + expect(alone.verdict.failures.join('\n'), alone.summary).toMatch( + /longest stall/, + ); + const proof = busyWindowProof(skipped, moving.window, { + axis: 'y', + presented: moving.presented, + }); + testInfo.annotations.push({ + type: 'busy-main-thread compositor record', + description: proof.summary, + }); + expect(proof.verdict.failures, proof.summary).toEqual([]); + expect(proof.recorderSkipped).toBeGreaterThanOrEqual(4); + + // A rAF animation presents no change for the whole block: the record + // excuses nothing, whatever the compositor drew meanwhile. + const frozen = await run('raf'); + const still = busyWindowProof(frozen.tracked, frozen.window, { + axis: 'y', + presented: frozen.presented, + }); + testInfo.annotations.push({ + type: 'busy-main-thread rAF control, compositor record', + description: `${still.summary}; compositor frames inside the block: ${frozen.presented.filter(t => t > frozen.window.from && t < frozen.window.to).length}`, + }); + expect(still.verdict.pass).toBe(false); + expect(still.recorderSkipped).toBe(0); + expect(still.verdict.failures.join('\n')).toMatch(/longest stall/); + }); + + test('a block that starts during a just-started transition still lets the compositor advance it', async ({ + page, + }, testInfo) => { + // `during` blocks in the frame the transition has run 16 ms. In WebKit + // the transition reaches the compositor in that frame's rendering update, + // so a loop inside the rAF callback froze it (2 of 29 frames moved). + // Linear and longer than the block, so every frame of the block moves two + // position steps or more and the transition is still running when it + // ends. The cast shows the box at rest first, so a transition that reaches + // the screen after the block began is judged from there (onScreenFromMs). + await busyFixture(page); + const box = page.locator('#box'); + await markElement(box, {mode: 'outline'}); + const cast = await startScreencast(page); + await expect.poll(() => cast.frameCount()).toBeGreaterThan(0); + await box.evaluate(el => { + el.style.transition = 'transform 1600ms linear'; + el.getBoundingClientRect(); + el.style.transform = 'translateY(560px)'; + }); + const window = await blockMainThread(page, 700, { + during: box, + minProgressMs: 16, + }); + await waitForRest(box); + const tracked = await trackMarker(page, await cast.stop()); + const proof = busyWindowProof(tracked, window, {axis: 'y'}); + testInfo.annotations.push({ + type: 'busy-main-thread transition', + description: proof.summary, + }); + expect(proof.verdict.failures, proof.summary).toEqual([]); + expect(proof.monotonic).toBe(true); + }); + + test('a block never starts on an animation that is not on the compositor yet', async ({ + page, + }, testInfo) => { + // An animation created in the frame the helper looks is still pending (not + // committed to the compositor). A block that started on it then froze it + // in Chromium: no presented frame moved. + await busyFixture(page); + const box = page.locator('#box'); + await markElement(box, {mode: 'outline'}); + await box.evaluate(el => { + el.style.willChange = 'transform'; + }); + const cast = await startScreencast(page); + // A rAF loop registered before the helper's tick runs ahead of it in every + // frame; on its 12th frame it creates the animation, and the helper's tick + // looks in that same frame. + await box.evaluate(el => { + let n = 0; + const loop = () => { + n += 1; + if (n === 12) { + el.animate( + [{transform: 'translateY(0px)'}, {transform: 'translateY(560px)'}], + {duration: 1600, easing: 'linear'}, + ); + return; + } + requestAnimationFrame(loop); + }; + requestAnimationFrame(loop); + }); + const window = await blockMainThread(page, 700, { + during: box, + minProgressMs: 0, + }); + await waitForRest(box); + const tracked = await trackMarker(page, await cast.stop()); + const proof = busyWindowProof(tracked, window, {axis: 'y'}); + testInfo.annotations.push({ + type: 'busy-main-thread pending animation', + description: proof.summary, + }); + expect(proof.verdict.failures, proof.summary).toEqual([]); + expect(proof.monotonic).toBe(true); + }); + + test('in the WebKit container, compiled shaders outlive the run', async ({ + browser, + }, testInfo) => { + // The official Playwright image keeps its browsers here. + test.skip( + testInfo.project.name !== 'webkit' || + process.env.PLAYWRIGHT_BROWSERS_PATH !== '/ms-playwright', + 'runs inside webkit-container.mjs', + ); + // A cache inside the container dies with it: every run then compiled each + // shader again, 450–700 ms in the first frame of a blurred layer. + const dir = process.env.MESA_SHADER_CACHE_DIR; + expect(dir, 'MESA_SHADER_CACHE_DIR names the host-kept cache').toBeTruthy(); + // Written on the host by the script before the container starts. + expect( + fs.readFileSync(path.join(dir ?? '', 'ASTRYX-SHADER-CACHE'), 'utf8'), + ).toContain('playwright'); + // Mesa writes what it compiles there once a page draws a blurred layer. + // Earlier runs have filled the cache already, so this run's browser gets + // an empty directory beside them, in the same host-kept mount, and the + // test watches its own writes arrive. + const fresh = fs.mkdtempSync(path.join(dir ?? '', 'self-test-')); + try { + const own = await browser.browserType().launch({ + env: {...process.env, MESA_SHADER_CACHE_DIR: fresh}, + }); + try { + const page = await own.newPage(); + await page.setContent( + '
', + ); + await page.evaluate( + () => + document + .getElementById('glass') + ?.animate([{scale: '1'}, {scale: '0.9'}], {duration: 200}) + .finished, + ); + } finally { + await own.close(); + } + expect(fs.readdirSync(fresh, {recursive: true})).not.toEqual([]); + } finally { + fs.rmSync(fresh, {recursive: true, force: true}); + } + }); + + test('rect sampling reads one rect per animation frame while the main thread is free', async ({ + page, + browserName, + }) => { + await fixture(page); + const box = page.locator('#box'); + await box.evaluate(el => { + el.animate( + [{transform: 'translateX(0px)'}, {transform: 'translateX(200px)'}], + {duration: 600, easing: 'linear'}, + ); + }); + const samples = await sampleRects(box, 400); + const xs = samples.map(s => s.box?.x ?? NaN); + const gaps = samples + .slice(1) + .map((s, i) => s.t - samples[i].t) + .sort((a, b) => a - b); + const cadence = gaps[Math.floor(gaps.length / 2)]; + expect(cadence).toBeLessThan(34); + expect(samples.length).toBeGreaterThanOrEqual(10); + if (browserName === 'chromium') { + // One sample per presented frame at 60 Hz. (Headless WebKit's animation + // frames are irregular, so only the cadence is checked there.) + expect(samples.length).toBeGreaterThanOrEqual(20); + } + expect(xs.every((x, i) => i === 0 || x >= xs[i - 1])).toBe(true); + expect(xs.at(-1) ?? 0).toBeGreaterThan(xs[0] + 80); + }); + + test('the trace counts no Layout for a transform animation and one per frame for `top`', async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'rendering traces are Chromium-only (README: engine limits)', + ); + await fixture(page); + const box = page.locator('#box'); + // Count the animation's steady middle: starting and ending it may lay out once each. + const animate = async (property: 'transform' | 'top') => { + let started = 0; + const trace = await traceRendering(page, async () => { + started = await box.evaluate((el, prop) => { + const keyframes = + prop === 'transform' + ? [ + {transform: 'translateY(0px)'}, + {transform: 'translateY(200px)'}, + ] + : [{top: '360px'}, {top: '560px'}]; + el.animate(keyframes, {duration: 600}); + return performance.timeOrigin + performance.now(); + }, property); + await page.waitForTimeout(700); + }); + return countBetween(trace, started + 100, started + 500); + }; + const transform = await animate('transform'); + const top = await animate('top'); + expect(transform.Layout ?? 0).toBe(0); + expect(transform.Paint ?? 0).toBe(0); + // ~24 frames in 400 ms: one layout per frame. + expect(top.Layout ?? 0).toBeGreaterThan(18); + }); + + test('with invalidations, a trace tells a compositor animation from one the compositor cannot run', async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'rendering traces are Chromium-only (README: engine limits)', + ); + await fixture(page); + const box = page.locator('#box'); + const animate = async (property: 'transform' | 'top') => { + let started = 0; + const trace = await traceRendering( + page, + async () => { + started = await box.evaluate((el, prop) => { + el.animate( + prop === 'transform' + ? [ + {transform: 'translateY(0px)'}, + {transform: 'translateY(200px)'}, + ] + : [{top: '360px'}, {top: '560px'}], + {duration: 600}, + ); + return performance.timeOrigin + performance.now(); + }, property); + await page.waitForTimeout(700); + }, + {invalidations: true}, + ); + return {trace, started}; + }; + const transform = await animate('transform'); + expect(transform.trace.animations).toEqual([ + expect.objectContaining({composited: true}), + ]); + expect( + transform.trace.styleInvalidations?.some( + inv => inv.reason === 'Animation', + ), + ).toBe(true); + const top = await animate('top'); + expect(top.trace.animations).toEqual([ + expect.objectContaining({composited: false}), + ]); + // Its per-frame layout is the page's: none of it is left out. + expect( + countBetween(top.trace, top.started + 100, top.started + 500, { + skipCompositorAnimationUpdates: true, + }).Layout ?? 0, + ).toBeGreaterThan(18); + }); + + test("the trace counts no Layout from the harness's own animation recorder, which stands down while it runs", async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'rendering traces are Chromium-only (README: engine limits)', + ); + await page.route('http://anchored.test/', route => + route.fulfill({contentType: 'text/html', body: ANCHORED}), + ); + await page.goto('http://anchored.test/'); + const pop = page.locator('#pop'); + // The recorder runs, as in any spec that waited for an animation first. + await waitForRest(pop); + const watch = await watchAnimations(pop); + const middle = async (askEveryFrame: boolean) => { + let started = 0; + const trace = await traceRendering(page, async () => { + started = await pop.evaluate((el, every) => { + const w = window as unknown as {stop?: boolean}; + w.stop = false; + if (every) { + const ask = () => { + if (!w.stop) { + requestAnimationFrame(ask); + } + }; + requestAnimationFrame(ask); + } + el.animate( + [ + {transform: 'translateY(0px)', opacity: 1}, + {transform: 'translateY(100px)', opacity: 0.5}, + ], + {duration: 600}, + ); + return performance.timeOrigin + performance.now(); + }, askEveryFrame); + await page.waitForTimeout(700); + await page.evaluate(() => { + (window as unknown as {stop?: boolean}).stop = true; + }); + }); + // The steady middle: starting and ending it may lay out once each. + return countBetween(trace, started + 100, started + 500); + }; + expect((await middle(false)).Layout ?? 0).toBe(0); + // Control: a page that asks for a frame every frame lays it out in each. + expect((await middle(true)).Layout ?? 0).toBeGreaterThan(18); + // Nothing the traces ran went unrecorded. + const started = await watch.started(); + expect(started.map(a => a.durationMs)).toEqual([600, 600]); + }); + + test('a waitForRest awaited inside a traced body stands down too, so the trace counts no Layout from it', async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'rendering traces are Chromium-only (README: engine limits)', + ); + await page.route('http://anchored.test/', route => + route.fulfill({contentType: 'text/html', body: ANCHORED}), + ); + await page.goto('http://anchored.test/'); + const pop = page.locator('#pop'); + await waitForRest(pop); + let started = 0; + const trace = await traceRendering(page, async () => { + started = await pop.evaluate(el => { + el.animate( + [ + {transform: 'translateY(0px)', opacity: 1}, + {transform: 'translateY(100px)', opacity: 0.5}, + ], + {duration: 600}, + ); + return performance.timeOrigin + performance.now(); + }); + // A per-frame wait here would make every frame a main-thread frame. + await waitForRest(pop); + }); + expect(countBetween(trace, started + 100, started + 500).Layout ?? 0).toBe( + 0, + ); + // It still waited for the curve to end. + expect(trace.window.to - started).toBeGreaterThanOrEqual(590); + }); + + test('a waitForAnimation or watch.started() awaited inside a traced body stands down too, so the trace counts no Layout from it', async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'rendering traces are Chromium-only (README: engine limits)', + ); + await page.route('http://anchored.test/', route => + route.fulfill({contentType: 'text/html', body: ANCHORED}), + ); + await page.goto('http://anchored.test/'); + const pop = page.locator('#pop'); + await waitForRest(pop); + const watch = await watchAnimations(pop); + let started = 0; + let waited: {durationMs: number}[] = []; + let watched: {durationMs: number}[] = []; + const trace = await traceRendering(page, async () => { + started = await pop.evaluate(el => { + el.animate( + [ + {transform: 'translateY(0px)', opacity: 1}, + {transform: 'translateY(100px)', opacity: 0.5}, + ], + {duration: 600}, + ); + return performance.timeOrigin + performance.now(); + }); + // Both poll until 450 ms of the curve has played, across the steady + // middle; a read of the page's animations each time would lay it out. + waited = await waitForAnimation(pop, {minProgressMs: 450}); + watched = await watch.started({minProgressMs: 450}); + await page.waitForTimeout(200); + }); + expect(countBetween(trace, started + 100, started + 500).Layout ?? 0).toBe( + 0, + ); + // Each still answered with the animation once 450 ms of it had played. + expect(waited.map(a => a.durationMs)).toEqual([600]); + expect(watched.map(a => a.durationMs)).toEqual([600]); + expect(trace.window.to - started).toBeGreaterThanOrEqual(590); + }); + + test('a waitForRest awaited inside a traced body waits for an animation the action starts two frames later', async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'rendering traces are Chromium-only (README: engine limits)', + ); + await page.route('http://anchored.test/', route => + route.fulfill({contentType: 'text/html', body: ANCHORED}), + ); + await page.goto('http://anchored.test/'); + const pop = page.locator('#pop'); + await waitForRest(pop); + const trace = await traceRendering(page, async () => { + // The action starts the animation two frames later, as a component's + // effect often does. + await pop.evaluate(el => { + requestAnimationFrame(() => + requestAnimationFrame(() => { + (window as unknown as {__started: number}).__started = + performance.timeOrigin + performance.now(); + el.animate( + [ + {transform: 'translateY(0px)'}, + {transform: 'translateY(100px)'}, + ], + {duration: 600}, + ); + }), + ); + }); + await waitForRest(pop); + }); + const started = await page.evaluate( + () => (window as unknown as {__started?: number}).__started ?? null, + ); + // The traced window holds the whole animation. + expect(started).not.toBeNull(); + expect(trace.window.to - (started as number)).toBeGreaterThanOrEqual(590); + }); + + test('a waitForRest awaited inside a traced body waits for a run going backward to its end, and no longer', async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'rendering traces are Chromium-only (README: engine limits)', + ); + await page.route('http://anchored.test/', route => + route.fulfill({contentType: 'text/html', body: ANCHORED}), + ); + await page.goto('http://anchored.test/'); + const pop = page.locator('#pop'); + await waitForRest(pop); + // How long a traced waitForRest holds after a 600 ms move starts. + const traced = async (reverse: boolean, endDelay = 0) => { + let started = 0; + const trace = await traceRendering(page, async () => { + started = await pop.evaluate( + (el, {reverse, endDelay}) => { + const run = el.animate( + [ + {transform: 'translateY(0px)'}, + {transform: 'translateY(100px)'}, + ], + {duration: 600, endDelay}, + ); + if (reverse) { + // Its current time falls from its end time to 0. + run.reverse(); + } + return performance.timeOrigin + performance.now(); + }, + {reverse, endDelay}, + ); + await waitForRest(pop); + }); + await waitForRest(pop); + return trace.window.to - started; + }; + // Reversed with a 200 ms end delay: the move plays from 200 to 800 ms. + expect(await traced(true, 200)).toBeGreaterThanOrEqual(790); + // A reversed run is waited for as long as the same run forward, not + // twice as long. + const forward = await traced(false); + const backward = await traced(true); + expect(backward).toBeGreaterThanOrEqual(590); + expect(backward).toBeLessThan(forward * 1.5); + }); + + test('a waitForRest awaited inside a traced body runs on the real timers, so a frozen timer gate cannot hold it', async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'rendering traces are Chromium-only (README: engine limits)', + ); + await page.route('http://anchored.test/', route => + route.fulfill({contentType: 'text/html', body: ANCHORED}), + ); + await page.goto('http://anchored.test/'); + const pop = page.locator('#pop'); + await waitForRest(pop); + await installTimerGate(page); + await freezeTimers(page); + let outcome = ''; + await traceRendering(page, async () => { + await pop.evaluate(el => { + el.animate( + [{transform: 'translateY(0px)'}, {transform: 'translateY(100px)'}], + {duration: 300}, + ); + }); + // Held by the gate, the wait would neither resolve nor reach its + // deadline: the race reads that as "held". + outcome = await Promise.race([ + waitForRest(pop, {timeout: 2000}).then( + () => 'rested', + () => 'rejected', + ), + new Promise(resolve => setTimeout(() => resolve('held'), 4000)), + ]); + }); + await thawTimers(page); + expect(outcome).toBe('rested'); + }); + + test('inside a trace, waitForRest rests beside a scroll-timeline animation', async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'rendering traces are Chromium-only (README: engine limits)', + ); + await fixture(page); + const scroller = page.locator('#scroller'); + await waitForRest(scroller); + let outcome = ''; + await traceRendering(page, async () => { + // A row driven by its scroller's timeline: it moves only on scroll. + await scroller.evaluate(el => { + const row = el.firstElementChild as HTMLElement; + row.animate([{opacity: 0.5}, {opacity: 1}], { + timeline: new ScrollTimeline({source: el}), + }); + }); + outcome = await waitForRest(scroller, {timeout: 2000}).then( + () => 'rested', + (error: Error) => error.message, + ); + }); + expect(outcome).toBe('rested'); + }); + + test('a CSS transition and animation that start and end inside a trace are reported, and lay out nothing in between', async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'rendering traces are Chromium-only (README: engine limits)', + ); + await page.route('http://anchored.test/', route => + route.fulfill({contentType: 'text/html', body: ANCHORED}), + ); + await page.goto('http://anchored.test/'); + await page.addStyleTag({ + content: + '@keyframes harness-fade { from { opacity: 1 } to { opacity: 0.6 } }', + }); + const pop = page.locator('#pop'); + // The recorder runs, and a watch is taken before the trace. + await waitForRest(pop); + const watch = await watchAnimations(pop); + let started = 0; + const trace = await traceRendering(page, async () => { + started = await pop.evaluate(el => { + const pop = el as HTMLElement; + pop.style.transition = 'transform 400ms linear'; + void pop.offsetWidth; + pop.style.transform = 'translateY(100px)'; + pop.style.animation = 'harness-fade 400ms linear'; + return performance.timeOrigin + performance.now(); + }); + await page.waitForTimeout(600); + }); + // Its reads at the two starts fall before the steady middle. + expect(countBetween(trace, started + 100, started + 300).Layout ?? 0).toBe( + 0, + ); + const ended = await watch.started({timeout: 1_000}); + expect(ended.map(a => a.kind).sort()).toEqual([ + 'CSSAnimation', + 'CSSTransition', + ]); + expect(ended.every(a => a.finishedAt != null)).toBe(true); + }); + + test('the compositor record refuses another open page, whose frames it could not tell apart', async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'the compositor record is a Chromium trace (README: engine limits)', + ); + await fixture(page); + // Another page animating: its presented frames would be credited to + // this one, so the record refuses to start. + const other = await page.context().newPage(); + await other.setContent( + '
', + ); + await expect(startCompositorTrace(page)).rejects.toThrow( + /another page is open as the trace starts/, + ); + await other.close(); + // A page opened mid-trace: stop() refuses too, whether it is still open + // or closed again before the record stops. + const record = await startCompositorTrace(page); + const late = await page.context().newPage(); + await expect(record.stop()).rejects.toThrow( + /another page opened during the record/, + ); + await late.close(); + const brief = await startCompositorTrace(page); + const spinner = await page.context().newPage(); + await spinner.setContent( + '
', + ); + await page.waitForTimeout(200); + await spinner.close(); + await expect(brief.stop()).rejects.toThrow( + /another page opened during the record/, + ); + // In another context too. + const browser = page.context().browser(); + if (browser != null) { + const elsewhere = await startCompositorTrace(page); + const context = await browser.newContext(); + await (await context.newPage()).close(); + await context.close(); + await expect(elsewhere.stop()).rejects.toThrow( + /another page opened during the record/, + ); + } + // Alone, it records. + const alone = await startCompositorTrace(page); + await page.waitForTimeout(100); + await expect(alone.stop()).resolves.toBeInstanceOf(Array); + }); + + test('the long-task log fails closed when it never ran on the document', async ({ + page, + }) => { + // No installLongTaskLog before navigating: an empty answer would read as + // "no long task", so it throws instead. + await fixture(page); + await expect(readLongTasks(page)).rejects.toThrow( + /not running on this document/, + ); + }); + + test("the long-task log records the page's own long task but not a DevTools evaluate, which the trace reports apart", async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'the Long Tasks API and rendering traces are Chromium-only', + ); + await installLongTaskLog(page); + await fixture(page); + // The page's own task: a timer that spins 120 ms. + const own = await page.evaluate( + () => + new Promise<{from: number; to: number}>(resolve => { + setTimeout(() => { + const from = performance.timeOrigin + performance.now(); + const end = performance.now() + 120; + while (performance.now() < end) { + // spin + } + resolve({from, to: performance.timeOrigin + performance.now()}); + }, 0); + }), + ); + await frames(page, 2); + const recorded = await readLongTasks(page, own); + expect(recorded).toHaveLength(1); + expect(recorded[0].durationMs).toBeGreaterThanOrEqual(110); + // A DevTools evaluate that spins 300 ms: the trace reports it apart from + // the page's long tasks, and the log never sees it. + let block = {from: 0, to: 0}; + const trace = await traceRendering(page, async () => { + block = await blockMainThread(page, 300); + }); + await frames(page, 2); + const inBlock = (t: {start: number}) => + t.start < block.to && t.start >= block.from - 20; + expect( + trace.devtoolsTasks.some(t => t.durationMs >= 290 && inBlock(t)), + ).toBe(true); + expect(trace.longTasks.filter(inBlock)).toEqual([]); + expect(await readLongTasks(page, block)).toEqual([]); + }); + + test("a harness read inside a trace is not the page's long task, while a test's own evaluate is, whether it calls the page directly or sets it off", async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'rendering traces are Chromium-only (README: engine limits)', + ); + // Enough animated rows that describing their keyframes takes a DevTools + // task over 50 ms. + const rows = Array.from( + {length: 1200}, + (_, i) => `
row ${i}
`, + ).join(''); + await page.setContent( + `
${rows}
`, + ); + const slow = page.locator('#slow'); + await slow.evaluate(el => + el.addEventListener('click', () => { + const end = performance.now() + 60; + while (performance.now() < end) { + // spin + } + }), + ); + // Page code a test can call directly, as through a story's own API. + await page.evaluate(() => { + (window as unknown as {pageWork: (ms: number) => void}).pageWork = ( + ms: number, + ) => { + const end = performance.now() + ms; + while (performance.now() < end) { + // spin + } + }; + }); + await frames(page, 3); + const now = () => + page.evaluate(() => performance.timeOrigin + performance.now()); + const read = {from: 0, to: 0}; + const click = {from: 0, to: 0}; + const direct = {from: 0, to: 0}; + let timer = {from: 0, to: 0}; + const trace = await traceRendering(page, async () => { + read.from = await now(); + await waitForAnimation(page.locator('#list')); + read.to = await now(); + // On a slow machine each frame of 1,200 running rows is a long task of + // the page's own; the windows below hold only their own work. + await page.evaluate(() => document.getElementById('list')?.remove()); + await frames(page, 2); + // The page's own task: a timer that spins 60 ms. + timer = await page.evaluate( + () => + new Promise<{from: number; to: number}>(resolve => { + setTimeout(() => { + const from = performance.timeOrigin + performance.now(); + const end = performance.now() + 60; + while (performance.now() < end) { + // spin + } + resolve({from, to: performance.timeOrigin + performance.now()}); + }, 0); + }), + ); + // Page script run inside a DevTools evaluate: a 60 ms click handler. + click.from = await now(); + await slow.evaluate(el => (el as HTMLElement).click()); + click.to = await now(); + // Page code a test's evaluate calls directly: 60 ms of the page's work. + direct.from = await now(); + await page.evaluate(() => + (window as unknown as {pageWork: (ms: number) => void}).pageWork(60), + ); + direct.to = await now(); + }); + const within = (w: {from: number; to: number}) => (t: {start: number}) => + t.start >= w.from - 20 && t.start < w.to; + // The read itself is set apart. The rows' frames around it may be long + // tasks too, the page's own: they stay in longTasks. + expect(trace.devtoolsTasks.some(within(read))).toBe(true); + expect( + trace.longTasks.some(t => within(timer)(t) && t.durationMs >= 55), + ).toBe(true); + expect( + trace.longTasks.some(t => within(click)(t) && t.durationMs >= 55), + ).toBe(true); + expect( + trace.longTasks.some(t => within(direct)(t) && t.durationMs >= 55), + ).toBe(true); + expect(trace.devtoolsTasks.filter(within(direct))).toEqual([]); + }); +}); + +test.describe('wheel and pen input', () => { + /** A non-passive wheel listener: Chromium then waits for the page itself. */ + const listenNonPassively = (page: Page) => + page.evaluate(() => + window.addEventListener('wheel', () => {}, {passive: false}), + ); + /** The wheel steps the page received. */ + const wheelsOf = async (page: Page) => + (await readInputLog(page)).filter(r => r.type === 'wheel'); + + test('a wheel step reaches the page as exactly its deltas, trusted, with Control when held, by the time wheel() resolves on a page whose wheel listeners are all passive', async ({ + page, + }) => { + await fixture(page); + const driver = await webInputDriver(page); + test.skip(!driver.wheels, WHEEL_SKIP); + await driver.wheel({x: 60, y: 520, dy: 120}); + // The input log listens passively, so Chromium answers the dispatch + // before the page has the step: read right after the call. + expect(await wheelsOf(page)).toHaveLength(1); + await driver.wheel({x: 60, y: 520, dx: 30, dy: -40, ctrl: true}); + const wheels = await wheelsOf(page); + expect(wheels).toHaveLength(2); + // Chromium's phone context has a device scale factor of 3: the driver + // multiplies back what Chromium divides by, so the page sees CSS px. + expect(wheels[0]).toMatchObject({ + isTrusted: true, + deltaX: 0, + deltaY: 120, + deltaMode: 0, + ctrlKey: false, + x: 60, + y: 520, + }); + expect(wheels[1]).toMatchObject({ + isTrusted: true, + deltaX: 30, + deltaY: -40, + deltaMode: 0, + ctrlKey: true, + }); + }); + + test('a wheel step reaches the page by the time wheel() resolves with a non-passive wheel listener too', async ({ + page, + }) => { + await fixture(page); + const driver = await webInputDriver(page); + test.skip(!driver.wheels, WHEEL_SKIP); + await listenNonPassively(page); + await driver.wheel({x: 60, y: 520, dy: 80}); + expect(await wheelsOf(page)).toHaveLength(1); + await driver.wheel({x: 60, y: 520, dy: -20, ctrl: true}); + const wheels = await wheelsOf(page); + expect(wheels).toHaveLength(2); + expect(wheels[0]).toMatchObject({ + isTrusted: true, + deltaY: 80, + ctrlKey: false, + }); + expect(wheels[1]).toMatchObject({ + isTrusted: true, + deltaY: -20, + ctrlKey: true, + }); + }); + + test('a pen stroke reaches the page as trusted pen pointers, with its pressure, at its scripted times', async ({ + page, + }) => { + await fixture(page); + const driver = await webInputDriver(page); + test.skip(!driver.pens, PEN_SKIP); + await driver.pen([ + {x: 100, y: 100, t: 0, pressure: 0.3}, + {x: 100, y: 140, t: 50, pressure: 0.6}, + {x: 100, y: 180, t: 100, pressure: 0.6}, + ]); + const log = await readInputLog(page); + const down = log.find(r => r.type === 'pointerdown'); + const moves = log.filter(r => r.type === 'pointermove'); + const up = log.find(r => r.type === 'pointerup'); + expect(down).toMatchObject({ + isTrusted: true, + pointerType: 'pen', + x: 100, + y: 100, + }); + expect(down?.pressure).toBeCloseTo(0.3, 2); + expect(moves.at(-1)).toMatchObject({pointerType: 'pen', y: 180}); + expect(moves.at(-1)?.pressure).toBeCloseTo(0.6, 2); + expect(up).toMatchObject({isTrusted: true, pointerType: 'pen'}); + // The scripted 100 ms between the first and last points reach the page. + expect( + (moves.at(-1)?.timeStamp ?? NaN) - (down?.timeStamp ?? NaN), + ).toBeCloseTo(100, 0); + }); + + test('a stroke sent right after a navigation presses where it is scripted', async ({ + page, + }) => { + // The driver exists before the page navigates, and the stroke goes out + // the moment the page has loaded: the press must not take the previous + // document's page scale (about:blank's, in the phone project). + const driver = await webInputDriver(page); + test.skip(!driver.pens, PEN_SKIP); + await fixture(page); + await driver.pen([ + {x: 100, y: 100, t: 0, pressure: 0.5}, + {x: 100, y: 120, t: 30, pressure: 0.5}, + ]); + const down = (await readInputLog(page)).find(r => r.type === 'pointerdown'); + expect(down).toMatchObject({pointerType: 'pen', x: 100, y: 100}); + }); + + test('says which inputs it has: WebKit in a phone context has no wheel and no pen', async ({ + page, + browserName, + }) => { + await fixture(page); + const driver = await webInputDriver(page); + if (browserName === 'chromium') { + expect({wheels: driver.wheels, pens: driver.pens}).toEqual({ + wheels: true, + pens: true, + }); + return; + } + expect({wheels: driver.wheels, pens: driver.pens}).toEqual({ + wheels: false, + pens: false, + }); + await expect(driver.wheel({x: 60, y: 520, dy: 120})).rejects.toThrow( + WHEEL_SKIP, + ); + await expect(driver.pen([{x: 100, y: 100, t: 0}])).rejects.toThrow( + PEN_SKIP, + ); + }); + + test.describe('in a desktop context', () => { + test.use({isMobile: false, hasTouch: false, deviceScaleFactor: 1}); + + test('a wheel step reaches the page as exactly its deltas in both engines', async ({ + page, + }) => { + await fixture(page); + const driver = await webInputDriver(page); + expect(driver.wheels).toBe(true); + await driver.wheel({x: 60, y: 520, dy: 120}); + expect(await wheelsOf(page)).toHaveLength(1); + await driver.wheel({x: 60, y: 520, dx: -25, dy: 0, ctrl: true}); + const wheels = await wheelsOf(page); + expect(wheels).toHaveLength(2); + expect(wheels[0]).toMatchObject({ + isTrusted: true, + deltaX: 0, + deltaY: 120, + deltaMode: 0, + ctrlKey: false, + }); + expect(wheels[1]).toMatchObject({ + isTrusted: true, + deltaX: -25, + deltaY: 0, + deltaMode: 0, + ctrlKey: true, + }); + }); + }); +}); + test.describe('curve sampling', () => { test('a CSS transition samples onto its cubic-bezier to 0.05 px', async ({ page, @@ -307,6 +1482,81 @@ test.describe('curve sampling', () => { }); }); +// A full-size veil on the fixture's box, drawn by its ::before. +const VEIL_STYLE = + "#box::before { content: ''; position: absolute; inset: 0; background: #000; opacity: 0; }"; + +test.describe('pseudo-element animations', () => { + test('sampleCurve reads a ::before animation when given its pseudo-element', async ({ + page, + }) => { + await fixture(page); + const box = page.locator('#box'); + await box.evaluate((el, veil) => { + const style = document.createElement('style'); + style.textContent = veil; + document.head.append(style); + el.animate( + [ + {opacity: 0, transform: 'translateY(0px)'}, + {opacity: 1, transform: 'translateY(40px)'}, + ], + {duration: 300, easing: 'linear', pseudoElement: '::before'}, + ); + }, VEIL_STYLE); + await holdAnimations(page, box); + const samples = await sampleCurve(box, [0, 75, 150, 225, 300], { + pseudoElement: '::before', + }); + // Labelled by the veil's own clock: the last read just inside its end. + expect(samples.at(-1)).toMatchObject({t: 300, presentedAt: 299.5}); + const fade = compareCurve(curveOf(samples, 'opacity'), t => t / 300, 0.005); + expect(fade.maxError, JSON.stringify(fade.worst)).toBeLessThanOrEqual( + 0.005, + ); + const move = compareCurve( + curveOf(samples, 'translateY'), + t => (40 * t) / 300, + 0.05, + ); + expect(move.maxError, JSON.stringify(move.worst)).toBeLessThanOrEqual(0.05); + // Without it, the box itself, which does not move. + const [boxAt150] = await sampleCurve(box, [150]); + expect(boxAt150.opacity).toBe(1); + expect(boxAt150.transform.translateY).toBe(0); + }); + + test('waitForRest waits out a ::before animation', async ({page}) => { + await fixture(page); + const box = page.locator('#box'); + await box.evaluate((el, veil) => { + const style = document.createElement('style'); + style.textContent = veil; + document.head.append(style); + (window as unknown as {__veil: Animation}).__veil = el.animate( + [{opacity: 0}, {opacity: 1}], + { + duration: 600, + easing: 'linear', + fill: 'forwards', + pseudoElement: '::before', + }, + ); + }, VEIL_STYLE); + // Its subtree read includes the element's pseudo-elements. + await waitForRest(box); + const veil = await box.evaluate(element => { + const a = (window as unknown as {__veil: Animation}).__veil; + return { + playState: a.playState, + currentTime: Number(a.currentTime), + opacity: getComputedStyle(element, '::before').opacity, + }; + }); + expect(veil).toMatchObject({playState: 'finished', opacity: '1'}); + }); +}); + test.describe('holding', () => { test('a sample at an end reports the time it was read at', async ({page}) => { await fixture(page); @@ -961,6 +2211,91 @@ test.describe('animation state', () => { expect(started.map(a => a.durationMs)).toEqual([300]); }); + test('while a rendering trace runs the recorder reads only at CSS starts, and catches up after it', async ({ + page, + }) => { + await fixture(page); + const box = page.locator('#box'); + const watch = await watchAnimations(box); + // Counts the recorder's reads of the page's animations (each one flushes + // style): the whole document's, and one element's. Then flags a trace, as + // traceRendering does. + await page.evaluate(() => { + const w = window as unknown as { + __reads: number; + __elementReads: number; + __harnessRenderingTrace?: boolean; + }; + w.__reads = 0; + w.__elementReads = 0; + const read = document.getAnimations.bind(document); + document.getAnimations = () => { + w.__reads += 1; + return read(); + }; + const readElement = Element.prototype.getAnimations; + Element.prototype.getAnimations = function ( + this: Element, + ...args: Parameters + ) { + w.__elementReads += 1; + return readElement.apply(this, args); + }; + w.__harnessRenderingTrace = true; + }); + // During the trace: a CSS transition and a CSS animation that both end + // inside it, and a script animation still running after it. + await page.addStyleTag({ + content: + '@keyframes harness-fade { from { opacity: 1 } to { opacity: 0.6 } }', + }); + await box.evaluate(el => { + const box = el as HTMLElement; + box.style.transition = 'transform 120ms linear'; + void box.offsetWidth; + box.style.transform = 'translateX(100px)'; + box.style.animation = 'harness-fade 120ms linear'; + box.animate([{scale: '1'}, {scale: '0.9'}], {duration: 600}); + }); + await page.waitForTimeout(300); + const reads = () => + page.evaluate(() => (window as unknown as {__reads: number}).__reads); + // No frame loop: no document-wide read. One element read per CSS start + // event (transitionrun, transitionstart, animationstart), not per frame. + expect(await reads()).toBe(0); + expect( + await page.evaluate( + () => (window as unknown as {__elementReads: number}).__elementReads, + ), + ).toBeLessThanOrEqual(3); + // The trace ends: traceRendering clears the flag and resumes the recorder, + // which reads again, every frame. + await page.evaluate(() => { + const w = window as unknown as { + __harnessRenderingTrace?: boolean; + __harnessAnimations: {resume: () => void}; + }; + w.__harnessRenderingTrace = false; + w.__harnessAnimations.resume(); + }); + await frames(page, 3); + expect(await reads()).toBeGreaterThan(2); + // All three are reported, the two CSS ones that ended inside the trace + // with the time they finished. + await waitForRest(box); + const started = await watch.started(); + expect(started.map(a => a.kind).sort()).toEqual([ + 'Animation', + 'CSSAnimation', + 'CSSTransition', + ]); + expect( + started + .filter(a => a.kind !== 'Animation') + .map(a => a.finishedAt != null), + ).toEqual([true, true]); + }); + test('a wait answers with the animation the action started, not one that finished before it', async ({ page, }) => { @@ -1025,6 +2360,28 @@ test.describe('animation state', () => { await box.evaluate(el => el.getAnimations().forEach(a => a.finish())); }); + test('inside a rendering trace too, a watch taken right after a style change does not report that change', async ({ + page, + browserName, + }) => { + test.skip(browserName !== 'chromium', 'rendering traces are Chromium-only'); + await fixture(page); + const box = page.locator('#box'); + await waitForRest(box); + await traceRendering(page, async () => { + // The step: a transition the next style read creates. + await box.evaluate(el => { + el.style.transition = 'opacity 300ms linear'; + el.style.opacity = '0.5'; + }); + const watch = await watchAnimations(box); + await expect(watch.started({timeout: 200})).rejects.toThrow( + /waitForAnimation/, + ); + }); + await box.evaluate(el => el.getAnimations().forEach(a => a.finish())); + }); + test('an exit whose element the page removes at its end is still reported', async ({ page, }) => { @@ -1076,6 +2433,43 @@ test.describe('animation state', () => { } }); + test('inside a rendering trace too, a wait on an element no longer in the document rejects as missing', async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'rendering traces are Chromium-only (README: engine limits)', + ); + await fixture(page); + await page.evaluate(() => { + const gone = document.createElement('div'); + gone.id = 'gone'; + document.body.append(gone); + }); + const gone = page.locator('#gone'); + const watch = await watchAnimations(gone); + await page.evaluate(() => document.getElementById('gone')?.remove()); + const outcomes: string[] = []; + await traceRendering(page, async () => { + for (const wait of [ + () => watch.started({timeout: 300}), + () => waitForAnimation(gone, {timeout: 300}), + ]) { + outcomes.push( + await wait().then( + () => 'resolved', + (error: Error) => error.message, + ), + ); + } + }); + expect(outcomes).toHaveLength(2); + for (const outcome of outcomes) { + expect(outcome).toMatch(/not in the document/); + } + }); + test('a watch reports only what started after it', async ({page}) => { await fixture(page); const box = page.locator('#box'); @@ -1144,45 +2538,64 @@ test.describe('animation state', () => { // A 300 ms delay plays last when going backward: the curve starts at once. let watch = await watchAnimations(box); - let readyAt = await box.evaluate(async (el, keyframes) => { + let begunAt = await box.evaluate(async (el, keyframes) => { const animation = el.animate(keyframes, {duration: 400, delay: 300}); animation.reverse(); await animation.ready; - return performance.timeOrigin + performance.now(); + // When it began, by its own clock read at one instant: a run going + // backward starts at its end, so its current time has come down from + // there by as much as the timeline has moved since (through the rate). + // performance.now() read after `ready` drifts later under load. + const end = Number(animation.effect?.getComputedTiming().endTime ?? 0); + return ( + performance.timeOrigin + + Number(document.timeline.currentTime) - + (end - Number(animation.currentTime)) / Math.abs(animation.playbackRate) + ); }, frames); const [delayed] = await watch.started(); - expect(Math.abs(delayed.from - readyAt)).toBeLessThan(50); + expect(Math.abs(delayed.from - begunAt)).toBeLessThan(50); expect(delayed.to - delayed.from).toBeCloseTo(400, 0); await box.evaluate(el => el.getAnimations().forEach(a => a.finish())); // A short one that ended before the look keeps its curve inside its life. watch = await watchAnimations(box); - readyAt = await box.evaluate(async (el, keyframes) => { + begunAt = await box.evaluate(async (el, keyframes) => { const animation = el.animate(keyframes, {duration: 60}); animation.reverse(); await animation.ready; - return performance.timeOrigin + performance.now(); + const end = Number(animation.effect?.getComputedTiming().endTime ?? 0); + return ( + performance.timeOrigin + + Number(document.timeline.currentTime) - + (end - Number(animation.currentTime)) / Math.abs(animation.playbackRate) + ); }, frames); await page.waitForTimeout(200); const [ended] = await watch.started(); expect(ended.finishedAt).not.toBeNull(); - expect(Math.abs(ended.from - readyAt)).toBeLessThan(50); + expect(Math.abs(ended.from - begunAt)).toBeLessThan(50); expect(ended.to).toBeLessThanOrEqual((ended.finishedAt ?? 0) + 1); // One the page cancels when it ends keeps its start. watch = await watchAnimations(box); - readyAt = await box.evaluate(async (el, keyframes) => { + begunAt = await box.evaluate(async (el, keyframes) => { const animation = el.animate(keyframes, {duration: 60}); animation.addEventListener('finish', () => animation.cancel(), { once: true, }); animation.reverse(); await animation.ready; - return performance.timeOrigin + performance.now(); + const end = Number(animation.effect?.getComputedTiming().endTime ?? 0); + return ( + performance.timeOrigin + + Number(document.timeline.currentTime) - + (end - Number(animation.currentTime)) / Math.abs(animation.playbackRate) + ); }, frames); await page.waitForTimeout(200); const [cancelled] = await watch.started(); - expect(Math.abs(cancelled.from - readyAt)).toBeLessThan(50); + expect(Math.abs(cancelled.from - begunAt)).toBeLessThan(50); expect(cancelled.to - cancelled.from).toBeCloseTo(60, 0); // Read right after finish(), before the finish event is delivered: the @@ -1429,6 +2842,34 @@ test.describe('storyboards', {tag: '@storyboard'}, () => { } }); + test("the harness's own waits run on the real timers: a frozen timer gate holds neither blockMainThread's block nor sampleRects", async ({ + page, + }) => { + await fixture(page); + await installTimerGate(page); + const box = page.locator('#box'); + await box.evaluate(el => { + el.animate( + [{transform: 'translateX(0px)'}, {transform: 'translateX(200px)'}], + {duration: 1_500, easing: 'linear'}, + ); + }); + await freezeTimers(page); + // Settled, or still held after 5 s. + const within = (work: Promise) => + Promise.race([ + work.then(() => 'settled'), + new Promise(resolve => + setTimeout(() => resolve('held'), 5_000), + ), + ]); + expect( + await within(blockMainThread(page, 100, {during: box, timeout: 2_000})), + ).toBe('settled'); + expect(await within(sampleRects(box, 200))).toBe('settled'); + await thawTimers(page); + }); + test('the timer gate holds a component timer until the filmstrip is done', async ({ page, }, testInfo) => { @@ -1614,4 +3055,58 @@ test.describe('storyboards', {tag: '@storyboard'}, () => { expect(state).toMatchObject({timer: false, frame: false}); expect(state.heldFor).toBeGreaterThanOrEqual(140); }); + + test('a cast filmstrip keeps exactly the presented frames of its window, labelled from its zero', async ({ + page, + }, testInfo) => { + await fixture(page); + const box = page.locator('#box'); + // JPEG frames (WebKit's are always JPEG) take the conversion path. + const cast = await startScreencast(page, {format: 'jpeg'}); + await box.evaluate(el => + el.animate( + [{transform: 'translateX(0px)'}, {transform: 'translateX(200px)'}], + {duration: 600}, + ), + ); + await page.waitForTimeout(500); + const presented = await cast.stop(); + expect(presented.length).toBeGreaterThan(8); + // A window of the middle frames, zeroed on one of them. + const zero = presented[2].t; + const from = presented[3].t; + const to = presented[presented.length - 3].t; + const inside = presented.filter(f => f.t >= from && f.t <= to); + const seen: number[] = []; + const film = await castStoryboard(page, presented, { + name: 'harness-self-test-cast', + zero, + from, + to, + data: t => { + seen.push(t); + return {t}; + }, + }); + expect(film.frames.map(f => f.t)).toEqual(inside.map(f => f.t - zero)); + expect(film.frames.map(f => f.label)).toEqual( + inside.map(f => `t=${(f.t - zero).toFixed(1)} ms`), + ); + expect(seen).toEqual(inside.map(f => f.t)); + // Every cell is a PNG, whatever the screencast delivered. + for (const frame of film.frames) { + expect([...frame.png.subarray(0, 4)]).toEqual([0x89, 0x50, 0x4e, 0x47]); + } + const files = await film.write(testInfo); + expect(files['sheet.png']).toMatch(/sheet\.png$/); + // A window with no presented frame in it films nothing. + const empty = await castStoryboard(page, presented, { + name: 'harness-self-test-cast-empty', + zero, + from: to + 1, + to: to + 2, + }); + expect(empty.frames).toEqual([]); + await expect(empty.write(testInfo)).rejects.toThrow(/no frames/); + }); }); diff --git a/internal/interaction-harness/src/index.ts b/internal/interaction-harness/src/index.ts index 537bfc4d4beca..73ba38af0d61f 100644 --- a/internal/interaction-harness/src/index.ts +++ b/internal/interaction-harness/src/index.ts @@ -12,10 +12,14 @@ * SYNC: When a module is added, export it here and describe it in README.md. */ +export * from './busy'; export * from './curves'; +export * from './frames'; export * from './math'; export * from './output'; export * from './scripts'; export * from './story'; export * from './storyboard'; export * from './touch'; +export * from './trace'; +export * from './webInput'; diff --git a/internal/interaction-harness/src/story.ts b/internal/interaction-harness/src/story.ts index 598f75d852cfe..cb57d59b50be2 100644 --- a/internal/interaction-harness/src/story.ts +++ b/internal/interaction-harness/src/story.ts @@ -6,7 +6,8 @@ * static server over a built Storybook * @output `phone` (the touch-phone browser context every interaction proof * runs in), `withStorybook` (serve the built Storybook for a spec file), and - * `openStory` (load one story and wait until it has rendered). + * `openStory` (load one story, with Storybook's highlight module off, and + * wait until it has rendered). * @position Stories are the reproduction path, as for every other browser * check in this repository: a scenario used for verification is checked in, * not built in a one-off page. @@ -59,13 +60,47 @@ export function withStorybook(test: TestType): () => string { }; } -/** Loads a story in the page and waits for its first render, its fonts and two frames. */ +/** Pages whose Storybook highlight module `openStory` turned off. */ +const highlightOff = new WeakSet(); + +export interface OpenStoryOptions { + /** + * Keeps Storybook's highlight module on. It is off by default: when on, it + * observes `#storybook-root` and, on every insertion, reads the computed + * style of every element on the page inside the inserting task, so a proof + * would measure Storybook as well as the component. + */ + readonly storybookHighlight?: boolean; +} + +/** + * Loads a story in the page and waits for its first render, its fonts and two + * frames. Storybook's highlight module is off (see `OpenStoryOptions`), and + * that is checked: its root element must be absent once the story renders. + */ export async function openStory( page: Page, origin: string, storyId: string, args?: Readonly>, + {storybookHighlight = false}: OpenStoryOptions = {}, ): Promise { + if (storybookHighlight) { + if (highlightOff.has(page)) { + throw new Error( + "openStory: this page already turned Storybook's highlight module off; open the story in a new page to keep it on", + ); + } + } else if (!highlightOff.has(page)) { + highlightOff.add(page); + // Storybook's highlight module returns at once when this is set before + // its preview runs. + await page.addInitScript(() => { + ( + globalThis as unknown as {__STORYBOOK_HIGHLIGHT_INITIALIZED?: boolean} + ).__STORYBOOK_HIGHLIGHT_INITIALIZED = true; + }); + } const params = new URLSearchParams({id: storyId, viewMode: 'story'}); if (args != null) { params.set( @@ -86,6 +121,14 @@ export async function openStory( requestAnimationFrame(() => requestAnimationFrame(() => resolve())), ); }); + if ( + !storybookHighlight && + (await page.locator('#storybook-highlights-root').count()) > 0 + ) { + throw new Error( + "openStory: Storybook's highlight module is on (#storybook-highlights-root exists), so every insertion would also read the style of every element on the page; this Storybook may no longer read __STORYBOOK_HIGHLIGHT_INITIALIZED", + ); + } } /** Resolves after `count` animation frames have been presented in the page. */ @@ -110,8 +153,9 @@ export async function frames(page: Page, count = 2): Promise { // A held callback is queued under its timer or frame id, so the page's own // clearTimeout and cancelAnimationFrame still cancel it while frozen; a held -// frame callback gets the time it actually runs at. `frame` stays the real -// requestAnimationFrame, for the harness's own waits. +// frame callback gets the time it actually runs at. `frame` and `timeout` +// stay the real requestAnimationFrame and setTimeout, for the harness's own +// waits. const TIMER_GATE = `(() => { if (window.__harnessTimerGate) return; const realTimeout = window.setTimeout.bind(window); @@ -119,7 +163,7 @@ const TIMER_GATE = `(() => { const realInterval = window.setInterval.bind(window); const realFrame = window.requestAnimationFrame.bind(window); const realCancelFrame = window.cancelAnimationFrame.bind(window); - const gate = {frozen: false, queue: [], frame: realFrame}; + const gate = {frozen: false, queue: [], frame: realFrame, timeout: realTimeout}; window.__harnessTimerGate = gate; const drop = key => { gate.queue = gate.queue.filter(entry => entry.key !== key); }; window.setTimeout = (fn, ms, ...args) => { diff --git a/internal/interaction-harness/src/storyboard.ts b/internal/interaction-harness/src/storyboard.ts index 2dc22fdc8cf04..a49207c4f81c4 100644 --- a/internal/interaction-harness/src/storyboard.ts +++ b/internal/interaction-harness/src/storyboard.ts @@ -336,7 +336,8 @@ export interface TouchStoryboardOptions extends StoryboardOptions { * photograph per event, about 100 ms between moves instead of the script's * 16.7. Its release velocity is that much lower, so a flick that dismisses * when played can settle back when filmed: a storyboard cannot show such a - * gesture; record it in real time instead. + * gesture; record it in real time instead (`startScreencast`, + * `castStoryboard`). */ export async function touchStoryboard( page: Page, @@ -364,3 +365,65 @@ export async function touchStoryboard( }); return storyboard; } + +/** + * A filmstrip of screencast frames (real presented frames, e.g. the frames of + * a busy-main-thread window), labelled with their time relative to `zero`. + * Evidence for what the compositor presented, not a deterministic storyboard. + */ +export async function castStoryboard( + page: Page, + frames: readonly { + readonly t: number; + readonly data: string; + readonly format: 'png' | 'jpeg'; + }[], + // A cast is already the screen's frames: no clip is applied to them. + options: Omit & { + readonly zero: number; + readonly from?: number; + readonly to?: number; + readonly data?: (t: number) => Readonly>; + }, +): Promise { + const storyboard = new Storyboard(page, options); + for (const frame of frames) { + if (options.from != null && frame.t < options.from) { + continue; + } + if (options.to != null && frame.t > options.to) { + continue; + } + const png = + frame.format === 'png' + ? Buffer.from(frame.data, 'base64') + : await toPng(page, frame.data); + const t = frame.t - options.zero; + storyboard.frames.push({ + label: `t=${t.toFixed(1)} ms`, + t, + data: options.data?.(frame.t) ?? {}, + png, + }); + } + return storyboard; +} + +async function toPng(page: Page, jpegBase64: string): Promise { + const scratch = await page.context().newPage(); + try { + const url = await scratch.evaluate(async data => { + const bitmap = await createImageBitmap( + await (await fetch(`data:image/jpeg;base64,${data}`)).blob(), + ); + const canvas = document.createElement('canvas'); + canvas.width = bitmap.width; + canvas.height = bitmap.height; + canvas.getContext('2d')?.drawImage(bitmap, 0, 0); + return canvas.toDataURL('image/png'); + }, jpegBase64); + return Buffer.from(url.slice(url.indexOf(',') + 1), 'base64'); + } finally { + await scratch.close(); + } +} diff --git a/internal/interaction-harness/src/storybook.interaction.browser.spec.ts b/internal/interaction-harness/src/storybook.interaction.browser.spec.ts new file mode 100644 index 0000000000000..a711feb62cb84 --- /dev/null +++ b/internal/interaction-harness/src/storybook.interaction.browser.spec.ts @@ -0,0 +1,104 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file storybook.interaction.browser.spec.ts + * @input The built Storybook (`withStorybook`) and one of its stories + * @output `openStory` turns Storybook's highlight module off, so an insertion + * in a story carries no `getComputedStyle` scan of the page; with the module + * on (the control), every insertion carries one. Both engines; Chromium's + * trace also counts the style recalculations the scan forces + * @position Self-test for story.ts; it needs the built Storybook, as every + * component proof does + */ + +import {expect, test, type Page} from '@playwright/test'; +import {openStory, withStorybook} from './story'; +import {countBetween, traceRendering} from './trace'; + +const storybook = withStorybook(test); + +/** Counts the page's `getComputedStyle` calls, from before Storybook loads. */ +function countStyleReads(): void { + const w = window as unknown as {__styleReads: number}; + w.__styleReads = 0; + const read = window.getComputedStyle.bind(window); + window.getComputedStyle = ((element: Element, pseudo?: string | null) => { + w.__styleReads += 1; + return read(element, pseudo); + }) as typeof window.getComputedStyle; +} + +/** + * Inserts an exit copy of the story's first element into `#storybook-root`, + * as an exit animation's copy is, lets the insertion's mutation microtasks run, + * and reports what ran between its marks. + */ +async function insertCopy(page: Page) { + return page.evaluate(async () => { + const w = window as unknown as {__styleReads: number}; + const root = document.getElementById('storybook-root') as HTMLElement; + const reads = w.__styleReads; + const from = + performance.timeOrigin + performance.mark('copy:start').startTime; + const copy = (root.firstElementChild as HTMLElement).cloneNode( + true, + ) as HTMLElement; + copy.setAttribute('aria-hidden', 'true'); + root.appendChild(copy); + // A MutationObserver's callback is a microtask the insertion queued: + // these run after it. + await Promise.resolve(); + await Promise.resolve(); + const to = performance.timeOrigin + performance.mark('copy:end').startTime; + const scanned = w.__styleReads - reads; + const elements = document.body.querySelectorAll('*').length; + copy.remove(); + return {from, to, scanned, elements}; + }); +} + +test("openStory turns Storybook's highlight module off: an insertion in a story carries no getComputedStyle scan of the page", async ({ + page, + browserName, +}) => { + // The insertion's reads in every engine, and in Chromium (whose trace the + // harness reads) the style recalculations between its marks. + const measure = async (p: Page) => { + if (browserName !== 'chromium') { + return {copy: await insertCopy(p), styleRecalc: null}; + } + let copy = {from: 0, to: 0, scanned: 0, elements: 0}; + const trace = await traceRendering(p, async () => { + copy = await insertCopy(p); + }); + return { + copy, + styleRecalc: + countBetween(trace, copy.from, copy.to).UpdateLayoutTree ?? 0, + }; + }; + // The harness default. + await page.addInitScript(countStyleReads); + await openStory(page, storybook(), 'core-button--primary'); + expect(await page.locator('#storybook-highlights-root').count()).toBe(0); + const off = await measure(page); + expect(off.copy.scanned).toBe(0); + if (off.styleRecalc != null) { + expect(off.styleRecalc).toBe(0); + } + + // Control: Storybook's module kept on reads the style of every element on + // the page inside the inserting task. + const control = await page.context().newPage(); + await control.addInitScript(countStyleReads); + await openStory(control, storybook(), 'core-button--primary', undefined, { + storybookHighlight: true, + }); + expect(await control.locator('#storybook-highlights-root').count()).toBe(1); + const on = await measure(control); + expect(on.copy.scanned).toBeGreaterThanOrEqual(on.copy.elements); + if (on.styleRecalc != null) { + expect(on.styleRecalc).toBeGreaterThan(0); + } + await control.close(); +}); diff --git a/internal/interaction-harness/src/touch.ts b/internal/interaction-harness/src/touch.ts index 1c16c4b1f82e0..3a5803d1e6cef 100644 --- a/internal/interaction-harness/src/touch.ts +++ b/internal/interaction-harness/src/touch.ts @@ -265,6 +265,13 @@ export interface InputRecord { readonly x?: number; readonly y?: number; readonly touches?: number; + /** A pointer event's pressure (a pen's, 0–1). */ + readonly pressure?: number; + /** A wheel step's deltas, mode and Control key (webInput.ts). */ + readonly deltaX?: number; + readonly deltaY?: number; + readonly deltaMode?: number; + readonly ctrlKey?: boolean; readonly target: string; } @@ -276,20 +283,21 @@ const INPUT_LOG_SCRIPT = `(() => { ? node.tagName.toLowerCase() + (node.id ? '#' + node.id : '') + (node.getAttribute('aria-label') ? '[' + node.getAttribute('aria-label') + ']' : '') : String(node && node.nodeName); const types = ['pointerdown', 'pointermove', 'pointerup', 'pointercancel', 'touchstart', 'touchmove', - 'touchend', 'touchcancel', 'click', 'contextmenu', 'gotpointercapture', 'lostpointercapture']; + 'touchend', 'touchcancel', 'click', 'contextmenu', 'gotpointercapture', 'lostpointercapture', 'wheel']; for (const type of types) { // Bubble phase on window, so defaultPrevented reflects the page's own handlers. window.addEventListener(type, event => { const record = {type, timeStamp: event.timeStamp, isTrusted: event.isTrusted, cancelable: event.cancelable, defaultPrevented: event.defaultPrevented, target: describe(event.target)}; - if ('pointerId' in event) Object.assign(record, {pointerId: event.pointerId, pointerType: event.pointerType, x: event.clientX, y: event.clientY}); + if ('pointerId' in event) Object.assign(record, {pointerId: event.pointerId, pointerType: event.pointerType, x: event.clientX, y: event.clientY, pressure: event.pressure}); + if (type === 'wheel') Object.assign(record, {deltaX: event.deltaX, deltaY: event.deltaY, deltaMode: event.deltaMode, ctrlKey: event.ctrlKey, x: event.clientX, y: event.clientY}); if (event.touches) Object.assign(record, {touches: event.touches.length, x: event.changedTouches[0] && event.changedTouches[0].clientX, y: event.changedTouches[0] && event.changedTouches[0].clientY}); log.push(record); }, {passive: true}); } })()`; -/** Records every pointer/touch/click the page receives (install before navigation). */ +/** Records every pointer, touch, click and wheel step the page receives (install before navigation). */ export async function installInputLog(page: Page): Promise { await page.addInitScript(INPUT_LOG_SCRIPT); } diff --git a/internal/interaction-harness/src/trace.popover.fixture.ts b/internal/interaction-harness/src/trace.popover.fixture.ts new file mode 100644 index 0000000000000..7023d8020a1c8 --- /dev/null +++ b/internal/interaction-harness/src/trace.popover.fixture.ts @@ -0,0 +1,333 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file trace.popover.fixture.ts + * @input A rendering trace recorded with `{invalidations: true}` while a + * Popover presented under a touch on a loaded machine (Chromium) + * @output POPOVER_PRESENT_UNDER_LOAD: its Animation events, rendering events, + * style recalculations and script starts, ms from the present's start + * @position Fixture for trace.test.ts + * + * The popover layer is trace node 6; its present runs two compositor + * animations under the ids 0x25 and 0x26, and the dismissal's exit copy + * (node 59) gets the same ids once they end. In the present's window + * [150, 482] Chrome restyles the layer on its own (`Animation`), then its + * `::backdrop` (`Conditional ::backdrop`), with one Layout and Paint and no + * script since the previous pass. + */ + +export const POPOVER_PRESENT_UNDER_LOAD = { + window: [150, 482], + animationEvents: [ + { + name: 'Animation', + ph: 'b', + ts: -86600, + id2: { + local: '0x25', + }, + args: { + data: { + name: '', + nodeId: 6, + state: 'running', + }, + }, + }, + { + name: 'Animation', + ph: 'e', + ts: -72600, + id2: { + local: '0x25', + }, + args: { + data: {}, + }, + }, + { + name: 'Animation', + ph: 'b', + ts: -54100, + id2: { + local: '0x25', + }, + args: { + data: { + name: '', + nodeId: 6, + state: 'running', + }, + }, + }, + { + name: 'Animation', + ph: 'b', + ts: -52800, + id2: { + local: '0x26', + }, + args: { + data: { + name: '', + nodeId: 6, + state: 'running', + }, + }, + }, + { + name: 'Animation', + ph: 'n', + ts: -1900, + id2: { + local: '0x25', + }, + args: { + data: { + compositeFailed: 0, + }, + }, + }, + { + name: 'Animation', + ph: 'n', + ts: -1900, + id2: { + local: '0x26', + }, + args: { + data: { + compositeFailed: 0, + }, + }, + }, + { + name: 'Animation', + ph: 'n', + ts: 17400, + id2: { + local: '0x25', + }, + args: { + data: { + state: 'running', + }, + }, + }, + { + name: 'Animation', + ph: 'n', + ts: 17400, + id2: { + local: '0x26', + }, + args: { + data: { + state: 'running', + }, + }, + }, + { + name: 'Animation', + ph: 'e', + ts: 533600, + id2: { + local: '0x25', + }, + args: { + data: {}, + }, + }, + { + name: 'Animation', + ph: 'e', + ts: 537900, + id2: { + local: '0x26', + }, + args: { + data: {}, + }, + }, + { + name: 'Animation', + ph: 'b', + ts: 649200, + id2: { + local: '0x25', + }, + args: { + data: { + name: '', + nodeId: 59, + state: 'running', + }, + }, + }, + { + name: 'Animation', + ph: 'b', + ts: 650200, + id2: { + local: '0x26', + }, + args: { + data: { + name: '', + nodeId: 59, + state: 'running', + }, + }, + }, + { + name: 'Animation', + ph: 'n', + ts: 660900, + id2: { + local: '0x25', + }, + args: { + data: { + compositeFailed: 0, + }, + }, + }, + { + name: 'Animation', + ph: 'n', + ts: 661000, + id2: { + local: '0x26', + }, + args: { + data: { + compositeFailed: 0, + }, + }, + }, + { + name: 'Animation', + ph: 'n', + ts: 668300, + id2: { + local: '0x25', + }, + args: { + data: { + state: 'running', + }, + }, + }, + { + name: 'Animation', + ph: 'n', + ts: 668300, + id2: { + local: '0x26', + }, + args: { + data: { + state: 'running', + }, + }, + }, + { + name: 'Animation', + ph: 'e', + ts: 1202300, + id2: { + local: '0x25', + }, + args: { + data: {}, + }, + }, + { + name: 'Animation', + ph: 'e', + ts: 1202400, + id2: { + local: '0x26', + }, + args: { + data: {}, + }, + }, + ], + events: [ + { + name: 'UpdateLayoutTree', + t: 118.6, + }, + { + name: 'Layout', + t: 119.6, + }, + { + name: 'UpdateLayoutTree', + t: 120.1, + }, + { + name: 'Layout', + t: 126, + }, + { + name: 'UpdateLayoutTree', + t: 151, + }, + { + name: 'Layout', + t: 151.6, + }, + { + name: 'UpdateLayoutTree', + t: 159.3, + }, + { + name: 'Layout', + t: 159.9, + }, + { + name: 'PrePaint', + t: 160.4, + }, + { + name: 'Paint', + t: 160.4, + }, + { + name: 'Paint', + t: 160.5, + }, + { + name: 'Layerize', + t: 160.5, + }, + ], + styleInvalidations: [ + { + t: 118.6, + reason: 'Animation', + nodeId: 6, + }, + { + t: 120.1, + reason: 'Conditional ::backdrop', + nodeId: 6, + }, + { + t: 150.9, + reason: 'Animation', + nodeId: 6, + }, + { + t: 157.4, + reason: 'Conditional ::backdrop', + nodeId: 6, + }, + ], + scripts: [ + 3.4, 3.5, 3.6, 6.7, 6.7, 13.8, 13.9, 13.9, 13.9, 14.1, 14.2, 14.3, 14.3, 72, + 72, + ], +} as const; diff --git a/internal/interaction-harness/src/trace.test.ts b/internal/interaction-harness/src/trace.test.ts new file mode 100644 index 0000000000000..4b4cc6b4c24b3 --- /dev/null +++ b/internal/interaction-harness/src/trace.test.ts @@ -0,0 +1,188 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file trace.test.ts + * @input countBetween, traceAnimations, hand-built rendering traces and one + * recorded trace (trace.popover.fixture.ts) + * @output Tests which rendering passes countBetween leaves out with + * skipCompositorAnimationUpdates: only Chrome's own update for compositor + * animations, never work the page caused + * @position Unit tests for internal/interaction-harness/src/trace.ts + */ + +import {describe, expect, it} from 'vitest'; +import {POPOVER_PRESENT_UNDER_LOAD} from './trace.popover.fixture'; +import { + countBetween, + traceAnimations, + type RenderingTrace, + type TraceAnimation, +} from './trace'; + +const PASS = ['UpdateLayoutTree', 'Layout', 'PrePaint', 'Paint', 'Paint']; + +function trace( + passes: number[], + extra: Pick, +): RenderingTrace { + const events = passes.flatMap(t => + PASS.map((name, k) => ({name, t: t + k * 0.1})), + ); + return { + counts: {}, + layout: 0, + paint: 0, + styleRecalc: 0, + prePaint: 0, + durationMs: 500, + longTasks: [], + devtoolsTasks: [], + events, + window: {from: 0, to: 500}, + ...extra, + }; +} + +const COMPOSITED = [ + {nodeId: 7, composited: true}, + {nodeId: 10, composited: true}, +]; + +describe('countBetween with skipCompositorAnimationUpdates', () => { + it('leaves out a pass Chrome made on its own for compositor animations', () => { + const t = trace([4, 154], { + animations: COMPOSITED, + styleInvalidations: [ + {t: 1, reason: 'Inline CSS style declaration was mutated', nodeId: 7}, + {t: 153.7, reason: 'Animation', nodeId: 7}, + {t: 153.8, reason: 'Animation', nodeId: 10}, + ], + scripts: [0.5], + }); + expect(countBetween(t, 150, 400)).toMatchObject({Layout: 1, Paint: 2}); + expect( + countBetween(t, 150, 400, {skipCompositorAnimationUpdates: true}), + ).toEqual({}); + // The start, which the page's style writes caused, still counts. + expect( + countBetween(t, 0, 400, {skipCompositorAnimationUpdates: true}), + ).toMatchObject({Layout: 1}); + }); + + it('counts the pass when script ran since the previous one', () => { + const t = trace([4, 154], { + animations: COMPOSITED, + styleInvalidations: [{t: 153.7, reason: 'Animation', nodeId: 7}], + scripts: [150], + }); + expect( + countBetween(t, 150, 400, {skipCompositorAnimationUpdates: true}), + ).toMatchObject({Layout: 1}); + }); + + it('counts the pass when any recalculation has another reason', () => { + const t = trace([4, 154], { + animations: COMPOSITED, + styleInvalidations: [ + {t: 153.7, reason: 'Animation', nodeId: 7}, + { + t: 153.8, + reason: 'Inline CSS style declaration was mutated', + nodeId: 10, + }, + ], + scripts: [], + }); + expect( + countBetween(t, 150, 400, {skipCompositorAnimationUpdates: true}), + ).toMatchObject({Layout: 1}); + }); + + it('counts every pass of an animation the compositor cannot run', () => { + const t = trace([4, 21, 38, 154], { + animations: [{nodeId: 7, composited: false}], + styleInvalidations: [20, 37, 153].map(at => ({ + t: at, + reason: 'Animation', + nodeId: 7, + })), + scripts: [], + }); + expect( + countBetween(t, 15, 400, {skipCompositorAnimationUpdates: true}), + ).toMatchObject({Layout: 3}); + }); + + it('needs a trace recorded with invalidations', () => { + const t = trace([4], {}); + expect(() => + countBetween(t, 0, 400, {skipCompositorAnimationUpdates: true}), + ).toThrow(/invalidations: true/); + }); +}); + +describe('a recorded trace: a popover presenting on a loaded machine', () => { + const recorded = POPOVER_PRESENT_UNDER_LOAD; + const animations = traceAnimations(recorded.animationEvents, ts => ts / 1000); + const withAnimations = (list: readonly TraceAnimation[]): RenderingTrace => ({ + ...trace([], { + animations: list, + styleInvalidations: recorded.styleInvalidations, + scripts: recorded.scripts, + }), + events: recorded.events, + }); + const [from, to] = recorded.window; + + it('keeps one entry per animation when Chrome reuses an id for a later one', () => { + // The layer's present (node 6) and, under the same two ids, the exit copy + // the dismissal plays on (node 59). + expect(animations.filter(a => a.nodeId === 6 && a.composited).length).toBe( + 2, + ); + expect(animations.filter(a => a.nodeId === 59 && a.composited).length).toBe( + 2, + ); + for (const a of animations.filter(a => a.nodeId === 6 && a.composited)) { + expect(a.from).toBeLessThan(from); + expect(a.to).toBeGreaterThan(to); + } + }); + + it("leaves out Chrome's own update of the layer and of its ::backdrop", () => { + const t = withAnimations(animations); + expect(countBetween(t, from, to)).toMatchObject({Layout: 2, Paint: 2}); + expect( + countBetween(t, from, to, {skipCompositorAnimationUpdates: true}), + ).toEqual({}); + }); + + it('counts the passes when script ran before them', () => { + const t = { + ...withAnimations(animations), + scripts: [...recorded.scripts, 140], + }; + expect( + countBetween(t, from, to, {skipCompositorAnimationUpdates: true}), + ).toMatchObject({Layout: 2}); + }); + + it('counts them when the layer has no compositor animation at the time', () => { + const ended = animations.map(a => + a.nodeId === 6 ? {...a, to: Math.min(a.to ?? Infinity, 100)} : a, + ); + expect( + countBetween(withAnimations(ended), from, to, { + skipCompositorAnimationUpdates: true, + }), + ).toMatchObject({Layout: 2}); + const mainThread = animations.map(a => + a.nodeId === 6 ? {...a, composited: false} : a, + ); + expect( + countBetween(withAnimations(mainThread), from, to, { + skipCompositorAnimationUpdates: true, + }), + ).toMatchObject({Layout: 2}); + }); +}); diff --git a/internal/interaction-harness/src/trace.ts b/internal/interaction-harness/src/trace.ts new file mode 100644 index 0000000000000..9e575692d6bb2 --- /dev/null +++ b/internal/interaction-harness/src/trace.ts @@ -0,0 +1,718 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file trace.ts + * @input Uses a Playwright `Page` (Chromium: CDP `Tracing`) and the page's + * PerformanceObserver for long tasks + * @output `traceRendering` — counts of the main thread's rendering work + * (Layout, Paint, UpdateLayoutTree, PrePaint, …) between two marks — and + * `installLongTaskLog` / `readLongTasks`. + * @position The "no Layout/Paint during a transform/opacity animation" proof. + * A compositor animation costs the main thread nothing per frame; one that + * animates a layout property, or writes styles from JavaScript each frame, + * shows a Layout (and usually a Paint) per frame. Chromium only: WebKit + * exposes no comparable trace to Playwright. + * + * Counting is scoped to the renderer main thread that ran the page's own + * `performance.mark` calls, and to the time between the start and end marks, + * so other pages, the browser process and setup work are excluded. + * + * With `{invalidations: true}` the trace also records why each style + * recalculation happened, which elements animations ran on and whether they + * ran on the compositor, and when script ran. `countBetween` can then leave + * out the one rendering pass Chrome makes on its own for a compositor + * animation (its start time arriving from the compositor, late on a loaded + * machine): a pass whose style recalculations all have the reason + * `Animation`, on elements with an animation on the compositor at the time + * (one entry per animation, as Chrome reuses an animation's trace id once it + * is gone), with no script since the previous pass. A pseudo-element's + * `Conditional ::backdrop` restyle on such an element counts with the + * element's own, in that pass or the one before. Anything the page causes (a + * style write, a layout read, an animation the compositor cannot run) is + * still counted. + * + * SYNC: Keep RENDERING_EVENTS equal to the README's list. + */ + +import type {CDPSession, Page} from '@playwright/test'; + +/** + * The mark a harness function puts on its own page-side work + * (`console.timeStamp('harness-read')` at the start of the function it + * evaluates in the page), so a trace can tell the harness's reads from page + * code a test's own evaluate runs. + */ +export const HARNESS_READ = 'harness-read'; + +/** Trace events that run page script: a listener, a timer, a frame callback. */ +const PAGE_SCRIPT_EVENTS = new Set([ + 'EventDispatch', + 'FunctionCall', + 'TimerFire', + 'FireAnimationFrame', + 'FireIdleCallback', +]); + +/** Main-thread trace events that are rendering work, by their trace names. */ +export const RENDERING_EVENTS = [ + 'UpdateLayoutTree', // style recalculation + 'Layout', + 'PrePaint', + 'Paint', + 'Layerize', + 'UpdateLayer', + 'ScrollLayer', + 'HitTest', +] as const; + +export interface RenderingTrace { + /** Count per trace event name inside the window, on the page's main thread. */ + readonly counts: Readonly>; + /** Shorthand for the four that matter most. */ + readonly layout: number; + readonly paint: number; + readonly styleRecalc: number; + readonly prePaint: number; + /** Window length, ms. */ + readonly durationMs: number; + /** + * The page's own main-thread tasks longer than 50 ms inside the window (long + * tasks), epoch ms. A task that ran a test's own DevTools command (a + * `page.evaluate`) is one, whether it calls page code directly or sets it + * off; only the harness's own marked reads are in `devtoolsTasks`. + */ + readonly longTasks: readonly { + readonly start: number; + readonly durationMs: number; + }[]; + /** + * Main-thread tasks longer than 50 ms inside the window that ran a DevTools + * command the harness marked as its own read (`HARNESS_READ`) and no page + * script, epoch ms: `waitForAnimation` describing every animation's + * keyframes, or `blockMainThread(page, ms)` without `during`. + */ + readonly devtoolsTasks: readonly { + readonly start: number; + readonly durationMs: number; + }[]; + /** Every counted rendering event with its epoch-ms time, for sub-window counts. */ + readonly events: readonly {readonly name: string; readonly t: number}[]; + /** The window, epoch ms on the page's clock. */ + readonly window: {readonly from: number; readonly to: number}; + /** + * With `invalidations`: each style recalculation Chrome scheduled, with its + * reason (`Animation`, `Inline CSS style declaration was mutated`, …) and + * the element's trace node id, epoch ms. + */ + readonly styleInvalidations?: readonly { + readonly t: number; + readonly reason: string; + readonly nodeId?: number; + }[]; + /** + * With `invalidations`: one entry per animation that ran, with the element + * it ran on, whether it ran on the compositor (Chrome reported no reason it + * could not) and when it ran, epoch ms (`to` absent while it still runs). + */ + readonly animations?: readonly TraceAnimation[]; + /** + * With `invalidations`: when script started on the main thread (function + * calls, animation frame callbacks, timers, event dispatch), epoch ms. + */ + readonly scripts?: readonly number[]; +} + +/** An animation as a trace shows it (see `traceAnimations`). */ +export interface TraceAnimation { + readonly nodeId: number; + readonly composited: boolean; + readonly from?: number; + readonly to?: number; +} + +export interface TraceRenderingOptions { + /** + * Also record style recalculation reasons, the animations and whether they + * ran on the compositor, and script, for + * `countBetween(…, {skipCompositorAnimationUpdates: true})`. + */ + invalidations?: boolean; +} + +export interface CountBetweenOptions { + /** + * Leave out a rendering pass Chrome makes on its own for compositor + * animations: every style recalculation since the previous pass has the + * reason `Animation` (or is a pseudo-element's `Conditional ::…` restyle + * following that element's) and is on an element with an animation on the + * compositor at the time, and no script ran in between. Needs a trace + * recorded with `{invalidations: true}`. + */ + skipCompositorAnimationUpdates?: boolean; +} + +// Rendering events closer than this belong to one pass (one frame's style, +// layout and paint); passes are a frame apart. +const PASS_GAP_MS = 4; + +/** + * The times of the rendering events that belong to passes Chrome made on its + * own for compositor animations (see `CountBetweenOptions`). + */ +function compositorAnimationUpdates(trace: RenderingTrace): Set { + const {styleInvalidations, animations, scripts} = trace; + if (styleInvalidations == null || animations == null || scripts == null) { + throw new Error( + 'countBetween: skipCompositorAnimationUpdates needs traceRendering(…, {invalidations: true})', + ); + } + // An element counts as composited at a time when an animation on it runs + // on the compositor then and none on the main thread does. + const alive = (a: TraceAnimation, t: number) => + (a.from == null || a.from <= t) && (a.to == null || t <= a.to); + const compositedAt = (nodeId: number | undefined, t: number) => + nodeId != null && + animations.some(a => a.nodeId === nodeId && a.composited && alive(a, t)) && + !animations.some(a => a.nodeId === nodeId && !a.composited && alive(a, t)); + const events = [...trace.events].sort((a, b) => a.t - b.t); + const skipped = new Set(); + let previousEnd = -Infinity; + // The elements Chrome restyled on its own in the previous pass. + let ownBefore = new Set(); + let i = 0; + while (i < events.length) { + let j = i; + while ( + j + 1 < events.length && + events[j + 1].t - events[j].t <= PASS_GAP_MS + ) { + j += 1; + } + const start = events[i].t; + const end = events[j].t; + const causes = styleInvalidations.filter( + inv => inv.t > previousEnd && inv.t <= end, + ); + // A pseudo-element (`::backdrop`) restyles after the element it belongs + // to: `Conditional ::backdrop` is Chrome's own when that element's + // restyle is, in this pass or the one before. + const own = new Set( + causes + .filter(inv => inv.reason === 'Animation' && inv.nodeId != null) + .map(inv => inv.nodeId as number), + ); + const isAnimationUpdate = + causes.length > 0 && + causes.every( + inv => + (inv.reason === 'Animation' || + (inv.reason.startsWith('Conditional ::') && + inv.nodeId != null && + (own.has(inv.nodeId) || ownBefore.has(inv.nodeId)))) && + compositedAt(inv.nodeId, start), + ) && + !scripts.some(t => t > previousEnd && t <= start); + if (isAnimationUpdate) { + for (let k = i; k <= j; k += 1) { + skipped.add(events[k].t); + } + } + ownBefore = isAnimationUpdate ? own : new Set(); + previousEnd = end; + i = j + 1; + } + return skipped; +} + +/** Counts rendering events by name inside `[from, to]` (epoch ms). */ +export function countBetween( + trace: RenderingTrace, + from: number, + to: number, + {skipCompositorAnimationUpdates = false}: CountBetweenOptions = {}, +): Record { + const skipped = skipCompositorAnimationUpdates + ? compositorAnimationUpdates(trace) + : new Set(); + const counts: Record = {}; + for (const e of trace.events) { + if (e.t >= from && e.t <= to && !skipped.has(e.t)) { + counts[e.name] = (counts[e.name] ?? 0) + 1; + } + } + return counts; +} + +interface TraceEvent { + name: string; + cat: string; + ph: string; + ts: number; + dur?: number; + pid: number; + tid: number; + args?: Record; +} + +/** A trace event as `traceAnimations` reads it. */ +export interface AnimationTraceEvent { + readonly name: string; + readonly ph: string; + readonly ts: number; + readonly id?: string; + readonly id2?: {readonly local?: string}; + readonly args?: Record; +} + +/** + * The animations a trace shows, one entry each, from Chrome's `Animation` + * events: the element (`nodeId`), whether it ran on the compositor and when. + * Chrome names an animation's events by an id it reuses once that animation + * is gone (a popover's exit copy gets its present's ids), so a begin event + * opens a new entry under its id and later events for the id add to it. + * `toMs` turns a trace timestamp into the times the trace reports. + */ +export function traceAnimations( + events: readonly AnimationTraceEvent[], + toMs: (ts: number) => number, +): TraceAnimation[] { + const open = new Map< + string, + {nodeId?: number; composited?: boolean; from: number; to?: number} + >(); + const all: { + nodeId?: number; + composited?: boolean; + from: number; + to?: number; + }[] = []; + for (const e of [...events].sort((a, b) => a.ts - b.ts)) { + if (e.name !== 'Animation') { + continue; + } + const id = e.id2?.local ?? e.id ?? ''; + const d = ((e.args as {data?: Record} | undefined)?.data ?? + {}) as Record; + let entry = open.get(id); + if (e.ph === 'b' || entry == null) { + entry = {from: toMs(e.ts)}; + open.set(id, entry); + all.push(entry); + } + if (typeof d.nodeId === 'number') { + entry.nodeId = d.nodeId; + } + if (typeof d.compositeFailed === 'number') { + entry.composited = (entry.composited ?? true) && d.compositeFailed === 0; + } + if (e.ph === 'e') { + entry.to = toMs(e.ts); + open.delete(id); + } + } + return all.flatMap(({nodeId, composited, from, to}) => + nodeId == null + ? [] + : [ + { + nodeId, + composited: composited ?? false, + from, + ...(to == null ? {} : {to}), + }, + ], + ); +} + +/** + * Traces the page while `body` runs and counts the rendering work its main + * thread did between a start mark (placed just before `body`) and an end mark + * (just after). The harness's animation recorder, `waitForRest`, + * `waitForAnimation` and a watch's `started()` stand down meanwhile, so a body + * may wait with any of them and add no frame of its own, with one gap: on a + * CSS `@keyframes` animation (and, for an unwatched `waitForAnimation`, a + * cancelled CSS transition) the wait reads its timing, which flushes that + * animation's style, so wait on such an animation with `waitForRest`, or + * after the trace. Taking a watch, or calling `waitForRest` while an + * animation already runs on the element, reads its animations once, which + * restyles it: do either before the window a proof counts. + */ +export async function traceRendering( + page: Page, + body: () => Promise, + {invalidations = false}: TraceRenderingOptions = {}, +): Promise { + const cdp: CDPSession = await page.context().newCDPSession(page); + const events: TraceEvent[] = []; + cdp.on('Tracing.dataCollected', ({value}) => { + events.push(...(value as unknown as TraceEvent[])); + }); + const complete = new Promise(resolve => + cdp.once('Tracing.tracingComplete', () => resolve()), + ); + await cdp.send('Tracing.start', { + transferMode: 'ReportEvents', + traceConfig: { + includedCategories: [ + 'devtools.timeline', + 'disabled-by-default-devtools.timeline', + 'blink.user_timing', + 'toplevel', + ...(invalidations + ? ['disabled-by-default-devtools.timeline.invalidationTracking'] + : []), + ], + excludedCategories: ['*'], + }, + }); + const tag = `harness-${Math.random().toString(36).slice(2, 8)}`; + const mark = (name: string) => + page.evaluate( + n => performance.timeOrigin + performance.mark(n).startTime, + name, + ); + // The harness's animation recorder (curves.ts) runs a callback every frame, + // which makes each frame a main-thread one where Blink restyles every running + // animation's target (and lays out an anchored one), and it reads the page's + // animations, which flushes style: exactly what this counts. It stands down + // while this flag is set, a recorder the body itself starts included, and + // resumes once the trace ends. + const standDown = (on: boolean) => + page.evaluate(value => { + const w = window as unknown as { + __harnessRenderingTrace?: boolean; + __harnessAnimations?: {resume?: () => void}; + }; + w.__harnessRenderingTrace = value; + if (!value) { + w.__harnessAnimations?.resume?.(); + } + }, on); + await standDown(true); + let fromEpoch: number; + let toEpoch: number; + try { + fromEpoch = await mark(`${tag}:start`); + await body(); + toEpoch = await mark(`${tag}:end`); + } finally { + // Its catch-up read lands after the end mark, outside the counted window. + // A failure here would hide the body's own error, which is the one to see. + await standDown(false).catch(() => {}); + } + await cdp.send('Tracing.end'); + await complete; + await cdp.detach().catch(() => {}); + + const start = events.find(e => e.name === `${tag}:start`); + const end = events.find(e => e.name === `${tag}:end`); + if (start == null || end == null) { + throw new Error('traceRendering: the window marks were not traced'); + } + const onMain = (e: TraceEvent) => + e.pid === start.pid && + e.tid === start.tid && + e.ts >= start.ts && + e.ts <= end.ts; + // Trace time (µs, monotonic) -> epoch ms, anchored on the start mark. + const toEpochMs = (ts: number) => fromEpoch + (ts - start.ts) / 1000; + const counts: Record = {}; + const counted: {name: string; t: number}[] = []; + for (const e of events) { + if (!onMain(e) || e.ph === 'E') { + continue; + } + if ((RENDERING_EVENTS as readonly string[]).includes(e.name)) { + counts[e.name] = (counts[e.name] ?? 0) + 1; + counted.push({name: e.name, t: toEpochMs(e.ts)}); + } + } + const extra: { + -readonly [ + K in 'styleInvalidations' | 'animations' | 'scripts' + ]?: RenderingTrace[K]; + } = {}; + if (invalidations) { + const data = (e: TraceEvent) => + ((e.args as {data?: Record} | undefined)?.data ?? + {}) as Record; + extra.animations = traceAnimations( + events.filter(e => e.pid === start.pid), + toEpochMs, + ); + extra.styleInvalidations = events + .filter(e => onMain(e) && e.name === 'StyleRecalcInvalidationTracking') + .map(e => { + const d = data(e); + return { + t: toEpochMs(e.ts), + reason: String(d.reason ?? ''), + nodeId: typeof d.nodeId === 'number' ? d.nodeId : undefined, + }; + }); + extra.scripts = events + .filter( + e => + onMain(e) && + [ + 'FunctionCall', + 'FireAnimationFrame', + 'TimerFire', + 'EventDispatch', + ].includes(e.name), + ) + .map(e => toEpochMs(e.ts)); + } + // A task that ran a DevTools command received it over the DevTools + // session's pipe; page script it set off shows as its listener, timer or + // frame callback. + const main = events.filter(e => onMain(e) && e.ph !== 'E'); + const longTasks: {start: number; durationMs: number}[] = []; + const devtoolsTasks: {start: number; durationMs: number}[] = []; + for (const task of main) { + if (task.name !== 'RunTask' || (task.dur ?? 0) <= 50_000) { + continue; + } + const end = task.ts + (task.dur ?? 0); + const inside = main.filter( + e => e !== task && e.ts >= task.ts && e.ts <= end, + ); + const devtools = inside.some( + e => + e.name === 'Receive mojo message' && + ( + e.args?.chrome_mojo_event_info as + {mojo_interface_tag?: unknown} | undefined + )?.mojo_interface_tag === 'blink.mojom.DevToolsSession', + ); + // Only the harness's own reads are set apart: a test's evaluate may call + // page code directly, which leaves no listener or timer in the trace. + const harnessRead = inside.some( + e => + e.name === 'TimeStamp' && + (e.args?.data as {message?: unknown} | undefined)?.message === + HARNESS_READ, + ); + const pageScript = inside.some(e => PAGE_SCRIPT_EVENTS.has(e.name)); + (devtools && harnessRead && !pageScript ? devtoolsTasks : longTasks).push({ + start: toEpochMs(task.ts), + durationMs: (task.dur ?? 0) / 1000, + }); + } + return { + counts, + layout: counts.Layout ?? 0, + paint: counts.Paint ?? 0, + styleRecalc: counts.UpdateLayoutTree ?? 0, + prePaint: counts.PrePaint ?? 0, + durationMs: (end.ts - start.ts) / 1000, + longTasks, + devtoolsTasks, + events: counted, + window: {from: fromEpoch, to: toEpoch}, + ...extra, + }; +} + +/** + * The browser's open pages. The compositor record is browser-wide: its + * `Display::FrameDisplayed` events carry no page identity. + */ +function openPages(page: Page): Page[] { + const browser = page.context().browser(); + return browser == null + ? page.context().pages() + : browser.contexts().flatMap(context => context.pages()); +} + +/** + * Watches for a page that opens while the record runs, in any context (it + * may close again before the record stops). Returns whether one opened. + */ +async function watchForOtherPages( + page: Page, +): Promise<{stop(): Promise}> { + let opened = false; + const onPage = (other: Page) => { + if (other !== page) { + opened = true; + } + }; + // The page's own context, through Playwright (a persistent context has no + // browser to ask). + page.context().on('page', onPage); + // Every context, through the browser's target discovery: a page in a + // context created during the record is a new target too. + let targets: CDPSession | null = null; + const browser = page.context().browser(); + if (browser != null) { + try { + targets = await browser.newBrowserCDPSession(); + const {targetInfos} = await targets.send('Target.getTargets'); + const known = new Set(targetInfos.map(info => info.targetId)); + targets.on('Target.targetCreated', ({targetInfo}) => { + if (targetInfo.type === 'page' && !known.has(targetInfo.targetId)) { + opened = true; + } + }); + await targets.send('Target.setDiscoverTargets', {discover: true}); + } catch { + targets = null; + } + } + return { + async stop() { + page.context().off('page', onPage); + await targets?.detach().catch(() => {}); + return opened; + }, + }; +} + +function assertOnlyPage(page: Page, when: string): void { + if (openPages(page).some(other => other !== page && !other.isClosed())) { + throw new Error( + `startCompositorTrace: another page is open ${when}; the compositor record is browser-wide (its frames carry no page identity), so another page's frames would be credited to this one`, + ); + } +} + +/** + * Records when the compositor presents each frame, for `busyWindowProof`'s + * `presented`: a Chromium trace of `Display::FrameDisplayed`, on the page + * clock. Start it with the screencast, before the block, and stop it after. + * Engines without a CDP trace (WebKit) resolve `stop()` to [], and the + * verdict then judges the screencast alone. It runs its own trace, so not + * inside `traceRendering`. The record is browser-wide, so it throws when + * another page is open as it starts or stops, or opened while it ran. + */ +export async function startCompositorTrace( + page: Page, +): Promise<{stop(): Promise}> { + let cdp: CDPSession; + try { + cdp = await page.context().newCDPSession(page); + } catch { + return {stop: async () => []}; + } + try { + assertOnlyPage(page, 'as the trace starts'); + } catch (error) { + await cdp.detach().catch(() => {}); + throw error; + } + const others = await watchForOtherPages(page); + const events: TraceEvent[] = []; + cdp.on('Tracing.dataCollected', ({value}) => { + events.push(...(value as unknown as TraceEvent[])); + }); + const complete = new Promise(resolve => + cdp.once('Tracing.tracingComplete', () => resolve()), + ); + await cdp.send('Tracing.start', { + transferMode: 'ReportEvents', + traceConfig: { + includedCategories: [ + 'disabled-by-default-display.framedisplayed', + 'blink.user_timing', + ], + excludedCategories: ['*'], + }, + }); + const tag = `harness-compositor-${Math.random().toString(36).slice(2, 8)}`; + const epoch = await page.evaluate( + n => performance.timeOrigin + performance.mark(n).startTime, + tag, + ); + return { + async stop() { + await cdp.send('Tracing.end'); + await complete; + await cdp.detach().catch(() => {}); + if (await others.stop()) { + throw new Error( + "startCompositorTrace: another page opened during the record; the compositor record is browser-wide (its frames carry no page identity), so that page's frames would be credited to this one", + ); + } + assertOnlyPage(page, 'as the trace stops'); + const mark = events.find(e => e.name === tag); + if (mark == null) { + throw new Error('startCompositorTrace: the clock mark was not traced'); + } + // Trace time (µs, monotonic) -> epoch ms, anchored on the mark. + return events + .filter(e => e.name === 'Display::FrameDisplayed') + .map(e => epoch + (e.ts - mark.ts) / 1000) + .sort((a, b) => a - b); + }, + }; +} + +const LONG_TASK_SCRIPT = `(() => { + if (window.__harnessLongTasks) return; + const tasks = []; + const log = {tasks, supported: false}; + window.__harnessLongTasks = log; + if (typeof PerformanceObserver === 'undefined' || !(PerformanceObserver.supportedEntryTypes || []).includes('longtask')) return; + try { + new PerformanceObserver(list => { + for (const entry of list.getEntries()) tasks.push({start: performance.timeOrigin + entry.startTime, durationMs: entry.duration, name: entry.name}); + }).observe({type: 'longtask', buffered: true}); + log.supported = true; + } catch {} +})()`; + +/** + * Records every long task (> 50 ms) the page runs (install before navigation; + * Chromium), through the Long Tasks API. That sees the tasks of the page's own + * event loop: a timer, an event handler, `blockMainThread(page, ms, {during})`. + * A DevTools `page.evaluate`, such as `blockMainThread(page, ms)` without + * `during`, is not one of them and is never recorded here; `traceRendering` + * reports it in `devtoolsTasks`, apart from the page's `longTasks`. + */ +export async function installLongTaskLog(page: Page): Promise { + await page.addInitScript(LONG_TASK_SCRIPT); +} + +export interface LongTask { + /** Epoch ms. */ + readonly start: number; + readonly durationMs: number; + readonly name: string; +} + +/** + * Long tasks recorded since navigation, optionally only those overlapping a + * window. Fails closed: it throws when the log never ran on this document + * (installed after the page navigated) or the engine has no Long Tasks API + * (WebKit), so an empty answer is evidence of no long task, never of no log. + */ +export async function readLongTasks( + page: Page, + range?: {from: number; to: number}, +): Promise { + const log = await page.evaluate( + () => + ( + window as unknown as { + __harnessLongTasks?: {tasks: LongTask[]; supported: boolean}; + } + ).__harnessLongTasks ?? null, + ); + if (log == null) { + throw new Error( + 'readLongTasks: the long-task log is not running on this document; call installLongTaskLog(page) before navigating', + ); + } + if (!log.supported) { + throw new Error( + 'readLongTasks: this engine has no Long Tasks API, so the log records nothing', + ); + } + const tasks = log.tasks; + return range == null + ? tasks + : tasks.filter( + t => t.start < range.to && t.start + t.durationMs > range.from, + ); +} diff --git a/internal/interaction-harness/src/webInput.ts b/internal/interaction-harness/src/webInput.ts new file mode 100644 index 0000000000000..267b4d7a02503 --- /dev/null +++ b/internal/interaction-harness/src/webInput.ts @@ -0,0 +1,250 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file webInput.ts + * @input Uses a Playwright `Page` (Chromium or WebKit) + * @output `webInputDriver(page)`: a wheel step (`wheel({x, y, dx, dy, ctrl})`) + * and a pen stroke (`pen(points)`) as REAL engine input, for proofs that + * the same motion works for every pointer, not only a finger + * (`spec:AST-100` FR4). + * @position The non-touch half of the input drivers; touch.ts is the touch + * half and owns the input log both are read through. Untrusted DOM events + * are not input (a component can tell them apart with `isTrusted`), so both + * go through the engine's own input pipeline. + * + * Chromium: CDP `Input.dispatchMouseEvent`. + * - A wheel step is a `mouseWheel` event at (x, y). Chromium divides the + * deltas it delivers by the device scale factor times the visual viewport's + * scale, so the driver multiplies them back: the page receives exactly `dx` + * and `dy` CSS px, in pixel mode (`deltaMode` 0). `ctrl` holds Control, as a + * trackpad pinch arrives. On a page whose wheel listeners are all passive + * (every page with the input log), Chromium answers the dispatch at once + * and delivers the step to the page afterwards, so `wheel` waits, through a + * one-shot passive listener, until the page has the step. + * - A pen stroke is pressed at its first point, moved through every point and + * released at the last, with `pointerType: 'pen'`, a pressure and an + * optional tilt, each event stamped with its scripted time: the page + * receives trusted pen pointer events (and the compatibility mouse events) + * with those timestamps. The stroke starts once the page has presented two + * frames: the press goes out with no page round trip, so right after a + * navigation it could carry the previous document's page scale (about:blank + * scales a press at 100 px to 41 px in the phone project). + * + * WebKit (Playwright's Linux build): `page.mouse.wheel` in a desktop context + * (`isMobile: false`, `hasTouch: false`), waiting the same way. The step + * moves the mouse to (x, y) first (a `pointermove`) and holds Control as a key, + * which Chromium's step does neither. WebKit's mobile emulation has no + * wheel input, and Playwright has no pen for WebKit: both throw, naming the + * reason, so a proof skips on the driver's flags instead. + * + * SYNC: Keep the engine limits above equal to the README's table and to the + * harness self-test (../src/harness.interaction.browser.spec.ts). + */ + +import type {CDPSession, JSHandle, Page} from '@playwright/test'; +import type {Engine} from './touch'; + +/** One wheel step: the page receives exactly `dx`, `dy` CSS px. */ +export interface WheelInput { + readonly x: number; + readonly y: number; + readonly dx?: number; + readonly dy?: number; + /** Hold Control during the step (a trackpad pinch arrives as a ctrl wheel). */ + readonly ctrl?: boolean; +} + +/** A point of a pen stroke. */ +export interface PenPoint { + readonly x: number; + readonly y: number; + /** Ms after the stroke's first point. */ + readonly t: number; + /** 0–1 (default 0.5) while the pen touches. */ + readonly pressure?: number; + /** Degrees, -90–90 (default 0). */ + readonly tiltX?: number; + readonly tiltY?: number; +} + +export interface WebInputDriver { + readonly engine: Engine; + /** Whether `wheel` works in this page (WebKit's mobile emulation has none). */ + readonly wheels: boolean; + /** Whether `pen` works in this page (Chromium only). */ + readonly pens: boolean; + wheel(input: WheelInput): Promise; + /** Presses at the first point, moves through all of them, releases at the last; paced on the wall clock. */ + pen(points: readonly PenPoint[]): Promise; +} + +/** Why a proof skips when the driver has no wheel in this page. */ +export const WHEEL_SKIP = + "WebKit's mobile emulation has no wheel input: run wheel proofs in a desktop context"; +/** Why a proof skips when the driver has no pen. */ +export const PEN_SKIP = + 'Playwright has no pen input for WebKit: pen proofs run in Chromium'; + +const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); + +/** How long `wheel` waits for the page to receive its step. */ +const WHEEL_DELIVERY_MS = 2000; + +/** + * Arms a one-shot passive capture listener for the page's next wheel step, + * before the step is dispatched. Passive, so the page's own listeners keep + * deciding whether the engine waits for it. (The promise sits in an object: + * a handle to a bare promise would wait for it to settle.) + */ +function armWheel(page: Page): Promise}>> { + return page.evaluateHandle(() => ({ + received: new Promise(resolve => { + window.addEventListener('wheel', () => resolve(), { + capture: true, + once: true, + passive: true, + }); + }), + })); +} + +/** Waits until the armed listener has seen the step; throws past the limit. */ +async function untilReceived( + page: Page, + armed: JSHandle<{received: Promise}>, +): Promise { + const received = await page.evaluate( + ([box, ms]) => + Promise.race([ + box.received.then(() => true), + new Promise(resolve => setTimeout(() => resolve(false), ms)), + ]), + [armed, WHEEL_DELIVERY_MS] as const, + ); + await armed.dispose(); + if (!received) { + throw new Error( + `webInputDriver: the page did not receive the wheel step within ${WHEEL_DELIVERY_MS} ms (is the point inside the viewport, over the main document?)`, + ); + } +} + +async function chromiumWebInput(page: Page): Promise { + const cdp: CDPSession = await page.context().newCDPSession(page); + return { + engine: 'chromium', + wheels: true, + pens: true, + async wheel({x, y, dx = 0, dy = 0, ctrl = false}) { + const scale = await page.evaluate( + () => window.devicePixelRatio * (window.visualViewport?.scale ?? 1), + ); + const armed = await armWheel(page); + await cdp.send('Input.dispatchMouseEvent', { + type: 'mouseWheel', + x, + y, + deltaX: dx * scale, + deltaY: dy * scale, + modifiers: ctrl ? 2 : 0, + }); + await untilReceived(page, armed); + }, + async pen(points) { + if (points.length === 0) { + return; + } + // Two presented frames first: the press reaches the compositor with no + // page round trip, and right after a navigation it could still carry + // the previous document's page scale. The harness's own frames, not a + // frozen timer gate's (story.ts). + await page.evaluate( + () => + new Promise(resolve => { + const gate = ( + window as unknown as { + __harnessTimerGate?: {frame?: typeof requestAnimationFrame}; + } + ).__harnessTimerGate; + const frame = + gate?.frame ?? window.requestAnimationFrame.bind(window); + frame(() => frame(() => resolve())); + }), + ); + const startWall = Date.now(); + const startStamp = Date.now() / 1000; + const events = [ + {type: 'mousePressed' as const, point: points[0]}, + ...points.slice(1).map(point => ({type: 'mouseMoved' as const, point})), + {type: 'mouseReleased' as const, point: points[points.length - 1]}, + ]; + for (const {type, point} of events) { + const wait = startWall + point.t - Date.now(); + if (wait > 0) { + await sleep(wait); + } + await cdp.send('Input.dispatchMouseEvent', { + type, + x: point.x, + y: point.y, + button: 'left', + buttons: type === 'mouseReleased' ? 0 : 1, + clickCount: type === 'mouseMoved' ? 0 : 1, + pointerType: 'pen', + force: type === 'mouseReleased' ? 0 : (point.pressure ?? 0.5), + tiltX: point.tiltX ?? 0, + tiltY: point.tiltY ?? 0, + timestamp: startStamp + point.t / 1000, + }); + } + }, + }; +} + +async function webkitWebInput(page: Page): Promise { + // Playwright refuses a wheel in a WebKit context that emulates a phone: read + // the option it reads, the context's `isMobile`. + const mobile = + (page.context() as unknown as {_options?: {isMobile?: boolean}})._options + ?.isMobile === true; + return { + engine: 'webkit', + wheels: !mobile, + pens: false, + async wheel({x, y, dx = 0, dy = 0, ctrl = false}) { + if (mobile) { + throw new Error(`webInputDriver: ${WHEEL_SKIP}`); + } + await page.mouse.move(x, y); + if (ctrl) { + await page.keyboard.down('Control'); + } + try { + const armed = await armWheel(page); + await page.mouse.wheel(dx, dy); + await untilReceived(page, armed); + } finally { + if (ctrl) { + await page.keyboard.up('Control'); + } + } + }, + async pen() { + throw new Error(`webInputDriver: ${PEN_SKIP}`); + }, + }; +} + +/** Wheel and pen input for the page's engine. Firefox is not supported (no project). */ +export async function webInputDriver(page: Page): Promise { + const name = page.context().browser()?.browserType().name(); + if (name === 'chromium') { + return chromiumWebInput(page); + } + if (name === 'webkit') { + return webkitWebInput(page); + } + throw new Error( + `webInputDriver: no wheel or pen input path for ${name ?? 'this browser'}`, + ); +} diff --git a/internal/test-utils/src/setup.ts b/internal/test-utils/src/setup.ts index 440b32ff22b3c..991bb056f0816 100644 --- a/internal/test-utils/src/setup.ts +++ b/internal/test-utils/src/setup.ts @@ -12,6 +12,8 @@ /// import '@testing-library/jest-dom/vitest'; import {configure} from '@testing-library/react'; +import {afterEach} from 'vitest'; +import {__resetClickGuardForTest} from '../../../packages/core/src/utils/clickGuard'; // Text queries (getByText/findByText/…) target VISIBLE text. Astryx's // `useAnnounce` renders a visually-hidden aria-live region @@ -25,6 +27,15 @@ import {configure} from '@testing-library/react'; // Keeps the jsdom defaults (script, style). configure({defaultIgnore: 'script, style, [data-astryx-live-region]'}); +// The click guard (packages/core/src/utils/clickGuard.ts) is armed per +// document by a gesture that already acted and disarms itself 400 ms later. +// A test that ends inside that window would leave the timer to fire after +// the environment tears its window down, during a later file. Disarm every +// guard as each test ends. +afterEach(() => { + __resetClickGuardForTest(); +}); + // Polyfill for matchMedia (not supported in jsdom). // // `(hover: hover)` reports TRUE: jsdom models a desktop browser driven by a diff --git a/packages/build/__fixtures__/stylex-consts/App.tsx b/packages/build/__fixtures__/stylex-consts/App.tsx new file mode 100644 index 0000000000000..5e839e5372912 --- /dev/null +++ b/packages/build/__fixtures__/stylex-consts/App.tsx @@ -0,0 +1,34 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * Reads `defineConsts` values from other files, as a media-query key and as a + * plain value, from both sides of the layer split and from a file compiled + * with the library class-name prefix. Only the constants are + * imported: once StyleX compiles `stylex.create`, nothing in this file uses + * the imports at runtime any more. + */ + +import * as stylex from '@stylexjs/stylex'; +import {appConsts} from './appConsts.stylex'; +import {libraryConsts} from './library/libraryConsts.stylex'; +import {packageConsts} from './packages/core/packageConsts.stylex'; + +const styles = stylex.create({ + row: { + minHeight: appConsts.rowHeight, + transitionProperty: {default: 'color', [appConsts.narrow]: 'none'}, + }, + panel: { + paddingTop: libraryConsts.inset, + paddingBottom: packageConsts.gap, + maxWidth: {default: null, [libraryConsts.wide]: '640px'}, + }, +}); + +export function App() { + return ( +
+
row
+
+ ); +} diff --git a/packages/build/__fixtures__/stylex-consts/appConsts.stylex.ts b/packages/build/__fixtures__/stylex-consts/appConsts.stylex.ts new file mode 100644 index 0000000000000..5e41328b69880 --- /dev/null +++ b/packages/build/__fixtures__/stylex-consts/appConsts.stylex.ts @@ -0,0 +1,9 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +import * as stylex from '@stylexjs/stylex'; + +/** Constants the app defines itself, beside the file that reads them. */ +export const appConsts = stylex.defineConsts({ + narrow: '@media (max-width: 599px)', + rowHeight: '52px', +}); diff --git a/packages/build/__fixtures__/stylex-consts/index.html b/packages/build/__fixtures__/stylex-consts/index.html new file mode 100644 index 0000000000000..ba86fc975ff35 --- /dev/null +++ b/packages/build/__fixtures__/stylex-consts/index.html @@ -0,0 +1,12 @@ + + + + + + stylex-consts fixture — defineConsts imported across files + + +
+ + + diff --git a/packages/build/__fixtures__/stylex-consts/library/libraryConsts.stylex.ts b/packages/build/__fixtures__/stylex-consts/library/libraryConsts.stylex.ts new file mode 100644 index 0000000000000..92dda6955e61f --- /dev/null +++ b/packages/build/__fixtures__/stylex-consts/library/libraryConsts.stylex.ts @@ -0,0 +1,12 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +import * as stylex from '@stylexjs/stylex'; + +/** + * Constants on the library side of the layer split (the build test points + * `libraryPattern` at this folder), read by an app file on the product side. + */ +export const libraryConsts = stylex.defineConsts({ + wide: '@media (min-width: 900px)', + inset: '13px', +}); diff --git a/packages/build/__fixtures__/stylex-consts/main.tsx b/packages/build/__fixtures__/stylex-consts/main.tsx new file mode 100644 index 0000000000000..5811fe017bb3e --- /dev/null +++ b/packages/build/__fixtures__/stylex-consts/main.tsx @@ -0,0 +1,6 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +import {createRoot} from 'react-dom/client'; +import {App} from './App'; + +createRoot(document.getElementById('root')!).render(); diff --git a/packages/build/__fixtures__/stylex-consts/packages/core/packageConsts.stylex.ts b/packages/build/__fixtures__/stylex-consts/packages/core/packageConsts.stylex.ts new file mode 100644 index 0000000000000..b4be85d7a3505 --- /dev/null +++ b/packages/build/__fixtures__/stylex-consts/packages/core/packageConsts.stylex.ts @@ -0,0 +1,13 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +import * as stylex from '@stylexjs/stylex'; + +/** + * Constants as an Astryx package defines them: the path holds `packages/core/`, + * so the babel wrapper compiles this file with the library class-name prefix + * (`astryx`), as it does a real package. The app file reading them compiles + * with its own prefix (`x`). + */ +export const packageConsts = stylex.defineConsts({ + gap: '17px', +}); diff --git a/packages/build/src/vite.build.test.ts b/packages/build/src/vite.build.test.ts index ee860275a2752..b0907675792c1 100644 --- a/packages/build/src/vite.build.test.ts +++ b/packages/build/src/vite.build.test.ts @@ -31,7 +31,7 @@ */ import {describe, it, expect, beforeAll, afterAll} from 'vitest'; -import {build} from 'vite'; +import {build, createLogger} from 'vite'; import react from '@vitejs/plugin-react'; import { mkdirSync, @@ -70,6 +70,21 @@ type Built = { async function buildFixture( name: string, htmlNames: string[] = ['index.html'], + { + libraryPattern = 'packages/core/', + zeroConfig = false, + stylexOverrides, + warnings = [], + }: { + /** Which source paths land in the library layer. */ + libraryPattern?: string; + /** Use `astryxStylex({...})` instead of the legacy `{stylexOptions}` form. */ + zeroConfig?: boolean; + /** Zero-config form only: `stylexOverrides`. */ + stylexOverrides?: Record; + /** Receives every warning the build logs. */ + warnings?: string[]; + } = {}, ): Promise { const root = path.join(FIXTURES, name); const outDir = mkdtempSync(path.join(tmpdir(), `astryx-${name}-`)); @@ -77,6 +92,11 @@ async function buildFixture( await build({ root, logLevel: 'error', + customLogger: { + ...createLogger('error'), + warn: (message: string) => void warnings.push(message), + warnOnce: (message: string) => void warnings.push(message), + }, build: { outDir, emptyOutDir: true, @@ -94,17 +114,27 @@ async function buildFixture( resolve: {alias: {'@astryxdesign/core': CORE_SRC}}, plugins: [ react(), - ...astryxStylex({ - stylexOptions: { - dev: false, - unstable_moduleResolution: {type: 'commonJS', rootDir: REPO_ROOT}, - aliases: { - '@astryxdesign/core/*': [path.join(CORE_SRC, '*')], - '@astryxdesign/core': [CORE_SRC], - }, - }, - libraryPattern: 'packages/core/', - }), + ...(zeroConfig + ? astryxStylex({ + dev: false, + rootDir: REPO_ROOT, + libraryPattern, + stylexOverrides, + }) + : astryxStylex({ + stylexOptions: { + dev: false, + unstable_moduleResolution: { + type: 'commonJS', + rootDir: REPO_ROOT, + }, + aliases: { + '@astryxdesign/core/*': [path.join(CORE_SRC, '*')], + '@astryxdesign/core': [CORE_SRC], + }, + }, + libraryPattern, + })), ], }); } catch (error) { @@ -338,6 +368,67 @@ describe('a production build separates Astryx and product styles by layer', () = }); }); +// `defineConsts` values are compiled into the reading file as placeholders, +// `var(--)`, and the defining file contributes the values; the +// final CSS pass substitutes one for the other. Both halves must reach that +// pass. Storybook builds through the legacy `{stylexOptions}` form, whose +// StyleX instance ran without `treeshakeCompensation`: once `stylex.create` +// was compiled, nothing used the import at runtime, the TypeScript transform +// dropped it, and the defining module was never loaded. A media-query key was +// left as `var(--x…){…}` and failed the build ("Invalid empty selector"); a +// value shipped as a custom property nothing defines. And a constant defined +// on one side of the layer split and read on the other was substituted only +// within its own side, and one an Astryx package defines (compiled with the +// library class-name prefix) never matched an app file's placeholder (written +// with the product prefix). +describe('a production build resolves defineConsts imported across files', () => { + /** A constant reference StyleX left unsubstituted. */ + const PLACEHOLDER = /var\(--(?:astryx|x)[a-z0-9]{4,}\)/; + + for (const zeroConfig of [false, true]) { + describe( + zeroConfig ? 'zero-config form' : 'legacy stylexOptions form (Storybook)', + () => { + let built: Built; + beforeAll(async () => { + built = await buildFixture('stylex-consts', ['index.html'], { + libraryPattern: '__fixtures__/stylex-consts/library/', + zeroConfig, + }); + }, 180_000); + afterAll( + () => built && rmSync(built.outDir, {recursive: true, force: true}), + ); + + it('substitutes a constant read as a value', () => { + expect(built.css).toMatch(/min-height:\s*52px/); + }); + + it('substitutes a constant read as a media-query key', () => { + expect(built.css).toMatch( + /@media \((?:max-width: ?599px|width ?<= ?599px)\)\s*\{[^}]*transition-property:\s*none/, + ); + }); + + it('substitutes a library-side constant read from the product side', () => { + expect(built.css).toMatch(/padding-top:\s*13px/); + expect(built.css).toMatch( + /@media \((?:min-width: ?900px|width ?>= ?900px)\)\s*\{[^}]*max-width:\s*640px/, + ); + }); + + it('substitutes a constant an Astryx package defines under the library prefix', () => { + expect(built.css).toMatch(/padding-bottom:\s*17px/); + }); + + it('leaves no constant placeholder behind', () => { + expect(built.css).not.toMatch(PLACEHOLDER); + }); + }, + ); + } +}); + describe('a production build validates StyleX declarations', () => { it('fails with longhand guidance instead of dropping a border shorthand', async () => { await expect(buildFixture('invalid-border')).rejects.toThrow( diff --git a/packages/build/src/vite.ts b/packages/build/src/vite.ts index 0581b58675edb..edaa88ca55a1d 100644 --- a/packages/build/src/vite.ts +++ b/packages/build/src/vite.ts @@ -136,6 +136,12 @@ export function astryxStylex( const libraryLayer = layers.library ?? 'astryx-base'; const productLayer = layers.product ?? 'product'; + // Library files compile with `stylexPrefix`, product files with StyleX's + // `classNamePrefix` (the babel wrapper routes them). + const classNamePrefixes = [ + stylexPrefix, + (stylexOverrides as {classNamePrefix?: string}).classNamePrefix ?? 'x', + ]; const overrideLightningcssOptions = ( stylexOverrides as {lightningcssOptions?: Record} @@ -284,6 +290,7 @@ export function astryxStylex( libraryPattern, libraryLayer, productLayer, + classNamePrefixes, }), ); }, @@ -302,6 +309,7 @@ export function astryxStylex( libraryLayer, productLayer, lightningcssOptions: stylexOptions.lightningcssOptions, + classNamePrefixes, }), ]; } @@ -330,24 +338,43 @@ function renderSplitLayers( libraryPattern: string; libraryLayer: string; productLayer: string; + /** The StyleX class-name prefixes in play: library, then product. */ + classNamePrefixes: readonly string[]; }, ): string { if (!rulesById || rulesById.size === 0) return ''; const libraryRules: unknown[] = []; const productRules: unknown[] = []; + // `defineConsts` values. A file that reads a constant compiles it to a + // placeholder, `var(--)`, and `processStylexRules` substitutes + // the value from the defining file's rules, but only among the rules it is + // given. The defining file and the reading one can sit on opposite sides of + // the split, so every layer gets every constant, under every class-name + // prefix (see constantRulesForEveryPrefix). Constant rules emit no CSS of + // their own, so repeating them adds nothing to the output. + const collectedConstants: unknown[] = []; for (const [filePath, rules] of rulesById.entries()) { - if (filePath.includes(options.libraryPattern)) { - libraryRules.push(...rules); - } else { - productRules.push(...rules); + const own = filePath.includes(options.libraryPattern) + ? libraryRules + : productRules; + for (const rule of rules) { + if (isConstantRule(rule)) { + collectedConstants.push(rule); + } else { + own.push(rule); + } } } + const constantRules = constantRulesForEveryPrefix( + collectedConstants, + options.classNamePrefixes, + ); const render = (rules: unknown[], layer: string) => rules.length - ? `@layer ${layer} {\n${stylexBabelPlugin.processStylexRules(rules as never, {useLayers: true})}\n}` + ? `@layer ${layer} {\n${stylexBabelPlugin.processStylexRules([...constantRules, ...rules] as never, {useLayers: true})}\n}` : ''; return [ @@ -358,6 +385,56 @@ function renderSplitLayers( .join('\n\n'); } +/** + * Each `defineConsts` rule once under every class-name prefix in play. + * + * StyleX keys a constant by its defining file's prefix, `` (or + * `-` with debug class names), where the hash depends on + * the export and key alone, and a file that reads it writes the placeholder + * with its own prefix. An Astryx package compiles with the library prefix and + * an app with the product one, so an app reading a package's constant writes + * `var(--x)` while the package's rule is keyed `astryx`; the + * copies make every prefix's placeholder resolve. A key StyleX took verbatim + * (`'--name'`) carries no prefix and is kept as it is. + */ +function constantRulesForEveryPrefix( + rules: readonly unknown[], + prefixes: readonly string[], +): unknown[] { + const ordered = [...new Set(prefixes)] + .filter(Boolean) + .sort((a, b) => b.length - a.length); + const out: unknown[] = []; + const keys = new Set(); + const add = (rule: unknown[], key: string) => { + if (keys.has(key)) return; + keys.add(key); + const [, body, ...rest] = rule; + out.push([key, {...(body as object), constKey: key}, ...rest]); + }; + for (const rule of rules as unknown[][]) { + const key = String((rule[1] as {constKey: unknown}).constKey); + add(rule, key); + for (const prefix of ordered) { + const match = new RegExp(`^(|.*-)${prefix}([0-9a-z]+)$`).exec(key); + if (match == null) continue; + for (const other of ordered) { + add(rule, `${match[1]}${other}${match[2]}`); + } + break; + } + } + return out; +} + +/** A `defineConsts` value as StyleX collects it: a rule with `constKey`/`constVal`. */ +function isConstantRule(rule: unknown): boolean { + const body = Array.isArray(rule) + ? (rule[1] as {constKey?: unknown; constVal?: unknown} | undefined) + : undefined; + return body?.constKey != null && body?.constVal != null; +} + /** * The build half of the split. StyleX appends one merged block of CSS to the * build's stylesheet, in its own top-level `@layer priority1…priorityN` — which @@ -387,6 +464,8 @@ function buildLayerSplitPlugin( libraryLayer: string; productLayer: string; lightningcssOptions?: unknown; + /** The StyleX class-name prefixes in play: library, then product. */ + classNamePrefixes: readonly string[]; }, ): Plugin { const stylex = basePlugin as { @@ -586,12 +665,25 @@ function astryxStylexLegacy(options: AstryxVitePluginLegacyOptions): Plugin[] { ).lightningcssOptions; const validatedStylexOptions = { propertyValidationMode: 'throw', + // The unplugin applies its Vite default (true) only to its own StyleX + // instance, which runs after the babel wrapper below has already compiled + // every StyleX call. Without it here, a file that reads only `defineConsts` + // from a `.stylex.ts` module loses that import once `stylex.create` is + // compiled (the TypeScript transform drops it as unused), the module is + // never loaded, and its constants never reach the CSS: a media-query key + // fails the build and a value ships as an undefined custom property. + treeshakeCompensation: true, ...(stylexOptions as any), lightningcssOptions: { targets: LIGHTNINGCSS_TARGETS, ...legacyLightningcssOptions, }, }; + const classNamePrefixes = [ + stylexPrefix, + (validatedStylexOptions as {classNamePrefix?: string}).classNamePrefix ?? + 'x', + ]; const astryxBabelPlugin = path.resolve(__dirname, 'babel.js'); const existingPlugins = validatedStylexOptions.babelConfig?.plugins ?? []; @@ -658,6 +750,7 @@ function astryxStylexLegacy(options: AstryxVitePluginLegacyOptions): Plugin[] { libraryPattern, libraryLayer, productLayer, + classNamePrefixes, }), ); }, @@ -677,6 +770,7 @@ function astryxStylexLegacy(options: AstryxVitePluginLegacyOptions): Plugin[] { lightningcssOptions: ( validatedStylexOptions as {lightningcssOptions?: unknown} ).lightningcssOptions, + classNamePrefixes, }), ]; } diff --git a/packages/core/package.json b/packages/core/package.json index 295c3d98433c4..3bba9ae4f24ac 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -864,6 +864,7 @@ "devDependencies": { "@astryxdesign/a11y-spec": "workspace:*", "@astryxdesign/cli": "workspace:*", + "@astryxdesign/interaction-harness": "workspace:*", "@babel/cli": "catalog:", "@babel/core": "catalog:", "@babel/preset-react": "catalog:", diff --git a/packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts b/packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts new file mode 100644 index 0000000000000..d276461150515 --- /dev/null +++ b/packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts @@ -0,0 +1,394 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file BottomSheetDetent.interaction.browser.spec.ts + * @input The built "Snap points" BottomSheet story, real touch input + * (Chromium CDP / WebKit's input command) and the interaction harness + * @output Interaction and animation proofs for dragging the sheet to a + * detent: the drag follows the finger and a slow release settles on the + * nearest detent; a flick dismisses; the settle's sampled curve matches its + * timing function; the settle keeps advancing every frame through a blocked + * main thread; the settle does no Layout or Paint per frame; a storyboard of + * the drag and the settle. + * @position The exemplar for `*.interaction.browser.spec.ts`: component lanes + * copy this shape. Each test names the instrument it uses from + * @astryxdesign/interaction-harness, so the evidence layer is explicit. + * + * The expected curve here is today's: the panel's `--ease-standard` CSS + * transition over `--duration-medium`, pinned below as SETTLE_EASING and + * SETTLE_MS rather than read from the animation under test. When the settle + * moves to the UIKit spring, `expectedSettle` becomes the spring from the + * motion constants, and the velocity hand-off reported below becomes an + * assertion. + */ + +import {expect, test, type Locator, type Page} from '@playwright/test'; +import { + animationStoryboard, + blockMainThread, + busyWindowProof, + castStoryboard, + compareCurve, + countBetween, + curveOf, + drag, + flick, + frames, + freezeTimers, + holdAnimations, + installLongTaskLog, + installTimerGate, + markElement, + openStory, + readLongTasks, + releaseAnimations, + releaseVelocity, + sampleCurve, + startScreencast, + thawTimers, + timeline, + timingFunction, + touchDriver, + toTouchEvents, + touchStoryboard, + traceRendering, + trackMarker, + withStorybook, + waitForAnimation, + waitForRest, + writeEvidence, + type TouchDriver, +} from '@astryxdesign/interaction-harness'; + +const storybook = withStorybook(test); + +// The settle's timing, from the theme the story renders in: `--ease-standard` +// is the default cubic-bezier(0.24, 1, 0.4, 1), and `--duration-medium` is the +// neutral theme's motion `medium` (300 ms; the base theme's is 410 ms). +const SETTLE_EASING = 'cubic-bezier(0.24, 1, 0.4, 1)'; +const SETTLE_MS = 300; + +interface OpenSheet { + driver: TouchDriver; + sheet: Locator; + /** Top of the sheet at its tall detent and at the half-viewport detent, CSS px. */ + tallTop: number; + halfTop: number; + handle: {x: number; y: number}; +} + +async function openSheet(page: Page): Promise { + await installLongTaskLog(page); + await openStory(page, storybook(), 'core-bottomsheet--snap-points'); + const driver = await touchDriver(page); + const trigger = page.getByRole('button', {name: 'Open nearby places'}); + const box = await trigger.boundingBox(); + if (box == null) { + throw new Error('no trigger'); + } + await driver.tap({x: box.x + box.width / 2, y: box.y + box.height / 2}); + const sheet = page + .getByRole('dialog', {name: 'Nearby places'}) + .locator('.astryx-bottom-sheet') + .first(); + await expect(sheet).toBeVisible(); + await waitForRest(sheet); + const rect = await sheet.boundingBox(); + if (rect == null) { + throw new Error('no sheet'); + } + const viewport = page.viewportSize(); + if (viewport == null) { + throw new Error('no viewport'); + } + return { + driver, + sheet, + tallTop: rect.y, + // snapPoints={[0.5]}: the extra stop shows half the viewport. + halfTop: viewport.height * 0.5, + handle: {x: rect.x + rect.width / 2, y: rect.y + 12}, + }; +} + +// A plain evaluate, not boundingBox(): it must also work while the page's timers are frozen. +const sheetTop = async (sheet: Locator) => + sheet.evaluate(el => el.getBoundingClientRect().y); + +/** A slow drag that rests before the lift: release velocity 0, so the release projects onto itself. */ +function slowDragTo(from: {x: number; y: number}, dy: number) { + return drag( + from, + {x: from.x, y: from.y + dy}, + {durationMs: 400, hz: 60, restMs: 150}, + ); +} + +test('a slow drag follows the finger and settles on the nearest detent', async ({ + page, + browserName, +}) => { + const {driver, sheet, tallTop, halfTop, handle} = await openSheet(page); + // Two thirds of the way to the half detent: nearer to it than to tall. + const dy = Math.round((halfTop - tallTop) * 0.66); + const script = slowDragTo(handle, dy); + const lastMove = + toTouchEvents(script).filter(e => e.type === 'touchMove').length - 1; + const tops: number[] = []; + let moves = 0; + let restingTop = NaN; + const started = Date.now(); + await driver.play(script, { + onEvent: async step => { + if (step.type !== 'touchMove') { + return; + } + // Read right away (moves are paced at 60 Hz), and once more after the + // last move has been presented: the finger rests 150 ms before lifting. + tops.push(await sheetTop(sheet)); + if (moves++ === lastMove) { + await frames(page, 1); + restingTop = await sheetTop(sheet); + } + }, + }); + // The sheet tracked the finger: monotonic, and 1:1 once out of the + // starting detent's 40 px magnet. + expect(tops.every((top, i) => i === 0 || top >= tops[i - 1] - 0.5)).toBe( + true, + ); + expect(restingTop).toBeCloseTo(tallTop + dy, 0); + await waitForRest(sheet); + expect(await sheetTop(sheet)).toBeCloseTo(halfTop, 0); + // No long task from the gesture or the settle (the story's own load is + // excluded). Only Chromium has the Long Tasks API, so only Chromium can + // check this; the drag and the settle above hold in both engines. + if (browserName === 'chromium') { + expect(await readLongTasks(page, {from: started, to: Date.now()})).toEqual( + [], + ); + } +}); + +test('a downward flick dismisses the sheet', async ({page}) => { + const {driver, handle} = await openSheet(page); + const script = flick(handle, { + velocity: {x: 0, y: 2000}, + durationMs: 80, + hz: 120, + }); + expect(releaseVelocity(script).y).toBeCloseTo(2000, -1); + await driver.play(script); + await expect(page.getByRole('dialog', {name: 'Nearby places'})).toBeHidden(); +}); + +test('the settle follows its timing function, sampled frame by frame', async ({ + page, +}, testInfo) => { + const {driver, sheet, tallTop, halfTop, handle} = await openSheet(page); + const dy = Math.round((halfTop - tallTop) * 0.66); + await driver.play(slowDragTo(handle, dy)); + const [settle] = await waitForAnimation(sheet); + const held = await holdAnimations(page, sheet); + const transition = held.find(a => a.name === 'transform'); + if (transition == null) { + throw new Error(`no transform transition held: ${JSON.stringify(held)}`); + } + // The settle runs the pinned timing: a changed curve or duration fails here, + // and the oracle below is built from the pinned values, not from it. + expect(transition.easing.replace(/\s+/g, '')).toBe( + SETTLE_EASING.replace(/\s+/g, ''), + ); + expect(transition.durationMs).toBe(SETTLE_MS); + // Seek the held settle to every frame time and read the presented transform. + const samples = curveOf( + await sampleCurve(sheet, timeline(SETTLE_MS)), + 'translateY', + ); + const from = samples[0].value; + const to = samples[samples.length - 1].value; + const ease = timingFunction(SETTLE_EASING); + const expectedSettle = (t: number) => + from + (to - from) * ease(Math.min(1, t / SETTLE_MS)); + const result = compareCurve(samples, expectedSettle, 0.5); + await releaseAnimations(page); + // UIKit parity metric, reported rather than asserted until the spring lands: + // the settle's initial velocity vs the finger's at release (here 0). + const initialVelocity = ((expectedSettle(1) - expectedSettle(0)) / 1) * 1000; // px/s over the first ms + testInfo.annotations.push({ + type: 'uikit-parity', + description: `settle starts at ${initialVelocity.toFixed(0)} px/s for a release at 0 px/s (UIKit hands the release velocity to the spring)`, + }); + await writeEvidence( + 'curves/bottomsheet-settle', + { + [`${testInfo.project.name}.json`]: JSON.stringify( + {settle, transition, comparison: result}, + null, + 2, + ), + }, + testInfo, + ); + expect(result.maxError, JSON.stringify(result.worst)).toBeLessThanOrEqual( + 0.5, + ); + expect(samples.at(-1)?.value).toBeCloseTo(halfTop - tallTop, 0); +}); + +test('the settle keeps advancing every frame while the main thread is blocked', async ({ + page, +}, testInfo) => { + const {driver, sheet, tallTop, halfTop, handle} = await openSheet(page); + await markElement(sheet); + const cast = await startScreencast(page); + // The settle must be running (and composited) before the block: an + // animation cannot start while the main thread is held. Armed before the + // release, the page waits for the settle itself (the sheet runs no other + // animation during the drag) and blocks once it has run 16 ms, however busy + // the machine; armed after, the round trip would leave little of the settle + // to judge. + const blocking = blockMainThread(page, 700, { + during: sheet, + minProgressMs: 16, + }); + await driver.play(slowDragTo(handle, Math.round((halfTop - tallTop) * 0.66))); + const block = await blocking; + // Where the settle ends, from the page: a frame after the block can still + // show a stale position (see `settledAt`). The frames track the marker + // chip, which sits inset in the sheet, so it is the chip's rest: the sheet's + // own top would end the judged window about 9 px before the settle does. + await waitForRest(sheet); + const settledAt = await sheet.evaluate(el => { + const chip = el.querySelector('[data-harness-marker-chip]'); + if (chip == null) { + throw new Error('the sheet lost its marker chip'); + } + return chip.getBoundingClientRect().y; + }); + const castFrames = await cast.stop(); + const tracked = await trackMarker(page, castFrames); + // Judge the part of the block during which the settle was still moving: + // until it is within 2 px of rest (the screencast rounds to whole pixels, + // so the easing tail shows plateaus there). + const proof = busyWindowProof(tracked, block, {settledWithin: 2, settledAt}); + await writeEvidence( + 'busy-main-thread/bottomsheet-settle', + { + [`${testInfo.project.name}.json`]: JSON.stringify( + { + block, + settledAt, + proof, + frames: tracked.map(f => ({ + t: +(f.t - block.from).toFixed(1), + y: f.box?.y ?? null, + h: f.box?.height ?? null, + })), + }, + null, + 1, + ), + }, + testInfo, + ); + const film = await castStoryboard(page, castFrames, { + name: 'bottomsheet-settle-busy-main-thread', + zero: block.from, + from: block.from - 40, + to: proof.judged.to + 40, + columns: 8, + cellScale: 0.35, + data: t => ({y: tracked.find(f => f.t === t)?.box?.y ?? -1}), + }); + await film.write(testInfo, {block, judged: proof.judged}); + testInfo.annotations.push({ + type: 'busy-main-thread', + description: proof.summary, + }); + expect(proof.verdict.failures, proof.summary).toEqual([]); + expect(proof.monotonic).toBe(true); + // The judged window runs until the marker is within 2 px of where it rests. + const lastJudged = tracked + .filter(f => f.box != null && f.t <= proof.judged.to) + .at(-1); + expect( + Math.abs((lastJudged?.box?.y ?? Number.NaN) - settledAt), + ).toBeLessThanOrEqual(2); +}); + +test('the settle does no Layout or Paint per frame', async ({ + page, + browserName, +}) => { + test.skip(browserName !== 'chromium', 'rendering traces are Chromium-only'); + const {driver, sheet, tallTop, halfTop, handle} = await openSheet(page); + let settle = {from: 0, to: 0}; + const trace = await traceRendering(page, async () => { + await driver.play( + slowDragTo(handle, Math.round((halfTop - tallTop) * 0.66)), + ); + [settle] = await waitForAnimation(sheet); + await waitForRest(sheet); + }); + // The steady part of the settle: its first and last frames may commit. + const during = countBetween(trace, settle.from + 50, settle.to - 50); + expect(during.Layout ?? 0, JSON.stringify(during)).toBe(0); + expect(during.Paint ?? 0, JSON.stringify(during)).toBe(0); + expect(trace.longTasks).toEqual([]); +}); + +test( + 'storyboard: drag to the half detent and settle', + {tag: '@storyboard'}, + async ({page}, testInfo) => { + const {driver, sheet, tallTop, halfTop, handle} = await openSheet(page); + // Gate the page's timers from here on, so the settle's completion timer can + // be held with its animation while the filmstrip is photographed. + await installTimerGate(page); + const viewport = page.viewportSize() ?? {width: 402, height: 714}; + const clip = {x: 0, y: 0, width: viewport.width, height: viewport.height}; + const dy = Math.round((halfTop - tallTop) * 0.66); + const measure = async () => ({top: await sheetTop(sheet)}); + const dragBoard = await touchStoryboard( + page, + driver, + drag( + handle, + {x: handle.x, y: handle.y + dy}, + {durationMs: 200, hz: 30, restMs: 150}, + ), + { + name: 'bottomsheet-drag-to-detent', + clip, + measure, + columns: 5, + cellScale: 0.4, + }, + ); + await waitForAnimation(sheet); + await freezeTimers(page); + const {storyboard: settleBoard} = await animationStoryboard(page, { + name: 'bottomsheet-settle', + clip, + scope: sheet, + measure, + columns: 7, + cellScale: 0.4, + }); + await releaseAnimations(page, {finish: true}); + await thawTimers(page); + await dragBoard.write(testInfo); + await settleBoard.write(testInfo); + // The trace is the assertion; the sheets are the evidence. + const dragTops = dragBoard.frames.map(f => f.data.top as number); + expect( + dragTops.every((top, i) => i === 0 || top >= dragTops[i - 1] - 0.5), + ).toBe(true); + const settleTops = settleBoard.frames.map(f => f.data.top as number); + expect( + settleTops.every((top, i) => i === 0 || top >= settleTops[i - 1] - 0.5), + ).toBe(true); + expect(settleTops.at(-1)).toBeCloseTo(halfTop, 0); + }, +); diff --git a/packages/core/src/Layer/__tests__/inputHold.fixture.ts b/packages/core/src/Layer/__tests__/inputHold.fixture.ts new file mode 100644 index 0000000000000..84cfc1b93f198 --- /dev/null +++ b/packages/core/src/Layer/__tests__/inputHold.fixture.ts @@ -0,0 +1,137 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file inputHold.fixture.ts + * @input Nothing: bundled by esbuild and injected into a blank page + * @output A page whose whole area is a button (`#under`), a modal dialog + * (`#dialog`: text, an action, and a gesture-host tap recognizer on the + * dialog itself, an ancestor of the hold's shield), a light-dismissing + * popover (`#pop`) and a full-viewport layer (`#lift`) whose content and + * tint take the hits while a test may make its root take none, plus + * `window.__hold`: open a layer, hold and + * release it, read what an element at a point is and what lies past a + * hold's shield there, and a log of every + * click, tap recognition and popover toggle + * @position Browser fixture of inputHold.interaction.browser.spec.ts; + * test-only + */ + +import {getGestureHost} from '../../gestures/GestureHost'; +import {TapGestureRecognizer} from '../../gestures/TapGestureRecognizer'; +import {holdInput, isInputShield, type InputHoldScope} from '../inputHold'; + +type LayerId = 'dialog' | 'pop' | 'lift'; + +interface HoldLog { + clicks: string[]; + taps: number; + toggles: string[]; +} + +declare global { + interface Window { + __hold: { + log: HoldLog; + open: (id: LayerId) => void; + hold: (id: LayerId, scope: InputHoldScope) => void; + release: () => void; + hitAt: (x: number, y: number) => string; + hitPast: (x: number, y: number) => string; + }; + } +} + +document.body.style.margin = '0'; +document.body.innerHTML = ` + + +

Delete this item?

+ + +
+ +
+ `; + +function element(id: string): HTMLElement { + const found = document.getElementById(id); + if (found == null) { + throw new Error(`missing #${id}`); + } + return found; +} + +const log: HoldLog = {clicks: [], taps: 0, toggles: []}; + +document.addEventListener( + 'click', + event => { + const target = event.target; + if (target instanceof Element) { + const button = target.closest('button'); + if (button != null) { + log.clicks.push(button.id); + } + } + }, + true, +); +element('pop').addEventListener('toggle', event => { + log.toggles.push(`pop:${event.newState}`); +}); + +// A recognizer on the dialog itself sees every touch inside it, the hold's +// shield included: only the gesture host's touch gate keeps it away. +const tap = new TapGestureRecognizer({name: 'dialogTap'}); +tap.addTransitionListener((_current, _from, to) => { + if (to === 'ended') { + log.taps += 1; + } +}); +getGestureHost(document).attach(element('dialog'), tap); + +let release: (() => void) | null = null; + +window.__hold = { + log, + open(id) { + if (id === 'dialog') { + (element('dialog') as HTMLDialogElement).showModal(); + } else if (id === 'pop') { + element('pop').showPopover(); + } else { + element('lift').style.display = 'block'; + } + }, + hold(id, scope) { + release = holdInput(element(id), { + until: new Promise(() => {}), + scope, + }); + }, + release() { + release?.(); + release = null; + }, + hitAt(x, y) { + return describe(document.elementFromPoint(x, y)); + }, + hitPast(x, y) { + return describe( + document.elementsFromPoint(x, y).find(el => !isInputShield(el)) ?? null, + ); + }, +}; + +function describe(hit: Element | null): string { + if (hit == null) { + return 'nothing'; + } + if (isInputShield(hit)) { + return `shield:${hit.parentElement?.id ?? ''}`; + } + return hit.id !== '' ? hit.id : hit.tagName.toLowerCase(); +} diff --git a/packages/core/src/Layer/__tests__/inputHold.interaction.browser.spec.ts b/packages/core/src/Layer/__tests__/inputHold.interaction.browser.spec.ts new file mode 100644 index 0000000000000..c31bc8fe820a5 --- /dev/null +++ b/packages/core/src/Layer/__tests__/inputHold.interaction.browser.spec.ts @@ -0,0 +1,348 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file inputHold.interaction.browser.spec.ts + * @input The input-hold fixture bundled with esbuild, a phone-sized + * touch screen, and real engine touch input (touchDriver) + * @output Browser proof of what the unit tests cannot see: the hold's shield + * is the hit target over the whole viewport, above a modal dialog in the + * top layer; a tap during a hold reaches no click and no recognizer on the + * layer (the gesture host's touch gate) and leaves a light-dismissing + * popover open; a layer root that takes no hits itself (pointer-events: + * none, or visibility: hidden, its content opting back in) is held all the + * same; keys still act during a presentation's input hold; a dismissal's + * hold makes the content inert and moves focus out of it, so no key acts + * on it either (a focused button that turns inert still takes Enter and + * Space in both engines); each release gives input back; a hit test + * that skips shields (isInputShield) finds what lies beneath, and + * a layer above the held one still wins; a one-finger + * pan on the shield scrolls nothing while a pinch on it still zooms the + * page (spec:AST-094 FR10, FR18, FR35; Chromium, which pinch-zooms from + * injected touches) + * @position Browser tests of packages/core/src/Layer/inputHold.ts, run + * by the interaction project (`pnpm test:interaction inputHold`) + */ + +import path from 'node:path'; +import {fileURLToPath} from 'node:url'; +import {build} from 'esbuild'; +import {expect, test, type Page} from '@playwright/test'; +import { + drag, + pinch, + touchDriver, + type TouchDriver, +} from '@astryxdesign/interaction-harness'; + +type Point = {x: number; y: number}; +type Scope = 'viewport' | 'layer'; +type LayerId = 'dialog' | 'pop' | 'lift'; + +interface HoldApi { + log: {clicks: string[]; taps: number; toggles: string[]}; + open: (id: LayerId) => void; + hold: (id: LayerId, scope: Scope) => void; + release: () => void; + hitAt: (x: number, y: number) => string; + hitPast: (x: number, y: number) => string; +} + +let bundle = ''; + +test.beforeAll(async () => { + const result = await build({ + entryPoints: [ + path.join( + path.dirname(fileURLToPath(import.meta.url)), + 'inputHold.fixture.ts', + ), + ], + bundle: true, + write: false, + format: 'iife', + platform: 'browser', + target: 'es2022', + }); + bundle = result.outputFiles[0].text; +}); + +async function load(page: Page): Promise { + // A real navigation first: touch emulation exposes the Touch Events API to + // documents it creates. + await page.goto('about:blank'); + await page.setContent( + '', + ); + await page.addScriptTag({content: bundle}); + return touchDriver(page); +} + +type HoldWindow = {__hold: HoldApi}; + +const open = async (page: Page, id: LayerId) => + page.evaluate(layer => { + (window as unknown as HoldWindow).__hold.open(layer); + }, id); +const hold = async (page: Page, id: LayerId, scope: Scope) => + page.evaluate( + ([layer, how]) => { + (window as unknown as HoldWindow).__hold.hold(layer, how); + }, + [id, scope] as const, + ); +const release = async (page: Page) => + page.evaluate(() => { + (window as unknown as HoldWindow).__hold.release(); + }); +const hitAt = async (page: Page, point: Point) => + page.evaluate( + at => (window as unknown as HoldWindow).__hold.hitAt(at.x, at.y), + point, + ); +const hitPast = async (page: Page, point: Point) => + page.evaluate( + at => (window as unknown as HoldWindow).__hold.hitPast(at.x, at.y), + point, + ); +const readLog = async (page: Page) => + page.evaluate(() => (window as unknown as HoldWindow).__hold.log); + +async function centre(page: Page, id: string): Promise { + const box = await page.locator(`#${id}`).boundingBox(); + if (box == null) { + throw new Error(`#${id} has no box`); + } + return {x: box.x + box.width / 2, y: box.y + box.height / 2}; +} + +// A tap's click (or its recognition) lands within a few frames. +const settle = async (page: Page) => page.waitForTimeout(250); + +test("during a presentation's input hold on a modal dialog, a tap on it or beside it reaches nothing, keys still act, and the release gives touch back", async ({ + page, +}) => { + const driver = await load(page); + await open(page, 'dialog'); + const action = await centre(page, 'action'); + const message = await centre(page, 'message'); + const beside = {x: 12, y: 12}; + await hold(page, 'dialog', 'viewport'); + // The shield is what a touch hits, on the dialog and over its backdrop. + expect(await hitAt(page, action)).toBe('shield:dialog'); + expect(await hitAt(page, beside)).toBe('shield:dialog'); + await driver.tap(action); + await driver.tap(message); + await settle(page); + // No click, and the recognizer on the dialog saw nothing: the gate. + expect(await readLog(page)).toEqual({clicks: [], taps: 0, toggles: []}); + // Keys still act during a presentation's input hold. + await page.locator('#action').focus(); + await page.keyboard.press('Enter'); + await expect + .poll(async () => (await readLog(page)).clicks) + .toEqual(['action']); + await release(page); + expect(await hitAt(page, action)).toBe('action'); + expect(await hitAt(page, beside)).toBe('dialog'); + await driver.tap(message); + await expect.poll(async () => (await readLog(page)).taps).toBe(1); +}); + +test('while a dismissal holds it, its content is inert, focus and keys leave it, and the shield still takes every tap; the release restores both', async ({ + page, +}) => { + const driver = await load(page); + await open(page, 'dialog'); + const action = await centre(page, 'action'); + const message = await centre(page, 'message'); + // The dialog moved focus to its action when it opened. + expect(await page.evaluate(() => document.activeElement?.id ?? null)).toBe( + 'action', + ); + await hold(page, 'dialog', 'layer'); + // A key reaches nothing: the focused action left the inert content. + await page.keyboard.press('Enter'); + await page.keyboard.press('Space'); + await settle(page); + expect((await readLog(page)).clicks).toEqual([]); + // An inert subtree is no hit target: the shield, in the dialog, is. + expect(await hitAt(page, action)).toBe('shield:dialog'); + const state = async () => + page.evaluate(() => { + const button = document.getElementById('action'); + button?.focus(); + return { + inert: button?.closest('[inert]') != null, + focused: document.activeElement === button, + }; + }); + expect(await state()).toEqual({inert: true, focused: false}); + await driver.tap(action); + await driver.tap(message); + await settle(page); + expect(await readLog(page)).toEqual({clicks: [], taps: 0, toggles: []}); + await release(page); + expect(await state()).toEqual({inert: false, focused: true}); + expect(await hitAt(page, action)).toBe('action'); + await driver.tap(message); + await expect.poll(async () => (await readLog(page)).taps).toBe(1); +}); + +test('a light-dismissing popover stays open when a hold takes a tap outside it; after the release the same tap dismisses it', async ({ + page, +}) => { + const driver = await load(page); + await open(page, 'pop'); + const viewport = page.viewportSize(); + if (viewport == null) { + throw new Error('no viewport'); + } + const outside = {x: viewport.width / 2, y: viewport.height - 40}; + expect(await hitAt(page, outside)).toBe('under'); + await hold(page, 'pop', 'viewport'); + expect(await hitAt(page, outside)).toBe('shield:pop'); + await driver.tap(outside); + await settle(page); + expect( + await page.evaluate(() => + document.getElementById('pop')?.matches(':popover-open'), + ), + ).toBe(true); + expect(await readLog(page)).toEqual({ + clicks: [], + taps: 0, + toggles: ['pop:open'], + }); + await release(page); + await driver.tap(outside); + await expect + .poll(async () => (await readLog(page)).toggles) + .toEqual(['pop:open', 'pop:closed']); +}); + +test('a hit test that skips shields sees past a hold: under a held modal dialog’s shield, its own action', async ({ + page, +}) => { + await load(page); + await open(page, 'dialog'); + const action = await centre(page, 'action'); + await hold(page, 'dialog', 'viewport'); + // The opening finger's press resolves its row past the shield. + expect(await hitAt(page, action)).toBe('shield:dialog'); + expect(await hitPast(page, action)).toBe('action'); + await release(page); + expect(await hitAt(page, action)).toBe('action'); +}); + +test('a layer opened above a held one is what that hit test finds, and past the shield the held layer’s content', async ({ + page, +}) => { + await load(page); + await open(page, 'lift'); + const liftAction = await centre(page, 'liftAction'); + await hold(page, 'lift', 'viewport'); + expect(await hitAt(page, liftAction)).toBe('shield:lift'); + expect(await hitPast(page, liftAction)).toBe('liftAction'); + await open(page, 'pop'); + const popAction = await centre(page, 'popAction'); + expect(await hitAt(page, popAction)).toBe('popAction'); + expect(await hitPast(page, popAction)).toBe('popAction'); +}); + +// A full-viewport layer whose root takes no hits itself and lets its tint +// and content take them (ContextMenu's lift layer is pointer-events: none): +// the shield, inside that root, must not inherit that. +for (const rootStyle of ['pointer-events: none', 'visibility: hidden']) { + test(`a layer root with ${rootStyle} (its content opting back in) is held all the same: a tap reaches neither the content nor the page`, async ({ + page, + }) => { + const driver = await load(page); + await page.evaluate(style => { + const lift = document.getElementById('lift'); + if (lift != null) { + lift.style.cssText += `; ${style}`; + } + }, rootStyle); + await open(page, 'lift'); + const action = await centre(page, 'liftAction'); + const tint = {x: 12, y: 12}; + // Unheld, the content and the tint take the hits. + expect(await hitAt(page, action)).toBe('liftAction'); + expect(await hitAt(page, tint)).toBe('tint'); + await hold(page, 'lift', 'layer'); + expect(await hitAt(page, action)).toBe('shield:lift'); + expect(await hitAt(page, tint)).toBe('shield:lift'); + await driver.tap(action); + await driver.tap(tint); + await settle(page); + expect((await readLog(page)).clicks).toEqual([]); + await release(page); + expect(await hitAt(page, action)).toBe('liftAction'); + }); +} + +// Page zoom stays the browser's while a layer is held (spec:AST-094 FR10, +// FR18 and FR35; WCAG 1.4.4): the shield starts no pan, but a pinch on it +// still zooms the page. +for (const [id, scope] of [ + ['pop', 'viewport'], + ['dialog', 'layer'], +] as const) { + test(`while a ${scope === 'viewport' ? 'presentation' : 'dismissal'} holds ${id === 'pop' ? 'a popover' : 'a modal dialog'}, a one-finger pan scrolls nothing and a pinch still zooms the page; neither reaches the layer or the page`, async ({ + page, + browserName, + }) => { + test.skip( + browserName === 'webkit', + 'needs native scrolling and pinch-zoom from injected touches (Mobile Safari on the simulator)', + ); + const driver = await load(page); + const viewport = page.viewportSize(); + if (viewport == null) { + throw new Error('no viewport'); + } + const middle = {x: viewport.width / 2, y: viewport.height / 2}; + const panUp = drag( + {x: middle.x, y: middle.y + 150}, + {x: middle.x, y: middle.y - 150}, + // A rest before the lift: no fling outlives the pan. + {durationMs: 200, restMs: 150}, + ); + // A page that scrolls under the layer: unheld, the same pan scrolls it. + await page.evaluate(() => { + const spacer = document.createElement('div'); + spacer.style.height = '300vh'; + document.body.append(spacer); + }); + await driver.play(panUp); + await expect + .poll(async () => page.evaluate(() => window.scrollY)) + .toBeGreaterThan(0); + await settle(page); + await page.evaluate(() => window.scrollTo(0, 0)); + await settle(page); + expect(await page.evaluate(() => window.scrollY)).toBe(0); + await open(page, id); + await hold(page, id, scope); + expect(await hitAt(page, middle)).toBe(`shield:${id}`); + await driver.play(panUp); + await settle(page); + expect(await page.evaluate(() => window.scrollY)).toBe(0); + await driver.play( + pinch({ + center: middle, + fromDistance: 60, + toDistance: 220, + durationMs: 200, + }), + ); + await expect + .poll(async () => page.evaluate(() => window.visualViewport?.scale ?? 1)) + .toBeGreaterThan(1.2); + expect(await readLog(page)).toEqual({ + clicks: [], + taps: 0, + toggles: id === 'pop' ? ['pop:open'] : [], + }); + }); +} diff --git a/packages/core/src/Layer/__tests__/layerExitCopy.interaction.browser.spec.ts b/packages/core/src/Layer/__tests__/layerExitCopy.interaction.browser.spec.ts new file mode 100644 index 0000000000000..c6af8c473c385 --- /dev/null +++ b/packages/core/src/Layer/__tests__/layerExitCopy.interaction.browser.spec.ts @@ -0,0 +1,189 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file layerExitCopy.interaction.browser.spec.ts + * @input The exit copy bundled with esbuild onto a blank fixture page: a + * popover layer inside a clipping, transformed panel, an entrance curve + * held halfway, and Chromium and WebKit + * @output Proof that the engine shows the copy where the layer was drawn: + * placed as the layer closes (from script and by the browser's own light + * dismiss), it is in the top layer as a manual popover with no `z-index`, + * uncut by the clipping panel, and its drawn box equals the layer's drawn + * box mid-entrance within half a px; a second copy for the same layer + * replaces the first while another layer's copy stays + * @position Browser proof for layerExitCopy.ts, run by `pnpm + * test:interaction`; jsdom has no layout and no top layer. The refusal + * path (Android Chromium 158 refuses a show during another popover's + * toggle) is proven on the Android gate emulator (pull request evidence): + * these engines do not refuse. + */ + +import {fileURLToPath} from 'node:url'; +import {expect, test, type Page} from '@playwright/test'; +import {build} from 'esbuild'; + +declare global { + interface Window { + ExitCopy: typeof import('../layerExitCopy'); + } +} + +let script = ''; + +test.beforeAll(async () => { + const result = await build({ + entryPoints: [ + fileURLToPath(new URL('../layerExitCopy.ts', import.meta.url)), + ], + bundle: true, + write: false, + format: 'iife', + globalName: 'ExitCopy', + target: 'es2022', + }); + script = result.outputFiles[0].text; +}); + +const ATTRIBUTE = 'data-exit-copy'; + +// A real navigation (not setContent): the page keeps its own window. The +// layer sits in a panel that clips and is transformed: a fixed box placed +// there would be cut and moved; the top layer is neither. +async function open(page: Page) { + const html = ` + +
+

One

+
Other
+
+`; + await page.route('http://exit-copy.test/', async route => + route.fulfill({contentType: 'text/html', body: html}), + ); + await page.goto('http://exit-copy.test/'); +} + +test('a copy placed as a layer closes mid-entrance is in the top layer, where the layer was drawn', async ({ + page, +}) => { + await open(page); + const result = await page.evaluate( + async ({attribute}) => { + const layer = document.getElementById('layer') as HTMLElement; + layer.showPopover(); + // An entrance halfway through: drawn at 60 % about its top-left corner. + const entrance = layer.animate( + [ + {scale: '0.2', transformOrigin: '0 0'}, + {scale: '1', transformOrigin: '0 0'}, + ], + {duration: 1000, fill: 'both'}, + ); + entrance.pause(); + entrance.currentTime = 500; + const drawn = layer.getBoundingClientRect(); + let copy: HTMLElement | null = null; + layer.addEventListener( + 'beforetoggle', + event => { + if ((event).newState === 'closed') { + copy = window.ExitCopy.placeLayerExitCopy(layer, attribute); + } + }, + {once: true}, + ); + layer.hidePopover(); + await new Promise(resolve => requestAnimationFrame(resolve)); + const placed = copy as HTMLElement | null; + if (placed == null) { + return null; + } + const rect = placed.getBoundingClientRect(); + return { + drawn: [drawn.left, drawn.top, drawn.width, drawn.height], + copy: [rect.left, rect.top, rect.width, rect.height], + open: placed.matches(':popover-open'), + popover: placed.getAttribute('popover'), + zIndex: getComputedStyle(placed).zIndex, + layerOpen: layer.matches(':popover-open'), + }; + }, + {attribute: ATTRIBUTE}, + ); + expect(result).not.toBeNull(); + expect(result!.layerOpen).toBe(false); + expect(result!.open).toBe(true); + expect(result!.popover).toBe('manual'); + expect(result!.zIndex).toBe('auto'); + // Mid-entrance the drawn box is not the layout box; the copy holds it. + expect(result!.drawn[2]).toBeLessThan(280); + result!.copy.forEach((value, index) => { + expect(Math.abs(value - result!.drawn[index])).toBeLessThanOrEqual(0.5); + }); +}); + +test('the browser\u2019s own light dismiss gets the copy too, uncut by the clipping panel', async ({ + page, +}) => { + await open(page); + await page.evaluate(attribute => { + const layer = document.getElementById('layer') as HTMLElement; + layer.showPopover(); + layer.addEventListener( + 'beforetoggle', + event => { + if ((event).newState === 'closed') { + window.ExitCopy.placeLayerExitCopy(layer, attribute); + } + }, + {once: true}, + ); + }, ATTRIBUTE); + // A press outside: the browser closes the layer by itself. + await page.mouse.click(360, 700); + const result = await page.evaluate(attribute => { + const copy = document.querySelector(`[${attribute}]`); + if (copy == null) { + return null; + } + const rect = copy.getBoundingClientRect(); + return { + open: copy.matches(':popover-open'), + rect: [rect.left, rect.top, rect.width, rect.height], + layerOpen: document.getElementById('layer')!.matches(':popover-open'), + }; + }, ATTRIBUTE); + expect(result).not.toBeNull(); + expect(result!.layerOpen).toBe(false); + expect(result!.open).toBe(true); + // Where the layer stood, at its full size: the 60 x 40 clipping panel and + // its 30 px transform neither cut nor moved it. + expect(result!.rect).toEqual([40, 120, 280, 200]); +}); + +test('a second copy for the same layer replaces the first; another layer\u2019s copy stays', async ({ + page, +}) => { + await open(page); + const result = await page.evaluate(attribute => { + const layer = document.getElementById('layer') as HTMLElement; + const other = document.getElementById('other') as HTMLElement; + layer.showPopover(); + other.showPopover(); + const otherCopy = window.ExitCopy.placeLayerExitCopy(other, attribute); + const first = window.ExitCopy.placeLayerExitCopy(layer, attribute); + const second = window.ExitCopy.placeLayerExitCopy(layer, attribute); + return { + first: first?.isConnected, + second: second?.isConnected && second.matches(':popover-open'), + other: otherCopy?.isConnected && otherCopy.matches(':popover-open'), + copies: document.querySelectorAll(`[${attribute}]`).length, + }; + }, ATTRIBUTE); + expect(result).toEqual({first: false, second: true, other: true, copies: 2}); +}); diff --git a/packages/core/src/Layer/inputHold.test.tsx b/packages/core/src/Layer/inputHold.test.tsx new file mode 100644 index 0000000000000..836f661c103c3 --- /dev/null +++ b/packages/core/src/Layer/inputHold.test.tsx @@ -0,0 +1,231 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +import {render} from '@testing-library/react'; +import {useRef} from 'react'; +import {afterEach, describe, expect, it, vi} from 'vitest'; +import {GestureHost} from '../gestures/GestureHost'; +import { + holdInput, + isInHeldLayer, + isInputShield, + useInputHold, +} from './inputHold'; + +function deferred() { + let resolve: () => void = () => {}; + let reject: (error: unknown) => void = () => {}; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return {promise, resolve, reject}; +} + +/** A layer root with a child, in the document. */ +function mountLayer(): {layer: HTMLElement; child: HTMLElement} { + const layer = document.createElement('div'); + const child = document.createElement('button'); + layer.append(child); + document.body.append(layer); + return {layer, child}; +} + +const flush = async () => { + await Promise.resolve(); + await Promise.resolve(); +}; + +afterEach(() => { + document.body.replaceChildren(); + vi.restoreAllMocks(); +}); + +describe('holdInput', () => { + it('holds a presenting layer with a shield over the viewport until its end settles', async () => { + const {layer, child} = mountLayer(); + const outside = document.createElement('div'); + document.body.append(outside); + const end = deferred(); + holdInput(layer, {until: end.promise, scope: 'viewport'}); + const shield = layer.lastElementChild as HTMLElement; + expect(shield).not.toBe(child); + expect(shield.getAttribute('popover')).toBe('manual'); + expect(shield.style.position).toBe('fixed'); + // No pan, and a pinch still zooms the page (spec:AST-094 FR10). + expect(shield.style.touchAction).toBe('pinch-zoom'); + // Its own hit-testing, whatever the layer root's (the browser proof holds + // a root that is pointer-events: none). + expect(shield.style.pointerEvents).toBe('auto'); + expect(shield.style.visibility).toBe('visible'); + expect(isInHeldLayer(child)).toBe(true); + expect(isInHeldLayer(shield)).toBe(true); + expect(isInHeldLayer(outside)).toBe(false); + // Keys and assistive technology stay live while presenting. + expect(layer.hasAttribute('inert')).toBe(false); + end.resolve(); + await flush(); + expect(shield.isConnected).toBe(false); + expect(isInHeldLayer(child)).toBe(false); + }); + + it('swallows a press and a click on the shield before they reach the page, and leaves a touch its start so a pinch can still zoom', () => { + const {layer} = mountLayer(); + holdInput(layer, {until: new Promise(() => {}), scope: 'viewport'}); + const shield = layer.lastElementChild as HTMLElement; + const seen: string[] = []; + for (const type of ['pointerdown', 'click', 'touchstart']) { + document.addEventListener(type, () => seen.push(type)); + } + const press = new Event('pointerdown', {bubbles: true, cancelable: true}); + const click = new MouseEvent('click', {bubbles: true, cancelable: true}); + const touch = new Event('touchstart', {bubbles: true, cancelable: true}); + shield.dispatchEvent(press); + shield.dispatchEvent(click); + shield.dispatchEvent(touch); + expect(seen).toEqual([]); + expect(press.defaultPrevented).toBe(true); + expect(click.defaultPrevented).toBe(true); + // Cancelling a touch's start would cancel the page's pinch-zoom too. + expect(touch.defaultPrevented).toBe(false); + }); + + it("holds a dismissing layer: its content inert, the layer itself live under the shield, each element's inert given back", async () => { + const {layer, child} = mountLayer(); + const already = document.createElement('div'); + already.setAttribute('inert', ''); + layer.append(already); + child.focus(); + expect(document.activeElement).toBe(child); + const end = deferred(); + holdInput(layer, {until: end.promise, scope: 'layer'}); + // Focus leaves the content: a focused element that turns inert would + // keep its key activation. + expect(document.activeElement).not.toBe(child); + const shield = layer.lastElementChild as HTMLElement; + expect(shield.getAttribute('popover')).toBe('manual'); + // The layer stays live so the shield inside it catches every hit: an + // inert subtree is no hit target. + expect(layer.hasAttribute('inert')).toBe(false); + expect(shield.hasAttribute('inert')).toBe(false); + expect(child.hasAttribute('inert')).toBe(true); + expect(isInHeldLayer(child)).toBe(true); + end.reject(new Error('interrupted')); + await flush(); + expect(child.hasAttribute('inert')).toBe(false); + expect(already.hasAttribute('inert')).toBe(true); + expect(shield.isConnected).toBe(false); + expect(isInHeldLayer(child)).toBe(false); + }); + + it('names a live hold’s shield, and only it, so a hit test can skip it', () => { + const {layer, child} = mountLayer(); + const popover = document.createElement('div'); + popover.setAttribute('popover', 'manual'); + document.body.append(popover); + const release = holdInput(layer, { + until: new Promise(() => {}), + scope: 'viewport', + }); + const shield = layer.lastElementChild as HTMLElement; + expect(isInputShield(shield)).toBe(true); + for (const other of [layer, child, popover, null]) { + expect(isInputShield(other)).toBe(false); + } + release(); + expect(isInputShield(shield)).toBe(false); + }); + + it('sees slotted content as held: the flat tree, a slotted node to its slot', () => { + const host = document.createElement('div'); + const slotted = document.createElement('button'); + host.append(slotted); + document.body.append(host); + const root = host.attachShadow({mode: 'open'}); + const layer = document.createElement('div'); + layer.append(document.createElement('slot')); + root.append(layer); + expect(isInHeldLayer(slotted)).toBe(false); + const release = holdInput(layer, { + until: new Promise(() => {}), + scope: 'viewport', + }); + expect(isInHeldLayer(slotted)).toBe(true); + release(); + expect(isInHeldLayer(slotted)).toBe(false); + }); + + it('takes an animator handle, ending with its finished', async () => { + const {layer, child} = mountLayer(); + const end = deferred(); + holdInput(layer, {until: {finished: end.promise}, scope: 'layer'}); + expect(isInHeldLayer(child)).toBe(true); + end.resolve(); + await flush(); + expect(isInHeldLayer(child)).toBe(false); + }); + + it('stacks holds on one layer, either scope, under one shield: it comes back when the last one ends', () => { + const {layer, child} = mountLayer(); + const first = holdInput(layer, { + until: new Promise(() => {}), + scope: 'viewport', + }); + const second = holdInput(layer, { + until: new Promise(() => {}), + scope: 'layer', + }); + expect(layer.querySelectorAll('[popover]')).toHaveLength(1); + expect(child.hasAttribute('inert')).toBe(true); + first(); + first(); + expect(isInHeldLayer(child)).toBe(true); + expect(layer.querySelectorAll('[popover]')).toHaveLength(1); + second(); + expect(isInHeldLayer(child)).toBe(false); + expect(child.hasAttribute('inert')).toBe(false); + expect(layer.querySelectorAll('[popover]')).toHaveLength(0); + }); + + it("registers one touch gate per document, which answers whether a touch's target is held", () => { + const addTouchGate = vi.spyOn(GestureHost.prototype, 'addTouchGate'); + const {layer, child} = mountLayer(); + holdInput(layer, {until: new Promise(() => {}), scope: 'layer'}); + const release = holdInput(layer, { + until: new Promise(() => {}), + scope: 'viewport', + }); + expect(addTouchGate.mock.calls.length).toBeLessThanOrEqual(1); + const registered = + addTouchGate.mock.calls[0]?.[0] ?? + ((target: Node) => isInHeldLayer(target)); + expect(registered(child)).toBe(true); + expect(registered(document.body)).toBe(false); + release(); + }); +}); + +describe('useInputHold', () => { + function Bar({isHeld}: {isHeld: boolean}) { + const ref = useRef(null); + useInputHold(ref, {isHeld, scope: 'layer'}); + return ( +
+ +
+ ); + } + + it('holds the layer while asked, and releases it when no longer asked or unmounted', () => { + const view = render(); + const bar = view.getByTestId('bar'); + const back = view.getByRole('button', {name: 'Back', hidden: true}); + expect(back.hasAttribute('inert')).toBe(true); + expect(isInHeldLayer(back)).toBe(true); + view.rerender(); + expect(back.hasAttribute('inert')).toBe(false); + view.rerender(); + expect(back.hasAttribute('inert')).toBe(true); + view.unmount(); + expect(isInHeldLayer(bar)).toBe(false); + }); +}); diff --git a/packages/core/src/Layer/inputHold.ts b/packages/core/src/Layer/inputHold.ts new file mode 100644 index 0000000000000..8dab031277803 --- /dev/null +++ b/packages/core/src/Layer/inputHold.ts @@ -0,0 +1,328 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +'use client'; + +/** + * @file inputHold.ts + * @input A layer's root element, what ends the hold (a promise, or an + * animator handle's `finished`), and its scope; the gesture host's touch + * gates + * @output holdInput, useInputHold, isInHeldLayer and + * isInputShield: input held away from a layer while its + * presentation runs + * @position The Layer module's one input hold, for every component + * whose layer presents or dismisses under a finger + * + * UIKit delivers no touch while a presentation runs: until it completes, a + * tap reaches nothing, and a dismissal drops every input until it ends + * (measured on iOS 26.2 with UIAlertController: taps 400 ms into its + * presentation ignored, 450 ms in handled). A hold covers one layer root + * until what ends it settles: + * + * - `scope: 'viewport'` (a presentation): a touch, a pointer press or a + * click anywhere reaches nothing. A transparent shield inside the layer + * covers the viewport from the top layer, so a tap on it lands inside the + * layer (a light-dismissing layer stays open) and starts no pan (iOS + * drops that input too while it presents), while a pinch on it still + * zooms the page, which stays the browser's (spec:AST-094 FR10, FR18, + * FR35; WCAG 1.4.4); the gesture host holds + * the layer's touches away from every recognizer. Keys and assistive + * technology stay live, as on iOS. + * - `scope: 'layer'` (a dismissal, an inert bar): the same shield, and the + * layer's content is inert (no focus, key or assistive-technology + * activation; a focused element inside it is blurred, since one that turns + * inert keeps its key activation). The layer element itself stays live, so + * the shield inside + * it still catches every hit: an inert subtree is no hit target, and a tap + * on it would otherwise land on whatever lies beneath. A modal dialog + * still receives its own `cancel` (Escape): the caller ignores it while + * held. + * + * The shield sets its own `pointer-events` and `visibility`, so a layer root + * that takes no hits itself (its content or a backdrop child opts back in) + * is held all the same. A hit test that must see past held input (a press + * the finger that opened the layer continues, captured before the hold + * began) skips shields and nothing else: the first element of + * `elementsFromPoint` for which `isInputShield` is false, so an + * overlay really above the layer still wins. The shield is a manual + * popover, so its `toggle` events reach a document listener that logs + * every popover's: such a log keeps to its own popovers. + * + * One hold per layer: holds on a held layer stack, and the layer comes back + * when the last one ends (when what ends it settles, resolved or rejected; + * when its release runs; on unmount, for the hook). Nothing is written to + * the DOM but the shield and the content's `inert` attribute; the host + * learns through one touch gate per document that asks `isInHeldLayer`, + * walking the flat tree as it does (a slotted node to its slot, a shadow + * root to its host). + * + * SYNC: When modified, update inputHold.test.tsx, + * __tests__/inputHold.interaction.browser.spec.ts and the components + * that hold their layer (holdInput and useInputHold callers). + */ + +import {useEffect, type RefObject} from 'react'; +import {getGestureHost} from '../gestures/GestureHost'; +import {parentAcrossShadow} from '../gestures/nativeScroll'; + +/** What a hold covers: the whole viewport, or the layer alone. */ +export type InputHoldScope = 'viewport' | 'layer'; + +/** What ends a hold: a promise, or an animator handle (its `finished`). */ +export type InputHoldEnd = + PromiseLike | {readonly finished: PromiseLike}; + +export interface InputHoldOptions { + /** The hold ends when this settles. */ + until: InputHoldEnd; + /** `'viewport'` while presenting, `'layer'` while dismissing. */ + scope: InputHoldScope; +} + +interface HeldLayer { + viewport: number; + layer: number; + shield: HTMLElement | null; + /** The content a dismissal made inert, and whether each was already. */ + inertContent: Map | null; +} + +const held = new WeakMap(); +const gatedDocuments = new WeakSet(); +const liveShields = new WeakSet(); + +/** + * Whether `node` is the shield of a live hold. A hit test that must see past + * held input takes the first element of `elementsFromPoint` that is not one. + */ +export function isInputShield(node: Node | null): boolean { + return node != null && liveShields.has(node); +} + +/** + * Whether `target` is inside a held layer, walking the flat tree as the + * gesture host does (parentAcrossShadow: a slotted node to its slot, a + * shadow root to its host). + */ +export function isInHeldLayer(target: Node | null): boolean { + for ( + let node: Node | null = target; + node != null; + node = parentAcrossShadow(node) + ) { + if (node instanceof Element && held.has(node)) { + return true; + } + } + return false; +} + +/** One gate per document: the host holds a held layer's touches away. */ +function gate(doc: Document): void { + if (gatedDocuments.has(doc)) { + return; + } + gatedDocuments.add(doc); + getGestureHost(doc).addTouchGate(isInHeldLayer); +} + +const SWALLOWED = [ + 'pointerdown', + 'pointerup', + 'touchstart', + 'touchend', + 'mousedown', + 'mouseup', + 'click', + 'contextmenu', +] as const; + +const swallow = (event: Event) => { + // A touch's start stays uncancelled: cancelling it would cancel the page's + // pinch-zoom too. The shield's `touch-action: pinch-zoom` keeps a finger + // from panning, and a tap's mouse events and click land on the shield, + // which swallows them. + if (event.cancelable && event.type !== 'touchstart') { + event.preventDefault(); + } + event.stopPropagation(); +}; + +/** A transparent top-layer cover of the viewport, inside `layer`. */ +function addShield(layer: Element): HTMLElement { + const shield = layer.ownerDocument.createElement('div'); + shield.setAttribute('popover', 'manual'); + shield.setAttribute('aria-hidden', 'true'); + Object.assign(shield.style, { + position: 'fixed', + inset: '0', + width: '100vw', + height: '100vh', + maxWidth: 'none', + maxHeight: 'none', + margin: '0', + padding: '0', + border: '0', + background: 'transparent', + overflow: 'hidden', + touchAction: 'pinch-zoom', + // The two inherited properties that decide hit-testing, set here so the + // shield takes hits whatever its layer root opts out of (a full-viewport + // layer that is `pointer-events: none` and lets a backdrop child take the + // hits): inherited, they would let a held tap fall through to the page. + pointerEvents: 'auto', + visibility: 'visible', + }); + for (const type of SWALLOWED) { + shield.addEventListener(type, swallow, {capture: true, passive: false}); + } + layer.append(shield); + liveShields.add(shield); + if (typeof shield.showPopover === 'function') { + try { + shield.showPopover(); + } catch { + // Not connected, or popovers unsupported: the shield still covers + // what it can as a fixed element. + } + } + return shield; +} + +/** + * Holds input away from `layer` until `until` settles. Returns the release + * (idempotent), for a hold that ends early. + */ +export function holdInput( + layer: Element, + {until, scope}: InputHoldOptions, +): () => void { + gate(layer.ownerDocument); + let state = held.get(layer); + if (state == null) { + state = {viewport: 0, layer: 0, shield: null, inertContent: null}; + held.set(layer, state); + } + if (state.shield == null) { + state.shield = addShield(layer); + } + if (scope === 'viewport') { + state.viewport += 1; + } else { + state.layer += 1; + if (state.layer === 1) { + state.inertContent = makeContentInert(layer, state.shield); + } + } + let isReleased = false; + const release = () => { + if (isReleased) { + return; + } + isReleased = true; + const current = held.get(layer); + if (current == null) { + return; + } + if (scope === 'viewport') { + current.viewport -= 1; + } else { + current.layer -= 1; + if (current.layer === 0 && current.inertContent != null) { + restoreInert(current.inertContent); + current.inertContent = null; + } + } + if (current.viewport === 0 && current.layer === 0) { + if (current.shield != null) { + liveShields.delete(current.shield); + current.shield.remove(); + } + held.delete(layer); + } + }; + const end = 'finished' in until ? until.finished : until; + void Promise.resolve(end).then(release, release); + return release; +} + +/** + * Makes the layer's content inert, the shield aside, and moves focus out of + * it: an element that turns inert while focused keeps its focus, and with it + * its key activation (Enter and Space still click a focused button), until + * the engine's focus fixup runs, if it runs at all (measured in Chromium and + * WebKit). Blurring it leaves focus on the document, as that fixup does. + */ +function makeContentInert( + layer: Element, + shield: Element, +): Map { + const content = new Map(); + for (const child of layer.children) { + if (child !== shield) { + content.set(child, child.hasAttribute('inert')); + child.setAttribute('inert', ''); + } + } + const focused = deepActiveElement(layer.ownerDocument); + if ( + focused instanceof HTMLElement && + [...content.keys()].some(child => isInFlatTree(child, focused)) + ) { + focused.blur(); + } + return content; +} + +/** The focused element, inside any shadow roots on the way down. */ +function deepActiveElement(doc: Document): Element | null { + let active = doc.activeElement; + while (active?.shadowRoot?.activeElement != null) { + active = active.shadowRoot.activeElement; + } + return active; +} + +/** Whether `node` is `root` or inside it, walking the flat tree. */ +function isInFlatTree(root: Node, node: Node): boolean { + for ( + let current: Node | null = node; + current != null; + current = parentAcrossShadow(current) + ) { + if (current === root) { + return true; + } + } + return false; +} + +function restoreInert(content: Map): void { + for (const [element, wasInert] of content) { + if (!wasInert) { + element.removeAttribute('inert'); + } + } +} + +/** + * Holds input away from the layer behind `ref` while `isHeld` is true (an + * inert bar, a presentation driven by state); released when `isHeld` turns + * false or the component unmounts. + */ +export function useInputHold( + ref: RefObject, + {isHeld, scope}: {isHeld: boolean; scope: InputHoldScope}, +): void { + useEffect(() => { + const layer = ref.current; + if (!isHeld || layer == null) { + return; + } + // Released by the cleanup alone. + return holdInput(layer, { + until: new Promise(() => {}), + scope, + }); + }, [ref, isHeld, scope]); +} diff --git a/packages/core/src/Layer/layerExitCopy.test.ts b/packages/core/src/Layer/layerExitCopy.test.ts new file mode 100644 index 0000000000000..ef0306fc46687 --- /dev/null +++ b/packages/core/src/Layer/layerExitCopy.test.ts @@ -0,0 +1,387 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file layerExitCopy.test.ts + * @input vitest (jsdom), layerExitCopy + * @output Checks the static copy a closing layer leaves for its close to + * play on: out of reach and out of the accessibility tree, no ids or form + * names, only in the top layer (hidden until a refused show succeeds, none + * without the popover API), on the layer's layout box with the transform + * it was drawn with, one per layer, scrolled where the layer was + * @position Testing; the browser proofs are in + * __tests__/layerExitCopy.interaction.browser.spec.ts + */ + +import {afterEach, describe, expect, it, vi} from 'vitest'; +import {placeLayerExitCopy} from './layerExitCopy'; + +const BOX = {left: 41, top: 204, width: 320, height: 253}; +const ATTRIBUTE = 'data-test-exit'; + +/** A layer drawn on BOX (jsdom has no layout). */ +function layer(): HTMLElement { + const host = document.createElement('section'); + document.body.appendChild(host); + const element = document.createElement('div'); + element.id = 'layer'; + element.setAttribute('popover', 'auto'); + element.style.setProperty('position-anchor', '--trigger'); + element.style.setProperty('scale', '0.5 0.4'); + element.style.setProperty('opacity', '0.3'); + element.style.setProperty('transition', 'opacity 1s'); + element.innerHTML = + ''; + host.appendChild(element); + element.getBoundingClientRect = () => ({ + ...BOX, + x: BOX.left, + y: BOX.top, + right: BOX.left + BOX.width, + bottom: BOX.top + BOX.height, + toJSON: () => ({}), + }); + // Its layout box, which a fixed layer's offset box reports in viewport px. + for (const [name, value] of [ + ['offsetLeft', BOX.left], + ['offsetTop', BOX.top], + ['offsetWidth', BOX.width], + ['offsetHeight', BOX.height], + ] as const) { + Object.defineProperty(element, name, {configurable: true, value}); + } + return element; +} + +describe('placeLayerExitCopy', () => { + const originalShow = Object.getOwnPropertyDescriptor( + HTMLElement.prototype, + 'showPopover', + ); + /** The popover API, stood in for (jsdom has none), or taken away. */ + const popoverApi = (present: boolean) => { + if (present) { + const show = vi.fn(); + Object.defineProperty(HTMLElement.prototype, 'showPopover', { + configurable: true, + writable: true, + value: show, + }); + return show; + } + Reflect.deleteProperty(HTMLElement.prototype, 'showPopover'); + return null; + }; + afterEach(() => { + if (originalShow == null) { + Reflect.deleteProperty(HTMLElement.prototype, 'showPopover'); + } else { + Object.defineProperty(HTMLElement.prototype, 'showPopover', originalShow); + } + vi.restoreAllMocks(); + document.body.innerHTML = ''; + }); + + const place = (element: HTMLElement) => { + const copy = placeLayerExitCopy(element, ATTRIBUTE); + if (copy == null) { + throw new Error('no copy'); + } + return copy; + }; + + it('puts a static copy right after the layer, out of reach and out of the accessibility tree', () => { + popoverApi(true); + const element = layer(); + const copy = place(element); + expect(copy.previousElementSibling).toBe(element); + expect(copy.hasAttribute(ATTRIBUTE)).toBe(true); + expect(copy.getAttribute('aria-hidden')).toBe('true'); + expect(copy.hasAttribute('inert')).toBe(true); + expect(copy.id).toBe(''); + expect(copy.querySelectorAll('[id]')).toHaveLength(0); + // The content is the layer's, classes and all. + expect(copy.querySelector('.list')?.textContent).toBe('One'); + expect(copy.style.getPropertyValue('pointer-events')).toBe('none'); + }); + + it('takes the copy out of every form and radio group, so the live selection stays', () => { + popoverApi(true); + const element = layer(); + element.insertAdjacentHTML( + 'beforeend', + '', + ); + const live = element.querySelector('input[value="m"]')!; + const copy = place(element); + expect(copy.querySelectorAll('[name]')).toHaveLength(0); + // Every cloned control names a form that cannot exist, so none has an + // owner, whatever the copy stands inside. + expect( + [...copy.querySelectorAll('input')].map(node => + node.getAttribute('form'), + ), + ).toEqual(['', '', '']); + expect(live.checked).toBe(true); + expect(live.getAttribute('name')).toBe('size'); + }); + + it('leaves a form around the layer valid while the copy stands in it', () => { + popoverApi(true); + const element = layer(); + const form = document.createElement('form'); + element.parentElement!.append(form); + form.append(element); + element.insertAdjacentHTML( + 'beforeend', + '', + ); + // The live layer's own fields are the form's while it is open. + expect(form.checkValidity()).toBe(false); + const copy = place(element); + // The copy stands right after the layer, so inside the form; its + // controls belong to no form, and the live fields are unchanged. + expect(copy.closest('form')).toBe(form); + expect([...form.elements]).toHaveLength(3); + for (const node of copy.querySelectorAll('input, select, textarea')) { + expect((node as HTMLInputElement).form).toBeNull(); + } + }); + + it('keeps one copy per layer: a second takes only that layer\u2019s first away', () => { + popoverApi(true); + const element = layer(); + // Another layer beside it, closing too, with its own copy. + const other = layer(); + element.parentElement!.append(other); + const otherCopy = place(other); + const first = place(element); + const second = place(element); + expect(first.isConnected).toBe(false); + expect(second.isConnected).toBe(true); + expect(otherCopy.isConnected).toBe(true); + expect(document.querySelectorAll(`[${ATTRIBUTE}]`)).toHaveLength(2); + }); + + it('stands on the layer\u2019s box, free of the live layer\u2019s anchoring and motion', () => { + popoverApi(true); + const element = layer(); + const copy = place(element); + expect(copy.style.getPropertyValue('position')).toBe('fixed'); + expect(copy.style.getPropertyValue('left')).toBe('41px'); + expect(copy.style.getPropertyValue('top')).toBe('204px'); + expect(copy.style.getPropertyValue('width')).toBe('320px'); + expect(copy.style.getPropertyValue('height')).toBe('253px'); + for (const property of [ + 'position-anchor', + 'scale', + 'opacity', + 'transition', + ]) { + expect(copy.style.getPropertyValue(property), property).toBe(''); + } + expect(copy.style.getPropertyValue('animation-name')).toBe('none'); + // No alignment against the anchor it left (a phone menu's anchor-center). + expect(copy.style.getPropertyValue('justify-self')).toBe('normal'); + expect(copy.style.getPropertyValue('align-self')).toBe('normal'); + }); + + it('holds where a layer still entering was drawn: its layout box, with the transform it was drawn with', () => { + popoverApi(true); + const element = layer(); + // Mid-entrance: drawn at half size about its top-left corner, so its + // drawn rect is not its layout box. + element.getBoundingClientRect = () => ({ + left: 41, + top: 204, + width: 160, + height: 126.5, + x: 41, + y: 204, + right: 201, + bottom: 330.5, + toJSON: () => ({}), + }); + for (const [name, value] of [ + ['offsetLeft', 41], + ['offsetTop', 204], + ['offsetWidth', 320], + ['offsetHeight', 253], + ] as const) { + Object.defineProperty(element, name, {configurable: true, value}); + } + element.getAnimations = () => + [ + {effect: {target: element, pseudoElement: null}}, + ] as unknown as Animation[]; + const presented: Record = { + scale: '0.5', + 'transform-origin': '0px 0px', + width: '320.4px', + height: '253px', + 'box-sizing': 'border-box', + }; + const original = window.getComputedStyle.bind(window); + vi.spyOn(window, 'getComputedStyle').mockImplementation((node, pseudo) => + node === element + ? ({ + getPropertyValue: (property: string) => presented[property] ?? '', + } as CSSStyleDeclaration) + : original(node, pseudo), + ); + const copy = place(element); + // Laid out where the layer is, at its used size, fractions kept... + expect(copy.style.getPropertyValue('left')).toBe('41px'); + expect(copy.style.getPropertyValue('top')).toBe('204px'); + expect(copy.style.getPropertyValue('width')).toBe('320.4px'); + expect(copy.style.getPropertyValue('height')).toBe('253px'); + // ...and drawn as it was drawn. + expect(copy.style.getPropertyValue('scale')).toBe('0.5'); + expect(copy.style.getPropertyValue('transform-origin')).toBe('0px 0px'); + }); + + it('shows the copy only in the top layer, as a manual popover, with no z-index', () => { + const show = popoverApi(true)!; + const element = layer(); + const copy = place(element); + expect(copy.getAttribute('popover')).toBe('manual'); + expect(show).toHaveBeenCalledTimes(1); + expect(show.mock.contexts[0]).toBe(copy); + expect(copy.style.getPropertyValue('z-index')).toBe(''); + expect(copy.style.getPropertyValue('display')).toBe(''); + // The UA's popover inset does not over-constrain the box. + expect(copy.style.getPropertyValue('right')).toBe('auto'); + expect(copy.style.getPropertyValue('bottom')).toBe('auto'); + }); + + it('waits, hidden, when the browser refuses the show during the closing toggle, and shows once it is over', async () => { + const show = popoverApi(true)!; + // Android Chromium 158 refuses a show inside another popover's toggle. + show.mockImplementationOnce(() => { + throw new DOMException( + 'Invalid to show a popover during another show operation', + 'InvalidStateError', + ); + }); + const element = layer(); + let copy: HTMLElement | null = null; + expect(() => { + copy = place(element); + }).not.toThrow(); + // Not shown yet, and nothing stands in for the top layer: the UA keeps a + // popover that is not open out of view. + expect(copy!.isConnected).toBe(true); + expect(copy!.style.getPropertyValue('display')).toBe(''); + expect(copy!.style.getPropertyValue('z-index')).toBe(''); + expect(show).toHaveBeenCalledTimes(1); + // The script that closed the layer is over: shown. + await Promise.resolve(); + expect(show).toHaveBeenCalledTimes(2); + expect(copy!.isConnected).toBe(true); + }); + + it('tries again in the next frame, and takes the copy away if the browser still refuses', async () => { + const show = popoverApi(true)!; + show.mockImplementation(() => { + throw new DOMException('refused', 'InvalidStateError'); + }); + const frames: FrameRequestCallback[] = []; + vi.spyOn(window, 'requestAnimationFrame').mockImplementation(callback => { + frames.push(callback); + return frames.length; + }); + const element = layer(); + const copy = place(element); + await Promise.resolve(); + expect(frames).toHaveLength(1); + frames[0](0); + expect(show).toHaveBeenCalledTimes(3); + expect(copy.isConnected).toBe(false); + }); + + it('does not reach for the top layer for a copy already taken away', async () => { + const show = popoverApi(true)!; + show.mockImplementationOnce(() => { + throw new DOMException('refused', 'InvalidStateError'); + }); + const copy = place(layer()); + copy.remove(); + await Promise.resolve(); + expect(show).toHaveBeenCalledTimes(1); + }); + + it('places no copy without the popover API: the close is immediate', () => { + popoverApi(false); + const element = layer(); + expect(placeLayerExitCopy(element, ATTRIBUTE)).toBeNull(); + expect(document.querySelector(`[${ATTRIBUTE}]`)).toBeNull(); + }); + + it('starts as the layer looks: values its running animations hold are written onto the copy', () => { + popoverApi(true); + const element = layer(); + const dialog = element.querySelector('[role="dialog"]')!; + const list = element.querySelector('.list')!; + // A compositor curve on the layer and one on its dialog; the list has none. + element.getAnimations = () => + [element, dialog].map(target => ({ + effect: {target, pseudoElement: null}, + })) as unknown as Animation[]; + const presented = new Map>([ + [element, {opacity: '0.7', 'transform-origin': '160px 0px'}], + [dialog, {translate: '0px 12px', opacity: '0.5'}], + ]); + const original = window.getComputedStyle.bind(window); + vi.spyOn(window, 'getComputedStyle').mockImplementation((node, pseudo) => { + const values = presented.get(node); + if (values == null) { + return original(node, pseudo); + } + return { + getPropertyValue: (property: string) => values[property] ?? '', + } as CSSStyleDeclaration; + }); + const copy = place(element); + expect(copy.style.getPropertyValue('opacity')).toBe('0.7'); + expect(copy.style.getPropertyValue('transform-origin')).toBe('160px 0px'); + const copyDialog = copy.querySelector('[role="dialog"]')!; + expect(copyDialog.style.getPropertyValue('translate')).toBe('0px 12px'); + expect(copyDialog.style.getPropertyValue('opacity')).toBe('0.5'); + // Nothing is written where nothing runs. + expect( + copy.querySelector('.list')!.getAttribute('style'), + ).toBe(list.getAttribute('style')); + }); + + it.each(['iframe', 'object', 'embed', 'video', 'audio', 'canvas'])( + 'copies a %s as an empty box of its size, so nothing loads or plays again', + tag => { + popoverApi(true); + const element = layer(); + const live = document.createElement(tag); + live.className = 'media'; + live.style.setProperty('margin', '4px'); + Object.defineProperty(live, 'offsetWidth', {value: 320}); + Object.defineProperty(live, 'offsetHeight', {value: 180}); + element.querySelector('.list')!.appendChild(live); + const copy = place(element); + expect(copy.querySelector(tag)).toBeNull(); + const box = copy.querySelector('.list > .media')!; + expect(box.tagName).toBe('SPAN'); + expect(box.childNodes).toHaveLength(0); + expect(box.style.getPropertyValue('width')).toBe('320px'); + expect(box.style.getPropertyValue('height')).toBe('180px'); + expect(box.style.getPropertyValue('margin')).toBe('4px'); + // The live one is untouched. + expect(element.querySelector(tag)).toBe(live); + }, + ); + + it('keeps scrolled content where it was scrolled to', () => { + popoverApi(true); + const element = layer(); + const list = element.querySelector('.list')!; + list.scrollTop = 120; + const copy = place(element); + expect(copy.querySelector('.list')!.scrollTop).toBe(120); + }); +}); diff --git a/packages/core/src/Layer/layerExitCopy.ts b/packages/core/src/Layer/layerExitCopy.ts new file mode 100644 index 0000000000000..024872cedf2ef --- /dev/null +++ b/packages/core/src/Layer/layerExitCopy.ts @@ -0,0 +1,335 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +'use client'; + +/** + * @file layerExitCopy.ts + * @input A layer's popover element as it closes + * @output Exports `placeLayerExitCopy`: a static, inert copy of the layer put + * where the layer is drawn, in the top layer, for a touch presentation's + * close to play on, and `LayerExitBox` + * @position Layer runtime helper for a touch presentation that animates a + * popover out: the one exit copy a layer may have + * (architecture:layer-runtime INV13, architecture:component-style-authoring + * INV7) + * + * The browser hides a popover the moment it closes (a pick, Escape, its own + * light dismiss), so a close that has to stay on screen plays on a copy taken + * as the popover closes: same place, same size, same inherited theme, and the + * same look, values its running animations hold included (a clone keeps only + * inline styles, and a compositor curve holds its value outside them). It is + * laid out on the layer's own box and carries the transform the layer was + * drawn with, so a copy taken mid-entrance holds where it was drawn. The copy + * has no ids, no form names and no form owner (a cloned checked radio would + * otherwise take the live group's selection, and a named field would + * submit), is `inert` and hidden from assistive technology, and takes no + * pointer events. Embedded content (an iframe, an object or embed, media, a + * canvas) stands in the copy as an empty box of its size: a deep copy would + * load or play it again (an iframe navigates, autoplay restarts, a canvas + * comes back blank). A layer has one copy: placing another for the same layer + * takes the previous one away; other layers' copies stay. + * + * It is shown only in the top layer, as a manual popover, so a clipping, + * transformed or sticky ancestor neither cuts nor moves it; a fixed box or a + * larger `z-index` is no substitute (spec:AST-027 FR8). Where the browser + * refuses the show during the closing popover's own toggle (Android Chromium + * 158: "Invalid to show a popover during another show operation"), the copy + * waits, hidden, and shows once the toggle is over: after the script that + * closed the layer or, failing that, in the next frame before it paints. A + * browser without the popover API, or one that still refuses, gets no copy, + * and the close is immediate. Its owner animates it and removes it when its + * spring settles; nothing on the main thread runs per frame while it moves. + * + * SYNC: When modified, update: + * - /packages/core/src/Layer/layerExitCopy.test.ts + * - /packages/core/src/Layer/__tests__/layerExitCopy.interaction.browser.spec.ts + * - /docs/architecture/layer-runtime.md + */ + +/** A box in viewport px. */ +export interface LayerExitBox { + left: number; + top: number; + width: number; + height: number; +} + +/** + * Properties a running animation may hold, which the copy takes from what + * the layer presents. + */ +const PRESENTED_PROPERTIES = [ + 'translate', + 'rotate', + 'scale', + 'transform', + 'transform-origin', + 'opacity', +] as const; + +/** Inline properties the copy must not keep from the live layer. */ +const LIVE_ONLY_PROPERTIES = [ + 'position-anchor', + 'position-area', + 'position-try-fallbacks', + 'inset', + 'right', + 'bottom', + 'scale', + 'opacity', + 'transition', +] as const; + +/** + * Writes onto `copy` what `element` and its subtree present through running + * animations: the copy starts as the layer looks, not as its inline styles + * say. + */ +function commitPresented(element: HTMLElement, copy: HTMLElement): void { + const view = element.ownerDocument.defaultView; + if (view == null || typeof element.getAnimations !== 'function') { + return; + } + const animated = new Set(); + for (const animation of element.getAnimations({subtree: true})) { + const effect = animation.effect as KeyframeEffect | null; + if (effect?.target != null && effect.pseudoElement == null) { + animated.add(effect.target); + } + } + if (animated.size === 0) { + return; + } + const sources = [element, ...element.querySelectorAll('*')]; + const copies = [copy, ...copy.querySelectorAll('*')]; + sources.forEach((source, index) => { + const target = copies[index]; + if (!animated.has(source) || !(target instanceof view.Element)) { + return; + } + const computed = view.getComputedStyle(source); + const {style} = target as HTMLElement; + for (const property of PRESENTED_PROPERTIES) { + style.setProperty(property, computed.getPropertyValue(property)); + } + }); +} + +/** The properties that move a box without moving its layout. */ +const TRANSFORM_PROPERTIES = [ + 'transform', + 'translate', + 'scale', + 'rotate', +] as const; + +/** Each layer's copy: placing another for the same layer replaces it. */ +/** The elements a `
` can own; `form=""` matches no form's id. */ +const FORM_CONTROLS = + 'button, fieldset, input, object, output, select, textarea'; + +const copies = new WeakMap(); + +/** A length in px, or 0. */ +const px = (value: string): number => parseFloat(value) || 0; + +/** + * Where `element` is laid out, in viewport px: its box without the transform + * a presentation in progress draws it with (the copy carries that transform + * itself). A popover's layer is fixed in the top layer, so its offset box is + * in viewport px. + */ +function layoutBox(element: HTMLElement, view: Window): LayerExitBox { + const style = view.getComputedStyle(element); + const isTransformed = TRANSFORM_PROPERTIES.some(property => { + const value = style.getPropertyValue(property); + return value !== '' && value !== 'none'; + }); + if (!isTransformed) { + const rect = element.getBoundingClientRect(); + return { + left: rect.left, + top: rect.top, + width: rect.width, + height: rect.height, + }; + } + // The used width and height keep their fractions, where the offset size + // rounds (a copy narrower by a fraction could wrap its text differently). + const isBorderBox = style.getPropertyValue('box-sizing') === 'border-box'; + const width = px(style.getPropertyValue('width')); + const height = px(style.getPropertyValue('height')); + return { + left: element.offsetLeft, + top: element.offsetTop, + width: + width === 0 + ? element.offsetWidth + : isBorderBox + ? width + : width + + px(style.getPropertyValue('padding-left')) + + px(style.getPropertyValue('padding-right')) + + px(style.getPropertyValue('border-left-width')) + + px(style.getPropertyValue('border-right-width')), + height: + height === 0 + ? element.offsetHeight + : isBorderBox + ? height + : height + + px(style.getPropertyValue('padding-top')) + + px(style.getPropertyValue('padding-bottom')) + + px(style.getPropertyValue('border-top-width')) + + px(style.getPropertyValue('border-bottom-width')), + }; +} + +/** Content a deep copy would load or play again. */ +const EMBEDDED_CONTENT = 'iframe, object, embed, video, audio, canvas'; + +/** + * An empty box standing where embedded content was drawn: the copied node's + * classes and inline style, sized to the live node's box, with nothing in it. + */ +function emptyBox( + copied: HTMLElement, + live: HTMLElement | undefined, + view: Window, +): HTMLElement { + const box = copied.ownerDocument.createElement('span'); + box.className = copied.className; + const inline = copied.getAttribute('style'); + if (inline != null) { + box.setAttribute('style', inline); + } + const display = live == null ? 'inline' : view.getComputedStyle(live).display; + box.style.setProperty( + 'display', + display === 'inline' || display === '' ? 'inline-block' : display, + ); + box.style.setProperty('box-sizing', 'border-box'); + box.style.setProperty('width', `${live?.offsetWidth ?? 0}px`); + box.style.setProperty('height', `${live?.offsetHeight ?? 0}px`); + return box; +} + +/** + * Copies `element` into a static box where it is drawn, marked with + * `attribute`, inserts it right after the element and shows it in the top + * layer. The copy keeps the element's scroll positions and presented look. + * Returns null where there is no top layer to show it in (no popover API): + * the close is then immediate. + */ +export function placeLayerExitCopy( + element: HTMLElement, + attribute: string, +): HTMLElement | null { + const view = element.ownerDocument.defaultView; + if (view == null || typeof element.showPopover !== 'function') { + return null; + } + // One copy per layer: a close during a close replaces it. + copies.get(element)?.remove(); + const box = layoutBox(element, view); + const copy = element.cloneNode(true) as HTMLElement; + copy.removeAttribute('id'); + for (const node of copy.querySelectorAll('[id]')) { + node.removeAttribute('id'); + } + // Out of every form and radio group: a cloned checked radio with the live + // group's name would take its selection, and a named field would submit. + for (const node of copy.querySelectorAll('[name], [form]')) { + node.removeAttribute('name'); + node.removeAttribute('form'); + } + // The copy stands right after the layer, which renders inline by default, + // so a layer inside a `` would leave the copy's controls owned by + // that form while it closes: a cloned empty `required` field makes the + // form invalid and blocks its submit. `form=""` matches no form's id, so + // each cloned control has no owner wherever the copy stands. + for (const node of copy.querySelectorAll(FORM_CONTROLS)) { + node.setAttribute('form', ''); + } + copy.setAttribute(attribute, ''); + copy.setAttribute('aria-hidden', 'true'); + copy.setAttribute('inert', ''); + copy.setAttribute('popover', 'manual'); + const {style} = copy; + for (const property of LIVE_ONLY_PROPERTIES) { + style.removeProperty(property); + } + commitPresented(element, copy); + // On the layer's box: in the top layer the viewport is its containing + // block. No alignment against an anchor or the viewport (a phone menu + // centered on its trigger with `anchor-center` would otherwise shift it), + // and the UA's popover inset would over-constrain the box (and win in RTL). + style.setProperty('position', 'fixed'); + style.setProperty('margin', '0'); + style.setProperty('left', `${box.left}px`); + style.setProperty('top', `${box.top}px`); + style.setProperty('right', 'auto'); + style.setProperty('bottom', 'auto'); + style.setProperty('width', `${box.width}px`); + style.setProperty('height', `${box.height}px`); + style.setProperty('box-sizing', 'border-box'); + style.setProperty('justify-self', 'normal'); + style.setProperty('align-self', 'normal'); + style.setProperty('pointer-events', 'none'); + style.setProperty('animation-name', 'none'); + style.setProperty('overflow', 'visible'); + // Scrolled content stays where it was scrolled to, once the copy is shown + // (a hidden box keeps no scroll position). + const scrolled: [Element, number, number][] = []; + const sources = element.querySelectorAll('*'); + const targets = copy.querySelectorAll('*'); + sources.forEach((source, index) => { + const target = targets[index]; + if (target != null && (source.scrollTop !== 0 || source.scrollLeft !== 0)) { + scrolled.push([target, source.scrollTop, source.scrollLeft]); + } + }); + // Embedded content would load or play again in a deep copy (an iframe + // navigates, autoplay media restarts, a canvas comes back blank): each one + // stands as an empty box of the same size where it was drawn. + const liveEmbeds = element.querySelectorAll(EMBEDDED_CONTENT); + copy + .querySelectorAll(EMBEDDED_CONTENT) + .forEach((node, index) => { + node.replaceWith(emptyBox(node, liveEmbeds[index], view)); + }); + element.after(copy); + copies.set(element, copy); + const show = (): boolean => { + try { + copy.showPopover(); + } catch { + return false; + } + for (const [target, top, left] of scrolled) { + target.scrollTop = top; + target.scrollLeft = left; + } + return true; + }; + const giveUp = () => { + copy.remove(); + if (copies.get(element) === copy) { + copies.delete(element); + } + }; + if (!show()) { + // Refused during the closing popover's toggle: hidden until it is over. + view.queueMicrotask(() => { + if (!copy.isConnected || show()) { + return; + } + view.requestAnimationFrame(() => { + if (copy.isConnected && !show()) { + giveUp(); + } + }); + }); + } + return copy; +} diff --git a/packages/core/src/Lightbox/Lightbox.doc.mjs b/packages/core/src/Lightbox/Lightbox.doc.mjs index 44e9cc55bcd82..84898eb0bdc66 100644 --- a/packages/core/src/Lightbox/Lightbox.doc.mjs +++ b/packages/core/src/Lightbox/Lightbox.doc.mjs @@ -68,7 +68,8 @@ export const docs = { { name: 'onOpenChange', type: '(isOpen: boolean) => void', - description: 'Callback when the lightbox open state changes.', + description: + 'Callback when the lightbox open state changes: the Close button, Escape, a tap beside the media, dragging a photo down from its fitted size (with a finger, a mouse, a pen or a trackpad), or pinching it in from there (or a ctrl-wheel out).', required: true, }, { @@ -80,7 +81,7 @@ export const docs = { // the description instead (#1645). type: 'LightboxMedia | LightboxMedia[]', description: - "Media to display. Pass a single object for one item, or an array for gallery mode with prev/next navigation. Each item is {src: string, alt: string, caption?: ReactNode, type?: 'image' | 'video'}; type defaults to 'image', and zoom/pan is disabled for 'video'.", + "Media to display. Pass a single object for one item, or an array for gallery mode with prev/next navigation (a swipe, a mouse drag or a trackpad's sideways scroll moves between items too). Each item is {src: string, alt: string, caption?: ReactNode, type?: 'image' | 'video'}; type defaults to 'image', and zoom/pan is disabled for 'video'.", required: true, }, { @@ -92,13 +93,13 @@ export const docs = { name: 'onIndexChange', type: '(index: number) => void', description: - 'Callback when the gallery index changes via prev/next navigation.', + 'Callback when the gallery index changes: prev/next, the arrow keys, or a swipe.', }, { name: 'hasZoom', type: 'boolean', description: - 'Enable zoom on double-click, or Enter/Space/+/- via keyboard (images only). When zoomed, drag or use arrow keys to pan.', + 'Enable zoom (images only): pinch (or ctrl-wheel), or double-tap or double-click to zoom in at that point and back; Enter/Space/+/- zoom 2× about the centre and back. When zoomed, drag or scroll to pan, with momentum, or use arrow keys. A tap or a click on the photo hides the controls and turns the background black (so does a zoom in by a finger or a pointer); the next tap or click, a return to the fitted size, or a keyboard focus on a control brings them back. A pinch in from the fitted size shrinks the photo, and closes the viewer once it goes below about 93.5 % (into its thumbnail on a touch screen) unless it is opened again by 5 % before the lift.', default: 'false', }, { @@ -115,6 +116,13 @@ export const docs = { 'Automatically start video playback when a video media item is shown.', default: 'false', }, + { + name: 'experimental', + type: '{sourceFor?: (index: number) => HTMLElement | null}', + stability: 'experimental', + description: + "Experimental options: anything in this object may change or be removed in a patch release. sourceFor: The element an item opens from and closes into, usually its thumbnail: on a touch screen the photo zooms out of it on open and back into it on close (for the item showing then), and a photo dragged down lands in it; a tap or a drag that lands while the photo zooms open is held, so it neither closes the viewer nor reaches the page, while keys and screen readers stay live. With a mouse the viewer opens and closes at once, as before. Return null when there is none or it is off screen. Without it, on a touch screen the element that opened the viewer is the opened item's source when it shows a picture. Proposed; pending owner review.", + }, { name: 'xstyle', type: 'StyleXStyles', @@ -128,7 +136,7 @@ export const docs = { usage: { anatomy, description: - 'A fullscreen overlay for viewing images and videos at full resolution. Supports single-item and gallery modes with prev/next navigation, optional zoom and pan for images, and native video controls.', + 'A fullscreen overlay for viewing images and videos at full resolution. Supports single-item and gallery modes with prev/next navigation and swiping, optional zoom and pan for images, and native video controls.', bestPractices: [ { guidance: true, @@ -188,14 +196,15 @@ export const docsZh = { { name: 'onOpenChange', type: '(isOpen: boolean) => void', - description: '灯箱打开状态变化时的回调。', + description: + '灯箱打开状态变化时的回调：关闭按钮、Esc 键、点按媒体旁空白处、从适合大小向下拖动照片（手指、鼠标、触控笔或触控板均可），或从适合大小向内捏合照片（或按住 Ctrl 向外滚动滚轮）。', required: true, }, { name: 'media', type: 'LightboxMedia | LightboxMedia[]', description: - "要显示的媒体。传入单个对象或数组（用于画廊模式的上一张/下一张导航）。每项为 {src, alt, caption?, type?: 'image' | 'video'}；type 默认为 'image'，'video' 禁用缩放/平移。", + "要显示的媒体。传入单个对象或数组（用于画廊模式的上一张/下一张导航；滑动、鼠标拖动或触控板横向滚动也可在各项之间切换）。每项为 {src, alt, caption?, type?: 'image' | 'video'}；type 默认为 'image'，'video' 禁用缩放/平移。", required: true, }, { @@ -211,9 +220,17 @@ export const docsZh = { { name: 'hasZoom', type: 'boolean', - description: '启用双击缩放（仅图片）。缩放后可拖动平移。', + description: + '启用缩放（仅图片）：双指捏合（或按住 Ctrl 滚动滚轮），或双点按、双击在该处放大并还原；Enter/Space/+/- 键以中心放大 2 倍并还原。缩放后可拖动或滚动平移（带惯性），或用方向键平移。点按或单击照片会隐藏控件并使背景变黑（手指或指针放大时也是如此）；再次点按或单击、还原到适合大小或用键盘聚焦控件会让控件重新出现。从适合大小双指向内捏合会缩小照片；一旦小于约 93.5%，除非松开前又张开 5%，查看器就会关闭（在触摸屏上缩回缩略图）。', default: 'false', }, + { + name: 'experimental', + type: '{sourceFor?: (index: number) => HTMLElement | null}', + stability: 'experimental', + description: + '实验性选项：此对象中的任何内容都可能在补丁版本中更改或移除。sourceFor：某一项打开时放大自、关闭时缩回的元素（通常是其缩略图），仅在触屏上；向下拖动的照片也会落入其中。照片放大打开期间，落下的点按或拖动会被拦截，既不会关闭查看器，也不会触及页面；键盘和读屏软件不受影响。使用鼠标时，查看器照常立即打开和关闭。没有或不在屏幕上时返回 null。未提供时，在触屏上，若打开查看器的元素显示图片，则它是所打开项的来源。', + }, { name: 'xstyle', type: 'StyleXStyles', @@ -227,7 +244,7 @@ export const docsZh = { usage: { anatomy, description: - 'A fullscreen overlay for viewing images and videos at full resolution. Supports single-item and gallery modes with prev/next navigation, optional zoom and pan for images, and native video controls.', + 'A fullscreen overlay for viewing images and videos at full resolution. Supports single-item and gallery modes with prev/next navigation and swiping, optional zoom and pan for images, and native video controls.', bestPractices: [ { guidance: true, @@ -265,7 +282,7 @@ export const docsDense = { usage: { anatomy, description: - 'A fullscreen overlay for viewing images and videos at full resolution. Supports single-item and gallery modes with prev/next navigation, optional zoom and pan for images, and native video controls.', + 'A fullscreen overlay for viewing images and videos at full resolution. Supports single-item and gallery modes with prev/next navigation and swiping, optional zoom and pan for images, and native video controls.', bestPractices: [ {guidance: true, description: 'Always provide alt text for every image.'}, { @@ -291,12 +308,16 @@ export const docsDense = { }, propDescriptions: { isOpen: 'Whether the lightbox is open.', - onOpenChange: 'Callback when open state changes.', + onOpenChange: + 'Callback when open state changes (including a photo dragged down or pinched in).', media: 'Single media object or array for gallery mode.', index: 'Current index in gallery mode.', - onIndexChange: 'Callback when gallery index changes.', + onIndexChange: + 'Callback when gallery index changes (buttons, keys or a swipe).', hasZoom: - 'Enable zoom (double-click, Enter/Space, or +/-) and pan (drag or arrow keys) for images.', + 'Enable zoom (pinch or ctrl-wheel, double-tap or double-click, Enter/Space, or +/-) and pan (drag, scroll or arrow keys) for images; a tap or click on the photo hides the controls, and a pinch in closes the viewer.', + experimental: + 'Experimental options, which may change in a patch release: sourceFor, the element an item zooms out of on open and back into on close on a touch screen (its thumbnail).', xstyle: 'StyleX styles for layout customization.', }, }; diff --git a/packages/core/src/Lightbox/Lightbox.test.tsx b/packages/core/src/Lightbox/Lightbox.test.tsx index f0dc0b3a67f6c..68946d3c46dd0 100644 --- a/packages/core/src/Lightbox/Lightbox.test.tsx +++ b/packages/core/src/Lightbox/Lightbox.test.tsx @@ -1,9 +1,10 @@ // Copyright (c) Meta Platforms, Inc. and affiliates. import {describe, it, expect, vi, beforeEach, afterEach} from 'vitest'; -import {render, screen, fireEvent, waitFor} from '@testing-library/react'; +import {act, render, screen, fireEvent, waitFor} from '@testing-library/react'; import {Profiler} from 'react'; import {Lightbox} from './Lightbox'; +import {getGestureHost} from '../gestures/GestureHost'; import {__resetLiveRegionsForTest} from '../hooks/useAnnounce'; import {InternationalizationProvider} from '../i18n'; @@ -29,6 +30,77 @@ function politeRegion(): HTMLElement | null { return document.querySelector('[data-astryx-live-region="polite"]'); } +/** + * jsdom has no layout, so give the zoom stage what a browser measures: a + * 400 × 600 stage showing a landscape photo fitted 400 × 200 at its vertical + * centre. Photos' aspect-fill double tap zooms that photo 3×. Returns the + * undo. + */ +function stubStageLayout(): () => void { + const undo: (() => void)[] = []; + const define = ( + proto: object, + key: string, + get: (element: HTMLElement) => number, + ) => { + const original = Object.getOwnPropertyDescriptor(proto, key); + Object.defineProperty(proto, key, { + configurable: true, + get(this: HTMLElement) { + return get(this); + }, + }); + undo.push(() => { + if (original) { + Object.defineProperty(proto, key, original); + } else { + Reflect.deleteProperty(proto, key); + } + }); + }; + const isPhoto = (element: Element) => element instanceof HTMLImageElement; + // The zoom stage: a single item's photo box, or a gallery page's. + const isStage = (element: Element) => + element.hasAttribute('data-lightbox-page-media') || + element.querySelector(':scope > img') != null; + define(HTMLElement.prototype, 'offsetWidth', element => + isPhoto(element) ? 400 : 0, + ); + define(HTMLElement.prototype, 'offsetHeight', element => + isPhoto(element) ? 200 : 0, + ); + define(Element.prototype, 'clientWidth', element => + isStage(element) ? 400 : 0, + ); + define(Element.prototype, 'clientHeight', element => + isStage(element) ? 600 : 0, + ); + return () => undo.reverse().forEach(restore => restore()); +} + +/** A phone: the primary pointer is coarse, so the touch layer is on. Returns the undo. */ +function stubCoarsePointer(): () => void { + const original = window.matchMedia; + window.matchMedia = (query: string) => ({ + matches: query === '(pointer: coarse)', + media: query, + onchange: null, + addEventListener: () => {}, + removeEventListener: () => {}, + addListener: () => {}, + removeListener: () => {}, + dispatchEvent: () => false, + }); + return () => { + window.matchMedia = original; + }; +} + +/** The photo's inline `translate` and `scale`, as the zoom stage wrote them. */ +function photoTransform(img: HTMLElement): {translate: string; scale: string} { + return {translate: img.style.translate, scale: img.style.scale}; +} + // Both position messages, supplied by the test. Overriding the pair means an // assertion also proves which of the two keys the component reached for. const POSITION_MESSAGES = { @@ -191,6 +263,203 @@ describe('Lightbox', () => { expect(screen.getByAltText('Image B')).toBeInTheDocument(); }); + it('renders the current item and its neighbours as pages, each with its own caption', () => { + render( + {}} + media={media} + index={1} + />, + ); + const pages = [ + ...document.querySelectorAll('[data-lightbox-page]'), + ]; + expect(pages).toHaveLength(3); + const current = pages.filter(page => page.hasAttribute('data-current')); + expect(current).toHaveLength(1); + expect(current[0]).toContainElement(screen.getByAltText('Image B')); + expect(current[0]).toHaveTextContent('Second'); + // The neighbours are off screen: hidden from assistive technology. + for (const page of pages.filter(p => p !== current[0])) { + const img = page.querySelector('img'); + expect(img).toHaveAttribute('aria-hidden', 'true'); + expect(img).toHaveAttribute('alt', ''); + } + expect(pages[0]).toHaveTextContent('First'); + }); + + it('closes on a click beside the photo on a page', () => { + const onOpenChange = vi.fn(); + render( + , + ); + const current = document.querySelector('[data-current]')!; + fireEvent.click(current.querySelector('[data-lightbox-page-media]')!); + expect(onOpenChange).toHaveBeenCalledWith(false); + onOpenChange.mockClear(); + fireEvent.click(screen.getByAltText('Image A')); + expect(onOpenChange).not.toHaveBeenCalled(); + }); + + it('shows the last item as the current one for an index past the end', () => { + render( + {}} + media={media} + index={media.length + 2} + />, + ); + const photo = screen.getByAltText('Image C'); + expect(photo).not.toHaveAttribute('aria-hidden'); + const page = photo.closest('[data-lightbox-page]')!; + expect(page).toHaveAttribute('data-current'); + expect(page).not.toHaveAttribute('inert'); + expect( + screen.getByText(`${media.length} / ${media.length}`), + ).toBeTruthy(); + }); + + it('pauses a playing video once the viewer moves to another item', () => { + const items = [ + {src: '/clip.mp4', alt: 'A clip', type: 'video' as const}, + {src: '/b.jpg', alt: 'Image B'}, + ]; + const {rerender} = render( + {}} + media={items} + index={0} + />, + ); + const video = document.querySelector('video')!; + // Started by someone: playing, with its sound. + const pause = vi.fn(); + video.pause = pause; + Object.defineProperty(video, 'paused', { + configurable: true, + get: () => pause.mock.calls.length > 0, + }); + rerender( + {}} + media={items} + index={1} + />, + ); + // Still mounted as the pager's neighbour, and stopped. + expect(document.querySelector('video')).toBe(video); + expect(pause).toHaveBeenCalledTimes(1); + }); + + it('stays open on a click on the photo, with zoom off', () => { + const onOpenChange = vi.fn(); + render( + , + ); + const current = document.querySelector('[data-current]')!; + const box = current.querySelector( + '[data-lightbox-page-media]', + )!; + // The photo takes no pointer events: a click on it lands on its box. + current.querySelector('img')!.getBoundingClientRect = () => ({ + x: 100, + y: 100, + left: 100, + top: 100, + right: 500, + bottom: 300, + width: 400, + height: 200, + toJSON: () => ({}), + }); + fireEvent.click(box, {clientX: 300, clientY: 200}); + expect(onOpenChange).not.toHaveBeenCalled(); + // Beside the photo, on the same box, the click is the backdrop's. + fireEvent.click(box, {clientX: 20, clientY: 200}); + expect(onOpenChange).toHaveBeenCalledWith(false); + }); + + it('asks sourceFor for the item opening and the item showing at close, on a touch screen', () => { + const restorePointer = stubCoarsePointer(); + const sourceFor = vi.fn((): HTMLElement | null => null); + const {rerender} = render( + {}} + media={media} + index={0} + experimental={{sourceFor}} + />, + ); + expect(sourceFor).not.toHaveBeenCalled(); + rerender( + {}} + media={media} + index={0} + experimental={{sourceFor}} + />, + ); + expect(sourceFor).toHaveBeenLastCalledWith(0); + rerender( + {}} + media={media} + index={2} + experimental={{sourceFor}} + />, + ); + rerender( + {}} + media={media} + index={2} + experimental={{sourceFor}} + />, + ); + expect(sourceFor).toHaveBeenLastCalledWith(2); + // No source box to zoom into: the dialog closes at once. + expect(HTMLDialogElement.prototype.close).toHaveBeenCalled(); + restorePointer(); + }); + + it('never asks sourceFor with a mouse: the viewer opens and closes at once, as before', () => { + // Touch work leaves fine-pointer behavior alone (spec:AST-017 FR59): + // the zoom transition is the touch layer's. + const thumbnail = document.createElement('img'); + document.body.append(thumbnail); + const sourceFor = vi.fn((): HTMLElement | null => thumbnail); + const props = { + onOpenChange: () => {}, + media, + index: 0, + experimental: {sourceFor}, + }; + const {rerender} = render(); + rerender(); + rerender(); + expect(sourceFor).not.toHaveBeenCalled(); + expect(HTMLDialogElement.prototype.close).toHaveBeenCalled(); + thumbnail.remove(); + }); + it('shows gallery counter', () => { render( { {src: '/b.jpg', alt: 'Image B'}, {src: '/c.jpg', alt: 'Image C'}, ]; + let restoreLayout: () => void = () => {}; + beforeEach(() => { + restoreLayout = stubStageLayout(); + }); + afterEach(() => { + restoreLayout(); + }); function zoomTarget(): HTMLElement { return screen.getByRole('button', {name: 'Zoom'}); @@ -593,24 +869,53 @@ describe('Lightbox', () => { const dialog = document.querySelector('dialog')!; fireEvent.keyDown(zoomTarget(), {key: 'Enter'}); const img = screen.getByAltText('Image B'); - expect(img.getAttribute('style') ?? '').toContain('translate(0px, 0px)'); + // Lightbox's released zoom (spec:AST-017 FR59): 2× about the stage's + // centre, 800 × 400, its centre on the stage's centre, whatever aspect + // fill would give (3× for this photo). + expect(photoTransform(img)).toEqual({ + translate: '-200px -100px', + scale: '2 2', + }); // ArrowRight reveals content to the right (image shifts left) and must // not fall through to gallery navigation. fireEvent.keyDown(dialog, {key: 'ArrowRight'}); expect(onIndexChange).not.toHaveBeenCalled(); - expect(img.getAttribute('style') ?? '').toContain( - 'translate(-25px, 0px)', - ); + expect(photoTransform(img).translate).toBe('-250px -100px'); + // At 400 px the photo is shorter than the 600 px stage, so it stays + // centred vertically. fireEvent.keyDown(dialog, {key: 'ArrowDown'}); - expect(img.getAttribute('style') ?? '').toContain( - 'translate(-25px, -25px)', - ); + expect(photoTransform(img).translate).toBe('-250px -100px'); fireEvent.keyDown(dialog, {key: 'ArrowLeft'}); fireEvent.keyDown(dialog, {key: 'ArrowUp'}); - expect(img.getAttribute('style') ?? '').toContain('translate(0px, 0px)'); + expect(photoTransform(img).translate).toBe('-200px -100px'); expect(onIndexChange).not.toHaveBeenCalled(); }); + it('stops panning at the edges of the photo', () => { + render( + {}} + media={{src: '/photo.jpg', alt: 'Photo'}} + hasZoom + />, + ); + const dialog = document.querySelector('dialog')!; + fireEvent.keyDown(dialog, {key: '+'}); + const img = screen.getByAltText('Photo'); + for (let i = 0; i < 12; i++) { + fireEvent.keyDown(dialog, {key: 'ArrowLeft'}); + } + // The photo's left edge rests on the stage's left edge, no gap: on main + // the arrows could carry the photo out of view. + expect(photoTransform(img).translate).toBe('0px -100px'); + for (let i = 0; i < 24; i++) { + fireEvent.keyDown(dialog, {key: 'ArrowRight'}); + } + // 800 px wide on a 400 px stage: its right edge on the stage's. + expect(photoTransform(img).translate).toBe('-400px -100px'); + }); + it('navigates the gallery with arrows when not zoomed, even with hasZoom', () => { const onIndexChange = vi.fn(); render( @@ -646,9 +951,7 @@ describe('Lightbox', () => { const img = screen.getByAltText('Image B'); // ArrowRight still reveals what is to the right of the photo. fireEvent.keyDown(dialog, {key: 'ArrowRight'}); - expect(img.getAttribute('style') ?? '').toContain( - 'translate(-25px, 0px)', - ); + expect(photoTransform(img).translate).toBe('-250px -100px'); expect(onIndexChange).not.toHaveBeenCalled(); }); @@ -727,6 +1030,33 @@ describe('Lightbox', () => { expect(container.querySelector('dialog')).not.toBeInTheDocument(); }); + describe('while a closing photo lands', () => { + it('Close, a backdrop click, a cancel and the arrow keys neither close it again nor page it', () => { + // The landing: the caller has closed the viewer, and its dialog stays + // open until the photo is back in its source. + HTMLDialogElement.prototype.close = vi.fn(); + const onOpenChange = vi.fn(); + const onIndexChange = vi.fn(); + const media = [ + {src: '/a.jpg', alt: 'Image A'}, + {src: '/b.jpg', alt: 'Image B'}, + {src: '/c.jpg', alt: 'Image C'}, + ]; + const props = {onOpenChange, onIndexChange, media, index: 1}; + const {rerender} = render(); + rerender(); + const dialog = document.querySelector('dialog')!; + expect(dialog.open).toBe(true); + fireEvent.click(screen.getByLabelText('Close')); + fireEvent.click(dialog.firstElementChild!); + dialog.dispatchEvent(new Event('cancel', {cancelable: true})); + fireEvent.keyDown(dialog, {key: 'ArrowRight'}); + fireEvent.keyDown(dialog, {key: 'ArrowLeft'}); + expect(onOpenChange).not.toHaveBeenCalled(); + expect(onIndexChange).not.toHaveBeenCalled(); + }); + }); + describe('backdrop dismiss', () => { it('calls onOpenChange(false) when the dark area around the media is clicked', () => { const onOpenChange = vi.fn(); @@ -761,7 +1091,458 @@ describe('Lightbox', () => { }); }); +describe('Lightbox gestures for a mouse and a pen (spec:AST-094 FR43)', () => { + // Gestures attach for every pointer: a mouse's or a pen's double-click is + // the stage's double tap, a zoom by any pointer hides the controls as a + // finger's does, and a zoomed photo pans once the pan begins. + let restoreLayout: () => void = () => {}; + beforeEach(() => { + restoreLayout = stubStageLayout(); + }); + afterEach(() => { + restoreLayout(); + }); + + /** A pointer event as a browser sends it, at `time` (ms). */ + function pointer( + type: 'pointerdown' | 'pointermove' | 'pointerup', + target: Element, + pointerType: 'mouse' | 'pen', + x: number, + y: number, + time: number, + ): void { + const event = new Event(type, {bubbles: true, cancelable: true}); + const fields: Record = { + pointerType, + pointerId: pointerType === 'pen' ? 7 : 1, + isPrimary: true, + button: 0, + buttons: type === 'pointerup' ? 0 : 1, + clientX: x, + clientY: y, + timeStamp: time, + }; + for (const [key, value] of Object.entries(fields)) { + Object.defineProperty(event, key, {value}); + } + act(() => { + target.dispatchEvent(event); + }); + } + + /** Two clicks within the double-tap interval, then the browser's dblclick. */ + function doubleClick( + target: Element, + pointerType: 'mouse' | 'pen', + x: number, + y: number, + ): void { + const t0 = performance.now(); + pointer('pointerdown', target, pointerType, x, y, t0); + pointer('pointerup', target, pointerType, x, y, t0 + 40); + pointer('pointerdown', target, pointerType, x, y, t0 + 120); + pointer('pointerup', target, pointerType, x, y, t0 + 160); + act(() => { + target.dispatchEvent( + new MouseEvent('dblclick', {bubbles: true, clientX: x, clientY: y}), + ); + }); + } + + /** A finger's double tap, on a phone (a coarse primary pointer). */ + function fingerDoubleTap(target: Element, x: number, y: number): void { + const t0 = performance.now(); + const at = (type: string, down: boolean, time: number) => { + const event = new Event(type, {bubbles: true, cancelable: true}); + const entry = {identifier: 1, clientX: x, clientY: y, target}; + Object.defineProperty(event, 'touches', {value: down ? [entry] : []}); + Object.defineProperty(event, 'changedTouches', {value: [entry]}); + Object.defineProperty(event, 'timeStamp', {value: time}); + act(() => { + target.dispatchEvent(event); + }); + }; + at('touchstart', true, t0); + at('touchend', false, t0 + 40); + at('touchstart', true, t0 + 120); + at('touchend', false, t0 + 160); + } + + function renderPhoto() { + const view = render( + {}} + media={{src: '/photo.jpg', alt: 'Photo'}} + hasZoom + />, + ); + return { + view, + stage: screen.getByRole('button', {name: 'Zoom'}), + img: screen.getByAltText('Photo'), + }; + } + + it("zooms a mouse double-click where a finger's double tap zooms, and back", () => { + const restoreCoarse = stubCoarsePointer(); + let finger: {translate: string; scale: string}; + try { + const {view, stage, img} = renderPhoto(); + fingerDoubleTap(stage, 20, 220); + finger = photoTransform(img); + view.unmount(); + } finally { + restoreCoarse(); + } + // Near the photo's top-left corner: Photos' double tap zooms the photo + // in about that point, whatever the pointer. + expect(finger.scale).not.toBe(''); + const {stage, img} = renderPhoto(); + doubleClick(stage, 'mouse', 20, 220); + expect(photoTransform(img)).toEqual(finger); + expect(stage).toHaveAttribute('aria-pressed', 'true'); + doubleClick(stage, 'mouse', 20, 220); + expect(stage).toHaveAttribute('aria-pressed', 'false'); + }); + + it('zooms a pen double-click the same way', () => { + const {stage} = renderPhoto(); + doubleClick(stage, 'pen', 200, 300); + expect(stage).toHaveAttribute('aria-pressed', 'true'); + }); + + it("hides the controls as a mouse's or a pen's zoom does a finger's; the keyboard's zoom leaves them", () => { + const container = () => + document.querySelector('dialog')!.firstElementChild!; + const mouse = renderPhoto(); + doubleClick(mouse.stage, 'mouse', 200, 300); + expect(mouse.stage).toHaveAttribute('aria-pressed', 'true'); + expect(container()).toHaveAttribute('data-chrome-hidden'); + mouse.view.unmount(); + const pen = renderPhoto(); + doubleClick(pen.stage, 'pen', 200, 300); + expect(container()).toHaveAttribute('data-chrome-hidden'); + pen.view.unmount(); + const keys = renderPhoto(); + fireEvent.keyDown(document.querySelector('dialog')!, {key: '+'}); + expect(keys.stage).toHaveAttribute('aria-pressed', 'true'); + expect(container()).not.toHaveAttribute('data-chrome-hidden'); + }); + + it('pans a zoomed photo under a mouse drag once the pan begins, with the grab cursor', async () => { + const {stage, img} = renderPhoto(); + fireEvent.keyDown(document.querySelector('dialog')!, {key: '+'}); + // A drag starts where the photo shows: let the 200 ms zoom land first. + await act(async () => { + await new Promise(resolve => setTimeout(resolve, 400)); + }); + const from = photoTransform(img).translate; + const t0 = performance.now(); + pointer('pointerdown', stage, 'mouse', 200, 300, t0); + // 2 px is under the pan's threshold: no drag, no grab. + pointer('pointermove', stage, 'mouse', 198, 300, t0 + 16); + expect(photoTransform(img).translate).toBe(from); + expect(stage).not.toHaveAttribute('data-zoom-dragging'); + pointer('pointermove', stage, 'mouse', 170, 300, t0 + 32); + expect(photoTransform(img).translate).not.toBe(from); + expect(stage).toHaveAttribute('data-zoom-dragging'); + pointer('pointerup', stage, 'mouse', 170, 300, t0 + 48); + expect(stage).not.toHaveAttribute('data-zoom-dragging'); + }); +}); + +describe('Lightbox double-click under the touch layer', () => { + let restoreLayout: () => void = () => {}; + const originalMatchMedia = window.matchMedia; + beforeEach(() => { + restoreLayout = stubStageLayout(); + window.matchMedia = (query: string) => ({ + matches: query === '(pointer: coarse)', + media: query, + onchange: null, + addEventListener: () => {}, + removeEventListener: () => {}, + addListener: () => {}, + removeListener: () => {}, + dispatchEvent: () => false, + }); + }); + afterEach(() => { + restoreLayout(); + window.matchMedia = originalMatchMedia; + }); + + it("leaves a finger's double-click to the touch layer's recognizers", () => { + render( + {}} + media={{src: '/photo.jpg', alt: 'Photo'}} + hasZoom + />, + ); + const stage = screen.getByRole('button', {name: 'Zoom'}); + const down = new Event('pointerdown', {bubbles: true, cancelable: true}); + Object.defineProperty(down, 'pointerType', {value: 'touch'}); + Object.defineProperty(down, 'button', {value: 0}); + act(() => { + stage.dispatchEvent(down); + stage.dispatchEvent( + new MouseEvent('dblclick', {bubbles: true, clientX: 200, clientY: 300}), + ); + }); + expect(stage).toHaveAttribute('aria-pressed', 'false'); + }); +}); + +describe('Lightbox recognizers attach for every pointer (spec:AST-094 FR43)', () => { + /** Names of the Lightbox recognizers attached while `body` runs. */ + function lightboxRecognizersAttached(body: () => void): string[] { + const attach = vi.spyOn(getGestureHost(document), 'attach'); + try { + body(); + return attach.mock.calls + .map(([, recognizer]) => recognizer.name) + .filter(name => /lightbox/i.test(name)); + } finally { + attach.mockRestore(); + } + } + + it('attaches the drag to dismiss where the primary pointer is fine too', () => { + expect( + lightboxRecognizersAttached(() => { + render( + {}} + media={{src: '/photo.jpg', alt: 'Photo'}} + />, + ); + }), + ).not.toEqual([]); + }); + + it('attaches the drag to dismiss under the touch layer', () => { + const originalMatchMedia = window.matchMedia; + window.matchMedia = (query: string) => ({ + matches: query === '(pointer: coarse)', + media: query, + onchange: null, + addEventListener: () => {}, + removeEventListener: () => {}, + addListener: () => {}, + removeListener: () => {}, + dispatchEvent: () => false, + }); + try { + expect( + lightboxRecognizersAttached(() => { + render( + {}} + media={{src: '/photo.jpg', alt: 'Photo'}} + />, + ); + }), + ).toContain('Lightbox.dismissPan'); + } finally { + window.matchMedia = originalMatchMedia; + } + }); +}); + +describe('Lightbox touch zoom (synthetic touches)', () => { + let restoreLayout: () => void = () => {}; + const originalMatchMedia = window.matchMedia; + beforeEach(() => { + restoreLayout = stubStageLayout(); + // A phone: the primary pointer is coarse, so the touch layer is on. + window.matchMedia = (query: string) => ({ + matches: query === '(pointer: coarse)', + media: query, + onchange: null, + addEventListener: () => {}, + removeEventListener: () => {}, + addListener: () => {}, + removeListener: () => {}, + dispatchEvent: () => false, + }); + }); + afterEach(() => { + restoreLayout(); + window.matchMedia = originalMatchMedia; + }); + + interface Contact { + id: number; + x: number; + y: number; + } + + /** A touch event as a browser sends it: who is down, and who changed. */ + function touch( + type: 'touchstart' | 'touchmove' | 'touchend', + target: Element, + down: Contact[], + changed: Contact[], + time: number, + ): void { + const event = new Event(type, {bubbles: true, cancelable: true}); + const entry = (contact: Contact) => ({ + identifier: contact.id, + clientX: contact.x, + clientY: contact.y, + target, + }); + Object.defineProperty(event, 'touches', {value: down.map(entry)}); + Object.defineProperty(event, 'changedTouches', {value: changed.map(entry)}); + Object.defineProperty(event, 'timeStamp', {value: time}); + act(() => { + target.dispatchEvent(event); + }); + } + + it('keeps panning with the finger left down after a pinch, without a jump', () => { + render( + {}} + media={{src: '/photo.jpg', alt: 'Photo'}} + hasZoom + />, + ); + const stage = screen.getByRole('button', {name: 'Zoom'}); + const img = screen.getByAltText('Photo'); + const x = () => parseFloat(photoTransform(img).translate || '0'); + // Two fingers land on the photo, spread (100 → 140 px) and drift left + // together (the midpoint 200 → 180). + let a: Contact = {id: 1, x: 150, y: 300}; + let b: Contact = {id: 2, x: 250, y: 300}; + touch('touchstart', stage, [a], [a], 0); + touch('touchstart', stage, [a, b], [b], 0); + for (let i = 1; i <= 10; i++) { + a = {...a, x: 150 - 4 * i}; + b = {...b, x: 250}; + touch('touchmove', stage, [a, b], [a, b], i * 16); + } + const scaled = parseFloat(photoTransform(img).scale); + expect(scaled).toBeGreaterThan(1.2); + // The right finger lifts: nothing moves. + const beforeLift = x(); + touch('touchend', stage, [a], [b], 11 * 16); + expect(x()).toBeCloseTo(beforeLift, 6); + // The left finger drags on 40 px: the photo follows it 1:1, zoom kept. + for (let i = 1; i <= 8; i++) { + a = {...a, x: a.x - 5}; + touch('touchmove', stage, [a], [a], (11 + i) * 16); + } + expect(x()).toBeCloseTo(beforeLift - 40, 6); + expect(parseFloat(photoTransform(img).scale)).toBeCloseTo(scaled, 6); + touch('touchend', stage, [], [a], 20 * 16); + }); + + // Times on the arena's clock: velocities read the gap since the last sample. + let clock = 0; + + /** A one-finger drag down by `dy` in 10 px steps. */ + function dragDown( + stage: Element, + dy: number, + ): {x: number; y: number; id: number} { + clock = performance.now(); + let finger = {id: 1, x: 200, y: 300}; + touch('touchstart', stage, [finger], [finger], clock); + for (let i = 1; i <= dy / 10; i++) { + finger = {...finger, y: 300 + 10 * i}; + touch('touchmove', stage, [finger], [finger], clock + i * 16); + } + clock += (dy / 10) * 16; + return finger; + } + + it('carries a photo dragged down, shrinking it, and the release dismisses', async () => { + const onOpenChange = vi.fn(); + render( + , + ); + const img = screen.getByAltText('Photo'); + const stage = img.parentElement!; + const finger = dragDown(stage, 100); + // The travel past the pan's 10 px threshold is 90 px: the photo's centre + // moved 90 px down, and it shrank to 1 − 4.91e-6 × 90². + const scale = 1 - 4.91e-6 * 90 * 90; + expect(parseFloat(photoTransform(img).scale)).toBeCloseTo(scale, 4); + const [tx, ty] = photoTransform(img).translate.split(' ').map(parseFloat); + expect(tx).toBeCloseTo(((1 - scale) * 400) / 2, 2); + expect(ty).toBeCloseTo(90 + ((1 - scale) * 200) / 2, 2); + expect(onOpenChange).not.toHaveBeenCalled(); + touch('touchend', stage, [], [finger], clock + 16); + await waitFor(() => expect(onOpenChange).toHaveBeenCalledWith(false)); + }); + + it('reports a drag-down close from the release, while the photo still leaves (architecture:react-component-runtime INV2)', () => { + const onOpenChange = vi.fn(); + render( + , + ); + const img = screen.getByAltText('Photo'); + const stage = img.parentElement!; + const finger = dragDown(stage, 100); + expect(onOpenChange).not.toHaveBeenCalled(); + touch('touchend', stage, [], [finger], clock + 16); + // From the release itself, not once the photo has left. + expect(onOpenChange).toHaveBeenCalledTimes(1); + expect(onOpenChange).toHaveBeenCalledWith(false); + expect(document.querySelector('dialog')!.open).toBe(true); + }); + + it('springs back when the finger lifts moving up', async () => { + const onOpenChange = vi.fn(); + render( + , + ); + const img = screen.getByAltText('Photo'); + const stage = img.parentElement!; + let finger = dragDown(stage, 100); + for (let i = 1; i <= 6; i++) { + finger = {...finger, y: finger.y - 10}; + touch('touchmove', stage, [finger], [finger], clock + i * 16); + } + touch('touchend', stage, [], [finger], clock + 7 * 16); + // The return's end state is in place at once (it springs there on the + // compositor); at rest the photo keeps no inline transform. + expect(photoTransform(img)).toEqual({translate: '', scale: ''}); + await new Promise(resolve => setTimeout(resolve, 50)); + expect(onOpenChange).not.toHaveBeenCalled(); + }); +}); + describe('Lightbox zoom reset timing', () => { + let restoreLayout: () => void = () => {}; + beforeEach(() => { + restoreLayout = stubStageLayout(); + }); + afterEach(() => { + restoreLayout(); + }); + it('shows the next image unzoomed in the same commit that changes it', () => { const media = [ {src: '/a.jpg', alt: 'Image A'}, @@ -796,6 +1577,13 @@ describe('Lightbox zoom reset timing', () => { 'aria-pressed', 'false', ); + // The photo element is reused for the next image: it is back at the + // fitted size before that image paints (an animator at rest at the + // identity leaves the element's own style). + expect(photoTransform(screen.getByAltText('Image B'))).toEqual({ + translate: '', + scale: '', + }); }); }); @@ -817,3 +1605,124 @@ describe('Lightbox with an unusable controlled index', () => { ).not.toThrow(); }); }); + +describe('Lightbox opener fallback listener (spec:AST-094 FR24)', () => { + // Counts the capture-phase `click` listeners Lightbox's own code adds to + // or removes from the document (the opener record), told apart from other + // modules' by the frame that calls the document. + function countLightboxCaptureClicks( + method: 'addEventListener' | 'removeEventListener', + ) { + let count = 0; + const original = document[method].bind(document); + const spy = vi + .spyOn(document, method) + .mockImplementation( + ( + type: string, + listener: EventListenerOrEventListenerObject | null, + options?: boolean | EventListenerOptions, + ) => { + const capture = + options === true || + (typeof options === 'object' && options.capture === true); + // [0] the message, [1] this function, [2] the spy, [3] the caller. + const caller = (new Error().stack ?? '').split('\n')[3] ?? ''; + if ( + type === 'click' && + capture && + /[\\/]Lightbox[\\/](?![^\s]*\.test\.)/.test(caller) + ) { + count += 1; + } + return original( + type, + listener as EventListenerOrEventListenerObject, + options, + ); + }, + ); + return {count: () => count, restore: () => spy.mockRestore()}; + } + + it('adds no document listener where the primary pointer is fine', () => { + const added = countLightboxCaptureClicks('addEventListener'); + const {unmount} = render( + {}} + media={{src: '/photo.jpg', alt: 'Photo'}} + />, + ); + expect(added.count()).toBe(0); + unmount(); + added.restore(); + }); + + it('shares the touch press controller across the viewers on a touch screen: its listeners once, none of their own (spec:AST-094 FR24)', () => { + const restorePointer = stubCoarsePointer(); + const added = vi.spyOn(document, 'addEventListener'); + const removed = vi.spyOn(document, 'removeEventListener'); + const viewers = (count: number) => + render( + <> + {Array.from({length: count}, (_, i) => ( + {}} + media={{src: `/photo-${i}.jpg`, alt: `Photo ${i}`}} + /> + ))} + , + ); + const one = viewers(1); + const forOne = added.mock.calls.length; + one.unmount(); + added.mockClear(); + removed.mockClear(); + const three = viewers(3); + const forThree = added.mock.calls.length; + // The press controller's listeners, installed once however many viewers. + expect(forThree).toBeGreaterThan(0); + expect(forThree).toBe(forOne); + expect(added.mock.calls.filter(([type]) => type === 'click')).toEqual([]); + // The last viewer to go takes them with it. + three.unmount(); + expect(removed.mock.calls.length).toBe(forThree); + added.mockRestore(); + removed.mockRestore(); + restorePointer(); + }); +}); + +describe('Lightbox chrome toggle listeners', () => { + it('listen on the viewer itself for every pointer (spec:AST-094 FR24, FR43)', () => { + const added = vi.spyOn(HTMLDialogElement.prototype, 'addEventListener'); + const viewer = ( + {}} + media={{src: '/photo.jpg', alt: 'Photo'}} + hasZoom + /> + ); + const focusIns = () => + added.mock.calls.filter(([type]) => type === 'focusin').length; + try { + const fine = render(viewer); + expect(focusIns()).toBe(1); + fine.unmount(); + added.mockClear(); + const restore = stubCoarsePointer(); + try { + render(viewer); + expect(focusIns()).toBe(1); + } finally { + restore(); + } + } finally { + added.mockRestore(); + } + }); +}); diff --git a/packages/core/src/Lightbox/Lightbox.tsx b/packages/core/src/Lightbox/Lightbox.tsx index 476487c352650..950c975c7e5de 100644 --- a/packages/core/src/Lightbox/Lightbox.tsx +++ b/packages/core/src/Lightbox/Lightbox.tsx @@ -5,13 +5,33 @@ /** * @file Lightbox.tsx * @input Uses React, native dialog, StyleX, IconButton, theme tokens, layerTextReset, layerStructureReset, - * isRtlElement (gallery arrow keys follow the rendered direction) - * @output Exports Lightbox component, LightboxProps, LightboxMedia + * isRtlElement (gallery arrow keys follow the rendered direction), useZoomCore (the zoom + * stage for every pointer: keys, a mouse double-click and drag) and useZoomStage (its touch + * layer: pinch, double tap and pan on the gesture recognizers), moved by the compositor + * animator, usePager (a gallery pages like Photos: a UIKit paging scroll view with Photos' + * 40 pt gap), and the compositor animator for Photos' drag to dismiss (the + * backdrop and the controls' layers fade with the drag), zoomFromSource (UIKit's + * zoom transition between a source, `experimental.sourceFor`, and the photo), zoomStage's chrome + * curves (a tap on the photo hides the controls, as Photos' chrome), pinch-close + * fades (a pinch in from the fitted size closes the viewer), holdInput (the Layer + * module's input hold: a touch opening holds taps and drags until it ends, + * spec:AST-094 FR35) and openerActivation (the click that opened a viewer, recorded by + * one document listener every closed touch viewer shares) + * @output Exports Lightbox component, LightboxProps, LightboxExperimentalOptions, LightboxMedia * @position Core implementation; consumed by index.ts * * SYNC: When modified, update these files to stay in sync: * - /packages/core/src/Lightbox/Lightbox.doc.mjs (props table, features, implementation notes) * - /packages/core/src/Lightbox/Lightbox.test.tsx (tests for new/changed behavior) + * - /packages/core/src/Lightbox/useZoomCore.ts (the zoom stage for every pointer) + * - /packages/core/src/Lightbox/useZoomStage.ts (the touch layer) + * - /packages/core/src/Lightbox/__tests__/LightboxZoom.interaction.browser.spec.ts (touch zoom proofs) + * - /packages/core/src/Lightbox/__tests__/LightboxPaging.interaction.browser.spec.ts (gallery paging proofs) + * - /packages/core/src/Lightbox/__tests__/LightboxDismiss.interaction.browser.spec.ts (drag to dismiss proofs) + * - /packages/core/src/Lightbox/__tests__/LightboxZoomTransition.interaction.browser.spec.ts (zoom transition proofs) + * - /packages/core/src/Lightbox/__tests__/LightboxChrome.interaction.browser.spec.ts (controls hidden by a tap) + * - /packages/core/src/Lightbox/__tests__/LightboxPinchClose.interaction.browser.spec.ts (pinch to close) + * - /packages/core/src/Lightbox/zoomFromSource.ts (the transition driver) * - /packages/core/src/Lightbox/index.ts (exports if types change) * - /packages/core/src/Layer/useLayerDismissal.ts (dismissal stack) * - /apps/storybook/stories/Lightbox.stories.tsx (storybook stories) @@ -22,6 +42,7 @@ import { useCallback, useEffect, useMemo, + useReducer, useRef, useState, type MouseEvent as ReactMouseEvent, @@ -44,11 +65,31 @@ import {LayerContentBoundary} from '../Layer/layerScopedContext'; import {ModalAnnounceProvider} from '../hooks/ModalAnnounceProvider'; import {overlayPaddingReset} from '../Layout/padding.stylex'; import {LayerDepthProvider} from '../Layer/LayerDepthContext'; +import {holdInput} from '../Layer/inputHold'; +import {takeOpenerActivation, useOpenerActivation} from './openerActivation'; import {useLayerDismissal} from '../Layer/useLayerDismissal'; import {useTranslator} from '../i18n'; import {useMergedRefs} from '../hooks/useMergedRefs'; import {isRtlElement} from '../hooks/isRtlElement'; +import {useZoomCore} from './useZoomCore'; +import {usePager, type Pager} from '../hooks/usePager'; +import {UIKIT_TAP} from '../motion/uikit/gestures'; +import {zoomMedia} from '../motion/uikit/zoomMedia'; +import {useZoomStage, type ZoomStage} from './useZoomStage'; +import {lightboxScope} from './lightbox.markers.stylex'; +import {useTouchPresentation} from '../utils/touchPresentation'; +import {animatorFor, type Animator} from '../motion/animator'; +import {liftCopy, type LiftedCopy} from '../liftFromTrigger/liftFromTrigger'; +import {sourceBoxOf, openingFadeStart, zoomFromSource} from './zoomFromSource'; +import { + chromeBackgroundCurve, + chromeFadeCurve, + dismissBackdropAlpha, + dismissChromeAlpha, + pinchCloseBackdropAlpha, + pinchCloseChromeAlpha, +} from './zoomStage'; /** * Media type for lightbox items. */ @@ -71,6 +112,18 @@ export interface LightboxMedia { type?: LightboxMediaType; } +/** + * Lightbox's experimental options (`experimental`): anything here may change or + * be removed in a patch release. + */ +export interface LightboxExperimentalOptions { + /** + * The element an item opens from and closes into, usually its thumbnail, or + * null for an item with none on screen. + */ + sourceFor?: (index: number) => HTMLElement | null; +} + export interface LightboxProps extends BaseProps { /** Ref forwarded to the root dialog element */ ref?: React.Ref; @@ -99,12 +152,15 @@ export interface LightboxProps extends BaseProps { */ defaultIndex?: number; /** - * Callback when the gallery index changes via prev/next navigation. + * Callback when the gallery index changes: the previous and next buttons, + * the arrow keys or a swipe. */ onIndexChange?: (index: number) => void; /** - * Enable zoom on double-click, or Enter/Space/`+`/`-` via keyboard - * (images only). When zoomed, drag or use arrow keys to pan. + * Enable zoom (images only): under a coarse primary pointer (a touch + * screen), pinch or double-tap to zoom in at that point; a double-click + * or Enter/Space/`+`/`-` zooms 2× about the centre. When zoomed, drag or + * use arrow keys to pan. * @default false */ hasZoom?: boolean; @@ -113,6 +169,17 @@ export interface LightboxProps extends BaseProps { * @default false */ hasAutoPlay?: boolean; + /** + * Experimental options: anything in this object may change or be removed in + * a patch release. `sourceFor` returns the element an item opens from and + * closes into, usually its thumbnail: on a touch screen the photo zooms out + * of it on open and back into it on close (UIKit's zoom transition), and a + * photo dragged down lands in it. It returns null for an item with none on + * screen. Without it, on a touch screen the element that opened the viewer + * is the opened item's source when it shows a picture. + * @experimental + */ + experimental?: LightboxExperimentalOptions; } // ============================================================================= @@ -137,9 +204,14 @@ const styles = stylex.create({ backgroundColor: 'transparent', overflow: 'hidden', outline: 'none', + // The backdrop dims and blurs the page as on main: the theme's overlay + // colour over a 2 px blur. While it fades (a drag to dismiss and its + // release) the blur comes off, through the dialog's + // --_lightbox-backdrop-filter: on a 2-core limit a full-viewport blur + // repainted every frame of the fade drops frames. '::backdrop': { backgroundColor: colorVars['--color-overlay'], - backdropFilter: 'blur(2px)', + backdropFilter: 'var(--_lightbox-backdrop-filter, blur(2px))', }, }, container: { @@ -150,6 +222,20 @@ const styles = stylex.create({ width: '100%', height: '100%', position: 'relative', + // The black the viewer turns as a tap hides the controls, as Photos' + // background does: under the photo (and a source's picture), over the + // backdrop. The colour that reads on light surfaces is black in every + // theme; a theme that retints it retints this. The container's own + // pseudo-element, no element of its own (architecture:layer-runtime INV7). + '::before': { + content: '""', + position: 'absolute', + inset: 0, + zIndex: -1, + backgroundColor: colorVars['--color-on-light'], + opacity: 0, + pointerEvents: 'none', + }, }, mediaGroup: { display: 'flex', @@ -157,17 +243,131 @@ const styles = stylex.create({ alignItems: 'center', maxWidth: '100%', maxHeight: '100%', - overflow: 'hidden', + // ZOOM_DISMISSING_ATTRIBUTE on the stage inside: a photo dragged to + // dismiss draws outside the box that hugs it. + overflow: {default: 'hidden', ':has([data-zoom-dismissing])': 'visible'}, + }, + // On a touch screen (the shared touch resolver, useTouchPresentation) a + // zoomable photo's stage fills the viewer, as a photo viewer's zooming + // scroll view does: a zoomed photo uses the whole screen and a pinch can + // start anywhere around it. Only for a zoom target: around any other item + // a tap on the dark area still closes the viewer. + mediaGroupZoom: { + alignSelf: 'stretch', + flexGrow: 1, + minHeight: 0, + }, + imageWrapperFill: { + alignSelf: 'stretch', + flexGrow: 1, + flexShrink: 1, + flexBasis: 0, }, imageWrapper: { display: 'flex', alignItems: 'center', justifyContent: 'center', - overflow: 'hidden', + // ZOOM_DISMISSING_ATTRIBUTE: a photo dragged to dismiss draws outside it. + overflow: {default: 'hidden', ':is([data-zoom-dismissing])': 'visible'}, cursor: 'default', userSelect: 'none', minHeight: 0, }, + // Under the touch layer, the stage's own pinch, pan and double tap take + // every touch on it. Only the stage: the controls and caption leave + // pinch-zoom to the page. On a touch screen it fills the viewer + // (mediaGroupZoom, imageWrapperFill). Where the primary pointer is fine no + // recognizer runs, so the browser keeps the touches (spec:AST-094 FR19). + imageWrapperZoomTarget: { + touchAction: 'none', + }, + // In a gallery the stage's touches go to the track instead, under the + // touch layer only (as imageWrapperZoomTarget): the pager's pan manages + // the stage's own `touch-action`. + trackZoomTarget: { + touchAction: 'none', + }, + // Under the touch layer, a photo without zoom leaves pinch-zoom to the page + // but takes one-finger drags: a drag down dismisses it. + imageWrapperDismissTarget: { + touchAction: 'pinch-zoom', + }, + // A gallery's stage is the pager's viewport: it fills the viewer on every + // presentation, so its pages have a size of their own. + mediaGroupGallery: { + alignSelf: 'stretch', + flexGrow: 1, + minHeight: 0, + }, + imageWrapperGallery: { + alignSelf: 'stretch', + flexGrow: 1, + flexShrink: 1, + flexBasis: 0, + position: 'relative', + }, + // Two rows, the photo's and the caption's, that every page shares + // (subgrid): the caption row is as tall as the gallery's tallest caption + // (the current page lays out, unseen, those no rendered page shows), so + // every photo centres on the same box, with or without a caption, the next + // photo slides in level with the current one, and paging never resizes it. + track: { + position: 'absolute', + inset: 0, + display: 'grid', + gridTemplateColumns: '100%', + gridTemplateRows: 'minmax(0, 1fr) auto', + }, + // One page per item, the stage's size, at index × (width + gap) (stacked + // in the track's one column, offset by its relative inset). Each on its + // own layer: through a busy main thread Mobile Safari moves the track but + // cannot paint what it had not painted, so a page that slides in must be + // painted whole before it moves. + page: { + position: 'relative', + gridColumn: '1', + gridRow: '1 / span 2', + display: 'grid', + gridTemplateColumns: '100%', + gridTemplateRows: 'subgrid', + minHeight: 0, + willChange: 'transform', + }, + // A page's photo box: the zoom stage of the current page, in the photo + // row. Its caption sits in the caption row below it, so the viewer's size + // never changes with the item. + pageMedia: { + gridRow: '1', + minHeight: 0, + minWidth: 0, + display: 'flex', + alignItems: 'center', + justifyContent: 'center', + overflow: 'hidden', + }, + pageCaption: { + gridRow: '2', + justifySelf: 'center', + }, + // The current page's caption row when it also lays out, unseen, the + // captions no rendered page shows (`unrenderedCaptions`): one grid cell + // holds its own caption and those, so the row is the tallest of them. + pageCaptionRow: { + gridRow: '2', + display: 'grid', + justifyItems: 'center', + minWidth: 0, + }, + captionStacked: { + gridArea: '1 / 1', + }, + captionSizer: { + display: 'grid', + justifyItems: 'center', + justifySelf: 'stretch', + visibility: 'hidden', + pointerEvents: 'none', + }, imageWrapperZoomable: { cursor: { default: 'zoom-in', @@ -178,12 +378,8 @@ const styles = stylex.create({ imageWrapperZoomed: { cursor: { default: 'grab', - ':is(:disabled,[aria-disabled="true"])': 'default', - }, - }, - imageWrapperDragging: { - cursor: { - default: 'grabbing', + // ZOOM_DRAGGING_ATTRIBUTE, set by the stage while a drag moves the photo. + ':is([data-zoom-dragging])': 'grabbing', ':is(:disabled,[aria-disabled="true"])': 'default', }, }, @@ -192,15 +388,9 @@ const styles = stylex.create({ maxHeight: '100%', objectFit: 'contain', pointerEvents: 'none', - transitionProperty: 'transform', - transitionDuration: { - default: '200ms', - '@media (prefers-reduced-motion: reduce)': '0ms', - }, - transitionTimingFunction: 'ease-out', - }, - imageDragging: { - transitionProperty: 'none', + // The zoom stage scales the photo about its top-left corner and moves it + // with the individual `scale` and `translate` properties. + transformOrigin: '0 0', }, video: { maxWidth: '100%', @@ -253,15 +443,46 @@ const styles = stylex.create({ }, controlButton: { color: colorVars['--color-on-dark'], + // Hidden by a tap on the photo, they take no taps: one where a control + // was reaches the viewer (and shows them again). + pointerEvents: { + default: 'auto', + [stylex.when.ancestor('[data-chrome-hidden]', lightboxScope)]: 'none', + }, + }, + // The controls sit in layers over the viewer that pass clicks through. A + // drag to dismiss fades the layers, which carry no transition of their own + // (a Button's opacity transition would trail the finger). + // Above the stage: while a fade writes their opacity they are stacking + // contexts of their own, which a gallery's positioned pages (later in tree + // order) would otherwise cover, taking the taps meant for the controls. + chromeLayer: { + position: 'absolute', + inset: 0, + zIndex: 1, + pointerEvents: 'none', }, }); -const dynamicStyles = stylex.create({ - imageTransform: (transform: string) => ({ - transform, +const pageStyles = stylex.create({ + position: (index: number) => ({ + insetInlineStart: `calc(${index} * (100% + ${zoomMedia.viewer.pageGap}px))`, }), }); +/** A control layer or a caption: what a tap on the photo hides. */ +function isChromeElement(element: HTMLElement | null): boolean { + return ( + element?.matches('[data-lightbox-chrome], [data-lightbox-caption]') === true + ); +} + +/** + * How long an opening waits for its photo to decode before it zooms (ms): + * the photo stays hidden meanwhile, and opens plainly past it. + */ +const OPEN_DECODE_WAIT_MS = 300; + /** * Pan distance (px) per arrow-key press while zoomed. Offsets move the * viewport in the arrow's direction — pressing ArrowRight reveals content to @@ -283,8 +504,11 @@ const KEYBOARD_PAN_OFFSETS: Record = { * prev/next navigation via buttons and arrow keys; the arrow keys follow * the writing direction (in RTL, ArrowLeft goes to the next item, like the * Next button on the left). Optionally supports - * zoom (double-click, Enter/Space on the image, or `+`/`-` to toggle 2x) - * and pan (drag or arrow keys when zoomed; arrows navigate the gallery + * zoom like a photo viewer: under a finger, pinch about the fingers (with the + * give past the limits and the eased return); double-tap or double-click to + * zoom in at that point and back, Enter/Space on the image or `+`/`-` from + * the keyboard; and pan (drag, under a finger with momentum and the rubber + * band at the edges, or arrow keys when zoomed; arrows navigate the gallery * when not zoomed). * * Uses the native `` element with `showModal()` for focus @@ -321,6 +545,7 @@ export function Lightbox({ onIndexChange, hasZoom = false, hasAutoPlay = false, + experimental, xstyle, className, style, @@ -331,15 +556,34 @@ export function Lightbox({ }: LightboxProps) { const t = useTranslator(); const dialogRef = useRef(null); - const mergedDialogRef = useMergedRefs(ref, dialogRef); const containerRef = useRef(null); const imageWrapperRef = useRef(null); + const trackRef = useRef(null); + // A gallery's zoom stage: the current page's photo box. + const pageMediaRef = useRef(null); const triggerElementRef = useRef(null); + /** The item the viewer opened on (the opener is its source). */ + const openedIndexRef = useRef(null); + /** The element whose activation opened the viewer (its source fallback). */ + const openerRef = useRef(null); + /** + * A dismissed photo landed in its thumbnail: the picture over it, and the + * hold that keeps the thumbnail hidden under it. + */ + const landedRef = useRef(null); + /** Lets go of a landed photo: the thumbnail shows again. */ + const restoreLanded = useCallback(() => { + const landed = landedRef.current; + if (landed != null) { + landedRef.current = null; + landed.drop(); + } + }, []); // Index state (controlled + uncontrolled) const isControlled = controlledIndex !== undefined; const [uncontrolledIndex, setUncontrolledIndex] = useState(defaultIndex); - const index = isControlled ? controlledIndex : uncontrolledIndex; + const requestedIndex = isControlled ? controlledIndex : uncontrolledIndex; const setIndex = useCallback( (value: number) => { @@ -351,11 +595,10 @@ export function Lightbox({ [isControlled, onIndexChange, setUncontrolledIndex], ); - // Zoom/pan state - const [zoom, setZoom] = useState(1); - const [pan, setPan] = useState({x: 0, y: 0}); - const [isDragging, setIsDragging] = useState(false); - const dragStartRef = useRef({x: 0, y: 0, panX: 0, panY: 0}); + // Whether the photo is zoomed: a discrete state for aria-pressed and the + // cursor. The zoom and pan themselves live outside React (useZoomCore, + // useZoomStage). + const [isZoomed, setIsZoomed] = useState(false); // Resolve current media item const mediaArray = useMemo( @@ -363,10 +606,11 @@ export function Lightbox({ [media], ); const isGallery = mediaArray.length > 1; - const currentItem = - mediaArray.length > 0 - ? mediaArray[Math.min(index, mediaArray.length - 1)] - : null; + // The item shown, current in every respect (its page, the counter, the + // pager, the announcement): an index past either end shows the nearest + // item. A NaN index stays NaN (the viewed-item check below). + const index = Math.max(0, Math.min(requestedIndex, mediaArray.length - 1)); + const currentItem = mediaArray.length > 0 ? mediaArray[index] : null; const currentType = currentItem?.type ?? 'image'; const isVideo = currentType === 'video'; const canPrev = isGallery && index > 0; @@ -376,7 +620,7 @@ export function Lightbox({ useScrollLock(isOpen); // A new media item starts unzoomed. Adjusted during render, so the new item - // never paints at the previous item's zoom and pan. + // never paints as zoomed; the stage resets the photo before it paints. const currentSrc = currentItem?.src; const [viewedItem, setViewedItem] = useState({index, src: currentSrc}); // Object.is, matching the dependency comparison of the effect this replaces: @@ -386,8 +630,7 @@ export function Lightbox({ !Object.is(viewedItem.src, currentSrc) ) { setViewedItem({index, src: currentSrc}); - setZoom(1); - setPan({x: 0, y: 0}); + setIsZoomed(false); } // Announce gallery navigation to screen readers. Moving between images only @@ -408,7 +651,7 @@ export function Lightbox({ if (!indexChanged || !isOpen || !wasOpen) { return; } - const item = mediaArray[Math.min(index, mediaArray.length - 1)]; + const item = mediaArray[index]; const position = {index: index + 1, total: mediaArray.length}; announce( item?.alt @@ -417,36 +660,69 @@ export function Lightbox({ ); }, [index, isOpen, announce, mediaArray, t]); - // Open/close dialog. The dialog renders only with an item, so a viewer - // opened before its first item arrives opens when that item does. - const hasItem = currentItem != null; - useIsomorphicLayoutEffect(() => { - const dialog = dialogRef.current; - if (!dialog || !hasItem) { + // A video left on another page stops: the pager keeps the neighbours + // mounted, and a playing one would go on, with its sound, off screen. + useEffect(() => { + const container = containerRef.current; + if (container == null) { return; } - - if (isOpen && !dialog.open) { - triggerElementRef.current = document.activeElement; - dialog.showModal(); - } else if (!isOpen && dialog.open) { - dialog.close(); - if (triggerElementRef.current instanceof HTMLElement) { - triggerElementRef.current.focus(); + for (const video of container.querySelectorAll( + '[data-lightbox-page]:not([data-current]) video', + )) { + if (!video.paused) { + video.pause(); } } - }, [isOpen, hasItem]); + }, [index]); + // A closed viewer is not closed again: while a closing photo lands, Close, + // a backdrop click and a platform cancel do nothing. const handleClose = useCallback(() => { - onOpenChange(false); - }, [onOpenChange]); + if (isOpen) { + onOpenChange(false); + } + }, [isOpen, onOpenChange]); + // An item's source: the caller's, or for the opened item the element that + // opened the viewer when it shows a picture (a thumbnail, or a button or + // link around one); a text button opens and closes plainly. + const sourceFor = experimental?.sourceFor; + const sourceElementFor = useCallback( + (i: number): HTMLElement | null => { + if (sourceFor != null) { + return sourceFor(i); + } + const opener = openerRef.current; + return i === openedIndexRef.current && + opener instanceof HTMLElement && + opener !== opener.ownerDocument.body && + (opener instanceof HTMLImageElement || + opener.querySelector('img') != null) + ? opener + : null; + }, + [sourceFor], + ); + + // The viewer stays presented while its dialog is open, also while a closing + // photo zooms back into its source: it stays in the dismissal stack and + // swallows an Escape (so the press cannot reach the layer behind it) until + // the photo lands and the dialog closes, as BottomSheet does through its + // exit. Latched when the viewer opens; cleared by the dialog's close event. + const [isPresented, setIsPresented] = useState(isOpen); + // Latched during render when the viewer opens, as `viewedItem` is. + if (isOpen && !isPresented) { + setIsPresented(true); + } const {shouldDismissOnCloseRequest} = useLayerDismissal({ - isActive: isOpen, + isActive: isOpen || isPresented, + escapeBehavior: isOpen ? 'close' : 'block', onDismiss: handleClose, // The dialog's document is this layer's: rendered into an iframe or a // Picture-in-Picture window, it ranks with the layers opened inside it. getContainer: () => dialogRef.current, + isPresent: () => dialogRef.current?.open ?? false, }); // The native `cancel` event is the browser's own close-watcher firing: an @@ -472,57 +748,963 @@ export function Lightbox({ // so clicks on the visual backdrop (the dark area around the media) land on // the container, never on the dialog element itself — treat both as the // backdrop. A pan drag that ends over the backdrop still fires a click on - // the common ancestor; ignore it so releasing a drag doesn't dismiss. + // the common ancestor; ignore it so releasing a drag doesn't dismiss (the + // flag is cleared when the next press lands on the photo). const didDragRef = useRef(false); + // Where the last press on the stage landed: a mouse drag released beside + // the photo still fires a click there, which is no click (UIKit's tap + // movement limit). + // And on which photo: a tap the stage recognizes after the double-tap + // interval belongs to the photo it landed on. + const pressRef = useRef<{x: number; y: number; index: number} | null>(null); + // With the controls hidden, a click or tap where the stage does not take + // it (a video page, the dark area beside a photo the stage only hugs) + // brings them back instead of closing the viewer unseen. + const showHiddenChromeRef = useRef<() => boolean>(() => false); + // The photo the zoom stage moves, in a gallery or alone: where a click + // counts as the photo's. + const currentPhotoRef = useRef(null); const handleBackdropClick = useCallback( (e: ReactMouseEvent) => { if (didDragRef.current) { didDragRef.current = false; return; } + if (showHiddenChromeRef.current()) { + return; + } if (e.target === e.currentTarget || e.target === containerRef.current) { handleClose(); + return; + } + // A gallery's stage fills the viewer: a click on it beside the media is + // a backdrop click too. With zoom on, the stage's own tap handles it + // for every pointer, after the double-tap interval (spec:AST-094 FR43). + const target = e.target as Element; + const stageTaps = hasZoom && !isVideo; + const press = pressRef.current; + const dragged = + hasZoom && + !isVideo && + press != null && + Math.hypot(e.clientX - press.x, e.clientY - press.y) > + UIKIT_TAP.allowableMovement.value; + // A click on the photo itself is the photo's, zoom or not (a + // double-click zooms it): the photo takes no pointer events, so the + // click lands on its box. The photo is the one the zoom stage moves, + // in a gallery or alone. + const photoBox = currentPhotoRef.current?.getBoundingClientRect(); + const onPhoto = + photoBox != null && + // A photo with no size yet (still loading) is not under the click. + photoBox.width > 0 && + photoBox.height > 0 && + e.clientX >= photoBox.left && + e.clientX <= photoBox.right && + e.clientY >= photoBox.top && + e.clientY <= photoBox.bottom; + if ( + isGallery && + !stageTaps && + !dragged && + !onPhoto && + (target === imageWrapperRef.current || + target === trackRef.current || + target.hasAttribute('data-lightbox-page') || + target.hasAttribute('data-lightbox-page-media')) + ) { + handleClose(); } }, - [handleClose], + [handleClose, hasZoom, isGallery, isVideo], ); - // Gallery navigation - const goToPrev = useCallback(() => { - if (canPrev) { - setIndex(index - 1); + // Zoom: pinch, double tap or double-click, Enter/Space on the image, or + // +/- keys. Zoom changes are silent to assistive tech (only the transform + // changes), so mirror them in the polite live region, including a hint that + // arrow keys pan while zoomed. A tap on the dark area around the photo + // closes, as a click on the backdrop does. + const isZoomTarget = hasZoom && !isVideo; + // What a dismissal or a transition faded (the backdrop, the controls, + // captions, neighbouring pages): exactly those go back to rest when the + // viewer next opens or closes, or stays open after a dismissal its caller + // kept. A page faded as a neighbour may be the current one by then. + const fadedRef = useRef(new Set()); + const markFaded = (animators: ReadonlyArray) => { + for (const animator of animators) { + fadedRef.current.add(animator); + } + }; + const restoreFaded = (): boolean => { + const faded = fadedRef.current; + if (faded.size === 0) { + return false; } - }, [canPrev, index, setIndex]); + for (const animator of faded) { + // The black layer rests where the controls' state puts it (restChrome). + if (animator.element?.hasAttribute('data-lightbox-blackout') !== true) { + animator.set({opacity: 1}); + } + } + faded.clear(); + return true; + }; + // The photo was dismissed: the viewer closes without zooming. + const dismissedRef = useRef(false); + // A dismissed photo still leaving or landing: the close waits for it. + const dismissLandingRef = useRef<{ + landing: Promise; + resolve: () => void; + } | null>(null); + // Set when a dismissal has asked the caller to close; the next render + // (one more is asked for) reads the caller's answer. + const dismissalAnsweredRef = useRef(false); + const [, renderAfterDismissal] = useReducer((n: number) => n + 1, 0); - const goToNext = useCallback(() => { - if (canNext) { - setIndex(index + 1); + // Photos hides its controls (the chrome) for a tap on the photo and as a + // zoom by touch passes the fitted size, and shows them for the next tap and + // as the photo returns to it. The background turns black with them and + // stays black while the photo is zoomed, when a tap toggles only the + // controls. Both run on Photos' measured curves (zoomMedia.chrome). The + // controls read the state from an attribute on the container, so a toggle + // in the middle of a gesture renders nothing. + const chromeHiddenRef = useRef(false); + const immersiveRef = useRef(false); + /** The last input: a pointer type, or 'keyboard'. */ + const lastInputRef = useRef('mouse'); + const chromeElements = useCallback((): HTMLElement[] => { + const container = containerRef.current; + return container == null + ? [] + : [ + ...container.querySelectorAll( + '[data-lightbox-chrome], [data-lightbox-caption]', + ), + ]; + }, []); + const blackoutAnimator = useCallback((): Animator | null => { + const container = containerRef.current; + return container == null + ? null + : animatorFor(container, { + pseudoElement: '::before', + initial: {opacity: 0}, + }); + }, []); + const setChrome = useCallback( + ({hidden, immersive}: {hidden: boolean; immersive: boolean}) => { + if (chromeHiddenRef.current !== hidden) { + chromeHiddenRef.current = hidden; + containerRef.current?.toggleAttribute('data-chrome-hidden', hidden); + for (const element of chromeElements()) { + const animator = animatorFor(element); + animator.play({ + opacity: chromeFadeCurve(animator.get('opacity'), hidden), + }); + } + } + if (immersiveRef.current !== immersive) { + immersiveRef.current = immersive; + const blackout = blackoutAnimator(); + blackout?.play({ + opacity: chromeBackgroundCurve(blackout.get('opacity'), immersive), + }); + } + }, + [blackoutAnimator, chromeElements], + ); + /** The controls and the background as an opened viewer has them. */ + const restChrome = useCallback(() => { + if (!chromeHiddenRef.current && !immersiveRef.current) { + return; + } + chromeHiddenRef.current = false; + immersiveRef.current = false; + containerRef.current?.removeAttribute('data-chrome-hidden'); + blackoutAnimator()?.set({opacity: 0}); + for (const element of chromeElements()) { + animatorFor(element).set({opacity: 1}); } - }, [canNext, index, setIndex]); + }, [blackoutAnimator, chromeElements]); + // Which input came last decides whether a zoom hides the controls (only + // touch does, as Photos); a keyboard focus on a hidden control shows them. + // Attached with the dialog node itself (a callback ref): a viewer whose + // first render had no item mounts its dialog later. They listen for every + // pointer, on the dialog itself (spec:AST-094 FR24, FR43). + const inputListenersRef = useRef<(() => void) | null>(null); + const attachInputListeners = useCallback( + (dialog: HTMLDialogElement | null) => { + inputListenersRef.current?.(); + inputListenersRef.current = null; + if (dialog == null) { + return; + } + const onPointerDown = (event: PointerEvent) => { + lastInputRef.current = event.pointerType; + }; + const onKeyDown = () => { + lastInputRef.current = 'keyboard'; + }; + const onFocusIn = (event: FocusEvent) => { + const target = event.target instanceof Element ? event.target : null; + if ( + chromeHiddenRef.current && + lastInputRef.current === 'keyboard' && + target?.closest('[data-lightbox-chrome]') != null + ) { + setChrome({hidden: false, immersive: immersiveRef.current}); + } + }; + // With the controls hidden, a tap where a gallery's caption sits + // brings them back, as a tap beside the photo does (`onTapOutside`): + // the caption sits below the photo's box, out of the stage's reach. + // The tap's click never comes (the gesture layer takes the touch's + // default), so the press itself decides: down and up on a caption + // within a tap's movement. + let captionPress: {id: number; x: number; y: number} | null = null; + const onCaptionPress = (event: PointerEvent) => { + const target = event.target instanceof Element ? event.target : null; + captionPress = + chromeHiddenRef.current && + target?.closest('[data-lightbox-caption]') != null + ? {id: event.pointerId, x: event.clientX, y: event.clientY} + : null; + }; + const onCaptionLift = (event: PointerEvent) => { + const press = captionPress; + captionPress = null; + if ( + press == null || + press.id !== event.pointerId || + !chromeHiddenRef.current || + Math.hypot(event.clientX - press.x, event.clientY - press.y) > + UIKIT_TAP.allowableMovement.value + ) { + return; + } + setChrome({ + hidden: false, + immersive: stageApiRef.current?.isZoomed() === true, + }); + }; + dialog.addEventListener('pointerdown', onPointerDown, true); + dialog.addEventListener('keydown', onKeyDown, true); + dialog.addEventListener('focusin', onFocusIn); + dialog.addEventListener('pointerdown', onCaptionPress, true); + dialog.addEventListener('pointerup', onCaptionLift, true); + inputListenersRef.current = () => { + dialog.removeEventListener('pointerdown', onPointerDown, true); + dialog.removeEventListener('keydown', onKeyDown, true); + dialog.removeEventListener('focusin', onFocusIn); + dialog.removeEventListener('pointerdown', onCaptionPress, true); + dialog.removeEventListener('pointerup', onCaptionLift, true); + }; + }, + [setChrome], + ); + const mergedDialogRef = useMergedRefs(ref, dialogRef, attachInputListeners); - // Zoom: double-click, Enter/Space on the image, or +/- keys toggle 1x ↔ 2x. - // Zoom changes are silent to assistive tech (only the transform changes), so - // mirror them in the polite live region, including a hint that arrow keys - // pan while zoomed. - const applyZoom = useCallback( - (next: number) => { - if (!hasZoom || isVideo || next === zoom) { + // An item that is not a zoom target (a video) has no stage to tap: the + // controls a photo hid come back when the viewer shows it. + useIsomorphicLayoutEffect(() => { + if (!isZoomTarget && (chromeHiddenRef.current || immersiveRef.current)) { + setChrome({hidden: false, immersive: false}); + } + }, [isZoomTarget, viewedItem, setChrome]); + + // Paging with the controls hidden: the next photo's caption arrives + // hidden too, before it paints. + useIsomorphicLayoutEffect(() => { + if (!chromeHiddenRef.current) { + return; + } + for (const element of chromeElements()) { + const animator = animatorFor(element); + // Without Web Animations the animator writes styles and nothing runs + // (the motion system's own reading, animatedLayout's movedByApp). + const fading = + typeof element.getAnimations === 'function' && + element.getAnimations().length > 0; + if (animator.get('opacity') !== 0 && !fading) { + animator.set({opacity: 0}); + } + } + }); + + // Drag to dismiss, as in Photos: the backdrop and, faster, the controls + // and the neighbouring photos fade with the drag's travel; a dismissal + // fades them out on its spring and closes once the photo has left. + // The backdrop's blur comes off while the backdrop moves and back once it + // rests (the dialog's style, '::backdrop'). Each move counts: a release's + // end restores it only if nothing has moved the backdrop since. + const backdropMovesRef = useRef(0); + const setBackdropMoving = useCallback((moving: boolean) => { + const dialog = dialogRef.current; + if (moving) { + dialog?.style.setProperty('--_lightbox-backdrop-filter', 'none'); + } else { + dialog?.style.removeProperty('--_lightbox-backdrop-filter'); + } + }, []); + + const dismissFades = useCallback((): { + backdrop: Animator | null; + chrome: Animator[]; + } => { + const dialog = dialogRef.current; + const container = containerRef.current; + if (dialog == null || container == null) { + return {backdrop: null, chrome: []}; + } + const chrome: HTMLElement[] = []; + for (const element of container.querySelectorAll( + '[data-lightbox-chrome], [data-lightbox-page]:not([data-current]), [data-lightbox-caption]', + )) { + if ( + element.parentElement?.closest( + '[data-lightbox-page]:not([data-current])', + ) == null + ) { + chrome.push(element); + } + } + return { + backdrop: animatorFor(dialog, {pseudoElement: '::backdrop'}), + chrome: chrome.map(element => animatorFor(element)), + }; + }, []); + + // The stage, for callbacks that run before this render returns it. + const stageApiRef = useRef(null); + // A dismissal in progress (a drag down or a pinch in): the backdrop, the + // black behind the photo when it shows, and the controls and neighbouring + // photos (hidden controls stay hidden). + const fadeForDismissal = (backdropAlpha: number, chromeAlpha: number) => { + const {backdrop, chrome} = dismissFades(); + markFaded(backdrop == null ? chrome : [backdrop, ...chrome]); + // The backdrop fades without its blur while a drag or a pinch moves it. + backdropMovesRef.current += 1; + setBackdropMoving(true); + backdrop?.set({opacity: backdropAlpha}); + const hidden = chromeHiddenRef.current; + for (const animator of chrome) { + animator.set({ + opacity: hidden && isChromeElement(animator.element) ? 0 : chromeAlpha, + }); + } + if (immersiveRef.current) { + blackoutAnimator()?.set({opacity: backdropAlpha}); + } + }; + // The zoom stage: the keyboard's zoom and pan, and the gestures of every + // pointer, a mouse, a pen and a finger alike (spec:AST-094 FR43), whose + // pinch (a ctrl-wheel's too) and pan (a wheel's too) give past their + // limits and bounce back. The touch presentation, which only the zoom + // transition and the opener record read, is the shared touch resolver's. + const {isTouch} = useTouchPresentation(); + // The opener is the element the viewer was opened from: a click or tap on + // it (which a plain image, or a button in Safari, does not focus), else + // the focused element. Only the touch presentation reads it, through one + // listener every closed touch viewer shares (spec:AST-094 FR24). + useOpenerActivation(!isOpen && isTouch); + const onDragStart = () => { + didDragRef.current = true; + }; + const zoom = useZoomCore({ + stageRef: isGallery ? pageMediaRef : imageWrapperRef, + surfaceRef: isGallery ? trackRef : imageWrapperRef, + isEnabled: isOpen && isZoomTarget, + itemKey: viewedItem, + onZoomedChange: next => { + setIsZoomed(next); + if (dialogRef.current?.open === true && isOpen) { + announce( + next + ? t('@astryx.lightbox.zoomedIn') + : t('@astryx.lightbox.zoomedOut'), + ); + // Photos hides the controls as a gesture zooms, a mouse's, a pen's + // or a finger's alike (spec:AST-094 FR43); a zoom from the keyboard + // leaves them, so the keys that zoomed can reach them. + if (lastInputRef.current !== 'keyboard') { + setChrome({hidden: next, immersive: next}); + } + } + }, + }); + const stage = useZoomStage(zoom, { + stageRef: isGallery ? pageMediaRef : imageWrapperRef, + surfaceRef: isGallery ? trackRef : imageWrapperRef, + isEnabled: isOpen && isZoomTarget, + itemKey: viewedItem, + onPinchZoomed: next => { + // Photos hides them as the pinch passes the fitted size, not at the + // lift; the return shows them once the photo is back at it. + if (next) { + setChrome({hidden: true, immersive: true}); + } + }, + onTapPhoto: () => { + // A page change before the tap was recognized calls it off: the next + // photo keeps its controls. + if (pressRef.current != null && pressRef.current.index !== index) { + return; + } + const hidden = !chromeHiddenRef.current; + // Zoomed, a tap toggles only the controls: the black stays. + setChrome({ + hidden, + immersive: stageApiRef.current?.isZoomed() === true ? true : hidden, + }); + }, + // With the controls hidden, a tap beside the photo brings them back + // (Photos' tap on the background) rather than closing unseen. + onTapOutside: () => { + if (pressRef.current != null && pressRef.current.index !== index) { return; } - setZoom(next); - setPan({x: 0, y: 0}); - announce( - next > 1 - ? t('@astryx.lightbox.zoomedIn') - : t('@astryx.lightbox.zoomedOut'), + if (chromeHiddenRef.current) { + setChrome({ + hidden: false, + immersive: stageApiRef.current?.isZoomed() === true, + }); + } else { + handleClose(); + } + }, + onDragStart, + dismiss: { + // Every pointer's, like the zoom's (spec:AST-094 FR43); a video keeps + // its own controls' drags. + isEnabled: isOpen && !isVideo, + onDrag: travel => { + fadeForDismissal( + dismissBackdropAlpha(travel), + dismissChromeAlpha(travel), + ); + }, + // Photos' pinch to close fades them by the pinch's scale instead, + // under any pinch, a ctrl-wheel's included (spec:AST-094 FR43). + onPinch: scale => { + fadeForDismissal( + pinchCloseBackdropAlpha(scale), + pinchCloseChromeAlpha(scale), + ); + }, + target: () => sourceBoxOf(sourceElementFor(index)), + onRelease: (dismisses, touch) => { + if (dismisses) { + dismissedRef.current = true; + let resolve = () => {}; + const landing = new Promise(done => { + resolve = done; + }); + dismissLandingRef.current = {landing, resolve}; + // A photo shrinking into its thumbnail lands on the thumbnail's + // picture, which fades in over it while the cell stays hidden. + const source = sourceElementFor(index); + const box = sourceBoxOf(source); + const container = containerRef.current; + const lifted = + source != null && box != null && container != null + ? liftCopy(source, { + host: container, + src: currentItem?.src ?? '', + attribute: 'data-lightbox-portal', + }) + : null; + if (lifted != null) { + // Under the photo, as the transition's copy (zoomFromSource). + lifted.element.style.zIndex = '-1'; + const portalAnimator = animatorFor(lifted.element); + portalAnimator.set({opacity: 0}); + portalAnimator.animate( + {opacity: 1}, + {spring: zoomMedia.dismiss.dismissSizeSpring}, + {touchRelease: touch}, + ); + restoreLanded(); + landedRef.current = lifted; + } + } + const {backdrop, chrome} = dismissFades(); + markFaded(backdrop == null ? chrome : [backdrop, ...chrome]); + const spring = dismisses + ? zoomMedia.dismiss.dismissSizeSpring + : zoomMedia.dismiss.cancelSizeSpring; + // A cancel brings hidden controls back as Photos does, on their own + // fade; the black stays. + const returning = !dismisses && chromeHiddenRef.current; + if (returning) { + chromeHiddenRef.current = false; + containerRef.current?.removeAttribute('data-chrome-hidden'); + } + const opacity = dismisses ? 0 : 1; + const move = (backdropMovesRef.current += 1); + setBackdropMoving(true); + const backdropMotion = backdrop?.animate( + {opacity}, + {spring}, + {touchRelease: touch}, + ); + for (const animator of chrome) { + if (returning && isChromeElement(animator.element)) { + animator.play( + {opacity: chromeFadeCurve(animator.get('opacity'), false)}, + {touchRelease: touch}, + ); + } else { + animator.animate({opacity}, {spring}, {touchRelease: touch}); + } + } + if (immersiveRef.current) { + blackoutAnimator()?.animate( + {opacity}, + {spring}, + {touchRelease: touch}, + ); + } + // Back at rest: the blur returns, unless the backdrop moved again. + void backdropMotion?.finished.then(() => { + if (!dismisses && backdropMovesRef.current === move) { + setBackdropMoving(false); + } + }); + if (dismisses) { + // The release decides the close and reports it from here, the + // handler that caused it (architecture:react-component-runtime + // INV2); the viewer stays until the photo has left or landed. A + // caller may keep it open (the close is its call): the render after + // its answer brings the photo, the backdrop and the controls back. + handleClose(); + dismissalAnsweredRef.current = true; + renderAfterDismissal(); + } + }, + onDismissed: () => { + // The photo has left or landed: the close it reported can end. + dismissLandingRef.current?.resolve(); + }, + }, + }); + const currentPhotoContentRef = useMergedRefs( + stage.contentRef, + currentPhotoRef, + ); + stageApiRef.current = stage; + showHiddenChromeRef.current = () => { + if (!chromeHiddenRef.current) { + return false; + } + setChrome({hidden: false, immersive: stage.isZoomed()}); + return true; + }; + + // Open and close the dialog. Under the touch layer, with a source + // (`experimental.sourceFor`, or the element that opened the viewer for the + // opened item), + // the photo zooms out of it on open and back into the current item's on + // close (UIKit's zoom transition); the dialog stays open, the page inert, + // until the photo has landed. Where the primary pointer is fine the viewer + // opens and closes at once, as it always has (spec:AST-017 FR59). Opening again after a dismissal or a transition puts the photo, + // the backdrop and the controls back first. Only then: a write promotes + // the layers, and their release later would repaint mid-gesture. + const transitionRef = useRef({ + stage, + dismissFades, + restChrome, + blackoutAnimator, + setBackdropMoving, + sourceElementFor, + index, + src: currentItem?.src ?? '', + isImage: !isVideo, + isTouch, + }); + transitionRef.current = { + stage, + dismissFades, + restChrome, + blackoutAnimator, + setBackdropMoving, + sourceElementFor, + index, + src: currentItem?.src ?? '', + isImage: !isVideo, + isTouch, + }; + const closeTokenRef = useRef(0); + const openTokenRef = useRef(0); + const openTimerRef = useRef>(undefined); + // Ends the touch entrance's input hold (spec:AST-094 FR35) while one runs. + const endEntranceRef = useRef<(() => void) | null>(null); + // Aborts the transition in flight: its thumbnail shows again and its + // portal goes (an unmount mid-transition). + const transitionAbortRef = useRef(null); + // The gallery's pager, for the opening (it is created further down). + const pagerRef = useRef(null); + // Unmounted (a caller may render the viewer only while it is open): the + // timers and continuations stop, and a thumbnail a landed photo or a + // transition hid shows again. + useEffect( + () => () => { + openTokenRef.current += 1; + closeTokenRef.current += 1; + clearTimeout(openTimerRef.current); + endEntranceRef.current?.(); + transitionAbortRef.current?.abort(); + restoreLanded(); + }, + [restoreLanded], + ); + // The dialog renders only with an item, so a viewer opened before its + // first item arrives opens when that item does. + const hasItem = currentItem != null; + useIsomorphicLayoutEffect(() => { + const dialog = dialogRef.current; + if (dialog == null || !hasItem) { + return; + } + const now = transitionRef.current; + // What fades with the transition: the backdrop and what shows over it + // (hidden controls stay hidden; the black, when shown, goes with the + // backdrop). + const fadesNow = () => { + const {backdrop, chrome} = now.dismissFades(); + const hidden = chromeHiddenRef.current; + const fades = chrome.filter( + animator => !(hidden && isChromeElement(animator.element)), ); + if (backdrop != null) { + fades.unshift(backdrop); + } + const blackout = immersiveRef.current ? now.blackoutAnimator() : null; + if (blackout != null) { + fades.push(blackout); + } + return fades; + }; + const restFades = () => { + const {backdrop, chrome} = now.dismissFades(); + for (const animator of backdrop == null + ? chrome + : [backdrop, ...chrome]) { + animator.set({opacity: 1}); + } + now.restChrome(); + }; + // The transition starting now takes over from the one in flight, which + // is let go once the new one has read where the photo shows (a close + // turns an opening around from its progress). + const transition = () => { + const previous = transitionAbortRef.current; + const controller = new AbortController(); + transitionAbortRef.current = controller; + return { + signal: controller.signal, + release: () => previous?.abort(), + }; + }; + if (isOpen) { + // A close still landing is called off: the viewer stays. + closeTokenRef.current += 1; + dismissedRef.current = false; + dismissLandingRef.current = null; + restoreLanded(); + if (restoreFaded()) { + now.stage.rest(); + restFades(); + } + // At rest: the backdrop's blur is back (setBackdropMoving). + backdropMovesRef.current += 1; + now.setBackdropMoving(false); + if (dialog.open) { + return; + } + triggerElementRef.current = document.activeElement; + const activation = takeOpenerActivation(); + // A click opened it (within this task's events): its target, or the + // button or link around it, is the opener. + openerRef.current = + activation != null && + activation.target.isConnected && + Date.now() - activation.time < 1000 + ? (activation.target.closest('button, a') ?? activation.target) + : document.activeElement; + openedIndexRef.current = now.index; + dialog.showModal(); + // Opened on another page: the pager is there at once, unanimated (its + // own sync, an Effect, would slide there during the opening). It lands + // the track at the shown viewer's width before the first frame. + const pagerNow = pagerRef.current; + if (pagerNow != null && pagerNow.index !== now.index) { + pagerNow.goTo(now.index, {animated: false, reason: 'controlled'}); + } + const source = + now.isImage && now.isTouch ? now.sourceElementFor(now.index) : null; + const container = containerRef.current; + // The current page's photo in a gallery, else the one photo: queried in + // that order, as the media group holds every page of a gallery. + const img = + container?.querySelector('[data-current] img') ?? + container?.querySelector( + ':scope > [data-lightbox-media] img', + ); + if (source != null && container != null && img != null) { + // Under a finger, a tap or a drag that lands while the viewer + // enters is held until the opening ends, as UIKit delivers no touch + // while a presentation runs (spec:AST-094 FR35): the Layer module's + // hold, which leaves keys and assistive technology live. + endEntranceRef.current?.(); + let endEntrance = () => {}; + const entering = new Promise(resolve => { + endEntrance = resolve; + }); + endEntranceRef.current = endEntrance; + holdInput(dialog, {until: entering, scope: 'viewport'}); + const token = ++openTokenRef.current; + const start = () => { + if (openTokenRef.current !== token) { + return; + } + openTokenRef.current += 1; + clearTimeout(openTimerRef.current); + img.style.opacity = ''; + // The opened page may not rest in the viewport yet (the pager lands + // it before the first frame): the opening aims where it will rest. + const page = img.closest('[data-lightbox-page]'); + const viewport = imageWrapperRef.current; + const offset = + page != null && viewport != null + ? viewport.getBoundingClientRect().left - + page.getBoundingClientRect().left + : 0; + const fades = fadesNow(); + const next = transition(); + const opening = zoomFromSource({ + kind: 'open', + source, + container, + stage: now.stage, + fades, + src: now.src, + signal: next.signal, + offset, + }); + next.release(); + if (opening != null) { + markFaded(fades); + // The backdrop fades without its blur until the photo lands. + const move = (backdropMovesRef.current += 1); + now.setBackdropMoving(true); + const landed = () => { + if (backdropMovesRef.current === move) { + now.setBackdropMoving(false); + } + endEntrance(); + }; + void opening.then(landed, landed); + } else { + // No zoom after all: what waited at its first value comes back. + restoreFaded(); + endEntrance(); + } + }; + if (img.complete && img.naturalWidth > 0 && img.offsetWidth > 0) { + start(); + } else { + // Not decoded, or not laid out yet (a page the opening mounts has + // no size until its picture applies, even from the cache): the + // photo stays hidden (the thumbnail shows) until it can zoom, at + // most a moment. + img.style.opacity = '0'; + // Meanwhile the backdrop and the controls wait at the opening's + // first value, so the dim never shows at full before it fades in. + const waiting = fadesNow(); + markFaded(waiting); + const first = openingFadeStart(container); + for (const animator of waiting) { + animator.set({opacity: first}); + } + if (typeof img.decode === 'function') { + img.decode().then(start, start); + } + openTimerRef.current = setTimeout(start, OPEN_DECODE_WAIT_MS); + } + } + return; + } + // A zoom-in still waiting for its photo is called off, and so is the + // entrance's hold: a close (an Escape mid-opening) ends the entrance. + openTokenRef.current += 1; + clearTimeout(openTimerRef.current); + endEntranceRef.current?.(); + endEntranceRef.current = null; + if (!dialog.open) { + return; + } + const finish = () => { + restoreLanded(); + dismissedRef.current = false; + dismissLandingRef.current = null; + dialog.close(); + if (restoreFaded()) { + now.stage.rest(); + restFades(); + } + now.restChrome(); + if (triggerElementRef.current instanceof HTMLElement) { + triggerElementRef.current.focus(); + } + }; + // A dismissed photo still leaving or landing: the viewer closes once it + // has (the release reported the close). + const landing = dismissedRef.current + ? dismissLandingRef.current?.landing + : undefined; + if (landing != null) { + const token = ++closeTokenRef.current; + void landing.then(() => { + if (closeTokenRef.current === token) { + finish(); + } + }); + return; + } + const container = containerRef.current; + const source = + !dismissedRef.current && now.isImage && now.isTouch + ? now.sourceElementFor(now.index) + : null; + const fades = fadesNow(); + const next = transition(); + const closing = + source != null && container != null + ? zoomFromSource({ + kind: 'close', + source, + container, + stage: now.stage, + fades, + src: now.src, + signal: next.signal, + }) + : null; + next.release(); + if (closing == null) { + finish(); + return; + } + markFaded(fades); + // The backdrop fades without its blur while the photo returns. + backdropMovesRef.current += 1; + now.setBackdropMoving(true); + const token = ++closeTokenRef.current; + void closing.then(() => { + if (closeTokenRef.current === token) { + finish(); + } + }); + }, [isOpen, hasItem]); + useIsomorphicLayoutEffect(() => { + if (!dismissalAnsweredRef.current) { + return; + } + dismissalAnsweredRef.current = false; + if (isOpen) { + // Kept open: the landing is called off. + dismissedRef.current = false; + dismissLandingRef.current = null; + restoreLanded(); + stage.rest(); + restoreFaded(); + restChrome(); + backdropMovesRef.current += 1; + setBackdropMoving(false); + } + }); + + // A gallery pages like Photos: a drag or flick moves between items on a + // UIKit paging scroll view's release (one page at most, Photos' 40 pt + // gap); the buttons and keys move there on `setContentOffset(animated:)`. + // A zoomed photo keeps the drag until its edge, then the pager takes the + // next one; the photo left behind returns to the fitted size once the + // pager rests (Photos does). + // A controlled parent can keep its index after a swipe (refuse it): the + // pager goes back to the photo the parent shows. A swipe's report renders + // once more so that check runs after the parent has answered. + const swipeReportRef = useRef(null); + const [, renderAfterSwipe] = useReducer((n: number) => n + 1, 0); + const pager = usePager({ + viewportRef: imageWrapperRef, + trackRef, + count: mediaArray.length, + gap: zoomMedia.viewer.pageGap, + index, + isPanEnabled: isOpen && isGallery, + // Every pointer pages, a mouse's drag and a trackpad's scroll included + // (spec:AST-094 FR43). + panOptions: { + allowedPointerTypes: ['touch', 'pen', 'mouse'], + maximumNumberOfTouches: 1, + }, + shouldBeginPan: direction => !stage.isZoomed() || stage.isAtEdge(direction), + onIndexChange: (next, reason) => { + if (reason === 'drag') { + setIndex(next); + if (isControlled) { + swipeReportRef.current = next; + renderAfterSwipe(); + } + } + }, + onSettle: () => { + for (const img of trackRef.current?.querySelectorAll( + '[data-lightbox-page]:not([data-current]) img', + ) ?? []) { + img.style.translate = ''; + img.style.scale = ''; + } + }, + }); + pagerRef.current = pager; + + useEffect(() => { + const reported = swipeReportRef.current; + swipeReportRef.current = null; + if (reported != null && index !== reported && pager.index !== index) { + pager.goTo(index, {reason: 'controlled'}); + } + }); + + // Gallery navigation. Under a finger the pager slides to the new page (a + // paging scroll view's setContentOffset). Where the primary pointer is + // fine, the buttons and the arrow keys show the new item at once, as they + // always have (spec:AST-017 FR59): the pager moves there first, so the + // controlled sync finds nothing to animate, and it returns to `index` if + // the owner refuses the change. While a closing photo lands, nothing pages + // the closed viewer. + const showIndex = useCallback( + (next: number) => { + if (!isTouch) { + pager.goTo(next, {animated: false}); + } + setIndex(next); }, - [hasZoom, isVideo, zoom, announce, t], + [isTouch, pager, setIndex], ); - const handleDoubleClick = useCallback(() => { - applyZoom(zoom === 1 ? 2 : 1); - }, [applyZoom, zoom]); + const goToPrev = useCallback(() => { + if (isOpen && canPrev) { + showIndex(index - 1); + } + }, [isOpen, canPrev, index, showIndex]); + + const goToNext = useCallback(() => { + if (isOpen && canNext) { + showIndex(index + 1); + } + }, [isOpen, canNext, index, showIndex]); // Keyboard navigation. While zoomed, arrows pan the image (matching common // lightbox conventions); when not zoomed they navigate the gallery. @@ -534,21 +1716,25 @@ export function Lightbox({ // its right. const handleKeyDown = useCallback( (e: React.KeyboardEvent) => { - if (hasZoom && !isVideo) { + // While a closing photo lands, keys neither page nor zoom it. + if (!isOpen) { + return; + } + if (isZoomTarget) { if (e.key === '+' || e.key === '=') { e.preventDefault(); - applyZoom(2); + stage.zoomIn(); return; } if (e.key === '-') { e.preventDefault(); - applyZoom(1); + stage.zoomOut(); return; } - if (zoom > 1 && KEYBOARD_PAN_OFFSETS[e.key] !== undefined) { + if (stage.isZoomed() && KEYBOARD_PAN_OFFSETS[e.key] !== undefined) { e.preventDefault(); const [dx, dy] = KEYBOARD_PAN_OFFSETS[e.key]; - setPan(prev => ({x: prev.x + dx, y: prev.y + dy})); + stage.panBy(dx, dy); return; } } @@ -563,80 +1749,72 @@ export function Lightbox({ } } }, - [hasZoom, isVideo, zoom, applyZoom, goToPrev, goToNext], + [isOpen, isZoomTarget, stage, goToPrev, goToNext], ); // Enter/Space on the focused image wrapper (role="button") toggles zoom. const handleImageKeyDown = useCallback( (e: React.KeyboardEvent) => { - if (e.key === 'Enter' || e.key === ' ') { + if (isOpen && (e.key === 'Enter' || e.key === ' ')) { e.preventDefault(); - handleDoubleClick(); + stage.toggle(); } }, - [handleDoubleClick], + [isOpen, stage], ); - // Pan: mouse drag when zoomed - const handlePointerDown = useCallback( - (e: ReactMouseEvent) => { - if (zoom <= 1 || !hasZoom) { - return; - } - setIsDragging(true); - didDragRef.current = false; - dragStartRef.current = { - x: e.clientX, - y: e.clientY, - panX: pan.x, - panY: pan.y, - }; - }, - [zoom, hasZoom, pan], - ); - - useEffect(() => { - if (!isDragging) { - return; - } - - const handlePointerMove = (e: PointerEvent) => { - didDragRef.current = true; - const dx = e.clientX - dragStartRef.current.x; - const dy = e.clientY - dragStartRef.current.y; - setPan({ - x: dragStartRef.current.panX + dx, - y: dragStartRef.current.panY + dy, - }); - }; - - const handlePointerUp = () => { - setIsDragging(false); - }; - - window.addEventListener('pointermove', handlePointerMove); - window.addEventListener('pointerup', handlePointerUp); - return () => { - window.removeEventListener('pointermove', handlePointerMove); - window.removeEventListener('pointerup', handlePointerUp); - }; - }, [isDragging]); - - const isZoomed = zoom > 1; - const isZoomTarget = hasZoom && !isVideo; - const imageTransform = - zoom === 1 - ? null - : `scale(${zoom}) translate(${pan.x / zoom}px, ${pan.y / zoom}px)`; - if (!currentItem) { return null; } + const renderMedia = ( + {src, alt, type: kind = 'image'}: LightboxMedia, + isCurrent: boolean, + ) => + kind === 'video' ? ( +