diff --git a/README.md b/README.md
index 98b8860..f244a06 100644
--- a/README.md
+++ b/README.md
@@ -364,7 +364,7 @@ const popup = await popupPromise; // already wrapped
await popup.getByRole("button", { name: "Approve" }).click();
```
-In video mode, the popup renders as an overlay **in the main page's video**: scaled to fit 90% of the frame over the dimmed page, faded in and out on open/close, with popup clicks pointer-annotated inside the overlay. One composed video per test, popups included. The popup's facts land in `video-mode.json` under `children`.
+In video mode, the popup renders as an overlay **in the main page's video**: scaled to fit 90% of the frame over the dimmed page, faded in and out on open/close, with popup clicks pointer-annotated inside the overlay. A popup the test leaves open slides away when the test goes back to the page that opened it. One composed video per test, popups included. The popup's facts land in `video-mode.json` under `children`.
Details and escape hatches:
diff --git a/spec/popup-video-left-open.spec.ts b/spec/popup-video-left-open.spec.ts
new file mode 100644
index 0000000..55d6274
--- /dev/null
+++ b/spec/popup-video-left-open.spec.ts
@@ -0,0 +1,189 @@
+import { execFile as execFileCallback } from "node:child_process";
+import { promisify } from "node:util";
+import { test, expect, type BrowserContext } from "@playwright/test";
+import { addPlugins, videoMode } from "../src/index.ts";
+
+// 1280x900 doesn't scale evenly into Playwright's 800x562 video, so the
+// recorder pads the screencast with a gray strip down the right edge — the
+// shape of recording that used to defeat videoMode's timeline calibration.
+test.use({ video: "on", viewport: { width: 1280, height: 900 } });
+
+// Seen in the wild on a sign-in flow: the popup hands over a code and stays
+// open, and the test carries on in the page that opened it. The render showed
+// the popup over the opener's FINAL (signed-in) state, then the popup turning
+// solid black, then a black last frame.
+test("a popup left open while the test carries on in its opener", async ({
+ page: basePage,
+ context,
+}, testInfo) => {
+ await routeCodeSignInApp(context);
+ const video = videoMode({ addressBar: false, highlight: { mode: "outline", duration: 400 } });
+ {
+ await using page = await addPlugins({ page: basePage, testInfo, plugins: [video] });
+ await page.goto("https://app.middlewright.test/");
+
+ const popupPromise = basePage.waitForEvent("popup");
+ await page.getByRole("button", { name: "Sign in" }).click();
+ const popup = await popupPromise;
+ await popup.getByRole("button", { name: "Authorize" }).click();
+ await popup.getByText("Your code is 1234").waitFor();
+
+ // The popup never closes: the test just goes back to its opener.
+ await page.getByLabel("Code").fill("1234");
+ await page.getByRole("button", { name: "Continue" }).click();
+ await page.getByText("Signed in as mmkal").waitFor();
+ }
+
+ expect(await videoScenes(video.outputPaths().rendered)).toEqual([
+ "signed-out app",
+ "popup over signed-out app",
+ "signed-out app",
+ "signed-in app",
+ ]);
+});
+
+// The popup's recorder is settled and closed during teardown, after the test
+// body: none of that — least of all its near-black calibration cover — belongs
+// in the video.
+test("a popup still open when the test ends", async ({ page: basePage, context }, testInfo) => {
+ await routeCodeSignInApp(context);
+ const video = videoMode({
+ addressBar: false,
+ finalHold: 0,
+ highlight: { mode: "outline", duration: 400 },
+ });
+ {
+ await using page = await addPlugins({ page: basePage, testInfo, plugins: [video] });
+ await page.goto("https://app.middlewright.test/");
+
+ const popupPromise = basePage.waitForEvent("popup");
+ await page.getByRole("button", { name: "Sign in" }).click();
+ const popup = await popupPromise;
+ await popup.getByRole("button", { name: "Authorize" }).click();
+ await popup.getByText("Your code is 1234").waitFor();
+ }
+
+ expect(await videoScenes(video.outputPaths().rendered)).toEqual([
+ "signed-out app",
+ "popup over signed-out app",
+ ]);
+ // Closing it was teardown's doing, so the popup is recorded as never closed.
+ const [child] = (await video.metadata()).children;
+ expect(child).not.toHaveProperty("closedAt");
+});
+
+/**
+ * app.middlewright.test opens an auth popup that hands over a code and stays
+ * open; typing the code back into the app signs in. Each state fills the
+ * viewport with its own flat color, and all content sits in the top-left
+ * corner, so a video frame's state is readable from two background pixels.
+ */
+const routeCodeSignInApp = async (context: BrowserContext) => {
+ const style = `
+
+ `;
+ await context.route("https://app.middlewright.test/**", async (route) => {
+ await route.fulfill({
+ body: `
+ ${style}
+
+
+
+
+
+
+
+ `,
+ contentType: "text/html",
+ });
+ });
+ await context.route("https://auth.middlewright.test/**", async (route) => {
+ await route.fulfill({
+ body: `
+ ${style}
+
+
+
+
+
+ `,
+ contentType: "text/html",
+ });
+ });
+};
+
+// `dimmed` is how the color reads under a popup's backdrop, which darkens and
+// desaturates the page beneath it.
+const sceneColors = {
+ "signed-out app": { dimmed: [47, 68, 63], lit: [13, 148, 136] },
+ "signed-in app": { dimmed: [89, 83, 66], lit: [202, 138, 4] },
+ popup: { dimmed: [49, 46, 129], lit: [49, 46, 129] },
+ black: { dimmed: [0, 0, 0], lit: [0, 0, 0] },
+};
+
+const execFile = promisify(execFileCallback);
+
+/**
+ * The video as a storyboard: each frame named by what it shows, consecutive
+ * repeats collapsed. A frame's edge pixel is always the page (the popup
+ * overlay is inset); its center pixel is the popup whenever one is up.
+ */
+const videoScenes = async (path: string) => {
+ const size = { height: 70, width: 100 };
+ const { stdout } = await execFile(
+ "ffmpeg",
+ [
+ "-hide_banner",
+ "-loglevel",
+ "error",
+ "-i",
+ path,
+ "-vf",
+ `fps=25,scale=${size.width}:${size.height}:flags=neighbor`,
+ "-f",
+ "rawvideo",
+ "-pix_fmt",
+ "rgb24",
+ "pipe:1",
+ ],
+ { encoding: "buffer", maxBuffer: 64 * 1024 * 1024 },
+ );
+ const frameSize = size.width * size.height * 3;
+ const scenes: string[] = [];
+
+ for (let offset = 0; offset + frameSize <= stdout.length; offset += frameSize) {
+ const sceneAt = (x: number, y: number) => {
+ const pixel = stdout.subarray(offset + (y * size.width + x) * 3);
+ const distances = Object.entries(sceneColors).map(([scene, { dimmed, lit }]) => ({
+ distance: Math.min(
+ Math.hypot(...lit.map((channel, index) => channel - pixel[index])),
+ Math.hypot(...dimmed.map((channel, index) => channel - pixel[index])),
+ ),
+ scene,
+ }));
+ return distances.sort((left, right) => left.distance - right.distance)[0].scene;
+ };
+ const edge = sceneAt(1, size.height / 2);
+ const center = sceneAt(size.width / 2, size.height / 2);
+ const scene = center === "popup" ? `popup over ${edge}` : center;
+ if (scenes.at(-1) !== scene) scenes.push(scene);
+ }
+
+ return scenes;
+};
diff --git a/src/plugins/video-mode.ts b/src/plugins/video-mode.ts
index 9891e1c..3d95cd8 100644
--- a/src/plugins/video-mode.ts
+++ b/src/plugins/video-mode.ts
@@ -317,6 +317,13 @@ type VideoModeState = {
outputs: VideoModeOutputs;
sourceRange: VideoModeSourceRange;
startedAt?: number;
+ /**
+ * When the test body finished, on this instance's clock. Stamped by the
+ * first finalizer to run — popup recorders finalize before their parent, and
+ * each spends over a second settling its recorder, which is teardown, not
+ * test time.
+ */
+ endedAt?: number;
};
type VideoModeDialogLayout = {
@@ -614,6 +621,14 @@ const calibrationCoverMarginMs = (frameDurationMs: number) =>
const CALIBRATION_COVER_CHANNEL_TOLERANCE = 8;
const CALIBRATION_COVER_RGB = [1, 2, 3];
+// Playwright's recorder scales each screencast frame to fit the video size and
+// pads the rest with gray (right and bottom). A viewport that doesn't scale
+// evenly into the video — 1280x900 into 800x562 leaves a 2px strip — therefore
+// has pixels the cover can never reach; the cover is looked for only inside
+// the screencast's own area, this far in from its computed edge (Chrome rounds
+// the frame size down to even numbers).
+const CALIBRATION_COVER_EDGE_INSET_PX = 4;
+
/**
* Raw-clock timestamp (ms) of the first frame of the recording's TRAILING
* calibration-cover run, or undefined when the recording doesn't end on the
@@ -621,9 +636,21 @@ const CALIBRATION_COVER_RGB = [1, 2, 3];
* backwards from the end means an app that legitimately paints near-black
* mid-test can never match — only the tail the cover owns.
*/
-const detectCalibrationCoverStartMs = async (inputPath: string): Promise => {
+const detectCalibrationCoverStartMs = async (options: {
+ inputPath: string;
+ video: { height: number; width: number };
+ /** The recorded page's viewport; undefined when it isn't emulated. */
+ viewport: VideoModeViewport | undefined;
+}): Promise => {
const size = VIDEO_ANALYSIS_SAMPLE_SIZE;
const frameSize = size * size * 3;
+ const screencast = options.viewport
+ ? scaledViewportSize(options.viewport, options.video)
+ : options.video;
+ const crop = {
+ height: Math.max(1, screencast.height - CALIBRATION_COVER_EDGE_INSET_PX),
+ width: Math.max(1, screencast.width - CALIBRATION_COVER_EDGE_INSET_PX),
+ };
let stdout: Buffer;
try {
const result = await execFile(
@@ -633,9 +660,9 @@ const detectCalibrationCoverStartMs = async (inputPath: string): Promise
+ freezes.reduce((ms, freeze) => (freeze.atMs <= rawMs ? ms + freeze.durationMs : ms), rawMs);
+
+/** Like `compositeTimeMs`, for where a span ENDS: before a freeze starting right there. */
+const compositeEndMs = (freezes: VideoModeFreeze[], rawMs: number) =>
+ freezes.reduce((ms, freeze) => (freeze.atMs < rawMs ? ms + freeze.durationMs : ms), rawMs);
+
+/**
+ * Move a parent-raw timeline onto the composite clock. Highlights and address
+ * bars shift whole (their `end - start` is a rendered duration, not footage);
+ * captions and dead air are footage spans, so one containing a freeze grows.
+ */
+const compositeVideoTimeline = (
+ timeline: ReturnType,
+ freezes: VideoModeFreeze[],
+): ReturnType => {
+ const span = (rawSpan: VideoModeSpan): VideoModeSpan => ({
+ end: compositeEndMs(freezes, rawSpan.end),
+ start: compositeTimeMs(freezes, rawSpan.start),
+ });
+ return {
+ addressBars: timeline.addressBars.map((addressBar) => {
+ const shiftMs = compositeTimeMs(freezes, addressBar.start) - addressBar.start;
+ return { ...addressBar, end: addressBar.end + shiftMs, start: addressBar.start + shiftMs };
+ }),
+ captions: timeline.captions.map((caption) => ({ ...caption, ...span(caption) })),
+ deadAir: timeline.deadAir.map(span),
+ highlights: timeline.highlights.map((highlight) => {
+ const shiftMs = compositeTimeMs(freezes, highlight.start) - highlight.start;
+ return {
+ ...highlight,
+ actionEnd: highlight.actionEnd === undefined ? undefined : highlight.actionEnd + shiftMs,
+ end: highlight.end + shiftMs,
+ sourceFrameAt:
+ highlight.sourceFrameAt === undefined ? undefined : highlight.sourceFrameAt + shiftMs,
+ start: highlight.start + shiftMs,
+ };
+ }),
+ };
+};
+
const childCompositeLayers = async (options: {
children: VideoModeChild[];
+ /** How long each exit freeze holds (ms): the animation plus settle time. */
+ exitFreezeMs: number;
+ /** Parent-raw time (ms) the rendered footage ends: test end, short of the cover. */
+ footageEndMs: number;
+ /** The opener's highlights on the parent raw clock. */
+ openerHighlights: VideoModeHighlight[];
outputDir: string;
timelineOffsetMs: number;
video: VideoInfo;
-}): Promise => {
- const layers: VideoModeChildLayer[] = [];
+}): Promise<{
+ freezes: VideoModeFreeze[];
+ layers: VideoModeChildLayer[];
+ /** Parent-raw spans whose opener frames are dropped (see CHILD_BIRTH_OPENER_HOLD_MS). */
+ openerHolds: VideoModeSpan[];
+}> => {
+ const placements: (Omit & {
+ closeRawMs: number;
+ openRawMs: number;
+ })[] = [];
for (const child of options.children) {
if (!child.raw) continue;
@@ -4601,7 +4706,11 @@ const childCompositeLayers = async (options: {
const detectedCoverStartMs =
child.calibrationCoverPaintedAt === undefined
? undefined
- : await detectCalibrationCoverStartMs(path);
+ : await detectCalibrationCoverStartMs({
+ inputPath: path,
+ video: rawInfo,
+ viewport: child.viewport,
+ });
// Same guard as the parent: a cover run reaching t=0 leaves nothing to
// calibrate against — fall back to endpoint arithmetic.
const coverStartMs =
@@ -4627,44 +4736,57 @@ const childCompositeLayers = async (options: {
);
const width = Math.max(2, 2 * Math.round((rawInfo.width * scale) / 2));
const height = Math.max(2, 2 * Math.round((rawInfo.height * scale) / 2));
- const enableFromMs = Math.max(0, child.openedAt + options.timelineOffsetMs);
- // The cover's composite-time position bounds every overlay window: from
- // that instant on, child footage is the calibration cover, not the popup.
- // A margin frame short, because ffmpeg's `between` enable windows include
- // their end timestamp and the detector's timestamp can trail the true
- // first cover frame by a tick.
- const coverCompositeMs =
+ // Never on the first frame: that one belongs to the opener, and whatever
+ // happened before the recorder started (the click that opened this popup,
+ // say) can only be shown on it.
+ const openRawMs = Math.max(
+ options.video.frameDurationMs,
+ child.openedAt + options.timelineOffsetMs,
+ );
+ // Where the popup's own footage ends. Covered at teardown, it is good
+ // right up to its cover (from there on, child footage is the calibration
+ // cover, not the popup) — a margin frame short, because the detector's
+ // timestamp can trail the true first cover frame by a tick. Without a
+ // detected cover, the end of the rendered footage is the best bound.
+ const lastFootageMs = Math.min(
+ options.video.durationMs,
coverStartMs === undefined
- ? undefined
+ ? options.footageEndMs
: coverStartMs -
- calibrationCoverMarginMs(rawInfo.frameDurationMs) +
- childOffsetMs +
- options.timelineOffsetMs;
- let closeMs = Math.min(
- options.video.durationMs,
- (child.closedAt === undefined ? child.openedAt + rawInfo.durationMs : child.closedAt) +
- options.timelineOffsetMs,
+ calibrationCoverMarginMs(rawInfo.frameDurationMs) +
+ childOffsetMs +
+ options.timelineOffsetMs,
);
- if (coverCompositeMs !== undefined) {
- closeMs = Math.min(closeMs, coverCompositeMs);
- }
- // The exit fade runs AFTER close (the screencast's padded final frame
- // supplies footage): a popup that closes itself right after a click would
- // otherwise put that click — and its hold's freeze frame — mid-fade.
- let enableToMs = Math.min(options.video.durationMs, closeMs + CHILD_OVERLAY_FADE_MS);
- if (coverCompositeMs !== undefined) {
- enableToMs = Math.min(enableToMs, coverCompositeMs);
- }
+ // A popup leaves the screen when it closes — or, still open, when the test
+ // goes back to the opener: the first opener highlight after the popup's
+ // last one. A popup the test never acted in stays up until it closes;
+ // there, only an opener highlight within a frame of the close counts (its
+ // hold would otherwise freeze the first frames of the exit).
+ const closedRawMs =
+ child.closedAt === undefined ? Infinity : child.closedAt + options.timelineOffsetMs;
+ const lastActivityRawMs =
+ child.highlights.length === 0
+ ? Math.min(closedRawMs, lastFootageMs) - options.video.frameDurationMs
+ : Math.max(
+ ...child.highlights.map((highlight) => highlight.actionEnd || highlight.start),
+ ) + options.timelineOffsetMs;
+ const returnedRawMs = Math.min(
+ Infinity,
+ ...options.openerHighlights
+ .map((highlight) => highlight.start)
+ .filter((start) => start >= lastActivityRawMs),
+ );
+ const closeRawMs = Math.min(lastFootageMs, closedRawMs, returnedRawMs);
- if (closeMs <= enableFromMs) continue;
+ if (closeRawMs <= openRawMs) continue;
- layers.push({
+ placements.push({
child,
- closeMs,
+ closeRawMs,
delayMs: childOffsetMs + options.timelineOffsetMs,
- enableFromMs,
- enableToMs,
+ exits: closeRawMs < lastFootageMs,
height,
+ openRawMs,
path,
rawInfo,
width,
@@ -4673,30 +4795,91 @@ const childCompositeLayers = async (options: {
});
}
- return layers;
+ // One freeze per exit instant: popups leaving together animate together.
+ const freezes = [
+ ...new Set(placements.filter((placement) => placement.exits).map(({ closeRawMs }) => closeRawMs)),
+ ]
+ .sort((left, right) => left - right)
+ .map((atMs): VideoModeFreeze => ({ atMs, durationMs: options.exitFreezeMs }));
+
+ return {
+ freezes,
+ openerHolds: placements.map(({ openRawMs }) => ({
+ end: openRawMs + CHILD_BIRTH_OPENER_HOLD_MS,
+ start: openRawMs - options.video.frameDurationMs,
+ })),
+ layers: placements.map(({ closeRawMs, openRawMs, ...layer }) => {
+ const closeMs = compositeEndMs(freezes, closeRawMs);
+ return {
+ ...layer,
+ closeMs,
+ enableFromMs: compositeTimeMs(freezes, openRawMs),
+ enableToMs: layer.exits ? closeMs + CHILD_OVERLAY_FADE_MS : closeMs,
+ };
+ }),
+ };
};
/**
* Pass A of the popup composite: overlay each popup screencast onto the
* parent's raw footage — dimmed backdrop, scaled to fit, alpha-faded in and
* out, windowed to the popup's open/close span, newest stacked on top. The
- * output shares the parent raw timeline exactly, so the annotation render
- * (pass B) runs on it unchanged; holds there freeze the composite, so it
- * never matters which source triggered them.
+ * output is the parent raw timeline plus the exit freezes (the composite
+ * clock, see `compositeTimeMs`); the annotation render (pass B) runs on it
+ * unchanged, and holds there freeze the composite, so it never matters which
+ * source triggered them.
*/
const compositeChildOverlays = async (options: {
/** Continuous frame rate for the overlay chains (see fps note below). */
fps: number;
+ freezes: VideoModeFreeze[];
inputPath: string;
layers: VideoModeChildLayer[];
+ /** How far the parent's footage is held back (ms); its first frame fills the gap. */
+ leadInMs: number;
+ openerHolds: VideoModeSpan[];
outputPath: string;
}) => {
const filters: string[] = [];
+ // Every source freezes together: frames from a freeze's instant on move
+ // later by its duration, and the fps resample below fills the hole with the
+ // last frame before it.
+ const freeze =
+ options.freezes.length === 0
+ ? []
+ : [
+ `setpts='PTS+(${options.freezes
+ .map(
+ ({ atMs, durationMs }) =>
+ `gte(T\\,${formatSeconds(atMs)})*${formatSeconds(durationMs)}`,
+ )
+ .join("+")})/TB'`,
+ ];
+ const openerHold =
+ options.openerHolds.length === 0
+ ? []
+ : [
+ `select='not(${options.openerHolds
+ .map(
+ ({ end, start }) =>
+ `between(t\\,${formatSeconds(Math.max(0, start))}\\,${formatSeconds(end)})`,
+ )
+ .join("+")})'`,
+ ];
// The parent screencast is as sparse as the child ones (a static page emits
// no frames), and overlay only emits output at primary-input frame times —
// without resampling, the whole popup window can contain zero composite
- // frames. A continuous base gives every enable window frames to land on.
- filters.push(`[0:v]fps=${formatFilterNumber(options.fps)}[base]`);
+ // frames. A continuous base gives every enable window frames to land on —
+ // from t=0, so its first frame stands in for the lead-in and for frames an
+ // opener hold dropped at the start.
+ filters.push(
+ `[0:v]${[
+ `setpts=PTS+${formatSeconds(options.leadInMs)}/TB`,
+ ...openerHold,
+ ...freeze,
+ `fps=${formatFilterNumber(options.fps)}:start_time=0`,
+ ].join(",")}[base]`,
+ );
let currentLabel = "base";
options.layers.forEach((layer, index) => {
@@ -4713,7 +4896,13 @@ const compositeChildOverlays = async (options: {
);
filters.push(
[
- `[${index + 1}:v]setpts=PTS+${formatSeconds(layer.delayMs)}/TB`,
+ // A popup that opened before the opener's first recorded frame has
+ // footage from before composite time 0. It can't be shown, and left
+ // in it blanks the whole overlay (a stream starting at negative time
+ // throws off the fps/fade chain below).
+ `[${index + 1}:v]trim=start=${formatSeconds(Math.max(0, -layer.delayMs))}`,
+ `setpts=PTS+${formatSeconds(layer.delayMs)}/TB`,
+ ...freeze,
// A mostly-static popup screencast has sparse frames; without
// resampling, the frame that happens to pass `fade` mid-ramp keeps
// its partial alpha while framesync repeats it for the whole window,
@@ -4795,7 +4984,10 @@ const projectChildHighlight = (options: {
// nudge the highlight a few ms earlier; order-preserving in practice.
const closeFloorMs = Math.floor(layer.closeMs);
const minSliceMs = options.video.frameDurationMs + 10;
- let { actionEnd, end, start } = highlight;
+ // A highlight with no action of its own (waitFor) still needs a source
+ // slice of composite frames to hold on, kept clear of close the same way.
+ let { end, start } = highlight;
+ let actionEnd = highlight.actionEnd === undefined ? start + minSliceMs : highlight.actionEnd;
// Entry pacing keeps actions clear of the enter animation at runtime; this
// is the degraded-mode backstop (paused pacing, custom flows): a highlight
// starting mid-slide defers past the animation so its hold can't freeze a
@@ -4808,20 +5000,16 @@ const projectChildHighlight = (options: {
);
start += deferMs;
end += deferMs;
- if (actionEnd !== undefined) {
- actionEnd += deferMs;
- }
+ actionEnd += deferMs;
}
- if (actionEnd !== undefined) {
- actionEnd = Math.min(actionEnd, closeFloorMs);
- if (actionEnd - start < minSliceMs) {
- actionEnd = Math.min(closeFloorMs, start + minSliceMs);
- }
- if (actionEnd - start < minSliceMs) {
- const shift = minSliceMs - (actionEnd - start);
- start -= shift;
- end -= shift;
- }
+ actionEnd = Math.min(actionEnd, closeFloorMs);
+ if (actionEnd - start < minSliceMs) {
+ actionEnd = Math.min(closeFloorMs, start + minSliceMs);
+ }
+ if (actionEnd - start < minSliceMs) {
+ const shift = minSliceMs - (actionEnd - start);
+ start -= shift;
+ end -= shift;
}
return {
@@ -5194,11 +5382,27 @@ export const videoMode = (options: VideoModeOptions = {}): VideoModePlugin => {
const offAfterTestFinalize = emitter.on(
"afterTestFinalize",
async ({ page, testInfo }) => {
+ if (state.endedAt === undefined) {
+ state.endedAt = getVideoTimestamp();
+ }
+ // A popup still open here outlived the test: closedAt stays unset
+ // and the close below is teardown, not something to render.
+ popupPage.off("close", onClose);
child.viewport = child.viewport || page.viewportSize() || undefined;
const video = page.video();
if (!page.isClosed()) {
let paintedAt: number | undefined;
if (video) {
+ const sinceLastHighlightMs =
+ getVideoTimestamp() -
+ Math.max(
+ ...child.highlights.map((highlight) => highlight.actionEnd || highlight.start),
+ );
+ if (sinceLastHighlightMs < CHILD_FINAL_STATE_HOLD_MS) {
+ await new Promise((resolve) =>
+ setTimeout(resolve, CHILD_FINAL_STATE_HOLD_MS - sinceLastHighlightMs),
+ );
+ }
paintedAt = await settleVideoRecorder(page);
}
const closeStartedAt = performance.now();
@@ -5213,7 +5417,6 @@ export const videoMode = (options: VideoModeOptions = {}): VideoModePlugin => {
);
}
}
- onClose();
if (video) {
const raw = suffixArtifactFileName(
VIDEO_MODE_RAW_FILE,
@@ -5432,7 +5635,7 @@ export const videoMode = (options: VideoModeOptions = {}): VideoModePlugin => {
const offAfterTestFinalize = emitter.on("afterTestFinalize", async ({ page, testInfo }) => {
await Promise.all(pendingDialogHighlights);
- const renderEndedAt = getVideoTimestamp();
+ const renderEndedAt = state.endedAt === undefined ? getVideoTimestamp() : state.endedAt;
const metadataBeforeVideo = metadataFor(state);
const addressBars = metadataBeforeVideo.addressBars;
const captions = metadataBeforeVideo.captions;
@@ -5490,6 +5693,7 @@ export const videoMode = (options: VideoModeOptions = {}): VideoModePlugin => {
let recordingEndedAt: number | undefined;
let coverPaintedAt: number | undefined;
+ const viewportAtEnd = page.viewportSize() || undefined;
if (!page.isClosed()) {
const needsTimelineCalibration =
addressBars.length > 0 ||
@@ -5525,15 +5729,18 @@ export const videoMode = (options: VideoModeOptions = {}): VideoModePlugin => {
const rawVideoInfo = await videoInfo(paths.raw);
// The wall→raw calibration marker is the cover itself: videoMode
// stamped when it painted it, and its first raw frame is findable by
- // color. The recorder ENDPOINT is only the fallback — Playwright
- // extends the final frame ~1s past the close instant
- // (version-dependent), so endpoint arithmetic skews every
- // translation by that much and lets the cover leak into the render
- // as a black flash.
+ // color. The recorder ENDPOINT can't carry the calibration alone —
+ // Playwright can extend the final frame ~1s past the close instant,
+ // which skews every translation by that much and lets the cover
+ // leak into the render as a black flash.
const detectedCoverStartMs =
coverPaintedAt === undefined
? undefined
- : await detectCalibrationCoverStartMs(paths.raw);
+ : await detectCalibrationCoverStartMs({
+ inputPath: paths.raw,
+ video: rawVideoInfo,
+ viewport: viewportAtEnd,
+ });
// A cover run reaching (within the cover margin of) t=0 means the
// recording holds no pre-cover footage: an instant test whose only
// captured frames ARE the cover. Nothing to calibrate against or
@@ -5545,92 +5752,148 @@ export const videoMode = (options: VideoModeOptions = {}): VideoModePlugin => {
detectedCoverStartMs > calibrationCoverMarginMs(rawVideoInfo.frameDurationMs)
? detectedCoverStartMs
: undefined;
- // Clamped at 0: the raw's first frame can lag videoMode's clock
- // zero slightly, making the true offset a hair negative — but range
- // starts clamp at footage 0 anyway, so an unclamped negative offset
- // would shift annotations relative to the range instead of with it.
- const sourceOffset =
+ // Two readings of the raw-minus-clock offset, and two ways to be
+ // wrong. The endpoint reading lands about a second LATE whenever the
+ // page was still producing frames at close (Playwright pads the last
+ // frame it received by >=1s); late is the harmful direction — an
+ // action's result plays before its own highlight — so it may only
+ // pull the offset earlier. The cover reading is the sharper one, but
+ // a negative one (the recorder's first frame landed after
+ // videoMode's clock zero: ordinary, ~100ms) moves the whole render
+ // off the test's clock, so it needs the endpoint to agree before it
+ // goes below zero.
+ const endpointOffset =
+ recordingEndedAt === undefined ? 0 : rawVideoInfo.durationMs - recordingEndedAt;
+ const coverOffset =
coverStartMs !== undefined && coverPaintedAt !== undefined
- ? Math.max(0, coverStartMs - coverPaintedAt)
- : recordingEndedAt === undefined
- ? 0
- : rawVideoInfo.durationMs - recordingEndedAt;
+ ? coverStartMs - coverPaintedAt
+ : undefined;
+ const sourceOffset =
+ coverOffset === undefined
+ ? endpointOffset
+ : Math.min(Math.max(0, coverOffset), endpointOffset);
const timelineOffset =
Math.floor(sourceOffset / rawVideoInfo.frameDurationMs) *
rawVideoInfo.frameDurationMs;
- const renderTimeline = translateVideoTimeline({
+ // What that leaves uncorrected is footage running ahead of its
+ // annotations by the part of a negative cover reading the endpoint
+ // didn't confirm. Under a popup that shows: the opener's state from
+ // a moment later. The composite re-times the footage anyway, so it
+ // holds the opener's back by the difference — its first frame fills
+ // the gap, and every raw time below shifts with it.
+ const footageLeadInMs =
+ coverOffset === undefined || !metadataBeforeVideo.children.some((child) => child.raw)
+ ? 0
+ : timelineOffset -
+ Math.floor(
+ Math.min(coverOffset, endpointOffset) / rawVideoInfo.frameDurationMs,
+ ) *
+ rawVideoInfo.frameDurationMs;
+ const footageInfo = {
+ ...rawVideoInfo,
+ durationMs: rawVideoInfo.durationMs + footageLeadInMs,
+ };
+ const rawTimeline = translateVideoTimeline({
addressBars,
captions,
deadAir,
highlights,
offsetMs: timelineOffset,
});
+ // Where the footage worth rendering ends: the end of the test,
+ // capped before the cover. The cover is never wanted footage — this
+ // cap is what stops the tail cover playing as a black flash. One
+ // frame of margin absorbs the detector's sampling granularity (its
+ // timestamp can land a tick late).
+ const endpointEnd = Math.round(renderEndedAt + sourceOffset);
+ const coverCap =
+ coverStartMs === undefined
+ ? undefined
+ : Math.max(
+ 0,
+ coverStartMs +
+ footageLeadInMs -
+ calibrationCoverMarginMs(rawVideoInfo.frameDurationMs),
+ );
+ const footageEndMs =
+ recordingEndedAt === undefined
+ ? footageInfo.durationMs
+ : coverCap === undefined
+ ? endpointEnd
+ : Math.min(endpointEnd, coverCap);
// Popup composite (pass A): overlay each popup's screencast onto the
// raw footage, then annotate that composite instead of the raw. The
- // composite shares the raw timeline, so nothing downstream changes.
+ // composite runs on the raw timeline plus the lead-in and a freeze
+ // per popup exit, so everything downstream moves onto that clock
+ // here and nowhere else.
let renderInputPath = paths.raw;
// Popup enter/exit animations must reach the output even when a
// hold's overlap-skip would jump across them.
const renderKeepSpans: VideoModeSpan[] = [];
- const childLayers = await childCompositeLayers({
+ // An exit freeze lasts the animation plus the fill stabilization
+ // window: the opener action it hands over to can open on a
+ // full-frame pre-action still (see preActionStabilizationMs), which
+ // must replace settled frames, not the tail of the slide. Whole
+ // frames, so footage after the freeze stays on its frame grid.
+ const exitFreezeMs =
+ Math.ceil(
+ (CHILD_OVERLAY_FADE_MS + Math.max(0, timelineOffset)) / rawVideoInfo.frameDurationMs +
+ VIDEO_MODE_FILL_PRE_ACTION_FRAME_PADDING +
+ 1,
+ ) * rawVideoInfo.frameDurationMs;
+ const {
+ freezes,
+ layers: childLayers,
+ openerHolds,
+ } = await childCompositeLayers({
children: metadataBeforeVideo.children,
+ exitFreezeMs,
+ footageEndMs,
+ openerHighlights: rawTimeline.highlights,
outputDir: testInfo.outputDir,
timelineOffsetMs: timelineOffset,
- video: rawVideoInfo,
+ video: footageInfo,
});
- if (childLayers.length > 0) {
+ const renderTimeline = compositeVideoTimeline(rawTimeline, freezes);
+ if (childLayers.length > 0 || footageLeadInMs > 0) {
const compositePath = join(
testInfo.outputDir,
suffixArtifactFileName(VIDEO_MODE_COMPOSITE_FILE, state.artifactSuffix),
);
await compositeChildOverlays({
fps: 1000 / rawVideoInfo.frameDurationMs,
+ freezes,
inputPath: paths.raw,
layers: childLayers,
+ leadInMs: footageLeadInMs,
+ openerHolds,
outputPath: compositePath,
});
renderInputPath = compositePath;
for (const layer of childLayers) {
- renderKeepSpans.push(
- { end: layer.enableFromMs + CHILD_OVERLAY_FADE_MS, start: layer.enableFromMs },
- { end: layer.enableToMs, start: layer.closeMs },
- );
+ renderKeepSpans.push({
+ end: layer.enableFromMs + CHILD_OVERLAY_FADE_MS,
+ start: layer.enableFromMs,
+ });
+ if (layer.exits) {
+ renderKeepSpans.push({ end: layer.closeMs + exitFreezeMs, start: layer.closeMs });
+ }
}
// Dead air recorded during an animation window (e.g. a child
// waitFor spanning the slide) must not compress it away.
renderTimeline.deadAir = subtractVideoSpans(renderTimeline.deadAir, renderKeepSpans);
- // A parent action right after a popup closes (waiting for the
- // signed-in state, say) would hold a freeze frame from inside the
- // overlay's exit animation — a ghost popup flashing back after it
- // disappeared. Shift such highlights past the fade window.
- for (const parentHighlight of renderTimeline.highlights) {
- for (const layer of childLayers) {
- const fadeEndMs = layer.enableToMs + rawVideoInfo.frameDurationMs;
- if (
- parentHighlight.start >= layer.closeMs - rawVideoInfo.frameDurationMs &&
- parentHighlight.start < fadeEndMs
- ) {
- const shift = fadeEndMs - parentHighlight.start;
- parentHighlight.start += shift;
- parentHighlight.end += shift;
- if (parentHighlight.actionEnd !== undefined) {
- parentHighlight.actionEnd += shift;
- }
- if (parentHighlight.sourceFrameAt !== undefined) {
- parentHighlight.sourceFrameAt += shift;
- }
- }
- }
- }
const projectedChildHighlights = childLayers.flatMap((layer) =>
- translateVideoTimeline({
- addressBars: [],
- captions: [],
- deadAir: [],
- highlights: layer.child.highlights,
- offsetMs: timelineOffset,
- }).highlights.map((highlight) =>
+ compositeVideoTimeline(
+ translateVideoTimeline({
+ addressBars: [],
+ captions: [],
+ deadAir: [],
+ highlights: layer.child.highlights,
+ offsetMs: timelineOffset,
+ }),
+ freezes,
+ ).highlights.map((highlight) =>
projectChildHighlight({ highlight, layer, video: rawVideoInfo }),
),
);
@@ -5697,14 +5960,18 @@ export const videoMode = (options: VideoModeOptions = {}): VideoModePlugin => {
detectedStart >= TRIM_START_MIN_LEAD_IN_MS &&
annotationSourceRange.start === undefined
) {
- state.sourceRange.start = detectedStart;
- sourceRange.start = detectedStart;
+ state.sourceRange.start = detectedStart + footageLeadInMs;
+ sourceRange.start = detectedStart + footageLeadInMs;
}
}
if (sourceRange.start === undefined) {
sourceRange.start = Math.max(0, timelineOffset);
}
+ sourceRange.start = compositeTimeMs(freezes, sourceRange.start);
+ if (sourceRange.end !== undefined) {
+ sourceRange.end = compositeEndMs(freezes, sourceRange.end);
+ }
if (sourceRange.end === undefined && recordingEndedAt !== undefined) {
// Each hold's freeze frame (sourceFrameAt) must be inside the
@@ -5726,25 +5993,18 @@ export const videoMode = (options: VideoModeOptions = {}): VideoModePlugin => {
Math.max(end, candidate.start + rawVideoInfo.frameDurationMs),
0,
);
- // The cover is never wanted footage, so it caps the endpoint term
- // — this is what stops the tail cover playing as a black flash.
- // One frame of margin absorbs the detector's sampling granularity
- // (its timestamp can land a tick late). The annotation minimums
- // still win when they need more: their holds freeze exactly the
- // frames they pull in, cover included on near-instant recordings
- // that never captured anything else.
- const endpointEnd = Math.round(renderEndedAt + sourceOffset);
- const coverCap =
- coverStartMs === undefined
- ? undefined
- : Math.max(
- 0,
- coverStartMs - calibrationCoverMarginMs(rawVideoInfo.frameDurationMs),
- );
+ // The annotation minimums win over the footage end when they need
+ // more: their holds freeze exactly the frames they pull in, cover
+ // included on near-instant recordings that never captured
+ // anything else. A popup still up at the end stays up to the last
+ // frame: the render stops where that popup's footage does.
sourceRange.end = Math.max(
minimumAddressBarEnd,
minimumHighlightEnd,
- coverCap === undefined ? endpointEnd : Math.min(endpointEnd, coverCap),
+ Math.min(
+ compositeEndMs(freezes, footageEndMs),
+ ...childLayers.filter((layer) => !layer.exits).map((layer) => layer.closeMs),
+ ),
);
}
@@ -5837,6 +6097,7 @@ export const videoMode = (options: VideoModeOptions = {}): VideoModePlugin => {
}
state.startedAt = undefined;
+ state.endedAt = undefined;
console.log(
`video-mode metadata written to ${videoModeOutputPaths(testInfo, state.artifactSuffix).metadata}`,
);
diff --git a/tasks/complete/2026-10-05-video-mode-popup-left-open.md b/tasks/complete/2026-10-05-video-mode-popup-left-open.md
new file mode 100644
index 0000000..b40d719
--- /dev/null
+++ b/tasks/complete/2026-10-05-video-mode-popup-left-open.md
@@ -0,0 +1,46 @@
+---
+status: done
+size: medium
+---
+
+# videoMode renders a popup that is left open wrongly
+
+**Status summary**: done. All three symptoms came from the timeline calibration failing on a padded screencast, plus the popup having no notion of "the test went back to the opener". Fixed: cover detection ignores the recorder's padding, a late endpoint reading can no longer skew the timeline, the popup composite holds the opener's footage back to where it belongs, a left-open popup slides away when the test returns to its opener (on a short freeze of the composite), and teardown no longer counts as test time. Nothing known missing; follow-ups listed at the bottom.
+
+Seen recording a sign-in spec in another repo. The main page clicks a button inside an iframe, a popup opens, the test acts in the popup, then `page.reload()`s the main page and carries on there. The popup is never closed. Viewport 1280x900.
+
+In `video-rendered.webm`:
+
+- while the popup is shown, the background behind it is the main page's final frame (already signed in), not the main page as it was at that moment
+- the popup overlay goes solid black for about a second before it closes
+- the video ends on a black frame
+
+Both raw recordings are fine on their own, so it is the composite step.
+
+## Checklist
+
+- [x] failing spec that mirrors the flow (popup left open, test goes back to the opener, 1280x900) _`spec/popup-video-left-open.spec.ts`: asserts the render's storyboard; a second test covers a popup still open at test end. Both fail on main with `"black"` scenes._
+- [x] find out why the popup lands late on the main timeline _Playwright pads a 1280x900 screencast with a 2px gray strip (800x562 video). `detectCalibrationCoverStartMs` wanted every pixel to be cover, never matched, and calibration fell back to endpoint arithmetic: ~0.9s late here. Detection now crops to the screencast's own area._
+- [x] find out why the popup's teardown footage (the calibration cover) reaches the render _Same failed detection (nothing capped the overlay before the popup's cover), plus `closedAt` and the render end being stamped after the popup's 1.1s recorder settle. The test end is now stamped by the first finalizer (`state.endedAt`); a popup closed by teardown keeps `closedAt` unset._
+- [x] decide what the overlay does when the test goes back to the opener with the popup still open _It exits: slide-out at the first opener highlight after the popup's last one. The footage has no spare time for the animation (the opener action follows within ms), so the composite splices in a freeze (`VideoModeFreeze`) and everything downstream moves onto the composite clock. Self-closing popups exit the same way; the old "shift opener highlights past the fade" loop is gone._
+- [x] todo-app baseline video still paces the same _20.5s before and after._
+
+## Implementation notes
+
+Log of what turned up along the way, not instructions.
+
+- With detection working, the cover reading came out around -100ms, which the old code clamped to 0. It is real: the recorder's first frame lands ~100ms after videoMode's clock zero (more on CI), so the opener's footage ran 100ms ahead of the popup and of every annotation.
+- First attempt: drop the clamp and move annotations earlier. Correct footage, but the render then starts that far into the test's clock, and the frame-exact specs sample rendered frames at clock-derived times (`renderedHighlightStartWithoutDeadAir`). On CI, where instant specs get ~80ms of footage before the cover, two failed outright. Second attempt: keep annotations on the clock and pad the footage's start instead. The filler frame is whatever the recorder caught first, which under load is a pan capture flash.
+- Landed: the main timeline keeps its old rule (cover reading clamped at 0), except that the endpoint reading may pull it earlier and can no longer push it later. That is what both kinds of spec ran on already: unpadded viewports on the clamped cover reading, padded ones (800x600) on the endpoint fallback. The popup composite, which re-times footage anyway, holds the opener's footage back by the part of the cover reading the main timeline ignored.
+- When the main timeline does go negative, a popup that opened before the first recorded frame would land at composite time 0, on top of everything before it. A popup never starts on the first frame, and its footage from before time 0 is trimmed (left in, ffmpeg dropped the whole overlay).
+- Chrome paints "Debugger paused in another tab" over the opener while Playwright holds a newborn popup paused. It sometimes reaches the screencast; when it was the first frame, pre-footage annotations held it. The composite now drops the opener's frames around a popup's birth.
+- A popup's last highlight can be within a frame of teardown. Popup holds freeze composite footage (no screenshot stand-in), so the popup recorder now waits until that state has been on screen 150ms before covering it, and every projected popup highlight gets a source slice inside the overlay window.
+- `spec/video-mode-ffmpeg.spec.ts` "returns from an offscreen blur pan…" fails now and then under heavy parallel load (once in a 3x full-suite run), on main too: the recording captured nothing but the cover.
+
+## Follow-ups (not done)
+
+- The popup backdrop desaturates the page as well as dimming it (`drawbox` blends chroma once per luma pixel in yuv420).
+- A popup still open at the end is cut away at the final hold: the final frame is a screenshot of the opener alone.
+- Step captions blank for the length of an exit freeze when one step ends before it and the next starts after.
+- Popup recordings are still calibrated from the cover alone; the opener's two-reading rule isn't applied to them.
+- Ordinary (no popup) videos still run their footage ahead of the annotations when the recorder starts late and the endpoint reading can't confirm it. Fixing that means the specs need the footage start from metadata instead of assuming clock zero.