From 4e197bd687e79d24edf7ad07de2dedcbbd2a3268 Mon Sep 17 00:00:00 2001 From: meetsu <96637888+klNuno@users.noreply.github.com> Date: Mon, 19 Oct 2026 00:20:58 +0200 Subject: [PATCH 01/13] ci: loop flaky tests on their runners with timing traces --- .github/workflows/flake-loop.yml | 127 +++++++++++++++++++++++++++++ docs/ci.md | 12 +++ packages/core/src/browser/cdp.ts | 47 ++++++++++- packages/core/src/client.ts | 7 +- packages/core/test/waits.test.ts | 2 + scripts/ci/flake-loop.test.ts | 45 +++++++++++ scripts/ci/flake-loop.ts | 135 +++++++++++++++++++++++++++++++ tests/e2e/lib/cdp.ts | 48 +++++++++-- tests/e2e/lib/ui.ts | 36 ++++++++- 9 files changed, 449 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/flake-loop.yml create mode 100644 scripts/ci/flake-loop.test.ts create mode 100644 scripts/ci/flake-loop.ts diff --git a/.github/workflows/flake-loop.yml b/.github/workflows/flake-loop.yml new file mode 100644 index 000000000..7ae16bca1 --- /dev/null +++ b/.github/workflows/flake-loop.yml @@ -0,0 +1,127 @@ +name: flake-loop + +# Runs flaky tests many times on the runners where they fail, with timing +# traces, to find a cause and to prove a fix (docs/ci.md#checks). A pull +# request runs it once it carries the `flake-loop` label. +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + workflow_dispatch: + inputs: + runs: + description: Runs per job. + type: number + default: 10 + +permissions: + contents: read + +concurrency: + group: flake-loop-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + RUNS: ${{ inputs.runs || 10 }} + +jobs: + browser: + name: Core browser files (${{ matrix.name }}) + if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'flake-loop') + strategy: + fail-fast: false + matrix: + include: + - { os: macos-15, name: macOS arm64 } + - { os: macos-15-intel, name: macOS x64 } + - { os: ubuntu-24.04, name: Linux x64 } + runs-on: ${{ matrix.os }} + timeout-minutes: 75 + env: + BOITE_CDP_TRACE: '1' + FLAKE_LOOP_TITLE: Core browser files, ${{ matrix.name }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: ./.github/actions/setup + - run: bun scripts/ci/flake-loop.ts --runs "$RUNS" --parallel 4 --cwd packages/core --out flake-loop test/browser-sharing.test.ts test/browser.test.ts test/browser-sleep.test.ts test/browser-profile-lifecycle.test.ts + - uses: ./.github/actions/upload-artifact + if: always() + with: + name: flake-loop-browser-${{ matrix.os }} + path: flake-loop/ + + cli: + name: cli.test.ts (${{ matrix.name }}) + if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'flake-loop') + strategy: + fail-fast: false + matrix: + include: + - { os: ubuntu-24.04, name: Linux x64 } + - { os: macos-15, name: macOS arm64 } + - { os: windows-2025, name: Windows x64 } + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + env: + FLAKE_LOOP_TITLE: cli.test.ts, ${{ matrix.name }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: ./.github/actions/setup + # Four copies at once load the runner as the core suite's four workers do. + - shell: bash + run: bun scripts/ci/flake-loop.ts --runs "$((RUNS * 2))" --parallel 4 --cwd packages/core --out flake-loop test/cli.test.ts test/cli.test.ts test/cli.test.ts test/cli.test.ts + - uses: ./.github/actions/upload-artifact + if: always() + with: + name: flake-loop-cli-${{ matrix.os }} + path: flake-loop/ + + e2e: + name: E2E coordination bridge (${{ matrix.mode }}) + if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'flake-loop') + strategy: + fail-fast: false + matrix: + mode: [alone, shard 3] + runs-on: windows-2025 + timeout-minutes: 90 + env: + BOITE_E2E_PREBUILT_UI: '1' + BOITE_E2E_FAKE_UI: tests/e2e/.artifacts/fake-ui + BOITE_E2E_TRACE: '1' + FLAKE_LOOP_TITLE: E2E coordination bridge, ${{ matrix.mode }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: ./.github/actions/setup + - run: bun run build:ui + - run: bun tests/e2e/lib/warm.ts --build-only $env:BOITE_E2E_FAKE_UI + # Same temp drive as the E2E job in ci.yml. + - name: Prepare end-to-end tests + env: + TEMP: ${{ runner.temp }} + TMP: ${{ runner.temp }} + run: bun tests/e2e/lib/warm.ts --prebuilt $env:BOITE_E2E_FAKE_UI + - name: Loop + env: + TEMP: ${{ runner.temp }} + TMP: ${{ runner.temp }} + MODE: ${{ matrix.mode }} + run: | + if ($env:MODE -eq 'alone') { + $files = @('tests/e2e/coordination-bridge-live.test.ts'); $runs = [int]$env:RUNS * 3 + } else { + $files = @(bun scripts/ci/e2e-shards.ts 3 3); $runs = [math]::Max(1, [int]([int]$env:RUNS / 4)) + } + if ($LASTEXITCODE -ne 0 -or $files.Count -eq 0) { throw 'No E2E files to loop' } + $together = @(if ($env:MODE -eq 'alone') { } else { '--together' }) + bun scripts/ci/flake-loop.ts --runs $runs --out flake-loop @together @files + - uses: ./.github/actions/upload-artifact + if: always() + with: + name: flake-loop-e2e-${{ matrix.mode == 'alone' && 'alone' || 'shard3' }} + path: flake-loop/ diff --git a/docs/ci.md b/docs/ci.md index d9b742093..310b2856c 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -112,6 +112,18 @@ crash, a timed-out file or any other failing test fails at once. The list started on 2026-10-18 from the 80 runs that passed on rerun since 2026-10-16: tests that failed at least three first attempts. A fixed test leaves the list; `flaky.test.ts` fails when an entry no longer exists in its file. +The `flake-loop` workflow measures a flaky test before and after its fix. It +runs on a pull request that carries the `flake-loop` label, or by hand: the +quarantined browser files four at a time on macOS arm64, macOS Intel and +Linux, `cli.test.ts` four copies at a time on every OS, and +`coordination-bridge-live.test.ts` alone and within E2E shard 3 on Windows. +`scripts/ci/flake-loop.ts` counts each test's passes and failures over the +runs, prints the error of each failure and writes a table to the job summary. +`BOITE_CDP_TRACE=1` makes a core DevTools timeout say how many bytes of the +browser's answers sat unread, the load and whether `Browser.getVersion` still +answered; `BOITE_E2E_TRACE=1` makes an E2E timeout or lost browser say how the +devtools socket closed, whether Chrome still ran, the test process's event +loop delays and what the UI server still owed. `scripts/ci/hdiutil-retry.ts` bundles the macOS application again, three attempts at most, when hdiutil reports `Resource busy`. That runner fault (actions/runner-images#7522) failed the Apple Silicon DMG of nightly diff --git a/packages/core/src/browser/cdp.ts b/packages/core/src/browser/cdp.ts index c955eadd6..07a463f99 100644 --- a/packages/core/src/browser/cdp.ts +++ b/packages/core/src/browser/cdp.ts @@ -10,12 +10,19 @@ */ import { fstatSync } from 'node:fs'; +import { loadavg } from 'node:os'; type Listener = (params: Record, sessionId: string | undefined) => void; interface Waiting { resolve(value: Record): void; reject(error: Error): void; timer: ReturnType; method: string } /** What a command may take before the page is called stuck. */ const COMMAND_TIMEOUT_MS = 30_000; +/** + * `BOITE_CDP_TRACE=1` (CI flake loops): a timed-out command also says how many + * bytes of the browser's answers sit unread on the replies descriptor, the + * machine's load, and whether the browser still answers `Browser.getVersion`. + */ +const TRACE = process.env.BOITE_CDP_TRACE === '1'; export class CdpError extends Error {} @@ -26,6 +33,10 @@ export class Cdp { #waiting = new Map(); #listeners = new Map>(); #closed: Error | null = null; + /** When the last message from the browser arrived, for a timeout to say how long it has been quiet. */ + #heardAt = performance.now(); + /** Bytes waiting unread on the replies descriptor, under `BOITE_CDP_TRACE` and the pipe only. */ + #unread: (() => number | null) | null = null; readonly closed: Promise; private constructor(write: (text: string) => void, stop: () => void) { @@ -88,6 +99,7 @@ export class Cdp { writer = await connect({ fd: commands, socket: { data() {}, drain: flush, close: ended, error: ended } }); } catch (error) { reader.end(); throw error; } flush(writer); + if (TRACE) cdp.#unread = await unreadBytes(replies); return cdp; } @@ -123,7 +135,10 @@ export class Cdp { return new Promise((resolve, reject) => { const timer = setTimeout(() => { this.#waiting.delete(id); - reject(new CdpError(`the browser did not answer ${method} within ${Math.round(timeoutMs / 1000)} seconds`)); + const quiet = `the last message from the browser came ${Math.round(performance.now() - this.#heardAt)} ms ago`; + const error = (detail: string) => new CdpError(`the browser did not answer ${method} within ${Math.round(timeoutMs / 1000)} seconds; ${quiet}${detail}`); + if (!TRACE || method === 'Browser.getVersion') { reject(error('')); return; } + void this.#trace().then(detail => reject(error(detail))); }, timeoutMs); this.#waiting.set(id, { resolve: resolve as (value: Record) => void, reject, timer, method }); this.#write(JSON.stringify({ id, method, params, ...(sessionId ? { sessionId } : {}) })); @@ -143,7 +158,19 @@ export class Cdp { this.#stop(); } + /** What `BOITE_CDP_TRACE` adds to a timeout: unread bytes, load, and a browser-level probe. */ + async #trace(): Promise { + const unread = this.#unread?.(); + const started = performance.now(); + const probe = await this.send('Browser.getVersion', {}, undefined, 5_000) + .then(() => `answered in ${Math.round(performance.now() - started)} ms`, (error: Error) => error.message.split(';')[0]); + const after = this.#unread?.(); + const bytes = unread === undefined || unread === null ? 'unread bytes unknown' : `${unread} bytes unread before the probe, ${after} after`; + return `; trace: ${bytes}, load ${loadavg().map(n => n.toFixed(1)).join(' ')}, Browser.getVersion ${probe}`; + } + #receive(text: string): void { + this.#heardAt = performance.now(); let message: { id?: number; method?: string; params?: Record; result?: Record; error?: { message?: string }; sessionId?: string }; try { message = JSON.parse(text); } catch { return; } if (typeof message.id === 'number') { @@ -168,6 +195,24 @@ export class Cdp { } } +/** + * Peeks the bytes a socket holds without reading them, `recv(MSG_PEEK | + * MSG_DONTWAIT)` through libc. Bytes there while a command times out mean the + * browser answered and nobody read it; none mean the browser stayed silent. + */ +async function unreadBytes(fd: number): Promise<(() => number | null) | null> { + if (process.platform !== 'darwin' && process.platform !== 'linux') return null; + try { + const { dlopen, FFIType, ptr } = await import('bun:ffi'); + const libc = dlopen(process.platform === 'darwin' ? '/usr/lib/libSystem.B.dylib' : 'libc.so.6', { + recv: { args: [FFIType.i32, FFIType.ptr, FFIType.u64, FFIType.i32], returns: FFIType.i64 }, + }); + const buffer = new Uint8Array(1 << 20); + const flags = 0x2 | (process.platform === 'darwin' ? 0x80 : 0x40); + return () => { const n = Number(libc.symbols.recv(fd, ptr(buffer), buffer.length, flags)); return n < 0 ? 0 : n; }; + } catch { return null; } +} + function concat(parts: Uint8Array[], size: number): Uint8Array { const whole = new Uint8Array(size); let at = 0; diff --git a/packages/core/src/client.ts b/packages/core/src/client.ts index 8b2567b58..310c0696a 100644 --- a/packages/core/src/client.ts +++ b/packages/core/src/client.ts @@ -91,6 +91,8 @@ export async function connect(url: string, token: string, options: ConnectOption const listeners = new Map void>>(); let nextId = 1; let closed = false; + /** Why the socket closed, so a later call names the code instead of only saying it is closed. */ + let closedBy: SocketClosedError | null = null; const send = (method: string, params: unknown, waitMs = options.requestTimeoutMs ?? 120_000): Promise => { const id = nextId; @@ -143,6 +145,7 @@ export async function connect(url: string, token: string, options: ConnectOption socket.addEventListener('close', (event: CloseEvent) => { closed = true; const error = new SocketClosedError(event.code, event.reason); + if (!ownerClosed) closedBy = error; for (const waiter of pending.values()) waiter.reject(error); pending.clear(); for (const watcher of [...watchers]) { @@ -200,7 +203,7 @@ export async function connect(url: string, token: string, options: ConnectOption threadId: hello.threadId ?? null, device: hello.device ?? null, async call(method: M, params: RpcMethods[M]['params']): Promise { - if (closed) throw new Error('the client is closed'); + if (closed) throw new Error(closedBy ? `the client is closed: ${closedBy.message}` : 'the client is closed'); const wait = method === 'delegation.wait' && options.requestTimeoutMs === undefined ? Math.min(3_600_000, Math.max(0, (params as RpcMethods['delegation.wait']['params']).timeoutMs ?? 600_000)) + 5000 : undefined; @@ -222,7 +225,7 @@ export async function connect(url: string, token: string, options: ConnectOption ): Promise { return new Promise((resolve, reject) => { if (closed) { - reject(new Error('the client is closed')); + reject(new Error(closedBy ? `the client is closed: ${closedBy.message}` : 'the client is closed')); return; } const watcher = { diff --git a/packages/core/test/waits.test.ts b/packages/core/test/waits.test.ts index 341de57ff..a1334f27e 100644 --- a/packages/core/test/waits.test.ts +++ b/packages/core/test/waits.test.ts @@ -55,4 +55,6 @@ test('a core that goes away rejects a pending event wait at once', async () => { await wait; expect((caught as Error).message).toStartWith('the socket closed'); expect(Date.now() - started).toBeLessThan(4000); + // A later call names how the core hung up, not only that the client is closed. + await expect(client.call('threads.list', {})).rejects.toThrow('the client is closed: the socket closed (1001: "core stopping")'); }); diff --git a/scripts/ci/flake-loop.test.ts b/scripts/ci/flake-loop.test.ts new file mode 100644 index 000000000..53e2edc5c --- /dev/null +++ b/scripts/ci/flake-loop.test.ts @@ -0,0 +1,45 @@ +import { expect, test } from 'bun:test'; +import { failureBlock, results, summary } from './flake-loop.ts'; + +const output = [ + '(pass) first [12.50ms]', + 'error: the client is closed: the socket closed (1002)', + ' at call (client.ts:203:29)', + '(fail) group > second [1.20s]', + '(pass) third', +].join('\n'); + +const junit = ` + + + + + + + + + + + +`; + +test('results reads each test the report names, its outcome and its duration, skipped ones left out', () => { + expect(results(junit)).toEqual([ + { file: 'a.test.ts', name: 'first', passed: true, ms: 12.5 }, + { file: 'a.test.ts', name: 'group > second "quoted"', passed: false, ms: 1200 }, + ]); +}); + +test('a failure block runs from the end of the test before to its fail line', () => { + expect(failureBlock(output, 'group > second')).toBe('error: the client is closed: the socket closed (1002)\n at call (client.ts:203:29)\n(fail) group > second [1.20s]'); + expect(failureBlock(output, 'missing')).toBe(''); +}); + +test('the summary puts the most failed test first and counts every failure', () => { + const table = summary(new Map([ + ['a.test.ts: steady', { pass: 3, fail: 0, ms: [10, 20, 30] }], + ['a.test.ts: flaky | piped', { pass: 1, fail: 2, ms: [5, 50, 500] }], + ]), 3); + expect(table.split('\n')[2]).toBe('| a.test.ts: flaky \\| piped | 2 | 1 | 50 | 500 | 500 |'); + expect(table).toEndWith('3 runs, 2 failures'); +}); diff --git a/scripts/ci/flake-loop.ts b/scripts/ci/flake-loop.ts new file mode 100644 index 000000000..ddc7fa4f3 --- /dev/null +++ b/scripts/ci/flake-loop.ts @@ -0,0 +1,135 @@ +import { appendFileSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { basename, resolve } from 'node:path'; + +// Runs test files again and again on one runner and counts, per test, how many +// runs passed and failed, to measure a flaky test before and after its fix. +// Each run starts every file in a fresh `bun test` process, `--parallel` at a +// time, as `packages/core/bin/test.ts` does. A failed file's whole output is +// kept in `--out`; the error block of each failed test is printed as it ends. +// +// Usage: bun scripts/ci/flake-loop.ts --runs 20 [--parallel 4] [--cwd packages/core] +// [--pattern ] [--out ] [--together] +const ROOT = resolve(import.meta.dir, '../..'); +/** A file that stops answering is ended, as the core runner's file deadline does. */ +const FILE_DEADLINE_MS = 300_000; + +export interface Tally { pass: number; fail: number; ms: number[] } + +const unescape = (text: string) => text.replace(/&(quot|apos|lt|gt|amp);/g, (_, entity: string) => ({ quot: '"', apos: "'", lt: '<', gt: '>', amp: '&' })[entity]!); + +/** + * The tests a `bun test --reporter=junit` file names, skipped ones left out. + * The console prints `(pass)` lines only in some environments; the report lists every test. + */ +export function results(junit: string): { file: string; name: string; passed: boolean; ms: number }[] { + return [...junit.matchAll(/]*?(?:\/>|>([\s\S]*?)<\/testcase>)/g)] + .filter(match => !match[5]?.includes(' ({ + file: basename(unescape(match[4]!).replaceAll('\\', '/')), + name: match[2] ? `${unescape(match[2])} > ${unescape(match[1]!)}` : unescape(match[1]!), + passed: !match[5]?.includes(' line.startsWith(`(fail) ${name}`)); + if (end < 0) return ''; + let start = end - 1; + while (start >= 0 && !/^\((pass|fail|skip|todo)\) /.test(lines[start]!)) start--; + return lines.slice(start + 1, end + 1).join('\n'); +} + +function percentile(values: number[], p: number): number { + if (!values.length) return 0; + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.min(sorted.length - 1, Math.floor(p * sorted.length))]!; +} + +export function summary(tallies: Map, runs: number): string { + const rows = [...tallies.entries()].sort(([, a], [, b]) => b.fail - a.fail); + return [ + `| test | failed | passed | median ms | p95 ms | max ms |`, + `|---|---|---|---|---|---|`, + ...rows.map(([name, t]) => `| ${name.replaceAll('|', '\\|')} | ${t.fail} | ${t.pass} | ${Math.round(percentile(t.ms, 0.5))} | ${Math.round(percentile(t.ms, 0.95))} | ${Math.round(Math.max(0, ...t.ms))} |`), + '', + `${runs} runs, ${rows.reduce((sum, [, t]) => sum + t.fail, 0)} failures`, + ].join('\n'); +} + +async function main(args: string[]): Promise { + const option = (name: string, fallback?: string) => { + const at = args.indexOf(name); + if (at < 0) return fallback; + const [, value] = args.splice(at, 2); + if (value === undefined) throw new Error(`${name}: expected a value`); + return value; + }; + const runs = Number(option('--runs', '10')); + const parallel = Number(option('--parallel', '1')); + const cwd = resolve(ROOT, option('--cwd', '.')!); + const pattern = option('--pattern'); + const out = resolve(ROOT, option('--out', 'flake-loop')!); + const together = args.includes('--together'); + if (together) args.splice(args.indexOf('--together'), 1); + if (!Number.isInteger(runs) || runs < 1) throw new Error(`--runs: expected a positive integer, received ${runs}`); + if (!Number.isInteger(parallel) || parallel < 1 || parallel > 8) throw new Error(`--parallel: expected 1 to 8, received ${parallel}`); + const files = args; + if (!files.length) throw new Error('expected at least one test file'); + mkdirSync(out, { recursive: true }); + + // `--together` runs every file in one process, as an E2E shard does. + const groups = together ? [files] : files.map(file => [file]); + const tallies = new Map(); + const count = (key: string, passed: boolean, ms: number) => { + const tally = tallies.get(key) ?? { pass: 0, fail: 0, ms: [] }; + tallies.set(key, tally); + if (passed) tally.pass++; else tally.fail++; + tally.ms.push(ms); + }; + const started = performance.now(); + for (let run = 1; run <= runs; run++) { + let next = 0; + await Promise.all(Array.from({ length: Math.min(parallel, groups.length) }, async () => { + for (let index = next++; index < groups.length; index = next++) { + const group = groups[index]!, label = `run-${run}-${index}-${basename(group[0]!)}`; + const report = resolve(out, `${label}.xml`); + const child = Bun.spawn([process.execPath, 'test', ...group, '--timeout', '60000', '--reporter=junit', `--reporter-outfile=${report}`, ...(pattern ? ['-t', pattern] : [])], { + cwd, stdin: 'ignore', stdout: 'pipe', stderr: 'pipe', windowsHide: true, + env: { ...process.env, NO_COLOR: '1', FORCE_COLOR: '0' }, + }); + const deadline = setTimeout(() => child.kill(), FILE_DEADLINE_MS * group.length); + const [stdout, stderr, code] = await Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text(), child.exited]); + clearTimeout(deadline); + const output = `${stdout}\n${stderr}`; + let junit = ''; + try { junit = readFileSync(report, 'utf8'); rmSync(report); } catch { /* killed before it wrote one */ } + const found = results(junit); + for (const { file, name, passed, ms } of found) { + const key = `${file}: ${name}`; + count(key, passed, ms); + if (!passed) console.log(`::group::run ${run} ${key}\n${failureBlock(output, name)}\n::endgroup::`); + } + if (code !== 0) { + writeFileSync(resolve(out, `${label}.log`), output); + if (!found.some(result => !result.passed)) { + const key = `${basename(group[0]!)}${group.length > 1 ? ` and ${group.length - 1} more` : ''}: (exit ${code ?? 'killed'} with no named failure)`; + count(key, false, 0); + console.log(`::group::run ${run} ${key}\n${output.slice(-4000)}\n::endgroup::`); + } + } + } + })); + const failed = [...tallies.values()].reduce((sum, t) => sum + t.fail, 0); + console.log(`run ${run}/${runs}: ${failed} failures so far, ${Math.round((performance.now() - started) / 1000)} s`); + } + const table = summary(tallies, runs); + console.log(`\n${table}`); + writeFileSync(resolve(out, 'summary.md'), table); + if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, `\n### ${process.env.FLAKE_LOOP_TITLE ?? files.join(' ')}\n\n${table}\n`); + return [...tallies.values()].some(t => t.fail > 0) ? 1 : 0; +} + +if (import.meta.main) process.exitCode = await main(process.argv.slice(2)); diff --git a/tests/e2e/lib/cdp.ts b/tests/e2e/lib/cdp.ts index f03659017..ec3a7ed65 100644 --- a/tests/e2e/lib/cdp.ts +++ b/tests/e2e/lib/cdp.ts @@ -25,6 +25,23 @@ const PROBE_TIMEOUT_MS = 5_000; /** How long a failed attach waits for its launcher's diagnosis. */ const DIAGNOSIS_TIMEOUT_MS = 20_000; const WAIT_DIAGNOSTIC_TIMEOUT_MS = 250; +/** + * `BOITE_E2E_TRACE=1` (CI flake loops): a timeout or a lost browser also says + * when devtools last spoke, how the socket closed, whether Chrome still runs, + * how late this process's event loop ran and what the UI server still owes. + */ +const TRACE = process.env.BOITE_E2E_TRACE === '1'; +/** Event loop delays over 50 ms in this process, as [when, ms], the last 30 s kept. */ +const loopLag: [number, number][] = []; +if (TRACE) { + let last = performance.now(); + setInterval(() => { + const now = performance.now(), late = now - last - 100; + last = now; + if (late > 50) loopLag.push([now, Math.round(late)]); + while (loopLag.length > 0 && now - loopLag[0]![0] > 30_000) loopLag.shift(); + }, 100).unref(); +} /** What a page that left an evaluate hanging is doing: frames drawn in a second, fonts and animations still pending. */ const HUNG_PAGE = `new Promise((resolve) => { let frames = 0; @@ -163,6 +180,10 @@ export class BrowserPage { #nextId = 1; #pending = new Map(); #closed = false; + /** When devtools last sent anything, and how its socket closed. */ + #heardAt = performance.now(); + #closedBy: string | null = null; + #stderr: () => string = () => ''; #pageErrors: string[] = []; /** Requests the page sent and has not finished loading, reported when a wait times out. */ #requests = new Map(); @@ -182,7 +203,8 @@ export class BrowserPage { socket.addEventListener('message', (event: MessageEvent) => { this.#receive(typeof event.data === 'string' ? event.data : ''); }); - socket.addEventListener('close', () => { + socket.addEventListener('close', (event: CloseEvent) => { + this.#closedBy = `code ${event.code}${event.reason ? ` ${JSON.stringify(event.reason)}` : ''}, ${Math.round(performance.now() - this.#heardAt)} ms after the last message`; for (const pending of this.#pending.values()) { clearTimeout(pending.timer); pending.reject(new Error('the devtools socket closed, the browser exited or was killed (bun test kills every process a file started when one of its tests or hooks times out)')); @@ -274,6 +296,7 @@ export class BrowserPage { const target = await waitForPageTarget(port, '', proc, true); const socket = await openSocket(target); page = new BrowserPage(socket, proc, ownsUserDataDir ? userDataDir : null, watch); + page.#stderr = () => stderr; open.add(page); await page.send('Page.enable', {}); await page.send('Runtime.enable', {}); @@ -387,7 +410,7 @@ export class BrowserPage { .catch(() => 'the page did not answer either: its main thread is busy'); throw new Error(`${error.message} Page: ${state} -In flight: ${JSON.stringify(this.#inFlight())}`); +In flight: ${JSON.stringify(this.#inFlight())}${this.#trace()}`); })) as { result?: { value?: unknown }; exceptionDetails?: { @@ -415,7 +438,7 @@ In flight: ${JSON.stringify(this.#inFlight())}`); const deadline = Date.now() + timeoutMs; let last = ''; while (Date.now() < deadline) { - if (this.#closed || this.#socket.readyState !== WebSocket.OPEN) throw new Error('the browser is closed'); + if (this.#closed || this.#socket.readyState !== WebSocket.OPEN) throw new Error(`the browser is closed${this.#trace()}`); try { const remaining = Math.max(1, deadline - Date.now()); const ok = await this.evaluate(`(async () => !!(await (${expression})))()`, Math.min(CALL_TIMEOUT_MS, remaining)); @@ -427,10 +450,10 @@ In flight: ${JSON.stringify(this.#inFlight())}`); } await Bun.sleep(Math.min(POLL_MS, Math.max(0, deadline - Date.now()))); } - if (this.#closed || this.#socket.readyState !== WebSocket.OPEN) throw new Error('the browser is closed'); + if (this.#closed || this.#socket.readyState !== WebSocket.OPEN) throw new Error(`the browser is closed${this.#trace()}`); // A separate, bounded grace preserves useful diagnostics after the condition consumes its budget. const state = await this.evaluate(`({ location: location.origin + location.pathname, ready: document.readyState, title: document.title, text: document.body?.innerText.slice(0, 500) })`, WAIT_DIAGNOSTIC_TIMEOUT_MS).catch(() => 'page diagnostic timed out'); - throw new Error(`waitFor timed out on ${expression}: ${last}\nPage: ${JSON.stringify(state)}\nErrors: ${JSON.stringify(this.#pageErrors)}\nIn flight: ${JSON.stringify(this.#inFlight())}\nFailed requests: ${JSON.stringify(this.#networkFailures)}`); + throw new Error(`waitFor timed out on ${expression}: ${last}\nPage: ${JSON.stringify(state)}\nErrors: ${JSON.stringify(this.#pageErrors)}\nIn flight: ${JSON.stringify(this.#inFlight())}\nFailed requests: ${JSON.stringify(this.#networkFailures)}${this.#trace()}`); } async navigate(url: string): Promise { @@ -566,6 +589,20 @@ In flight: ${JSON.stringify(this.#inFlight())}`); else if (this.#userDataDir !== null) await removeDirectory(this.#userDataDir); } + /** What `BOITE_E2E_TRACE` adds to a failure, on its own lines; empty without it. */ + #trace(): string { + if (!TRACE) return ''; + const now = performance.now(); + const lags = loopLag.filter(([at]) => now - at < 30_000).map(([at, ms]) => `${ms} ms ${Math.round((now - at) / 1000)} s ago`); + const ui = (globalThis as { __boiteUiTrace?: () => string }).__boiteUiTrace?.() ?? 'not traced'; + const stderr = this.#stderr().trim().split('\n').slice(-12).join('\n'); + return `\nTrace: last devtools message ${Math.round(now - this.#heardAt)} ms ago; socket ${this.#closedBy ?? `state ${this.#socket.readyState}`}; browser ${this.#proc === null ? 'attached' : this.#proc.exitCode === null ? `pid ${this.#proc.pid} running` : `exited ${this.#proc.exitCode} (signal ${this.#proc.signalCode})`} +Test loop delays over 50 ms in 30 s: ${lags.length ? lags.slice(-8).join(', ') : 'none'} +UI server: ${ui} +Browser stderr tail: +${stderr || '(empty)'}`; + } + /** What a stalled page is still waiting on: whether the server never answered or the body never arrived. */ #inFlight(): string[] { const now = Date.now(); @@ -574,6 +611,7 @@ In flight: ${JSON.stringify(this.#inFlight())}`); } #receive(raw: string): void { + this.#heardAt = performance.now(); if (raw === '') return; let frame: { id?: unknown; method?: string; params?: { exceptionDetails?: { text?: string; exception?: { description?: string } }; requestId?: string; request?: { url?: string }; response?: { status?: number }; errorText?: string }; result?: unknown; error?: { message?: string } }; try { diff --git a/tests/e2e/lib/ui.ts b/tests/e2e/lib/ui.ts index fe046efc4..1214aa19f 100644 --- a/tests/e2e/lib/ui.ts +++ b/tests/e2e/lib/ui.ts @@ -18,6 +18,38 @@ export const fixtureBridge = { }, }; +/** + * `BOITE_E2E_TRACE=1`: the preview server times every request, and a failing + * page's trace (`lib/cdp.ts`) says which ones it still owes and which took + * over a second, to tell a server that never answered from a browser that + * never read the answer. + */ +const traced = { open: new Map(), slow: [] as string[], next: 0, served: 0 }; +const requestTrace = { + name: 'boite-e2e-request-trace', + configurePreviewServer(server: { middlewares: { use(handler: (req: { url?: string }, res: { once(event: 'close', listener: () => void): void }, next: () => void) => void): void } }) { + server.middlewares.use((req, res, next) => { + const id = traced.next++, at = Date.now(), url = req.url ?? '?'; + traced.open.set(id, { url, at }); + res.once('close', () => { + traced.open.delete(id); + traced.served++; + const ms = Date.now() - at; + if (ms > 1_000) { traced.slow.push(`${url} ${ms} ms`); if (traced.slow.length > 10) traced.slow.shift(); } + }); + next(); + }); + }, +}; +if (process.env.BOITE_E2E_TRACE === '1') { + (globalThis as { __boiteUiTrace?: () => string }).__boiteUiTrace = () => { + const now = Date.now(); + const open = [...traced.open.values()].map(request => `${request.url} for ${now - request.at} ms`).slice(0, 8); + return `${traced.served} served; open: ${open.length ? open.join(', ') : 'none'}; over 1 s: ${traced.slow.length ? traced.slow.join(', ') : 'none'}`; + }; +} +const previewPlugins = process.env.BOITE_E2E_TRACE === '1' ? [requestTrace] : []; + /** Build fake-client fixtures once in CI, keeping cold transforms outside browser interaction deadlines. */ export async function startUi(port: number, options: { development?: boolean } = {}): Promise<{ close(): Promise }> { const address = { host: '127.0.0.1', port, strictPort: true }; @@ -25,14 +57,14 @@ export async function startUi(port: number, options: { development?: boolean } = if (shared && !options.development) { // Built once by `lib/warm.ts` before a parallel run. if (!existsSync(join(shared, 'index.html'))) throw new Error(`BOITE_E2E_FAKE_UI=${shared} holds no index.html; build it with bun tests/e2e/lib/warm.ts ${shared}`); - return preview({ root, build: { outDir: resolve(shared) }, preview: address, clearScreen: false }); + return preview({ root, plugins: previewPlugins, build: { outDir: resolve(shared) }, preview: address, clearScreen: false }); } if (process.env.BOITE_E2E_PREBUILT_UI === '1' && !options.development) { // Production intentionally excludes ?fake=1. This separate test bundle // enables it without changing the UI staged in the installer. fixtureBuild ??= build({ root, plugins: [fixtureBridge], define: { 'import.meta.env.DEV': 'true' }, build: { outDir: fixtureDir, emptyOutDir: true }, logLevel: 'warn' }); await fixtureBuild; - return preview({ root, build: { outDir: fixtureDir }, preview: address, clearScreen: false }); + return preview({ root, plugins: previewPlugins, build: { outDir: fixtureDir }, preview: address, clearScreen: false }); } return startDevUi(port); } From 67abc8626e146e047a66afd216a8b3f9c431b314 Mon Sep 17 00:00:00 2001 From: meetsu <96637888+klNuno@users.noreply.github.com> Date: Mon, 19 Oct 2026 00:38:20 +0200 Subject: [PATCH 02/13] ci(flake-loop): build the alone E2E file list without a stale exit code --- .github/workflows/flake-loop.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/flake-loop.yml b/.github/workflows/flake-loop.yml index 7ae16bca1..8eb9006e1 100644 --- a/.github/workflows/flake-loop.yml +++ b/.github/workflows/flake-loop.yml @@ -113,12 +113,12 @@ jobs: MODE: ${{ matrix.mode }} run: | if ($env:MODE -eq 'alone') { - $files = @('tests/e2e/coordination-bridge-live.test.ts'); $runs = [int]$env:RUNS * 3 + $files = @('tests/e2e/coordination-bridge-live.test.ts'); $runs = [int]$env:RUNS * 3; $together = @() } else { - $files = @(bun scripts/ci/e2e-shards.ts 3 3); $runs = [math]::Max(1, [int]([int]$env:RUNS / 4)) + $files = @(bun scripts/ci/e2e-shards.ts 3 3) + if ($LASTEXITCODE -ne 0 -or $files.Count -eq 0) { throw 'No E2E files for shard 3' } + $runs = [math]::Max(1, [int]([int]$env:RUNS / 4)); $together = @('--together') } - if ($LASTEXITCODE -ne 0 -or $files.Count -eq 0) { throw 'No E2E files to loop' } - $together = @(if ($env:MODE -eq 'alone') { } else { '--together' }) bun scripts/ci/flake-loop.ts --runs $runs --out flake-loop @together @files - uses: ./.github/actions/upload-artifact if: always() From 01ef58772612d1563d8f373beb6be5bb92184019 Mon Sep 17 00:00:00 2001 From: meetsu <96637888+klNuno@users.noreply.github.com> Date: Mon, 19 Oct 2026 00:39:06 +0200 Subject: [PATCH 03/13] ci(flake-loop): let a running loop finish when a new head arrives --- .github/workflows/flake-loop.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/flake-loop.yml b/.github/workflows/flake-loop.yml index 8eb9006e1..6fe578c52 100644 --- a/.github/workflows/flake-loop.yml +++ b/.github/workflows/flake-loop.yml @@ -18,7 +18,8 @@ permissions: concurrency: group: flake-loop-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true + # A loop already running keeps its data; a newer head waits for it. + cancel-in-progress: false env: RUNS: ${{ inputs.runs || 10 }} From 990ea7d630192fd4d9394c444655c7ab0887f575 Mon Sep 17 00:00:00 2001 From: meetsu <96637888+klNuno@users.noreply.github.com> Date: Mon, 19 Oct 2026 01:11:10 +0200 Subject: [PATCH 04/13] ci(flake-loop): name the busiest processes when Chromium stops answering, and loop the invitation E2E --- .github/workflows/flake-loop.yml | 2 +- docs/ci.md | 3 ++- packages/core/src/browser/cdp.ts | 12 +++++++++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/flake-loop.yml b/.github/workflows/flake-loop.yml index 6fe578c52..c94d3b926 100644 --- a/.github/workflows/flake-loop.yml +++ b/.github/workflows/flake-loop.yml @@ -114,7 +114,7 @@ jobs: MODE: ${{ matrix.mode }} run: | if ($env:MODE -eq 'alone') { - $files = @('tests/e2e/coordination-bridge-live.test.ts'); $runs = [int]$env:RUNS * 3; $together = @() + $files = @('tests/e2e/coordination-bridge-live.test.ts', 'tests/e2e/machines.test.ts'); $runs = [int]$env:RUNS * 2; $together = @() } else { $files = @(bun scripts/ci/e2e-shards.ts 3 3) if ($LASTEXITCODE -ne 0 -or $files.Count -eq 0) { throw 'No E2E files for shard 3' } diff --git a/docs/ci.md b/docs/ci.md index 310b2856c..7e8c39293 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -116,7 +116,8 @@ The `flake-loop` workflow measures a flaky test before and after its fix. It runs on a pull request that carries the `flake-loop` label, or by hand: the quarantined browser files four at a time on macOS arm64, macOS Intel and Linux, `cli.test.ts` four copies at a time on every OS, and -`coordination-bridge-live.test.ts` alone and within E2E shard 3 on Windows. +`coordination-bridge-live.test.ts` and `machines.test.ts` alone and shard 3 +whole on Windows. `scripts/ci/flake-loop.ts` counts each test's passes and failures over the runs, prints the error of each failure and writes a table to the job summary. `BOITE_CDP_TRACE=1` makes a core DevTools timeout say how many bytes of the diff --git a/packages/core/src/browser/cdp.ts b/packages/core/src/browser/cdp.ts index 07a463f99..78fd81d7f 100644 --- a/packages/core/src/browser/cdp.ts +++ b/packages/core/src/browser/cdp.ts @@ -166,7 +166,7 @@ export class Cdp { .then(() => `answered in ${Math.round(performance.now() - started)} ms`, (error: Error) => error.message.split(';')[0]); const after = this.#unread?.(); const bytes = unread === undefined || unread === null ? 'unread bytes unknown' : `${unread} bytes unread before the probe, ${after} after`; - return `; trace: ${bytes}, load ${loadavg().map(n => n.toFixed(1)).join(' ')}, Browser.getVersion ${probe}`; + return `; trace: ${bytes}, load ${loadavg().map(n => n.toFixed(1)).join(' ')}, Browser.getVersion ${probe}; busiest: ${busiest()}`; } #receive(text: string): void { @@ -195,6 +195,16 @@ export class Cdp { } } +/** The ten processes using the most CPU, with their state and memory, and how many Chromium processes run. */ +function busiest(): string { + if (process.platform === 'win32') return 'not read on Windows'; + const ps = Bun.spawnSync(['ps', '-A', '-o', 'pid=,pcpu=,rss=,state=,comm='], { stdout: 'pipe', stderr: 'ignore' }); + const rows = ps.stdout.toString().trim().split('\n').map(line => line.trim().split(/\s+/)) + .map(([pid, cpu, rss, state, ...command]) => ({ pid, cpu: Number(cpu), rss: Math.round(Number(rss) / 1024), state, name: command.join(' ').split('/').pop() ?? '' })); + const chromium = rows.filter(row => /chrom/i.test(row.name)).length; + return `${chromium} Chromium processes; ` + rows.sort((a, b) => b.cpu - a.cpu).slice(0, 10).map(row => `${row.name} ${row.pid} ${row.cpu}% ${row.rss} MB ${row.state}`).join(', '); +} + /** * Peeks the bytes a socket holds without reading them, `recv(MSG_PEEK | * MSG_DONTWAIT)` through libc. Bytes there while a command times out mean the From f334fbae4a1037abfe739f6991b043b3c043e8e1 Mon Sep 17 00:00:00 2001 From: meetsu <96637888+klNuno@users.noreply.github.com> Date: Mon, 19 Oct 2026 01:43:00 +0200 Subject: [PATCH 05/13] ci(flake-loop): check Chrome's devtools frames through a relay and keep them on a 1002 close --- .github/workflows/flake-loop.yml | 2 + docs/ci.md | 5 +- scripts/ci/ws-relay.test.ts | 59 +++++++++++ tests/e2e/lib/cdp.ts | 19 +++- tests/e2e/lib/ws-relay.ts | 161 +++++++++++++++++++++++++++++++ 5 files changed, 243 insertions(+), 3 deletions(-) create mode 100644 scripts/ci/ws-relay.test.ts create mode 100644 tests/e2e/lib/ws-relay.ts diff --git a/.github/workflows/flake-loop.yml b/.github/workflows/flake-loop.yml index c94d3b926..3cd968dbc 100644 --- a/.github/workflows/flake-loop.yml +++ b/.github/workflows/flake-loop.yml @@ -120,6 +120,8 @@ jobs: if ($LASTEXITCODE -ne 0 -or $files.Count -eq 0) { throw 'No E2E files for shard 3' } $runs = [math]::Max(1, [int]([int]$env:RUNS / 4)); $together = @('--together') } + New-Item -ItemType Directory -Force flake-loop/devtools | Out-Null + $env:BOITE_E2E_TRACE_DIR = (Resolve-Path flake-loop/devtools).Path bun scripts/ci/flake-loop.ts --runs $runs --out flake-loop @together @files - uses: ./.github/actions/upload-artifact if: always() diff --git a/docs/ci.md b/docs/ci.md index 7e8c39293..8b20bfc86 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -124,7 +124,10 @@ runs, prints the error of each failure and writes a table to the job summary. browser's answers sat unread, the load and whether `Browser.getVersion` still answered; `BOITE_E2E_TRACE=1` makes an E2E timeout or lost browser say how the devtools socket closed, whether Chrome still ran, the test process's event -loop delays and what the UI server still owed. +loop delays and what the UI server still owed. It also passes each devtools +socket through a loopback relay (`tests/e2e/lib/ws-relay.ts`) that checks +every frame Chrome sends, and on a 1002 close keeps Chrome's bytes and read +lengths in `BOITE_E2E_TRACE_DIR` for a replay against Bun's client. `scripts/ci/hdiutil-retry.ts` bundles the macOS application again, three attempts at most, when hdiutil reports `Resource busy`. That runner fault (actions/runner-images#7522) failed the Apple Silicon DMG of nightly diff --git a/scripts/ci/ws-relay.test.ts b/scripts/ci/ws-relay.test.ts new file mode 100644 index 000000000..5faaae555 --- /dev/null +++ b/scripts/ci/ws-relay.test.ts @@ -0,0 +1,59 @@ +import { expect, test } from 'bun:test'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join as joinPath } from 'node:path'; +import { FrameCheck } from '../../tests/e2e/lib/ws-relay.ts'; + +const handshake = new TextEncoder().encode('HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\n\r\n'); +function frame(opcode: number, length: number, fin = true): Uint8Array { + const head = length < 126 ? [length] : length < 65536 ? [126, length >> 8, length & 255] : [127, 0, 0, 0, 0, (length >>> 24) & 255, (length >> 16) & 255, (length >> 8) & 255, length & 255]; + const out = new Uint8Array(1 + head.length + length).fill(0x78); + out[0] = (fin ? 0x80 : 0) | opcode; out.set(head, 1); + return out; +} +function join(parts: Uint8Array[]): Uint8Array { + const out = new Uint8Array(parts.reduce((sum, part) => sum + part.length, 0)); + let at = 0; + for (const part of parts) { out.set(part, at); at += part.length; } + return out; +} + +test('the relay check reads a well-formed browser stream however the reads cut it', () => { + const stream = join([handshake, frame(1, 5), frame(1, 126), frame(1, 70_000), frame(9, 0), frame(1, 0, false), frame(0, 3)]); + for (const cut of [1, 2, 3, 7, 125, 4096, stream.length]) { + const check = new FrameCheck(); + for (let at = 0; at < stream.length; at += cut) check.push(stream.subarray(at, at + cut)); + expect(check.report()).toStartWith(`browser stream (extensions none) well-formed over 6 frames and ${stream.length} bytes`); + } +}); + +test('the relay check names the first byte a malformed stream breaks at', () => { + const fragmented = new FrameCheck(); + fragmented.push(join([handshake, frame(1, 10), frame(9, 2, false)])); + expect(fragmented.report()).toStartWith(`browser stream (extensions none) malformed: a fragmented or long control frame at byte ${handshake.length + 12}`); + const opcode = new FrameCheck(); + opcode.push(join([handshake, frame(3, 1)])); + expect(opcode.report()).toStartWith(`browser stream (extensions none) malformed: opcode 3 at byte ${handshake.length}`); +}); + +test('the relay check lets a deflated message set RSV1 only once the browser accepted permessage-deflate', () => { + const deflated = frame(1, 4); deflated[0] = deflated[0]! | 0x40; + const accepted = new FrameCheck(); + accepted.push(join([new TextEncoder().encode('HTTP/1.1 101 Switching Protocols\r\nSec-WebSocket-Extensions: permessage-deflate; client_max_window_bits=15\r\n\r\n'), deflated])); + expect(accepted.report()).toContain('well-formed over 1 frames'); + const refused = new FrameCheck(); + refused.push(join([handshake, deflated])); + expect(refused.report()).toContain(`malformed: reserved bits set at byte ${handshake.length}`); +}); + +test('the relay check keeps the browser bytes and read lengths for a replay', () => { + const dir = mkdtempSync(joinPath(tmpdir(), 'ws-relay-')); + try { + const check = new FrameCheck(); + const stream = join([handshake, frame(1, 300)]); + check.push(stream.subarray(0, 40)); check.push(stream.subarray(40)); + expect(check.dump(joinPath(dir, 'devtools'))).toEndWith(`(${stream.length} of ${stream.length} bytes)`); + expect(new Uint8Array(readFileSync(joinPath(dir, 'devtools.bin')))).toEqual(stream); + expect(JSON.parse(readFileSync(joinPath(dir, 'devtools.reads.json'), 'utf8'))).toEqual([40, stream.length - 40]); + } finally { rmSync(dir, { recursive: true, force: true }); } +}); diff --git a/tests/e2e/lib/cdp.ts b/tests/e2e/lib/cdp.ts index ec3a7ed65..1b40f0819 100644 --- a/tests/e2e/lib/cdp.ts +++ b/tests/e2e/lib/cdp.ts @@ -5,6 +5,7 @@ import { dirname, join } from 'node:path'; import { ONBOARDING_STORAGE_KEY, ONBOARDING_VERSION } from '../../../packages/ui/src/lib/onboarding.ts'; import type { Subprocess } from 'bun'; import { E2E_DIR_PREFIX, killProcessTreeAsync, removeDirectory, watchParentDeath, type ParentDeathWatch } from './cleanup.ts'; +import { relay } from './ws-relay.ts'; /** * Marks the tour seen before any script of the page runs, and only where no @@ -599,6 +600,7 @@ In flight: ${JSON.stringify(this.#inFlight())}`); return `\nTrace: last devtools message ${Math.round(now - this.#heardAt)} ms ago; socket ${this.#closedBy ?? `state ${this.#socket.readyState}`}; browser ${this.#proc === null ? 'attached' : this.#proc.exitCode === null ? `pid ${this.#proc.pid} running` : `exited ${this.#proc.exitCode} (signal ${this.#proc.signalCode})`} Test loop delays over 50 ms in 30 s: ${lags.length ? lags.slice(-8).join(', ') : 'none'} UI server: ${ui} +Relay: ${relays.get(this.#socket)?.report() ?? 'none'} Browser stderr tail: ${stderr || '(empty)'}`; } @@ -744,11 +746,24 @@ export async function probeDebuggingPort(port: number): Promise { } } -function openSocket(target: TargetInfo): Promise { +/** Under `BOITE_E2E_TRACE`, each devtools socket's relay, which checks the frames the browser sent. */ +const relays = new WeakMap(); + +async function openSocket(target: TargetInfo): Promise { const url = target.webSocketDebuggerUrl; if (url === undefined) throw new Error('the page target has no debugger url'); + const traced = TRACE ? await relay(url) : null; return new Promise((resolve, reject) => { - const socket = new WebSocket(url); + const socket = new WebSocket(traced?.url ?? url); + if (traced) { + relays.set(socket, traced); + socket.addEventListener('close', (event) => { + // A protocol error keeps the browser's bytes for a replay against Bun's client (`BOITE_E2E_TRACE_DIR`). + const dir = process.env.BOITE_E2E_TRACE_DIR; + if (event.code === 1002 && dir) console.warn(`[e2e] devtools socket closed 1002, kept ${traced.dump(join(dir, `devtools-${Date.now()}`))}`); + setTimeout(() => traced.close(), 100); + }, { once: true }); + } // A socket that never opened is closed with its refusal: nothing else // holds it, and a late open would keep its connection for the whole run. const fail = (message: string) => { diff --git a/tests/e2e/lib/ws-relay.ts b/tests/e2e/lib/ws-relay.ts new file mode 100644 index 000000000..ac8ceefc5 --- /dev/null +++ b/tests/e2e/lib/ws-relay.ts @@ -0,0 +1,161 @@ +/* + * `BOITE_E2E_TRACE=1` (CI flake loops): a loopback relay between the test's + * devtools WebSocket and the browser. It checks every frame the browser sends + * against RFC 6455 as the bytes pass, so a socket Bun closes with 1002 + * ("control frame is fragmented", "unexpected opcode") can be told apart: a + * malformed stream from the browser, or a well-formed one Bun misread. + */ + +import { writeFileSync } from 'node:fs'; + +/** A devtools stream past this stops being kept for a replay; the check goes on. */ +const KEEP_BYTES = 64 << 20; + +interface Frame { at: number; opcode: number; fin: boolean; compressed: boolean; length: number } + +/** Reads frame headers as bytes arrive and skips payloads; keeps the first error and the last frames and reads. */ +export class FrameCheck { + #buffer = new Uint8Array(0); + #skip = 0; + /** Stream bytes before the buffer, skipped payloads included, and all bytes seen. */ + #offset = 0; + #total = 0; + #handshake = true; + /** The `Sec-WebSocket-Extensions` the browser accepted; `permessage-deflate` lets a message's first frame set RSV1. */ + #extensions = ''; + #frames = 0; + #recent: Frame[] = []; + #chunks: Array<{ at: number; length: number; ms: number }> = []; + #error: string | null = null; + #started = performance.now(); + /** Every read as it arrived, up to `KEEP_BYTES`, for `dump` to replay. */ + #kept: Uint8Array[] = []; + #keptBytes = 0; + + push(bytes: Uint8Array): void { + this.#chunks.push({ at: this.#total, length: bytes.length, ms: Math.round(performance.now() - this.#started) }); + this.#total += bytes.length; + if (this.#keptBytes + bytes.length <= KEEP_BYTES) { this.#kept.push(bytes); this.#keptBytes += bytes.length; } + if (this.#chunks.length > 12) this.#chunks.shift(); + if (this.#error) return; + let data = bytes; + if (this.#skip > 0) { + const used = Math.min(this.#skip, data.length); + this.#skip -= used; this.#offset += used; data = data.subarray(used); + } + const joined = new Uint8Array(this.#buffer.length + data.length); + joined.set(this.#buffer); joined.set(data, this.#buffer.length); + let at = 0; + if (this.#handshake) { + const end = indexOfHeaderEnd(joined); + if (end === -1) { this.#buffer = joined; return; } + at = end; this.#handshake = false; + this.#extensions = /^sec-websocket-extensions:\s*(.*)$/im.exec(new TextDecoder().decode(joined.subarray(0, end)))?.[1]?.trim() ?? ''; + } + while (!this.#error) { + const rest = joined.length - at; + if (rest < 2) break; + const first = joined[at]!, second = joined[at + 1]!; + let length = second & 0x7f, header = 2; + if (length === 126) { if (rest < 4) break; length = (joined[at + 2]! << 8) | joined[at + 3]!; header = 4; } + else if (length === 127) { + if (rest < 10) break; + length = 0; + for (let i = 2; i < 10; i++) length = length * 256 + joined[at + i]!; + header = 10; + } + const frame = { at: this.#offset + at, opcode: first & 0x0f, fin: (first & 0x80) !== 0, compressed: (first & 0x40) !== 0, length }; + this.#error = problem(first, second, frame, this.#extensions.includes('permessage-deflate')); + this.#frames++; + this.#recent.push(frame); + if (this.#recent.length > 12) this.#recent.shift(); + const payload = Math.min(length, rest - header); + at += header + payload; + if (payload < length) { this.#skip = length - payload; break; } + } + this.#offset += at; + this.#buffer = joined.slice(at); + } + + /** What the browser sent, as one file of bytes and one of read lengths, so a replay cuts it the same way. */ + dump(path: string): string { + const whole = new Uint8Array(this.#keptBytes); + let at = 0; + for (const part of this.#kept) { whole.set(part, at); at += part.length; } + writeFileSync(`${path}.bin`, whole); + writeFileSync(`${path}.reads.json`, JSON.stringify(this.#kept.map(part => part.length))); + return `${path}.bin (${whole.length} of ${this.#total} bytes)`; + } + + report(): string { + const frames = this.#recent.map(frame => `${frame.at}:op${frame.opcode}${frame.compressed ? ' deflated' : ''}${frame.fin ? '' : ' unfinished'} ${frame.length} B`).join(', '); + const reads = this.#chunks.map(chunk => `${chunk.at}+${chunk.length} at ${chunk.ms} ms`).join(', '); + return `browser stream (extensions ${this.#extensions || 'none'}) ${this.#error ? `malformed: ${this.#error}` : 'well-formed'} over ${this.#frames} frames and ${this.#total} bytes; last frames ${frames || 'none'}; last reads ${reads}`; + } +} + +function problem(first: number, second: number, frame: Frame, deflate: boolean): string | null { + if (first & (deflate && (frame.opcode === 1 || frame.opcode === 2) ? 0x30 : 0x70)) return `reserved bits set at byte ${frame.at}`; + if (second & 0x80) return `a masked server frame at byte ${frame.at}`; + if (![0, 1, 2, 8, 9, 10].includes(frame.opcode)) return `opcode ${frame.opcode} at byte ${frame.at}`; + if (frame.opcode >= 8 && (!frame.fin || frame.length > 125)) return `a fragmented or long control frame at byte ${frame.at}`; + return null; +} + +function indexOfHeaderEnd(bytes: Uint8Array): number { + for (let i = 3; i < bytes.length; i++) if (bytes[i - 3] === 13 && bytes[i - 2] === 10 && bytes[i - 1] === 13 && bytes[i] === 10) return i + 1; + return -1; +} + +/** Each side writes what fits now and the rest once it drains, in order. */ +function sender(): { write(socket: { write(bytes: Uint8Array): number }, bytes: Uint8Array): void; drain(socket: { write(bytes: Uint8Array): number }): void } { + const queue: Uint8Array[] = []; + const drain = (socket: { write(bytes: Uint8Array): number }) => { + while (queue.length) { + const chunk = queue[0]!, written = socket.write(chunk); + if (written < 0) { queue.length = 0; return; } + if (written < chunk.length) { queue[0] = chunk.subarray(written); return; } + queue.shift(); + } + }; + return { write: (socket, bytes) => { queue.push(bytes); if (queue.length === 1) drain(socket); }, drain }; +} + +/** A `ws://` url on a loopback relay to `url`, and the check of what the browser sent through it. */ +export async function relay(url: string): Promise<{ url: string; report(): string; dump(path: string): string; close(): void }> { + const target = new URL(url); + const check = new FrameCheck(); + const toBrowser = sender(), toTest = sender(); + let upstream: Bun.Socket | null = null; + let early: Uint8Array[] = []; + let test: Bun.Socket | null = null; + const server = Bun.listen({ + hostname: '127.0.0.1', port: 0, + socket: { + async open(socket) { + test = socket; + upstream = await Bun.connect({ + hostname: target.hostname, port: Number(target.port), + socket: { + data(_upstream, bytes) { const copy = bytes.slice(); check.push(copy); toTest.write(socket, copy); }, + drain(own) { toBrowser.drain(own); }, + close() { socket.end(); }, + error() { socket.end(); }, + }, + }); + for (const bytes of early) toBrowser.write(upstream, bytes); + early = []; + }, + data(_socket, bytes) { const copy = bytes.slice(); if (upstream) toBrowser.write(upstream, copy); else early.push(copy); }, + drain(socket) { toTest.drain(socket); }, + close() { upstream?.end(); }, + error() { upstream?.end(); }, + }, + }); + return { + url: `ws://127.0.0.1:${server.port}${target.pathname}`, + report: () => check.report(), + dump: path => check.dump(path), + close: () => { test?.end(); upstream?.end(); server.stop(true); }, + }; +} From 9a007c10b2fbc986f1afdb44edb3e357b140d7c2 Mon Sep 17 00:00:00 2001 From: meetsu <96637888+klNuno@users.noreply.github.com> Date: Mon, 19 Oct 2026 01:55:27 +0200 Subject: [PATCH 06/13] ci(flake-loop): loop with #623 applied, reverted before merge --- packages/core/src/browser/cdp.ts | 3 ++- packages/core/src/client.ts | 15 +++++++++--- packages/core/src/group/relay.ts | 3 ++- packages/core/test/browser/fixture.ts | 4 ++-- packages/core/test/socket-options.test.ts | 28 +++++++++++++++++++++++ tests/e2e/group-devices.test.ts | 4 ++-- tests/e2e/lib/cdp.ts | 3 ++- 7 files changed, 50 insertions(+), 10 deletions(-) create mode 100644 packages/core/test/socket-options.test.ts diff --git a/packages/core/src/browser/cdp.ts b/packages/core/src/browser/cdp.ts index 78fd81d7f..9f9eae001 100644 --- a/packages/core/src/browser/cdp.ts +++ b/packages/core/src/browser/cdp.ts @@ -11,6 +11,7 @@ import { fstatSync } from 'node:fs'; import { loadavg } from 'node:os'; +import { SOCKET_OPTIONS } from '../client.ts'; type Listener = (params: Record, sessionId: string | undefined) => void; interface Waiting { resolve(value: Record): void; reject(error: Error): void; timer: ReturnType; method: string } @@ -106,7 +107,7 @@ export class Cdp { /** The Windows transport: the browser's WebSocket endpoint on 127.0.0.1. */ static connect(url: string, timeoutMs = 10_000): Promise { return new Promise((resolve, reject) => { - const socket = new WebSocket(url); + const socket = new WebSocket(url, SOCKET_OPTIONS as unknown as string[]); const timer = setTimeout(() => { socket.close(); reject(new CdpError(`the browser did not accept a DevTools connection within ${timeoutMs / 1000} seconds`)); }, timeoutMs); socket.addEventListener('error', () => { clearTimeout(timer); reject(new CdpError(`could not connect to the browser's DevTools at ${url}`)); }, { once: true }); socket.addEventListener('open', () => { diff --git a/packages/core/src/client.ts b/packages/core/src/client.ts index 310c0696a..5fc3726d7 100644 --- a/packages/core/src/client.ts +++ b/packages/core/src/client.ts @@ -81,12 +81,21 @@ export class SocketClosedError extends Error { } } +/** + * What every WebSocket this code opens with Bun's client passes: no + * `permessage-deflate` offer. With the extension on, Bun 1.4.2 and 1.4.3 + * sometimes close a sound stream with 1002 ("control frame is fragmented", + * "unsupported control frame", "unexpected opcode"): against the core's own + * server and against Chrome's DevTools, whose bytes a relay checked frame by + * frame and found well-formed. Uncompressed, a message is larger on the wire; + * compressed, the connection drops. + */ +export const SOCKET_OPTIONS = { perMessageDeflate: false } as const; + export async function connect(url: string, token: string, options: ConnectOptions = {}): Promise { const timeoutMs = options.timeoutMs ?? 5000; const deadline = Date.now() + timeoutMs; - const socket = options.headers === undefined - ? new WebSocket(socketUrl(url)) - : new WebSocket(socketUrl(url), { headers: options.headers } as unknown as string[]); + const socket = new WebSocket(socketUrl(url), { ...SOCKET_OPTIONS, ...(options.headers === undefined ? {} : { headers: options.headers }) } as unknown as string[]); const pending = new Map(); const listeners = new Map void>>(); let nextId = 1; diff --git a/packages/core/src/group/relay.ts b/packages/core/src/group/relay.ts index afd1b2c9f..203fc27c2 100644 --- a/packages/core/src/group/relay.ts +++ b/packages/core/src/group/relay.ts @@ -29,6 +29,7 @@ import type { ServerWebSocket } from 'bun'; import { FILE_ROUTE, GROUP_RELAY_ROUTE, PROTOCOL_VERSION, RPC_PATH, RpcErrorCode, VIEW_ROUTE } from '@boite/contracts'; +import { SOCKET_OPTIONS } from '../client.ts'; import type { Core } from '../core.ts'; import { messageOf } from '../errors.ts'; import { OUTBOUND_CONGESTION_TIMEOUT_MS, OUTBOUND_MAX_BUFFERED_BYTES } from '../server/connection.ts'; @@ -388,7 +389,7 @@ export class RelayPipe { const opened = await new Promise((resolve) => { let socket: Upstream; try { - socket = new WebSocket(`${target.replace(/^http/, 'ws')}${RPC_PATH}`); + socket = new WebSocket(`${target.replace(/^http/, 'ws')}${RPC_PATH}`, SOCKET_OPTIONS as unknown as string[]); } catch { resolve(null); return; diff --git a/packages/core/test/browser/fixture.ts b/packages/core/test/browser/fixture.ts index 03d59b68b..8bd8a837c 100644 --- a/packages/core/test/browser/fixture.ts +++ b/packages/core/test/browser/fixture.ts @@ -3,7 +3,7 @@ import { mkdirSync, mkdtempSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { RemoteBrowserFrame, RemoteBrowserUnchanged } from '@boite/contracts'; -import { connect, type CoreClient } from '../../src/client.ts'; +import { connect, SOCKET_OPTIONS, type CoreClient } from '../../src/client.ts'; import { chromiumArgs, findChromium, waitForEndpoint } from '../../src/browser/chromium.ts'; import { echoThread, startTestCore, type TestCore } from '../harness.ts'; @@ -148,7 +148,7 @@ export function createBrowserFixture() { try { const args = chromiumArgs(dir, 'linux').filter(arg => arg !== '--remote-debugging-pipe'); browser = Bun.spawn([findChromium().path!, ...args, '--remote-debugging-port=0', '--remote-debugging-address=127.0.0.1', ...extraArgs], { stdout: 'ignore', stderr: 'ignore' }); - const connectedSocket = socket = new WebSocket(await waitForEndpoint(dir, browser.exited)); + const connectedSocket = socket = new WebSocket(await waitForEndpoint(dir, browser.exited), SOCKET_OPTIONS as unknown as string[]); await new Promise((resolve, reject) => { const finish = (error?: Error) => { clearTimeout(timer); diff --git a/packages/core/test/socket-options.test.ts b/packages/core/test/socket-options.test.ts new file mode 100644 index 000000000..10fa3fa75 --- /dev/null +++ b/packages/core/test/socket-options.test.ts @@ -0,0 +1,28 @@ +import { expect, test } from 'bun:test'; +import { Cdp } from '../src/browser/cdp.ts'; +import { connect } from '../src/client.ts'; + +// Bun's client with permessage-deflate on sometimes closes a sound stream with +// 1002 (see SOCKET_OPTIONS): no socket this code opens may offer it. +test('the core client and the DevTools socket never offer permessage-deflate', async () => { + const offers: Array = []; + const server = Bun.serve({ + hostname: '127.0.0.1', port: 0, + fetch(request, self) { + offers.push(request.headers.get('sec-websocket-extensions')); + return self.upgrade(request) ? undefined : new Response('upgrade required'); + }, + websocket: { perMessageDeflate: true, message() {} }, + }); + try { + const url = `http://127.0.0.1:${server.port}`; + for (const headers of [undefined, { origin: url }]) { + await expect(connect(url, 'token', { timeoutMs: 200, headers })).rejects.toThrow(); + } + const cdp = await Cdp.connect(`ws://127.0.0.1:${server.port}/devtools/browser/test`); + cdp.close(); + expect(offers).toEqual([null, null, null]); + } finally { + server.stop(true); + } +}); diff --git a/tests/e2e/group-devices.test.ts b/tests/e2e/group-devices.test.ts index e978aca31..2c28d5968 100644 --- a/tests/e2e/group-devices.test.ts +++ b/tests/e2e/group-devices.test.ts @@ -8,7 +8,7 @@ import { startUi } from './lib/ui.ts'; import { startCore, type RunningCore } from './lib/core.ts'; import { mobileAction } from './lib/mobile.ts'; import { codeVideo } from './lib/qr-video.ts'; -import { connect, type CoreClient } from '../../packages/core/src/client.ts'; +import { connect, SOCKET_OPTIONS, type CoreClient } from '../../packages/core/src/client.ts'; /** * A phone is a device of the group, not of one machine. @@ -71,7 +71,7 @@ function proxy(core: RunningCore): ReturnType { websocket: { open(socket: ServerWebSocket) { const headers: Record = { host: socket.data.host, ...(socket.data.origin === null ? {} : { origin: socket.data.origin }) }; - const upstream = new WebSocket(`${core.url.replace(/^http/, 'ws')}${socket.data.path}`, { headers } as unknown as string[]); + const upstream = new WebSocket(`${core.url.replace(/^http/, 'ws')}${socket.data.path}`, { ...SOCKET_OPTIONS, headers } as unknown as string[]); upstream.binaryType = 'arraybuffer'; socket.data.upstream = upstream; upstream.onopen = () => { diff --git a/tests/e2e/lib/cdp.ts b/tests/e2e/lib/cdp.ts index 1b40f0819..233326d70 100644 --- a/tests/e2e/lib/cdp.ts +++ b/tests/e2e/lib/cdp.ts @@ -4,6 +4,7 @@ import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { ONBOARDING_STORAGE_KEY, ONBOARDING_VERSION } from '../../../packages/ui/src/lib/onboarding.ts'; import type { Subprocess } from 'bun'; +import { SOCKET_OPTIONS } from '../../../packages/core/src/client.ts'; import { E2E_DIR_PREFIX, killProcessTreeAsync, removeDirectory, watchParentDeath, type ParentDeathWatch } from './cleanup.ts'; import { relay } from './ws-relay.ts'; @@ -754,7 +755,7 @@ async function openSocket(target: TargetInfo): Promise { if (url === undefined) throw new Error('the page target has no debugger url'); const traced = TRACE ? await relay(url) : null; return new Promise((resolve, reject) => { - const socket = new WebSocket(traced?.url ?? url); + const socket = new WebSocket(traced?.url ?? url, SOCKET_OPTIONS as unknown as string[]); if (traced) { relays.set(socket, traced); socket.addEventListener('close', (event) => { From 3f7178d9d542c888ae4143e19078d34159e3357e Mon Sep 17 00:00:00 2001 From: meetsu <96637888+klNuno@users.noreply.github.com> Date: Mon, 19 Oct 2026 02:16:38 +0200 Subject: [PATCH 07/13] ci(flake-loop): loop macOS arm64 browser files two at once beside four --- .github/workflows/flake-loop.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/flake-loop.yml b/.github/workflows/flake-loop.yml index 3cd968dbc..9f39cf0c2 100644 --- a/.github/workflows/flake-loop.yml +++ b/.github/workflows/flake-loop.yml @@ -26,30 +26,32 @@ env: jobs: browser: - name: Core browser files (${{ matrix.name }}) + name: Core browser files (${{ matrix.name }}, ${{ matrix.parallel }} at once) if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'flake-loop') strategy: fail-fast: false matrix: include: - - { os: macos-15, name: macOS arm64 } - - { os: macos-15-intel, name: macOS x64 } - - { os: ubuntu-24.04, name: Linux x64 } + # Two at once against four: does the load of four Chromiums explain the macOS arm64 timeouts? + - { os: macos-15, name: macOS arm64, parallel: 4 } + - { os: macos-15, name: macOS arm64, parallel: 2 } + - { os: macos-15-intel, name: macOS x64, parallel: 4 } + - { os: ubuntu-24.04, name: Linux x64, parallel: 4 } runs-on: ${{ matrix.os }} timeout-minutes: 75 env: BOITE_CDP_TRACE: '1' - FLAKE_LOOP_TITLE: Core browser files, ${{ matrix.name }} + FLAKE_LOOP_TITLE: Core browser files, ${{ matrix.name }}, ${{ matrix.parallel }} at once steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: persist-credentials: false - uses: ./.github/actions/setup - - run: bun scripts/ci/flake-loop.ts --runs "$RUNS" --parallel 4 --cwd packages/core --out flake-loop test/browser-sharing.test.ts test/browser.test.ts test/browser-sleep.test.ts test/browser-profile-lifecycle.test.ts + - run: bun scripts/ci/flake-loop.ts --runs "$RUNS" --parallel ${{ matrix.parallel }} --cwd packages/core --out flake-loop test/browser-sharing.test.ts test/browser.test.ts test/browser-sleep.test.ts test/browser-profile-lifecycle.test.ts - uses: ./.github/actions/upload-artifact if: always() with: - name: flake-loop-browser-${{ matrix.os }} + name: flake-loop-browser-${{ matrix.os }}-${{ matrix.parallel }} path: flake-loop/ cli: From a1e598510744ea3254e73a7d3a6fc491967f892a Mon Sep 17 00:00:00 2001 From: meetsu <96637888+klNuno@users.noreply.github.com> Date: Mon, 19 Oct 2026 02:19:53 +0200 Subject: [PATCH 08/13] ci(flake-loop): count the devtools messages Chrome sent against those Bun delivered --- scripts/ci/ws-relay.test.ts | 2 +- tests/e2e/lib/cdp.ts | 10 +++++++++- tests/e2e/lib/ws-relay.ts | 5 ++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/scripts/ci/ws-relay.test.ts b/scripts/ci/ws-relay.test.ts index 5faaae555..7e6465ab2 100644 --- a/scripts/ci/ws-relay.test.ts +++ b/scripts/ci/ws-relay.test.ts @@ -23,7 +23,7 @@ test('the relay check reads a well-formed browser stream however the reads cut i for (const cut of [1, 2, 3, 7, 125, 4096, stream.length]) { const check = new FrameCheck(); for (let at = 0; at < stream.length; at += cut) check.push(stream.subarray(at, at + cut)); - expect(check.report()).toStartWith(`browser stream (extensions none) well-formed over 6 frames and ${stream.length} bytes`); + expect(check.report()).toStartWith(`browser stream (extensions none) well-formed over 6 frames, 4 messages and ${stream.length} bytes`); } }); diff --git a/tests/e2e/lib/cdp.ts b/tests/e2e/lib/cdp.ts index 233326d70..c9ae2ec41 100644 --- a/tests/e2e/lib/cdp.ts +++ b/tests/e2e/lib/cdp.ts @@ -185,6 +185,10 @@ export class BrowserPage { /** When devtools last sent anything, and how its socket closed. */ #heardAt = performance.now(); #closedBy: string | null = null; + /** Messages Bun handed over, and those that were not devtools JSON: a lost frame shows as a gap with the relay's count. */ + #heard = 0; + #unread = 0; + #firstUnread = ''; #stderr: () => string = () => ''; #pageErrors: string[] = []; /** Requests the page sent and has not finished loading, reported when a wait times out. */ @@ -599,6 +603,7 @@ In flight: ${JSON.stringify(this.#inFlight())}`); const ui = (globalThis as { __boiteUiTrace?: () => string }).__boiteUiTrace?.() ?? 'not traced'; const stderr = this.#stderr().trim().split('\n').slice(-12).join('\n'); return `\nTrace: last devtools message ${Math.round(now - this.#heardAt)} ms ago; socket ${this.#closedBy ?? `state ${this.#socket.readyState}`}; browser ${this.#proc === null ? 'attached' : this.#proc.exitCode === null ? `pid ${this.#proc.pid} running` : `exited ${this.#proc.exitCode} (signal ${this.#proc.signalCode})`} +Messages: ${this.#heard} delivered, ${this.#unread} unreadable${this.#firstUnread ? ` (first ${this.#firstUnread})` : ''} Test loop delays over 50 ms in 30 s: ${lags.length ? lags.slice(-8).join(', ') : 'none'} UI server: ${ui} Relay: ${relays.get(this.#socket)?.report() ?? 'none'} @@ -615,11 +620,14 @@ ${stderr || '(empty)'}`; #receive(raw: string): void { this.#heardAt = performance.now(); - if (raw === '') return; + this.#heard++; + if (raw === '') { this.#unread++; this.#firstUnread ||= 'a binary or empty message'; return; } let frame: { id?: unknown; method?: string; params?: { exceptionDetails?: { text?: string; exception?: { description?: string } }; requestId?: string; request?: { url?: string }; response?: { status?: number }; errorText?: string }; result?: unknown; error?: { message?: string } }; try { frame = JSON.parse(raw) as typeof frame; } catch { + this.#unread++; + this.#firstUnread ||= JSON.stringify(raw.slice(0, 80)); return; } if (typeof frame.id !== 'number') { diff --git a/tests/e2e/lib/ws-relay.ts b/tests/e2e/lib/ws-relay.ts index ac8ceefc5..218e25237 100644 --- a/tests/e2e/lib/ws-relay.ts +++ b/tests/e2e/lib/ws-relay.ts @@ -24,6 +24,8 @@ export class FrameCheck { /** The `Sec-WebSocket-Extensions` the browser accepted; `permessage-deflate` lets a message's first frame set RSV1. */ #extensions = ''; #frames = 0; + /** Complete data messages, to compare with what the test's socket delivered. */ + #messages = 0; #recent: Frame[] = []; #chunks: Array<{ at: number; length: number; ms: number }> = []; #error: string | null = null; @@ -67,6 +69,7 @@ export class FrameCheck { const frame = { at: this.#offset + at, opcode: first & 0x0f, fin: (first & 0x80) !== 0, compressed: (first & 0x40) !== 0, length }; this.#error = problem(first, second, frame, this.#extensions.includes('permessage-deflate')); this.#frames++; + if (frame.fin && frame.opcode <= 2) this.#messages++; this.#recent.push(frame); if (this.#recent.length > 12) this.#recent.shift(); const payload = Math.min(length, rest - header); @@ -90,7 +93,7 @@ export class FrameCheck { report(): string { const frames = this.#recent.map(frame => `${frame.at}:op${frame.opcode}${frame.compressed ? ' deflated' : ''}${frame.fin ? '' : ' unfinished'} ${frame.length} B`).join(', '); const reads = this.#chunks.map(chunk => `${chunk.at}+${chunk.length} at ${chunk.ms} ms`).join(', '); - return `browser stream (extensions ${this.#extensions || 'none'}) ${this.#error ? `malformed: ${this.#error}` : 'well-formed'} over ${this.#frames} frames and ${this.#total} bytes; last frames ${frames || 'none'}; last reads ${reads}`; + return `browser stream (extensions ${this.#extensions || 'none'}) ${this.#error ? `malformed: ${this.#error}` : 'well-formed'} over ${this.#frames} frames, ${this.#messages} messages and ${this.#total} bytes; last frames ${frames || 'none'}; last reads ${reads}`; } } From e50f0b09f08beac5854a5189662993179efaaf75 Mon Sep 17 00:00:00 2001 From: meetsu <96637888+klNuno@users.noreply.github.com> Date: Mon, 19 Oct 2026 05:42:01 +0200 Subject: [PATCH 09/13] Revert "ci(flake-loop): loop with #623 applied, reverted before merge" This reverts commit 9a007c10b2fbc986f1afdb44edb3e357b140d7c2. --- packages/core/src/browser/cdp.ts | 3 +-- packages/core/src/client.ts | 15 +++--------- packages/core/src/group/relay.ts | 3 +-- packages/core/test/browser/fixture.ts | 4 ++-- packages/core/test/socket-options.test.ts | 28 ----------------------- tests/e2e/group-devices.test.ts | 4 ++-- tests/e2e/lib/cdp.ts | 3 +-- 7 files changed, 10 insertions(+), 50 deletions(-) delete mode 100644 packages/core/test/socket-options.test.ts diff --git a/packages/core/src/browser/cdp.ts b/packages/core/src/browser/cdp.ts index 9f9eae001..78fd81d7f 100644 --- a/packages/core/src/browser/cdp.ts +++ b/packages/core/src/browser/cdp.ts @@ -11,7 +11,6 @@ import { fstatSync } from 'node:fs'; import { loadavg } from 'node:os'; -import { SOCKET_OPTIONS } from '../client.ts'; type Listener = (params: Record, sessionId: string | undefined) => void; interface Waiting { resolve(value: Record): void; reject(error: Error): void; timer: ReturnType; method: string } @@ -107,7 +106,7 @@ export class Cdp { /** The Windows transport: the browser's WebSocket endpoint on 127.0.0.1. */ static connect(url: string, timeoutMs = 10_000): Promise { return new Promise((resolve, reject) => { - const socket = new WebSocket(url, SOCKET_OPTIONS as unknown as string[]); + const socket = new WebSocket(url); const timer = setTimeout(() => { socket.close(); reject(new CdpError(`the browser did not accept a DevTools connection within ${timeoutMs / 1000} seconds`)); }, timeoutMs); socket.addEventListener('error', () => { clearTimeout(timer); reject(new CdpError(`could not connect to the browser's DevTools at ${url}`)); }, { once: true }); socket.addEventListener('open', () => { diff --git a/packages/core/src/client.ts b/packages/core/src/client.ts index 5fc3726d7..310c0696a 100644 --- a/packages/core/src/client.ts +++ b/packages/core/src/client.ts @@ -81,21 +81,12 @@ export class SocketClosedError extends Error { } } -/** - * What every WebSocket this code opens with Bun's client passes: no - * `permessage-deflate` offer. With the extension on, Bun 1.4.2 and 1.4.3 - * sometimes close a sound stream with 1002 ("control frame is fragmented", - * "unsupported control frame", "unexpected opcode"): against the core's own - * server and against Chrome's DevTools, whose bytes a relay checked frame by - * frame and found well-formed. Uncompressed, a message is larger on the wire; - * compressed, the connection drops. - */ -export const SOCKET_OPTIONS = { perMessageDeflate: false } as const; - export async function connect(url: string, token: string, options: ConnectOptions = {}): Promise { const timeoutMs = options.timeoutMs ?? 5000; const deadline = Date.now() + timeoutMs; - const socket = new WebSocket(socketUrl(url), { ...SOCKET_OPTIONS, ...(options.headers === undefined ? {} : { headers: options.headers }) } as unknown as string[]); + const socket = options.headers === undefined + ? new WebSocket(socketUrl(url)) + : new WebSocket(socketUrl(url), { headers: options.headers } as unknown as string[]); const pending = new Map(); const listeners = new Map void>>(); let nextId = 1; diff --git a/packages/core/src/group/relay.ts b/packages/core/src/group/relay.ts index 203fc27c2..afd1b2c9f 100644 --- a/packages/core/src/group/relay.ts +++ b/packages/core/src/group/relay.ts @@ -29,7 +29,6 @@ import type { ServerWebSocket } from 'bun'; import { FILE_ROUTE, GROUP_RELAY_ROUTE, PROTOCOL_VERSION, RPC_PATH, RpcErrorCode, VIEW_ROUTE } from '@boite/contracts'; -import { SOCKET_OPTIONS } from '../client.ts'; import type { Core } from '../core.ts'; import { messageOf } from '../errors.ts'; import { OUTBOUND_CONGESTION_TIMEOUT_MS, OUTBOUND_MAX_BUFFERED_BYTES } from '../server/connection.ts'; @@ -389,7 +388,7 @@ export class RelayPipe { const opened = await new Promise((resolve) => { let socket: Upstream; try { - socket = new WebSocket(`${target.replace(/^http/, 'ws')}${RPC_PATH}`, SOCKET_OPTIONS as unknown as string[]); + socket = new WebSocket(`${target.replace(/^http/, 'ws')}${RPC_PATH}`); } catch { resolve(null); return; diff --git a/packages/core/test/browser/fixture.ts b/packages/core/test/browser/fixture.ts index 8bd8a837c..03d59b68b 100644 --- a/packages/core/test/browser/fixture.ts +++ b/packages/core/test/browser/fixture.ts @@ -3,7 +3,7 @@ import { mkdirSync, mkdtempSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { RemoteBrowserFrame, RemoteBrowserUnchanged } from '@boite/contracts'; -import { connect, SOCKET_OPTIONS, type CoreClient } from '../../src/client.ts'; +import { connect, type CoreClient } from '../../src/client.ts'; import { chromiumArgs, findChromium, waitForEndpoint } from '../../src/browser/chromium.ts'; import { echoThread, startTestCore, type TestCore } from '../harness.ts'; @@ -148,7 +148,7 @@ export function createBrowserFixture() { try { const args = chromiumArgs(dir, 'linux').filter(arg => arg !== '--remote-debugging-pipe'); browser = Bun.spawn([findChromium().path!, ...args, '--remote-debugging-port=0', '--remote-debugging-address=127.0.0.1', ...extraArgs], { stdout: 'ignore', stderr: 'ignore' }); - const connectedSocket = socket = new WebSocket(await waitForEndpoint(dir, browser.exited), SOCKET_OPTIONS as unknown as string[]); + const connectedSocket = socket = new WebSocket(await waitForEndpoint(dir, browser.exited)); await new Promise((resolve, reject) => { const finish = (error?: Error) => { clearTimeout(timer); diff --git a/packages/core/test/socket-options.test.ts b/packages/core/test/socket-options.test.ts deleted file mode 100644 index 10fa3fa75..000000000 --- a/packages/core/test/socket-options.test.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { expect, test } from 'bun:test'; -import { Cdp } from '../src/browser/cdp.ts'; -import { connect } from '../src/client.ts'; - -// Bun's client with permessage-deflate on sometimes closes a sound stream with -// 1002 (see SOCKET_OPTIONS): no socket this code opens may offer it. -test('the core client and the DevTools socket never offer permessage-deflate', async () => { - const offers: Array = []; - const server = Bun.serve({ - hostname: '127.0.0.1', port: 0, - fetch(request, self) { - offers.push(request.headers.get('sec-websocket-extensions')); - return self.upgrade(request) ? undefined : new Response('upgrade required'); - }, - websocket: { perMessageDeflate: true, message() {} }, - }); - try { - const url = `http://127.0.0.1:${server.port}`; - for (const headers of [undefined, { origin: url }]) { - await expect(connect(url, 'token', { timeoutMs: 200, headers })).rejects.toThrow(); - } - const cdp = await Cdp.connect(`ws://127.0.0.1:${server.port}/devtools/browser/test`); - cdp.close(); - expect(offers).toEqual([null, null, null]); - } finally { - server.stop(true); - } -}); diff --git a/tests/e2e/group-devices.test.ts b/tests/e2e/group-devices.test.ts index 2c28d5968..e978aca31 100644 --- a/tests/e2e/group-devices.test.ts +++ b/tests/e2e/group-devices.test.ts @@ -8,7 +8,7 @@ import { startUi } from './lib/ui.ts'; import { startCore, type RunningCore } from './lib/core.ts'; import { mobileAction } from './lib/mobile.ts'; import { codeVideo } from './lib/qr-video.ts'; -import { connect, SOCKET_OPTIONS, type CoreClient } from '../../packages/core/src/client.ts'; +import { connect, type CoreClient } from '../../packages/core/src/client.ts'; /** * A phone is a device of the group, not of one machine. @@ -71,7 +71,7 @@ function proxy(core: RunningCore): ReturnType { websocket: { open(socket: ServerWebSocket) { const headers: Record = { host: socket.data.host, ...(socket.data.origin === null ? {} : { origin: socket.data.origin }) }; - const upstream = new WebSocket(`${core.url.replace(/^http/, 'ws')}${socket.data.path}`, { ...SOCKET_OPTIONS, headers } as unknown as string[]); + const upstream = new WebSocket(`${core.url.replace(/^http/, 'ws')}${socket.data.path}`, { headers } as unknown as string[]); upstream.binaryType = 'arraybuffer'; socket.data.upstream = upstream; upstream.onopen = () => { diff --git a/tests/e2e/lib/cdp.ts b/tests/e2e/lib/cdp.ts index c9ae2ec41..b1a9280e8 100644 --- a/tests/e2e/lib/cdp.ts +++ b/tests/e2e/lib/cdp.ts @@ -4,7 +4,6 @@ import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { ONBOARDING_STORAGE_KEY, ONBOARDING_VERSION } from '../../../packages/ui/src/lib/onboarding.ts'; import type { Subprocess } from 'bun'; -import { SOCKET_OPTIONS } from '../../../packages/core/src/client.ts'; import { E2E_DIR_PREFIX, killProcessTreeAsync, removeDirectory, watchParentDeath, type ParentDeathWatch } from './cleanup.ts'; import { relay } from './ws-relay.ts'; @@ -763,7 +762,7 @@ async function openSocket(target: TargetInfo): Promise { if (url === undefined) throw new Error('the page target has no debugger url'); const traced = TRACE ? await relay(url) : null; return new Promise((resolve, reject) => { - const socket = new WebSocket(traced?.url ?? url, SOCKET_OPTIONS as unknown as string[]); + const socket = new WebSocket(traced?.url ?? url); if (traced) { relays.set(socket, traced); socket.addEventListener('close', (event) => { From 2cce06d183fff6c2af697f0a35a16831d1b83928 Mon Sep 17 00:00:00 2001 From: meetsu <96637888+klNuno@users.noreply.github.com> Date: Mon, 19 Oct 2026 05:56:48 +0200 Subject: [PATCH 10/13] ci(flake-loop): loop the macOS browser files with the temp folder on disk and in RAM --- .github/workflows/flake-loop.yml | 33 ++++++++++++++++++++------------ docs/ci.md | 5 +++-- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/.github/workflows/flake-loop.yml b/.github/workflows/flake-loop.yml index dc2ca1834..6d1b51530 100644 --- a/.github/workflows/flake-loop.yml +++ b/.github/workflows/flake-loop.yml @@ -26,34 +26,43 @@ env: jobs: browser: - name: Core browser files (${{ matrix.name }}, ${{ matrix.parallel }} at once) - # Paused while the E2E loop measures the UI stalls: run 39180308980 already - # compared two at once (1/10) with four (3/10) on macOS arm64. - if: false && (github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'flake-loop')) + name: Core browser files (${{ matrix.name }}, ${{ matrix.parallel }} at once, temp on ${{ matrix.temp }}) + if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'flake-loop') strategy: fail-fast: false matrix: include: - # Two at once against four: does the load of four Chromiums explain the macOS arm64 timeouts? - - { os: macos-15, name: macOS arm64, parallel: 4 } - - { os: macos-15, name: macOS arm64, parallel: 2 } - - { os: macos-15-intel, name: macOS x64, parallel: 4 } - - { os: ubuntu-24.04, name: Linux x64, parallel: 4 } + # Chromium stopped answering with its processes idle and a load of 9 to 29: + # does the runner's disk stall it? The same loop with the temp folder in RAM. + - { os: macos-15, name: macOS arm64, parallel: 4, temp: disk } + - { os: macos-15, name: macOS arm64, parallel: 4, temp: ram } runs-on: ${{ matrix.os }} timeout-minutes: 75 env: BOITE_CDP_TRACE: '1' - FLAKE_LOOP_TITLE: Core browser files, ${{ matrix.name }}, ${{ matrix.parallel }} at once + FLAKE_LOOP_TITLE: Core browser files, ${{ matrix.name }}, ${{ matrix.parallel }} at once, temp on ${{ matrix.temp }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: persist-credentials: false - uses: ./.github/actions/setup - - run: bun scripts/ci/flake-loop.ts --runs "$RUNS" --parallel ${{ matrix.parallel }} --cwd packages/core --out flake-loop test/browser-sharing.test.ts test/browser.test.ts test/browser-sleep.test.ts test/browser-profile-lifecycle.test.ts + - name: Temp folder in RAM + if: matrix.temp == 'ram' + run: | + # 2 GiB in 512-byte sectors. + device=$(hdiutil attach -nomount ram://4194304 | tr -d '[:space:]') + diskutil erasevolume APFS boite-ram "$device" + mkdir -p /Volumes/boite-ram/tmp + echo "TMPDIR=/Volumes/boite-ram/tmp/" >> "$GITHUB_ENV" + - run: | + mkdir -p flake-loop + # Disk throughput and load each second, to line a stall up with the disk. + (while true; do echo "$(date -u +%H:%M:%S) $(iostat -d -c 2 -w 1 disk0 | tail -1) $(sysctl -n vm.loadavg)"; done > flake-loop/iostat.txt 2>&1 &) + bun scripts/ci/flake-loop.ts --runs "$RUNS" --parallel ${{ matrix.parallel }} --cwd packages/core --out flake-loop test/browser-sharing.test.ts test/browser.test.ts test/browser-sleep.test.ts test/browser-profile-lifecycle.test.ts - uses: ./.github/actions/upload-artifact if: always() with: - name: flake-loop-browser-${{ matrix.os }}-${{ matrix.parallel }} + name: flake-loop-browser-${{ matrix.os }}-${{ matrix.parallel }}-${{ matrix.temp }} path: flake-loop/ cli: diff --git a/docs/ci.md b/docs/ci.md index 31f5bffc6..f03e4bb31 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -123,8 +123,9 @@ tests that failed at least three first attempts. A fixed test leaves the list; `flaky.test.ts` fails when an entry no longer exists in its file. The `flake-loop` workflow measures a flaky test before and after its fix. It runs on a pull request that carries the `flake-loop` label, or by hand: the -quarantined browser files four at a time on macOS arm64, macOS Intel and -Linux, `cli.test.ts` four copies at a time on every OS, and +quarantined browser files four at a time on macOS arm64, once with the temp +folder on the disk and once on a RAM disk, with disk throughput logged each +second (`iostat.txt`), `cli.test.ts` four copies at a time on every OS, and `coordination-bridge-live.test.ts` and `machines.test.ts` alone and each E2E shard whole on Windows, with the runner's disk latency, queue length and Defender CPU logged every second (`perf.csv`). From f55219939cfa85980a8c5523a733cdddd2aa5f77 Mon Sep 17 00:00:00 2001 From: meetsu <96637888+klNuno@users.noreply.github.com> Date: Mon, 19 Oct 2026 06:57:11 +0200 Subject: [PATCH 11/13] ci(flake-loop): loop the macOS browser files with Spotlight on and off --- .github/workflows/flake-loop.yml | 28 ++++++++++++++-------------- docs/ci.md | 6 +++--- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/flake-loop.yml b/.github/workflows/flake-loop.yml index 6d1b51530..891534ede 100644 --- a/.github/workflows/flake-loop.yml +++ b/.github/workflows/flake-loop.yml @@ -26,43 +26,43 @@ env: jobs: browser: - name: Core browser files (${{ matrix.name }}, ${{ matrix.parallel }} at once, temp on ${{ matrix.temp }}) + name: Core browser files (${{ matrix.name }}, ${{ matrix.parallel }} at once, Spotlight ${{ matrix.spotlight }}) if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'flake-loop') strategy: fail-fast: false matrix: include: - # Chromium stopped answering with its processes idle and a load of 9 to 29: - # does the runner's disk stall it? The same loop with the temp folder in RAM. - - { os: macos-15, name: macOS arm64, parallel: 4, temp: disk } - - { os: macos-15, name: macOS arm64, parallel: 4, temp: ram } + # Chromium stopped answering at a load of 25 to 53, with Spotlight's mdworker_shared and mds + # the busiest processes, temp folder on disk or in RAM alike (run 39197082818). The same loop + # with indexing off. + - { os: macos-15, name: macOS arm64, parallel: 4, spotlight: on } + - { os: macos-15, name: macOS arm64, parallel: 4, spotlight: off } runs-on: ${{ matrix.os }} timeout-minutes: 75 env: BOITE_CDP_TRACE: '1' - FLAKE_LOOP_TITLE: Core browser files, ${{ matrix.name }}, ${{ matrix.parallel }} at once, temp on ${{ matrix.temp }} + FLAKE_LOOP_TITLE: Core browser files, ${{ matrix.name }}, ${{ matrix.parallel }} at once, Spotlight ${{ matrix.spotlight }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: persist-credentials: false - uses: ./.github/actions/setup - - name: Temp folder in RAM - if: matrix.temp == 'ram' + - name: Spotlight off + if: matrix.spotlight == 'off' run: | - # 2 GiB in 512-byte sectors. - device=$(hdiutil attach -nomount ram://4194304 | tr -d '[:space:]') - diskutil erasevolume APFS boite-ram "$device" - mkdir -p /Volumes/boite-ram/tmp - echo "TMPDIR=/Volumes/boite-ram/tmp/" >> "$GITHUB_ENV" + sudo mdutil -a -i off + mdutil -a -s - run: | mkdir -p flake-loop # Disk throughput and load each second, to line a stall up with the disk. (while true; do echo "$(date -u +%H:%M:%S) $(iostat -d -c 2 -w 1 disk0 | tail -1) $(sysctl -n vm.loadavg)"; done > flake-loop/iostat.txt 2>&1 &) + # The five busiest processes every 5 s, to see what indexes or scans while Chromium stalls. + (while true; do echo "== $(date -u +%H:%M:%S)"; ps -Aro pcpu,rss,comm | head -6; sleep 5; done > flake-loop/top.txt 2>&1 &) bun scripts/ci/flake-loop.ts --runs "$RUNS" --parallel ${{ matrix.parallel }} --cwd packages/core --out flake-loop test/browser-sharing.test.ts test/browser.test.ts test/browser-sleep.test.ts test/browser-profile-lifecycle.test.ts - uses: ./.github/actions/upload-artifact if: always() with: - name: flake-loop-browser-${{ matrix.os }}-${{ matrix.parallel }}-${{ matrix.temp }} + name: flake-loop-browser-${{ matrix.os }}-${{ matrix.parallel }}-spotlight-${{ matrix.spotlight }} path: flake-loop/ cli: diff --git a/docs/ci.md b/docs/ci.md index f03e4bb31..427643e27 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -123,9 +123,9 @@ tests that failed at least three first attempts. A fixed test leaves the list; `flaky.test.ts` fails when an entry no longer exists in its file. The `flake-loop` workflow measures a flaky test before and after its fix. It runs on a pull request that carries the `flake-loop` label, or by hand: the -quarantined browser files four at a time on macOS arm64, once with the temp -folder on the disk and once on a RAM disk, with disk throughput logged each -second (`iostat.txt`), `cli.test.ts` four copies at a time on every OS, and +quarantined browser files four at a time on macOS arm64, once with Spotlight +indexing and once without, with disk throughput and load logged each second +(`iostat.txt`) and the busiest processes every 5 s (`top.txt`), `cli.test.ts` four copies at a time on every OS, and `coordination-bridge-live.test.ts` and `machines.test.ts` alone and each E2E shard whole on Windows, with the runner's disk latency, queue length and Defender CPU logged every second (`perf.csv`). From b0872c34dc3ee036b250ef2a965288407cd9190f Mon Sep 17 00:00:00 2001 From: meetsu <96637888+klNuno@users.noreply.github.com> Date: Mon, 19 Oct 2026 06:58:46 +0200 Subject: [PATCH 12/13] ci(flake-loop): loop cli.test.ts with the pinned Bun and canary --- .github/workflows/flake-loop.yml | 21 +++++++++++++-------- docs/ci.md | 2 +- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/flake-loop.yml b/.github/workflows/flake-loop.yml index 891534ede..7d9a6df03 100644 --- a/.github/workflows/flake-loop.yml +++ b/.github/workflows/flake-loop.yml @@ -66,31 +66,36 @@ jobs: path: flake-loop/ cli: - name: cli.test.ts (${{ matrix.name }}) + name: cli.test.ts (${{ matrix.name }}, Bun ${{ matrix.bun }}) if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'flake-loop') strategy: fail-fast: false matrix: + # The 1002 closes: 1 in 480 runs before #623, 3 in 720 with it, never on Windows. The pinned Bun + # against canary and its WebSocket rewrite, ten times longer for a rate that small. include: - - { os: ubuntu-24.04, name: Linux x64 } - - { os: macos-15, name: macOS arm64 } - - { os: windows-2025, name: Windows x64 } + - { os: ubuntu-24.04, name: Linux x64, bun: pinned } + - { os: ubuntu-24.04, name: Linux x64, bun: canary } + - { os: macos-15, name: macOS arm64, bun: pinned } + - { os: macos-15, name: macOS arm64, bun: canary } runs-on: ${{ matrix.os }} - timeout-minutes: 30 + timeout-minutes: 45 env: - FLAKE_LOOP_TITLE: cli.test.ts, ${{ matrix.name }} + FLAKE_LOOP_TITLE: cli.test.ts, ${{ matrix.name }}, Bun ${{ matrix.bun }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: persist-credentials: false - uses: ./.github/actions/setup + - if: matrix.bun == 'canary' + run: bun upgrade --canary && bun --revision # Four copies at once load the runner as the core suite's four workers do. - shell: bash - run: bun scripts/ci/flake-loop.ts --runs "$((RUNS * 2))" --parallel 4 --cwd packages/core --out flake-loop test/cli.test.ts test/cli.test.ts test/cli.test.ts test/cli.test.ts + run: bun scripts/ci/flake-loop.ts --runs "$((RUNS * 20))" --parallel 4 --cwd packages/core --out flake-loop test/cli.test.ts test/cli.test.ts test/cli.test.ts test/cli.test.ts - uses: ./.github/actions/upload-artifact if: always() with: - name: flake-loop-cli-${{ matrix.os }} + name: flake-loop-cli-${{ matrix.os }}-${{ matrix.bun }} path: flake-loop/ e2e: diff --git a/docs/ci.md b/docs/ci.md index 427643e27..3a5d6a730 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -125,7 +125,7 @@ The `flake-loop` workflow measures a flaky test before and after its fix. It runs on a pull request that carries the `flake-loop` label, or by hand: the quarantined browser files four at a time on macOS arm64, once with Spotlight indexing and once without, with disk throughput and load logged each second -(`iostat.txt`) and the busiest processes every 5 s (`top.txt`), `cli.test.ts` four copies at a time on every OS, and +(`iostat.txt`) and the busiest processes every 5 s (`top.txt`), `cli.test.ts` four copies at a time on Linux and macOS, with the pinned Bun and with canary, and `coordination-bridge-live.test.ts` and `machines.test.ts` alone and each E2E shard whole on Windows, with the runner's disk latency, queue length and Defender CPU logged every second (`perf.csv`). From 5149d5542a546f1afcd8105193dc9fed66d7820d Mon Sep 17 00:00:00 2001 From: meetsu <96637888+klNuno@users.noreply.github.com> Date: Mon, 19 Oct 2026 08:49:55 +0200 Subject: [PATCH 13/13] ci(flake-loop): loop only cli.test.ts on macOS until #623 is reverted --- .github/workflows/flake-loop.yml | 15 ++++++--------- docs/ci.md | 2 +- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/flake-loop.yml b/.github/workflows/flake-loop.yml index 7d9a6df03..85e006197 100644 --- a/.github/workflows/flake-loop.yml +++ b/.github/workflows/flake-loop.yml @@ -27,7 +27,8 @@ env: jobs: browser: name: Core browser files (${{ matrix.name }}, ${{ matrix.parallel }} at once, Spotlight ${{ matrix.spotlight }}) - if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'flake-loop') + # Parked while the cli loop runs alone after #635: set the FLAKE_LOOP_ALL variable to 1 to run it. + if: vars.FLAKE_LOOP_ALL == '1' strategy: fail-fast: false matrix: @@ -71,13 +72,10 @@ jobs: strategy: fail-fast: false matrix: - # The 1002 closes: 1 in 480 runs before #623, 3 in 720 with it, never on Windows. The pinned Bun - # against canary and its WebSocket rewrite, ten times longer for a rate that small. + # The 1002 closes on macOS, 200 runs four at a time: 5 with #623, 11 (as 1006) on Bun canary. + # The same loop once #623 is reverted (#635). include: - - { os: ubuntu-24.04, name: Linux x64, bun: pinned } - - { os: ubuntu-24.04, name: Linux x64, bun: canary } - { os: macos-15, name: macOS arm64, bun: pinned } - - { os: macos-15, name: macOS arm64, bun: canary } runs-on: ${{ matrix.os }} timeout-minutes: 45 env: @@ -87,8 +85,6 @@ jobs: with: persist-credentials: false - uses: ./.github/actions/setup - - if: matrix.bun == 'canary' - run: bun upgrade --canary && bun --revision # Four copies at once load the runner as the core suite's four workers do. - shell: bash run: bun scripts/ci/flake-loop.ts --runs "$((RUNS * 20))" --parallel 4 --cwd packages/core --out flake-loop test/cli.test.ts test/cli.test.ts test/cli.test.ts test/cli.test.ts @@ -100,7 +96,8 @@ jobs: e2e: name: E2E ${{ matrix.mode }} - if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'flake-loop') + # Parked while the cli loop runs alone after #635: set the FLAKE_LOOP_ALL variable to 1 to run it. + if: vars.FLAKE_LOOP_ALL == '1' strategy: fail-fast: false matrix: diff --git a/docs/ci.md b/docs/ci.md index 3a5d6a730..d99034d3a 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -125,7 +125,7 @@ The `flake-loop` workflow measures a flaky test before and after its fix. It runs on a pull request that carries the `flake-loop` label, or by hand: the quarantined browser files four at a time on macOS arm64, once with Spotlight indexing and once without, with disk throughput and load logged each second -(`iostat.txt`) and the busiest processes every 5 s (`top.txt`), `cli.test.ts` four copies at a time on Linux and macOS, with the pinned Bun and with canary, and +(`iostat.txt`) and the busiest processes every 5 s (`top.txt`), `cli.test.ts` four copies at a time on macOS (the browser and E2E jobs run only with the `FLAKE_LOOP_ALL` variable set to 1), and `coordination-bridge-live.test.ts` and `machines.test.ts` alone and each E2E shard whole on Windows, with the runner's disk latency, queue length and Defender CPU logged every second (`perf.csv`).