From 001328823111bf5fd372f310b58d8e7bf066465e Mon Sep 17 00:00:00 2001 From: Exoridus Date: Wed, 9 Sep 2026 03:13:09 +0200 Subject: [PATCH 1/5] feat(site): lead the benchmarks page with a scoreboard and drawn ratios The page opened with a paragraph of generated prose and then repeated every comparison as a wide table whose mechanism column wrapped over several lines, so the numbers were the hardest thing on it to find. It ran to about 10500 px at 1440 px wide, and the practices it is published under sat above the results. Rearrange it around the figures. A scoreboard leads with one line per arm - a rendering backend against one opponent, or one physics arm - showing how that pair's comparisons came out as a strip and as counts; nothing is summed across lines, because the arms answer different questions. Under it, one table per domain whose columns are the pairs and whose rows are archetypes, so a single archetype reads across both backends and every opponent on one line. The ratio is drawn as a bar on a log axis with the published factor beside it, and the medians, the p95 of the same window and the range the pooled runs observed sit under it on lines that never wrap. Methodology, reproduction and fairness move below the tables. A comparison whose runs disagreed no longer prints a verdict placeholder and a sentence about what each run concluded. It keeps its numbers and shows the ratio band those runs spanned, drawn without a side - the one figure the page derives rather than reads, from the published extremes of both arms, so a cell the measurement could not settle still says what was measured. The docs shell caps its width for prose, which a table page has no use for, so DocsLayout gains a wide variant. The benchmark hues become their own tokens: the shared semantic palette is tuned for the dark surface and washes out under a 6 px bar on the light one. Claude-Session: https://claude.ai/code/session_01Y3YY9obdzhWK7nBPby2jK5 --- .../src/components/BenchComparisonTable.astro | 538 +++++++++++ site/src/components/BenchMeasurement.astro | 93 -- site/src/components/BenchProfileReport.astro | 887 +++++------------- site/src/components/BenchScoreboard.astro | 214 +++++ .../src/components/pages/BenchmarksPage.astro | 512 +++++----- site/src/layouts/DocsLayout.astro | 21 +- site/src/lib/bench-profiles.ts | 297 +++--- site/src/lib/bench-tables.ts | 257 +++++ site/src/styles/tokens.scss | 13 + 9 files changed, 1632 insertions(+), 1200 deletions(-) create mode 100644 site/src/components/BenchComparisonTable.astro delete mode 100644 site/src/components/BenchMeasurement.astro create mode 100644 site/src/components/BenchScoreboard.astro create mode 100644 site/src/lib/bench-tables.ts diff --git a/site/src/components/BenchComparisonTable.astro b/site/src/components/BenchComparisonTable.astro new file mode 100644 index 000000000..ded1934b4 --- /dev/null +++ b/site/src/components/BenchComparisonTable.astro @@ -0,0 +1,538 @@ +--- +/** + * BenchComparisonTable - one archetype per row, one comparison pair per column. + * + * The ratio is the statement, so it is drawn: a bar on a log axis that runs left + * from the centre when ExoJS leads and right when the arm does, with the + * published factor beside it. The milliseconds, the p95 of the same window and + * the range the pooled runs observed sit under it as the evidence, never behind + * a toggle. Both sides of a comparison get the same bar, the same type and the + * same weight; only the hue differs. + * + * A pair whose runs reached different verdicts publishes none: it keeps its + * numbers and shows the ratio band those runs spanned, drawn without a side, so + * a cell the measurement could not settle still says what was measured. A pair + * that produced no comparable cell says so in place of the bar rather than + * leaving the column blank. + * + * The axis is scaled to the widest ratio the table itself contains, up to a cap, + * and printed with it: a table whose axis had to hold a thousand-fold outlier + * would draw every ordinary row as a few pixels. A bar past the cap is drawn + * cut, so it still reads as an outlier, and its exact factor is printed beside + * it. Because each table states its own axis, two tables are never read against + * each other by bar length alone. + */ + +import { + describeRungs, + formatBand, + formatFactor, + formatMs, + formatRange, + isWideSpread, + outcomeOf, + ratioBand, + type ProfileCell, + type ProfileSpread, +} from '../lib/bench-profiles'; +import type { ComparisonTable } from '../lib/bench-tables'; + +interface Props { + table: ComparisonTable; + /** Accessible name for the table, since the caption carries the reading terms rather than a title. */ + label: string; +} + +const { table, label } = Astro.props; +const { columns, rows, unit, countColumn } = table; + +/** Every ratio the table draws, including both ends of a disagreeing pair's band. */ +const ratios = rows + .flatMap(row => + row.entries.flatMap(entry => { + const cell = entry.cell; + + if (cell === null) return []; + + if (!cell.aggregate.stable) { + const band = ratioBand(cell); + + return band === null ? [] : [band.low, band.high]; + } + + return cell.verdict.ratio === null ? [] : [cell.verdict.ratio]; + }), + ) + .filter(ratio => Number.isFinite(ratio) && ratio > 0); + +/** + * Widest half-axis the table will draw, in decades. + * + * A single outlying comparison can be three orders of magnitude wide, and an + * axis stretched to hold it would compress every ordinary row into a few + * pixels. Past this point a bar is drawn cut rather than scaled down, so the + * rows a reader is comparing stay legible and the outlier still reads as one. + * Its exact factor is printed beside it either way. + */ +const AXIS_CAP_DECADES = 2; + +/** Half-width of the axis in decades: the widest ratio present, up to the cap. */ +const decades = Math.min(AXIS_CAP_DECADES, Math.max(0.5, ...ratios.map(ratio => Math.abs(Math.log10(ratio))))); + +/** True where at least one bar runs past the axis and is drawn cut. */ +const clips = ratios.some(ratio => Math.abs(Math.log10(ratio)) > decades); + +/** Where a ratio sits on the axis, as a percentage from its left edge. */ +const positionOf = (ratio: number): number => { + const clamped = Math.max(-decades, Math.min(decades, Math.log10(ratio))); + + return 50 + (clamped / decades) * 50; +}; + +interface BarShape { + readonly outcome: string; + readonly style: string; + readonly band: boolean; + /** Which end of the axis the bar was cut at, if it ran past it. */ + readonly clipped: 'left' | 'right' | null; +} + +const shapeOf = (cell: ProfileCell): BarShape | null => { + const outcome = outcomeOf(cell); + + if (outcome === 'unstable') { + const band = ratioBand(cell); + + if (band === null) return null; + + const from = positionOf(band.low); + const to = positionOf(band.high); + + return { + outcome, + style: `left:${String(Math.min(from, to))}%;width:${String(Math.max(1.5, Math.abs(to - from)))}%`, + band: true, + clipped: Math.abs(Math.log10(band.low)) > decades ? 'left' : Math.abs(Math.log10(band.high)) > decades ? 'right' : null, + }; + } + + const ratio = cell.verdict.ratio; + + if (ratio === null || !Number.isFinite(ratio) || ratio <= 0) return null; + + const at = positionOf(ratio); + const style = at < 50 ? `right:50%;width:${String(50 - at)}%` : `left:50%;width:${String(at - 50)}%`; + const past = Math.abs(Math.log10(ratio)) > decades; + + return { outcome, style, band: false, clipped: past ? (ratio < 1 ? 'left' : 'right') : null }; +}; + +/** The figure printed beside the bar: the published factor, or the band a disagreeing pair spanned. */ +const figureOf = (cell: ProfileCell): string => { + if (outcomeOf(cell) !== 'unstable') return formatFactor(cell.verdict.factor); + + const band = ratioBand(cell); + + return band === null ? 'runs split' : formatBand(band); +}; + +/** Consecutive columns that share a heading, such as the backend they were measured on. */ +const groups = columns.reduce<{ label: string; span: number }[]>((built, column) => { + if (column.group === null) return built; + + const last = built.at(-1); + + if (last !== undefined && last.label === column.group) { + last.span += 1; + + return built; + } + + return [...built, { label: column.group, span: 1 }]; +}, []); + +const rangeOf = (spread: ProfileSpread | undefined): { text: string; wide: boolean } => ({ + text: spread === undefined || formatRange(spread) === '' ? '-' : formatRange(spread), + wide: spread !== undefined && isWideSpread(spread), +}); +--- + +
+
+ + + {groups.length > 0 && ( + + + ))} + + )} + + + {countColumn && } + {columns.map(column => ( + + ))} + + + + {rows.map((row, index) => ( + <> + {row.section !== null && row.section !== rows[index - 1]?.section && ( + + + + )} + + + {countColumn && } + {row.entries.map(entry => { + if (entry.kind === 'reference') { + const reference = entry.reference; + const spread = rangeOf(reference?.spread); + + return ( + + ); + } + + const cell = entry.cell; + + if (cell === null) { + return ( + + ); + } + + const shape = shapeOf(cell); + const outcome = outcomeOf(cell); + const spread = rangeOf(cell.aggregate.competitor); + + return ( + + ); + })} + + + ))} + +
+ {groups.map(group => ( + + {group.label} +
Archetype{unit === 'bodies' ? 'Bodies' : 'Nodes'} + {column.overline !== '' && {column.overline}} + {column.label} +
{row.section}
{row.archetype}{row.count} + + + {formatMs(reference?.ms ?? null)} ms + + + p95 {formatMs(reference?.p95Ms ?? null)} · {spread.text} + + + + + - + + + {entry.count === null ? 'not measured here' : `not comparable at ${String(entry.count)} ${unit}`} + + + + + + {shape !== null && ( + + )} + + {figureOf(cell)} + + + {formatMs(cell.competitorMs)} ms + + + p95 {formatMs(cell.competitorP95Ms)} · {spread.text} + +
+
+
+ The bar is log-scaled and reaches {formatFactor(Math.pow(10, decades))} to each side of the centre, which is 1.00x + {clips ? '; a bar past that is drawn cut and keeps its exact factor beside it' : ''}. Under every time: the p95 of the same window, and the range the + pooled runs observed for the median. +
+
+ + diff --git a/site/src/components/BenchMeasurement.astro b/site/src/components/BenchMeasurement.astro deleted file mode 100644 index 96d8f579a..000000000 --- a/site/src/components/BenchMeasurement.astro +++ /dev/null @@ -1,93 +0,0 @@ ---- -/** - * BenchMeasurement - one published median together with the p95 of the same - * timed window and the range the pooled runs observed behind the median. - * - * All three are part of the value rather than annotations on it. A median whose - * runs agreed to three decimals and one whose runs moved by half again as much - * as the band the verdict ladder calls noise are different kinds of claim, and a - * reader shown only the median cannot tell them apart. The p95 makes the same - * kind of distinction inside a single run: a workload that is periodically - * expensive has a comfortable median and a hitch the player feels. - * - * Two conditions are marked, in one visual language: a range that reaches into - * the ladder's own noise band, and a median past a whole 60 fps frame. Both - * print the fact rather than only colouring the number, so a reader scanning the - * column does not have to divide two numbers or hold a threshold in their head. - * - * A value no run produced prints a dash and nothing else: an absence, never a - * zero. - */ - -import { formatFactor, formatMs, formatRange, FRAME_BUDGET_MS, isWideSpread, type ProfileSpread } from '../lib/bench-profiles'; - -interface Props { - /** The published value: median of the per-run medians. */ - ms: number | null; - /** The 95th percentile of the same timed window, pooled the same way. */ - p95Ms?: number | null; - /** True when the median is past a whole 60 fps frame, as the profile recorded it. */ - overFrameBudget?: boolean; - /** Spread of the per-run medians behind it; omitted where the arm produced no cell. */ - spread?: ProfileSpread | undefined; -} - -const { ms, p95Ms = null, overFrameBudget = false, spread } = Astro.props; -const range = spread === undefined ? '' : formatRange(spread); -const wide = spread !== undefined && isWideSpread(spread); ---- - -{ - ms === null ? ( - - - ) : ( - <> - {formatMs(ms)} - {p95Ms !== null && p95 {formatMs(p95Ms)}} - {overFrameBudget && over the {FRAME_BUDGET_MS} ms frame} - {range !== '' && ( - - {range} - {wide && {formatFactor(spread?.ratio ?? null)} spread} - - )} - - ) -} - - diff --git a/site/src/components/BenchProfileReport.astro b/site/src/components/BenchProfileReport.astro index c1f23506d..06defa9f4 100644 --- a/site/src/components/BenchProfileReport.astro +++ b/site/src/components/BenchProfileReport.astro @@ -2,45 +2,32 @@ /** * BenchProfileReport - one published machine profile, rendered whole. * - * Everything on screen is read out of the profile document: the provenance, the - * per-backend tables, the physics table, the disclosed caveats and the rows the - * harness measured but kept out of the comparison. Nothing is authored here and - * nothing is filtered out, so a re-measurement changes the page and no prose is - * left behind that the numbers no longer support. + * The order is the argument: how the comparisons came out, then the tables they + * came out of, then everything a reader needs only once they want to check a + * row - the structural evidence, the rows the harness kept out, the caveats the + * runs disclosed, and the provenance. Nothing is authored here and nothing is + * filtered out, so a re-measurement changes the page and leaves no prose behind + * that the numbers no longer support. * - * A cell where ExoJS trails carries the same weight as one where it leads: the - * two sides differ by hue and label, never by prominence. A missing cell is - * shown as "not comparable" rather than dropped - an arm that sat an archetype - * out or exceeded the harness's runaway budget produced no number, which is a - * result and not a defect. - * - * Every published number is pooled from several separate runs, so every number - * is printed with the p95 of the same timed window and the range those runs - * observed underneath it. Two conditions are marked rather than left for the - * reader to compute: a range wide enough to reach into the verdict ladder's own - * noise band, and a median past a whole 60 fps frame. A cell whose runs reached - * different verdicts prints no verdict: it keeps its numbers and its range and - * states what each run concluded instead. - * - * The physics table gives the body count its own column, because the physics - * archetypes carry per-archetype ladders and its rows are therefore measured at - * different sizes. The rendering tables state their single node count in the - * block note instead, which is where it belongs when every row shares it. + * A cell where ExoJS trails carries the same weight as one where it leads, and + * a pair whose pooled runs disagreed publishes no verdict at all - it keeps its + * numbers and the band those runs spanned. An arm that produced no comparable + * cell is shown as such rather than dropped: sitting an archetype out, or + * exceeding the harness's runaway budget, is a result and not a defect. */ -import BenchMeasurement from './BenchMeasurement.astro'; +import BenchComparisonTable from './BenchComparisonTable.astro'; +import BenchScoreboard from './BenchScoreboard.astro'; import { - armsOfSection, BACKEND_LABELS, - describeRungs, + comparisonTallies, formatDay, formatRunTime, FRAME_BUDGET_MS, + WIDE_SPREAD_RATIO, type BenchProfileDocument, - type ProfileCell, - type ProfileRow, - type ProfileSection, } from '../lib/bench-profiles'; +import { physicsComparison, renderingComparison, webgl1Comparison } from '../lib/bench-tables'; interface Props { document: BenchProfileDocument; @@ -55,87 +42,33 @@ const { profile, rendering, physics } = profileDocument; const Heading = `h${String(headingLevel)}` as 'h2' | 'h3'; const SubHeading = `h${String(headingLevel + 1)}` as 'h3' | 'h4'; -/** - * What a cell publishes in its verdict column. - * - * The three outcomes are decided here and nowhere else, so no table can print a - * verdict for a cell whose runs did not agree on one. An unstable cell gets its - * own tone rather than a side, which keeps it out of the win/loss hues as well - * as out of the win/loss words; its evidence is what each run concluded. - */ -const outcomeOf = (cell: ProfileCell | null) => { - if (cell === null) { - return { tone: 'absent', label: 'not comparable', rungs: null }; - } - - if (!cell.aggregate.stable) { - return { tone: 'unstable', label: 'no verdict - the runs disagreed', rungs: describeRungs(cell.aggregate.rungs) }; - } - - return { tone: cell.verdict.side, label: cell.verdict.label, rungs: null }; -}; +const tallies = comparisonTallies(profileDocument); +const renderingTable = renderingComparison(profileDocument); +const physicsTable = physicsComparison(profileDocument); +const backends = rendering?.backends ?? []; +const webgl1Blocks = backends.flatMap(backend => { + const table = webgl1Comparison(backend); -/** - * One archetype flattened into the table rows it occupies: one per arm the block - * has a column for, with the arm's value left null where it produced no - * comparable number. The ExoJS median is lifted onto the archetype because it is - * the same measurement for every arm in the row, which also keeps it readable - * when one arm is missing. - */ -const tableRows = (sections: readonly ProfileSection[], arms: readonly string[]) => - sections.flatMap(section => - section.rows.map((row: ProfileRow) => { - const reference = row.cells.find(cell => cell.referenceMs !== null) ?? null; - - return { - section: section.title, - archetype: row.archetype, - count: row.count, - exojsMs: reference?.referenceMs ?? null, - exojsP95Ms: reference?.referenceP95Ms ?? null, - exojsOverFrameBudget: reference?.referenceOverFrameBudget ?? false, - exojsSpread: reference?.aggregate.reference, - entries: arms.map(arm => { - const cell = row.cells.find(candidate => candidate.competitor === arm) ?? null; - - return { - arm, - armMs: cell?.competitorMs ?? null, - armP95Ms: cell?.competitorP95Ms ?? null, - armOverFrameBudget: cell?.competitorOverFrameBudget ?? false, - armSpread: cell?.aggregate.competitor, - outcome: outcomeOf(cell), - mechanism: cell?.mechanism ?? null, - present: cell !== null, - }; - }), - }; - }), - ); + return table === null ? [] : [{ backend, table }]; +}); /** - * The mechanism sentence the harness wrote, shortened for the table cell. The - * equivalent-counters case is the bulk of the rows and says the same thing - * every time, so it becomes a label and the full sentence moves to the title. + * The node count the rendering rows were measured at. + * + * A backend picks its own count from the archetype ladders, so the note only + * states one figure where the backends agreed on it and names them separately + * where they did not. */ -const mechanismCell = (mechanism: string): { label: string; title: string | null } => - mechanism.startsWith('structurally equivalent') - ? { label: 'CPU-side; counters level', title: mechanism } - : { label: mechanism, title: null }; - -const renderingBlocks = (rendering?.backends ?? []).map(backend => { - const webgl1Arms = [...new Set(backend.webgl1.flatMap(row => row.cells.map(cell => cell.competitor)))].sort(); - - return { - backend, - rows: tableRows(backend.sections, backend.competitors), - webgl1Arms, - webgl1Rows: tableRows([{ title: 'WebGL1', rows: backend.webgl1 }], webgl1Arms), - }; -}); +const headlineCounts = [...new Set(backends.map(backend => backend.headlineCount).filter((count): count is number => count !== null))]; +const renderingCountNote = + headlineCounts.length === 1 + ? `Every row is measured at ${String(headlineCounts[0])} nodes` + : `Rows are measured at ${backends + .map(backend => `${String(backend.headlineCount ?? 0)} nodes on ${BACKEND_LABELS[backend.backend]}`) + .join(' and ')}`; -const physicsArms = physics === undefined ? [] : armsOfSection(physics.section); -const physicsRows = physics === undefined ? [] : tableRows([physics.section], physicsArms); +const mechanisms = [...renderingTable.mechanisms, ...(physicsTable?.mechanisms ?? [])]; +const excluded = backends.flatMap(backend => backend.excluded.map(entry => ({ backend: backend.backend, ...entry }))); /** The machine is one across the pooled runs, so the first run's stamps name it for all of them. */ const renderingStamps = rendering?.runs[0]?.provenance ?? []; @@ -146,8 +79,8 @@ const physicsHost = physics?.runs[0]; * * The times are the point of the entry. A profile that named only its newest * measurement would read as one run, which is the reading the pooling exists to - * prevent; seeing three separate stamps is what tells a reader the number above - * them is a median over three measurements and not a single lucky one. + * prevent; seeing the separate stamps is what tells a reader the number above + * them is a median over several measurements and not a single lucky one. */ const runTimes = [ ...(rendering === undefined ? [] : [{ domain: 'Rendering', times: rendering.runs.map(run => run.provenance[0]?.timestamp ?? null) }]), @@ -156,573 +89,239 @@ const runTimes = [ domain: entry.domain, times: entry.times.filter((time): time is string => time !== null).map(formatRunTime), })); + +const libraries = [...(rendering?.libraries ?? []), ...(physics?.libraries ?? [])].map(library => `${library.name} ${library.version}`); ---
{showTitle && {profile.gpu} / {profile.os} / {profile.browser}} -
- - ExoJS {profile.engineVersion} - {profile.runs} runs, {formatDay(profile.measuredAt)} - {rendering !== undefined && {rendering.libraries.map(library => `${library.name} ${library.version}`).join(', ')}} - {physics !== undefined && {physics.libraries.map(library => `${library.name} ${library.version}`).join(', ')}} - machine, adapters, signature - -
-
ExoJS
{profile.engineVersion}
-
Measured
{formatDay(profile.measuredAt)}
-
-
Pooled runs
-
- {profile.runs} separate runs - {runTimes.map(entry => ( - - {entry.domain}: {entry.times.join('; ')} - - ))} -
-
-
Profile
{profile.slug}
-
Signature
{profileDocument.signature.algorithm}:{profileDocument.signature.value.slice(0, 16)}
- {rendering !== undefined && ( -
-
Rendering arms
-
{rendering.libraries.map(library => `${library.name} ${library.version}`).join(', ')}
-
- )} - {physics !== undefined && ( -
-
Physics arms
-
{physics.libraries.map(library => `${library.name} ${library.version}`).join(', ')}
-
- )} - {renderingStamps.map(stamp => ( -
-
{BACKEND_LABELS[stamp.backend]} adapter
-
- {stamp.adapter} - - {stamp.headless ? 'headless' : 'headed'}, {stamp.software ? 'software rasterizer - timings untrusted' : 'hardware adapter'} - {stamp.slotTier === undefined ? '' : `, texture-slot tier ${String(stamp.slotTier)}`} - {stamp.flags.length === 0 ? '' : `, flags: ${stamp.flags.join(' ')}`} - -
-
- ))} - {physicsHost !== undefined && ( -
-
Physics host
-
- {physicsHost.host.cpu} - - {physicsHost.host.cpuCount} logical cores, {physicsHost.host.os} {physicsHost.host.arch}, {physicsHost.browser}{' '} - {physicsHost.browserVersion}, fixed step {(physicsHost.fixedDelta * 1000).toFixed(2)} ms - -
-
- )} -
-
- - {renderingBlocks.map(block => ( + + + {renderingTable.rows.length > 0 && (
- Rendering on {BACKEND_LABELS[block.backend.backend]} -

- {block.backend.headlineCount === null - ? 'No single node count qualified for a headline table on this backend, so no rows are published for it.' - : `${String(block.backend.headlineCount)} nodes per row. CPU ms per frame, median of ${String(profile.runs)} runs, lower is better.`} + Rendering +

+ {renderingCountNote}, chosen from the archetype ladders before any timing was read. CPU milliseconds per frame, pooled over {profile.runs} + {' '}separate runs as the median of their per-run medians; lower is better.

- - {block.rows.length > 0 && ( -
- - - - - - - - - - - - - {block.rows.map(row => - row.entries.map((entry, index) => ( - - {index === 0 && ( - - )} - {index === 0 && ( - - )} - - - - - - )), - )} - -
ArchetypeExoJS (ms)AgainstArm (ms)VerdictMechanism
- {row.archetype} - {row.section} - - - {entry.arm} - - - {entry.outcome.label} - {entry.outcome.rungs !== null && {entry.outcome.rungs}} - - {!entry.present ? ( - `no comparable cell at ${String(row.count)} nodes` - ) : entry.mechanism === null ? ( - 'no structural counters reported' - ) : ( - {mechanismCell(entry.mechanism).label} - )} -
-
- )} - - {block.webgl1Rows.length > 0 && ( -
-
WebGL1 arms, CPU time only
-

WebGL1 context: CPU time only, no mechanism, an observation rather than a finding.

-
- - - - - - - - - - - - {block.webgl1Rows.map(row => - row.entries.map((entry, index) => ( - - {index === 0 && ( - - )} - {index === 0 && ( - - )} - - - - - )), - )} - -
ArchetypeExoJS (ms)AgainstArm (ms)Verdict
- {row.archetype} - - - {entry.arm} - - - {entry.outcome.label} - {entry.outcome.rungs !== null && {entry.outcome.rungs}} -
-
-
- )} - - {block.backend.excluded.length > 0 && ( -
- {block.backend.excluded.length} archetypes measured but left out of this table -
    - {block.backend.excluded.map(entry => ( -
  • - {entry.archetype} - {entry.reason} -
    • - ))} -
-
- )} +
- ))} + )} - {physics !== undefined && ( + {physicsTable !== null && physicsTable.rows.length > 0 && (
Physics -

- {physicsRows.length === 0 - ? 'No physics archetype produced a valid cell on every arm at any rung of its ladder, so no physics rows are published.' - : `Body count per row, each archetype on its own ladder. CPU ms per fixed step, median of ${String(profile.runs)} runs, lower is better.`} +

+ Each archetype sits on its own body-count ladder, so every row states the count it was measured at and rows are not comparable with one + another - only the arms within one row are. CPU milliseconds per fixed step, pooled the same way.

- - {physicsRows.length > 0 && ( -
- - - - - - - - - - - - - - {physicsRows.map(row => - row.entries.map((entry, index) => ( - - {index === 0 && ( - - )} - {index === 0 && ( - - )} - {index === 0 && ( - - )} - - - - - - )), - )} - -
ArchetypeBodiesExoJS (ms)AgainstArm (ms)VerdictMechanism
- {row.archetype} - - {row.count} - - - {entry.arm} - - - {entry.outcome.label} - {entry.outcome.rungs !== null && {entry.outcome.rungs}} - - {!entry.present - ? `This arm produced no comparable cell at ${String(row.count)} bodies - it either sits this archetype out or exceeded the harness's runaway step budget there.` - : (entry.mechanism ?? 'No structural counters were reported for this pair.')} -
-
- )} - - {physicsHost !== undefined && physicsHost.caveats.length > 0 && ( -
- {physicsHost.caveats.length} caveats the physics runs disclosed -
    - {physicsHost.caveats.map(caveat => ( -
  • {caveat}
    • - ))} -
-
- )} +
)} -

- ExoJS leads - the arm leads - level, inside the noise band - no verdict - the runs disagreed - not comparable +

+ ExoJS leads + the arm leads + level, inside the noise band + the runs disagreed: the band is the ratio they spanned + not comparable + a time past the {FRAME_BUDGET_MS} ms frame + a range spanning {WIDE_SPREAD_RATIO}x or more

-
- How to read a cell -

- Each time is published with the p95 of the same timed window under it and, under that, the range the {profile.runs} pooled runs - observed for the median. Verdicts are computed from the medians alone; the p95 is the step or frame a player feels, so a pair far apart hitches even - where the median reads as comfortable. A range marked in amber spans a factor of 1.2 or more - as wide as the band the verdict ladder treats as noise - - so the number above it is soft on this machine. A median marked in amber as over the {FRAME_BUDGET_MS} ms frame costs more than a whole 60 fps - frame on its own: how much of a frame this work may take is your decision, but one that exceeds the entire frame is unplayable whatever you decide. A - cell whose runs reached different verdicts publishes none of them and lists what each run concluded instead. -

-
-
- - diff --git a/site/src/components/BenchScoreboard.astro b/site/src/components/BenchScoreboard.astro new file mode 100644 index 000000000..a07bd85be --- /dev/null +++ b/site/src/components/BenchScoreboard.astro @@ -0,0 +1,214 @@ +--- +/** + * BenchScoreboard - how one profile's comparisons came out, one line per arm. + * + * The line is per arm and never per backend or per domain: pooling two + * opponents into one strip would show a reader a mix that belongs to neither of + * them. Nothing is summed across lines and no line is ranked against another, + * so the block counts outcomes without producing a score or an overall winner. + * + * Leads and losses are drawn with the same weight and differ only in hue, and + * the two outcomes that carry no verdict - runs that disagreed, and arms that + * produced no comparable cell - keep neither side's hue so a reader scanning + * the strip cannot read one as a quiet win or a quiet loss. + */ + +import { OUTCOME_LABELS, OUTCOME_ORDER, type ComparisonTally } from '../lib/bench-profiles'; + +interface Props { + tallies: readonly ComparisonTally[]; +} + +const { tallies } = Astro.props; +--- + +{ + tallies.length > 0 && ( +
+
+ Pair + + leads left, losses right + + rows +
+ {tallies.map(entry => ( +
+ {entry.label} + {entry.meta} + + {OUTCOME_ORDER.filter(outcome => entry.counts[outcome] > 0).map(outcome => ( + + ))} + + + {OUTCOME_ORDER.filter(outcome => entry.counts[outcome] > 0).map(outcome => ( + + {entry.counts[outcome]} {OUTCOME_LABELS[outcome]} + + ))} + + {entry.total} +
+ ))} +
+ ) +} + + diff --git a/site/src/components/pages/BenchmarksPage.astro b/site/src/components/pages/BenchmarksPage.astro index 6c8a7c5e7..d88f0d34c 100644 --- a/site/src/components/pages/BenchmarksPage.astro +++ b/site/src/components/pages/BenchmarksPage.astro @@ -4,34 +4,30 @@ * * The page is generated at build time from the machine profiles under * `packages/exojs-bench/results/` and from nothing else, so it cannot drift from - * the harness: a re-measurement rewrites the headline paragraph, the tables and - * the provenance together. Losses are published on the same terms as wins, and - * the rows the harness left out of a comparison are shown with their reasons. + * the harness: a re-measurement rewrites the scoreboard, the tables and the + * provenance together. Losses are published on the same terms as wins, nothing + * is aggregated into a score or an overall winner, and the rows the harness left + * out of a comparison are listed with their reasons. * - * When the directory holds no profile - a fresh clone, or the state a release - * branch sits in before its reference run - the measured sections disappear and - * the methodology, reproduction and fairness sections stay, because those are - * what a reader can act on when there is nothing to read yet. + * The numbers come first and the practices behind them last. Methodology, + * reproduction and fairness are what a reader consults once a row surprises + * them, so they sit under the tables rather than in front of them - and they + * stay on the page when the directory holds no profile at all, which is the + * state a fresh clone and a release branch before its reference run are in. */ import BenchProfileReport from '../BenchProfileReport.astro'; import DocsLayout from '../../layouts/DocsLayout.astro'; import EnglishFallbackNotice from '../EnglishFallbackNotice.astro'; import { appInfo } from '../../lib/app-info'; -import { formatDay, FRAME_BUDGET_MS, furtherProfiles, headlineFindings, olderThanReference, referenceProfile } from '../../lib/bench-profiles'; +import { formatDay, FRAME_BUDGET_MS, furtherProfiles, olderThanReference, profileScope, referenceProfile } from '../../lib/bench-profiles'; interface Props { locale: 'en' | 'de'; } const { locale } = Astro.props; -const allFindings = referenceProfile === undefined ? [] : headlineFindings(referenceProfile); -/** The scope sentence names the measurement; it sits under the title, the verdict sentences are the findings. */ -const scope = allFindings.find(finding => finding.tone === 'scope'); -const findings = allFindings.filter(finding => finding.tone !== 'scope'); - -/** The word next to a headline sentence, so a lead and a loss are labelled rather than coloured into a ranking. */ -const TONE_LABELS = { scope: 'scope', lead: 'lead', level: 'level', loss: 'loss', unstable: 'unstable' } as const; +const scope = referenceProfile === undefined ? '' : profileScope(referenceProfile); /** * Runs the reproduction below pools. A published profile states the number it @@ -43,62 +39,39 @@ const REPRODUCTION_RUNS = 3; const pooledRuns = referenceProfile?.profile.runs ?? REPRODUCTION_RUNS; --- - - - +
{locale === 'de' && }

Benchmarks

- ExoJS against Pixi, Phaser and Excalibur for rendering, and against matter.js, planck and Rapier for physics. Every number comes from the - harness, pooled over {pooledRuns} runs; losses are published like wins. How the numbers are made is at the end of the page. + ExoJS against Pixi, Phaser and Excalibur for rendering, and against matter.js, planck and Rapier for physics. Every number is written by the + harness into a signed profile in the repository, pooled over {pooledRuns} separate runs; a comparison the runs did not agree on publishes no + verdict, and every cell where ExoJS trails is published like the ones where it leads.

- {scope !== undefined &&

{scope.text}

} + {scope !== '' &&

{scope}

}
-
-

Headline findings

- {findings.length === 0 ? ( -

- No measurements are published yet. The reference measurement is {REPRODUCTION_RUNS} separate runs on one machine after a release is - tagged, pooled into a single profile file in the repository; until then there is nothing here to read. The methodology, the reproduction - steps and the fairness terms below already apply. -

- ) : ( -
    - {findings.map(finding => ( -
  • - {TONE_LABELS[finding.tone]} -

    {finding.text}

    -
    • - ))} -
- )} -
- - {referenceProfile !== undefined && ( -
-

Reference profile

- -
+ {referenceProfile === undefined ? ( +

+ No measurements are published yet. The reference measurement is {REPRODUCTION_RUNS} separate runs on one machine after a release is tagged, + pooled into a single profile file in the repository; until then there is nothing here to read. The practices below already apply. +

+ ) : ( + )} {furtherProfiles.length > 0 && (

Further machines

-

Contributed profiles for other hardware, newest engine version first: do the ratios hold on another GPU, driver and JavaScript engine?

+

+ Contributed profiles for other hardware, newest engine version first: do the ratios hold on another GPU, driver and JavaScript engine? +

{furtherProfiles.map(document => { const newer = olderThanReference(document); @@ -125,105 +98,111 @@ const pooledRuns = referenceProfile?.profile.runs ?? REPRODUCTION_RUNS;
)} -
-

Methodology

+
+

How these numbers are made

+

+ The terms the tables above are published under. They are here rather than in front of the numbers because they are what a reader consults + once a row surprises them. +

+
- How a number becomes a verdict: pooling, medians, ranges, the ladder, the frame budget -
    -
  • - A published profile pools {pooledRuns} separate runs of the same matrix. One run cannot support a ratio: the same code - measured twice on one idle machine moves a cell's median far enough to reverse which library a cell favours - a reversal this harness - has produced in practice, which is why a single run is not published as a reference measurement. -
    • -
  • - The published value is the median of the per-run medians - each run's own median over its timed window, then the median - across the runs - so one unlucky run cannot set a number. Nothing is ever a mean, and nothing is aggregated across archetypes. -
    • -
  • - Every number carries the range its runs observed, printed under it. A range spanning a factor of 1.2 or more is marked: - the measurement's own noise then reaches as wide as the band the ladder below treats as no difference at all, which is something to know - before reading the number, not after. -
    • -
  • - Every median is published with the p95 of the same timed window beside it. The median is the amortised cost and the - number every verdict is computed from - it is also what independent published comparisons state - while the p95 is the step or frame a - player feels as a hitch. A pair far apart describes work that is periodically expensive, which a median alone would report as cheap. There - is no p99: the largest cells time 120 steps, so a p99 there is the second-worst sample, an outlier rather than a percentile. -
    • -
  • - A published median past {FRAME_BUDGET_MS} ms - a whole 60 fps frame - is marked. The line is the entire frame and not a - fraction of it on purpose: how much of a frame you may spend on physics, or on the CPU side of rendering, depends on everything else your - frame does and is your decision, while a single step or frame that costs more than the frame it has to fit in is unplayable whatever you - decide. Nothing is derived from the mark - there is no "how many bodies at N ms" capacity figure, because that would interpolate between - the ladder's rungs rather than report something measured. -
    • -
  • - A verdict is published only when all {pooledRuns} runs independently reach it. Each run is placed on the ladder on its - own, and the pooled medians have to land on the same rung as well. A cell whose runs disagreed publishes no verdict: it keeps - its numbers and its range, states what each run concluded, and is counted nowhere - not as a win, not as a loss, and not as a level row. - That the pair cannot be separated on this machine is the finding. -
    • -
  • - A ratio between 0.8 and 1.2 is reported as level: inside that band the difference is not distinguishable - from machine mood. -
    • -
  • - A gap of 5x or more is reported as leads clearly rather than leads. Below that factor a driver state - or an unlucky window could still explain the result; at or above it, the difference is attributable to how the two libraries are built. -
    • -
  • - Each row names the mechanism behind its difference, drawn from structural counters the harness collects - draw calls, - texture binds, buffer uploads, solved contacts. A row whose difference could not be evidenced that way is not published as a - comparison; it appears among the omissions instead. -
    • -
  • - One profile describes one machine measured in one browser. Numbers from different machines are never mixed into one - table or averaged together, and neither are numbers from different browsers - including for physics, which touches no GPU but whose - per-step times depend on which JavaScript engine executed the steps. -
    • -
  • - Physics is measured in the browser, not in the process that drives the harness. Nobody runs ExoJS physics in Node, and - the runtime is part of the measurement: heap limits, garbage collection and WebAssembly compilation are the browser's. The fastest cells - step in microseconds, close to the browser's clock resolution, so a step too fast to time on its own is timed in batches large enough to - clear that resolution, with the batch published beside the number and the timed-step budget unchanged. -
    • -
  • - Phaser renders through a WebGL1 context, so its rows sit in their own block, compare CPU time only, and carry no - mechanism - the structural probe cannot attach to that context to say whether the backend generation or the engine caused the gap. -
    • -
  • - Rapier is a Rust/WASM reference ceiling, not a pure-JS peer. It measures what leaving JavaScript buys; matter.js and - planck are the peers ExoJS is compared against. -
    • -
  • Culling is configured to match across arms, so no arm wins a row by drawing less of the same scene.
    • -
  • - A row's node or body count is chosen from the archetype ladders before any timing is read - the largest rung at which - every arm produced a valid cell - so it can never be picked to suit the outcome. A rendering table goes further and uses one node count - for all of its rows, because its archetypes share their ladders. An arm that produced no comparable cell at that count - because it sits - the archetype out, or because it exceeded the runaway step budget - is shown as not comparable rather than omitted. -
    • -
  • - Each physics archetype has its own body-count ladder, placed so its rungs straddle the frame budget: two inside the - frame and one past it, so you can see both the slope and where the archetype stops being viable. They reach a frame at sizes that differ - by nearly an order of magnitude, so one shared count would put most rows at a size chosen to suit a different scene. Every physics row - therefore states its own count, and physics rows are not comparable with one another - only the arms within one row are, - which is what the table was ever for. Two rows are two different scenes at two different sizes. -
    • -
+ Methodology: pooling, medians, ranges, the ladder, the frame budget +
    +
  • + A published profile pools {pooledRuns} separate runs of the same matrix. One run cannot support a ratio: the same + code measured twice on one idle machine moves a cell's median far enough to reverse which library a cell favours - a reversal this + harness has produced in practice, which is why a single run is not published as a reference measurement. +
    • +
  • + The published value is the median of the per-run medians - each run's own median over its timed window, then the + median across the runs - so one unlucky run cannot set a number. Nothing is ever a mean, and nothing is aggregated across archetypes. +
    • +
  • + Every number carries the range its runs observed, printed under it. A range spanning a factor of 1.2 or more is + marked: the measurement's own noise then reaches as wide as the band the ladder treats as no difference at all, which is something to + know before reading the number, not after. +
    • +
  • + Every median is published with the p95 of the same timed window beside it. The median is the amortised cost and the + number every verdict is computed from - it is also what independent published comparisons state - while the p95 is the step or frame + a player feels as a hitch. A pair far apart describes work that is periodically expensive, which a median alone would report as + cheap. There is no p99: the largest cells time 120 steps, so a p99 there is the second-worst sample, an outlier rather than a + percentile. +
    • +
  • + A published median past {FRAME_BUDGET_MS} ms - a whole 60 fps frame - is marked. The line is the entire frame and + not a fraction of it on purpose: how much of a frame you may spend on physics, or on the CPU side of rendering, depends on everything + else your frame does and is your decision, while a single step or frame that costs more than the frame it has to fit in is unplayable + whatever you decide. Nothing is derived from the mark - there is no "how many bodies at N ms" capacity figure, because that would + interpolate between the ladder's rungs rather than report something measured. +
    • +
  • + A verdict is published only when all {pooledRuns} runs independently reach it. Each run is placed on the ladder on + its own, and the pooled medians have to land on the same rung as well. A cell whose runs disagreed publishes no verdict: it + keeps its numbers and shows the ratio band those runs spanned, and it is counted nowhere - not as a win, not as a loss, and not as a + level row. That the pair cannot be separated on this machine is the finding. The band is the one figure the page computes rather than + reads, from the published extremes of both arms, and it is drawn without a side. +
    • +
  • + A ratio between 0.8 and 1.2 is reported as level: inside that band the difference is not + distinguishable from machine mood. +
    • +
  • + A gap of 5x or more is reported as leads clearly rather than leads. Below that factor a driver + state or an unlucky window could still explain the result; at or above it, the difference is attributable to how the two libraries + are built. +
    • +
  • + Each comparison names the mechanism behind its difference, drawn from structural counters the harness collects - + draw calls, texture binds, buffer uploads, solved contacts - and they are listed under the tables. A row whose difference could not + be evidenced that way is not published as a comparison; it appears among the omissions instead. +
    • +
  • + One profile describes one machine measured in one browser. Numbers from different machines are never mixed into one + table or averaged together, and neither are numbers from different browsers - including for physics, which touches no GPU but whose + per-step times depend on which JavaScript engine executed the steps. +
    • +
  • + Physics is measured in the browser, not in the process that drives the harness. Nobody runs ExoJS physics in Node, + and the runtime is part of the measurement: heap limits, garbage collection and WebAssembly compilation are the browser's. The + fastest cells step in microseconds, close to the browser's clock resolution, so a step too fast to time on its own is timed in + batches large enough to clear that resolution, with the timed-step budget unchanged. +
    • +
  • + Phaser renders through a WebGL1 context, so its rows sit in their own block, compare CPU time only, and carry no + mechanism - the structural probe cannot attach to that context to say whether the backend generation or the engine caused the gap. +
    • +
  • + Rapier is a Rust/WASM reference ceiling, not a pure-JS peer. It measures what leaving JavaScript buys; matter.js and + planck are the peers ExoJS is compared against. +
    • +
  • Culling is configured to match across arms, so no arm wins a row by drawing less of the same scene.
    • +
  • + A row's node or body count is chosen from the archetype ladders before any timing is read - the largest rung at + which every arm produced a valid cell - so it can never be picked to suit the outcome. A rendering backend goes further and uses one + node count for all of its rows, because its archetypes share their ladders. An arm that produced no comparable cell at that count - + because it sits the archetype out, or because it exceeded the runaway step budget - is shown as not comparable rather than + omitted. +
    • +
  • + Each physics archetype has its own body-count ladder, placed so its rungs straddle the frame budget: two inside the + frame and one past it, so you can see both the slope and where the archetype stops being viable. They reach a frame at sizes that + differ by nearly an order of magnitude, so one shared count would put most rows at a size chosen to suit a different scene. Every + physics row therefore states its own count, and physics rows are not comparable with one another - only the arms + within one row are, which is what the table was ever for. +
    • +
-
-
-

Reproduce

- Clone, bootstrap, run the matrix {REPRODUCTION_RUNS} times, pool into a profile -

- The harness is in the repository, and so are the adapters. A reference profile is {REPRODUCTION_RUNS} separate runs, not one - run repeated inside itself: repeating a matrix in one process shares JIT and heap state between the repetitions and measures the same warm - state {REPRODUCTION_RUNS} times, which is the very effect the repetition exists to expose. Each run is its own invocation with its own output - directory, and bench:compare pools them by taking each input flag once per run. -

- 
git clone {appInfo.repositoryUrl}
+                Reproduce: run the harness yourself and publish a profile for your machine
+                

+                    The harness is in the repository, and so are the adapters. A reference profile is {REPRODUCTION_RUNS} separate runs, not
+                    one run repeated inside itself: repeating a matrix in one process shares JIT and heap state between the repetitions and measures the same
+                    warm state {REPRODUCTION_RUNS} times, which is the very effect the repetition exists to expose. Each run is its own invocation with its
+                    own output directory, and bench:compare pools them by taking each input flag once per run.
+                

+                
git clone {appInfo.repositoryUrl}
 cd exojs
 pnpm bootstrap
 cd packages/exojs-bench
@@ -240,52 +219,52 @@ pnpm bench:compare -- \
   --rendering=a/results.json --rendering=b/results.json --rendering=c/results.json \
   --physics=a-physics/results.json --physics=b-physics/results.json --physics=c-physics/results.json \
   --profile

-            

-                The harness lives in packages/exojs-bench and its scripts are run from there; the root pnpm bench is the engine's own
-                micro-benchmark suite and not this one. The -- separates the harness's flags from pnpm's own. Output directories are relative to
-                the package. pnpm bootstrap installs the competitor libraries alongside the workspace, each pnpm bench runs one
-                matrix into its own directory, and bench:compare --profile pools them into a profile named after your machine under
-                packages/exojs-bench/results/. Repeating an input flag is what pools: a profile built from fewer than {REPRODUCTION_RUNS} runs is
-                rejected by the repository's own check on this directory. Pooling is refused outright for runs that do not repeat one measurement - a different engine version, different library
-                arms, a different matrix - because a spread over those would describe the difference between two runs rather than the noise of one. Measure on
-                an otherwise idle machine: these are wall-clock comparisons and background load moves them.
-            

-            

-                pnpm bench -- --browser=webkit measures in WebKit instead of the default Chromium, for both domains - a physics run takes the
-                flag too, and the two domains of one profile have to agree on it. The browser is part of the profile's name, so the same machine measured in
-                both publishes two files rather than overwriting one with the other, and the harness refuses to pool runs taken in different browsers. A
-                backend the browser does not expose is published as not measured, never as a number: WebKit reaches WebGPU on macOS alone. A physics arm the
-                browser cannot load is published the same way, with the reason it failed. WebKit also reports a constant in place of the GPU, so a WebKit
-                profile is named after the machine's CPU model and needs a physics measurement of the same machine alongside the rendering one.
-            

-            

-                On macOS and Linux, add --platform=26 to every run - or --platform=26-beta on a pre-release build. The profile's
-                name carries the operating system's major version, so a beta measurement and the shipping platform's later one do not overwrite each other,
-                and neither the version nor the beta status is readable at runtime there: os.release() reports the kernel version. Windows
-                reports its own version and needs no flag. Every stamp records whether the version was read or declared, and whether the platform was
-                pre-release and how that was established.
-            

-            

-                A profile for a machine that is not published yet is welcome as a pull request containing that one file. Nothing else needs to change - this
-                page picks it up from the directory.
-            

+                

+                    The harness lives in packages/exojs-bench and its scripts are run from there; the root pnpm bench is the
+                    engine's own micro-benchmark suite and not this one. The -- separates the harness's flags from pnpm's own. Output
+                    directories are relative to the package. pnpm bootstrap installs the competitor libraries alongside the workspace, each
+                    pnpm bench runs one matrix into its own directory, and bench:compare --profile pools them into a profile named
+                    after your machine under packages/exojs-bench/results/. Repeating an input flag is what pools: a profile built from fewer
+                    than {REPRODUCTION_RUNS} runs is rejected by the repository's own check on this directory. Pooling is refused outright for runs that do
+                    not repeat one measurement - a different engine version, different library arms, a different matrix - because a spread over those would
+                    describe the difference between two runs rather than the noise of one. Measure on an otherwise idle machine: these are wall-clock
+                    comparisons and background load moves them.
+                

+                

+                    pnpm bench -- --browser=webkit measures in WebKit instead of the default Chromium, for both domains - a physics run takes
+                    the flag too, and the two domains of one profile have to agree on it. The browser is part of the profile's name, so the same machine
+                    measured in both publishes two files rather than overwriting one with the other, and the harness refuses to pool runs taken in different
+                    browsers. A backend the browser does not expose is published as not measured, never as a number: WebKit reaches WebGPU on macOS alone. A
+                    physics arm the browser cannot load is published the same way, with the reason it failed. WebKit also reports a constant in place of the
+                    GPU, so a WebKit profile is named after the machine's CPU model and needs a physics measurement of the same machine alongside the
+                    rendering one.
+                

+                

+                    On macOS and Linux, add --platform=26 to every run - or --platform=26-beta on a pre-release build. The profile's
+                    name carries the operating system's major version, so a beta measurement and the shipping platform's later one do not overwrite each
+                    other, and neither the version nor the beta status is readable at runtime there: os.release() reports the kernel version.
+                    Windows reports its own version and needs no flag. Every stamp records whether the version was read or declared, and whether the platform
+                    was pre-release and how that was established.
+                

+                

+                    A profile for a machine that is not published yet is welcome as a pull request containing that one file. Nothing else needs to change -
+                    this page picks it up from the directory.
+
-
-
-

Fairness

- Adapters at library defaults, no score, no overall winner -

- The adapter for every competitor is in the repository next to the ExoJS one, and each is configured at that library's own defaults with the - deviations disclosed in the run's caveats. If a maintainer of one of these libraries thinks an adapter misrepresents it, an issue about that - adapter is the right place to say so, and a fix changes the published numbers the next time the harness runs. -

-

- Nothing here is aggregated into a score and nothing is declared an overall winner. Every cell where ExoJS trails is published with the same - weight as one where it leads, with the same mechanism evidence attached, and the rows the harness could not compare are listed with the - reason it could not. -

+ Fairness: how the arms are configured and what is never derived +

+ The adapter for every competitor is in the repository next to the ExoJS one, and each is configured at that library's own defaults with + the deviations disclosed in the run's caveats. If a maintainer of one of these libraries thinks an adapter misrepresents it, an issue + about that adapter is the right place to say so, and a fix changes the published numbers the next time the harness runs. +

+

+ Nothing here is aggregated into a score and nothing is declared an overall winner. The scoreboard counts outcomes per arm and never sums + them across arms, because the arms answer different questions. Every cell where ExoJS trails is published with the same weight as one + where it leads, with the same mechanism evidence attached, and the rows the harness could not compare are listed with the reason it could + not. +

@@ -296,8 +275,13 @@ pnpm bench:compare -- \ max-width: none; } + /* The docs reading measure belongs to prose pages; here it would wrap short notes beside full-width tables. */ + .bench-prose :where(p, li) { + max-width: none; + } + .bench-head { - margin-bottom: var(--s-6); + margin-bottom: var(--s-5); } .bench-head h1 { @@ -306,37 +290,17 @@ pnpm bench:compare -- \ .bench-head p { margin: 0; - max-width: 68ch; color: var(--fg-muted); - line-height: 1.65; + line-height: 1.6; } .scope { - margin-top: var(--s-3) !important; + margin-top: var(--s-2) !important; font-family: var(--f-mono); - font-size: 0.78rem !important; + font-size: 0.74rem; color: var(--fg-faint) !important; } - .section-note { - margin: 0 0 var(--s-4); - max-width: 72ch; - color: var(--fg-muted); - font-size: 0.9rem; - } - .practice { - border: 1px solid var(--line-soft); - border-radius: var(--r-3); - padding: var(--s-3) var(--s-4); - background: var(--bg-elevated); - } - .practice summary { - cursor: pointer; - color: var(--fg-muted); - } - .practice[open] summary { - margin-bottom: var(--s-3); - color: var(--fg); - } + .empty { margin: 0; padding: var(--s-4); @@ -345,73 +309,21 @@ pnpm bench:compare -- \ border-radius: var(--r-3); background: var(--bg-elevated); color: var(--fg-muted); - max-width: 68ch; } - .findings { - list-style: none; - margin: 0; - padding: 0; - display: flex; - flex-direction: column; - gap: var(--s-3); - } - - /* - * A lead and a loss get the same box, the same type and the same border - * weight; the tone chip only names which one it is. Styling a win louder - * than a loss would make the paragraph an argument rather than a report. - */ - .finding { - display: grid; - grid-template-columns: 5.5rem minmax(0, 1fr); - gap: var(--s-3); - align-items: start; - padding: var(--s-3) var(--s-4); - border: 1px solid var(--line-soft); - border-radius: var(--r-3); - background: var(--bg-elevated); - } - - .finding p { - margin: 0; - line-height: 1.6; - } - - .tone { - display: inline-block; - padding: 0.15rem 0.5rem; - border: 1px solid currentcolor; - border-radius: var(--r-pill); - font-family: var(--f-mono); - font-size: 0.7rem; - font-weight: 600; - text-align: center; - letter-spacing: 0.04em; - } - - .finding.scope .tone { - color: var(--fg-faint); - } - - .finding.lead .tone { - color: var(--sky); - } - - .finding.loss .tone { - color: var(--violet); - } - - .finding.level .tone { + .section-note { + margin: 0 0 var(--s-3); color: var(--fg-muted); + font-size: 0.85rem; } - .finding.unstable .tone { - color: var(--amber); + #further, + #practices { + margin-top: var(--s-8); } .machine { - margin-top: var(--s-4); + margin-top: var(--s-3); border: 1px solid var(--line-soft); border-radius: var(--r-3); padding: var(--s-3) var(--s-4); @@ -443,12 +355,28 @@ pnpm bench:compare -- \ font-size: 0.85rem; } + .practice { + margin-top: var(--s-3); + border: 1px solid var(--line-soft); + border-radius: var(--r-3); + padding: var(--s-3) var(--s-4); + } + + .practice summary { + cursor: pointer; + font-weight: 600; + } + + .practice p, .method { - margin: 0; - padding-left: 1.15rem; - max-width: 74ch; color: var(--fg-muted); line-height: 1.6; + font-size: 0.88rem; + } + + .method { + margin: var(--s-3) 0 0; + padding-left: 1.15rem; } .method li { @@ -459,26 +387,8 @@ pnpm bench:compare -- \ color: var(--fg); } - #reproduce pre { + .practice pre { overflow-x: auto; max-width: 100%; } - - #reproduce p, - #fairness p { - max-width: 72ch; - color: var(--fg-muted); - line-height: 1.65; - } - - @media (max-width: 760px) { - .finding { - grid-template-columns: 1fr; - gap: var(--s-2); - } - - .tone { - justify-self: start; - } - } diff --git a/site/src/layouts/DocsLayout.astro b/site/src/layouts/DocsLayout.astro index e7fab65f3..e4658e839 100644 --- a/site/src/layouts/DocsLayout.astro +++ b/site/src/layouts/DocsLayout.astro @@ -4,7 +4,7 @@ import AppShell from './AppShell.astro'; interface Props { title: string; description?: string; - variant?: 'default' | 'guide' | 'api'; + variant?: 'default' | 'guide' | 'api' | 'wide'; } const { title, description = 'ExoJS documentation', variant = 'default' } = Astro.props; @@ -13,6 +13,7 @@ const hasRail = Astro.slots.has('rail'); const sidebarToggleLabel = variant === 'api' ? 'Browse API' : 'Browse chapters'; const isGuideVariant = variant === 'guide'; const isApiVariant = variant === 'api'; +const isWideVariant = variant === 'wide'; --- @@ -22,6 +23,7 @@ const isApiVariant = variant === 'api'; !hasSidebar && 'docs-shell--full', isGuideVariant && 'docs-shell--guide', isApiVariant && 'docs-shell--api', + isWideVariant && 'docs-shell--wide', hasRail && 'docs-shell--with-rail', ]} > @@ -510,6 +512,23 @@ const isApiVariant = variant === 'api'; grid-template-columns: minmax(0, 1fr); } + /* + * Wide pages carry data tables rather than prose, so the shell's reading + * width would push their columns into a horizontal scroller for no reason. + */ + .docs-shell--wide { + width: min(1720px, 100% - 2.5rem); + } + + .docs-shell--wide .docs-shell__content { + border: 0; + border-radius: 0; + box-shadow: none; + background: transparent; + padding: 0; + max-width: none; + } + .docs-shell--full .docs-shell__sidebar { display: none; } diff --git a/site/src/lib/bench-profiles.ts b/site/src/lib/bench-profiles.ts index 4d5a18257..7c65b94ee 100644 --- a/site/src/lib/bench-profiles.ts +++ b/site/src/lib/bench-profiles.ts @@ -451,7 +451,7 @@ interface FlatCell { * * The WebGL1 block is deliberately left out. Those arms report no structural * counters, so their rows are an observation about a different backend - * generation rather than a finding, and a headline sentence must not rest on one. + * generation rather than a finding, and the scope sentence must not count one. */ const renderingCells = (document: BenchProfileDocument): readonly FlatCell[] => (document.rendering?.backends ?? []).flatMap(backend => @@ -464,30 +464,6 @@ const renderingCells = (document: BenchProfileDocument): readonly FlatCell[] => const physicsCells = (document: BenchProfileDocument): readonly FlatCell[] => (document.physics?.section.rows ?? []).flatMap(row => row.cells.map(cell => ({ backend: null, archetype: row.archetype, count: row.count, cell }))); -/** - * The cells a sentence may be built on: the ones whose pooled runs all reached - * the same verdict. - * - * Every count, comparison and superlative below draws from this set and never - * from the full one. An unstable cell's verdict is a placeholder whose `side` is - * `neither`, so counting it as a row that "reads as level" would turn a - * measurement the runs contradicted into a published claim - which is exactly - * the laundering the stability rule exists to prevent. - */ -const settled = (cells: readonly FlatCell[]): readonly FlatCell[] => cells.filter(entry => entry.cell.aggregate.stable); - -/** The cell with the widest computed factor, or `undefined` when none carries one. */ -const widest = (cells: readonly FlatCell[]): FlatCell | undefined => - cells.reduce((best, candidate) => { - const factor = candidate.cell.verdict.factor; - - if (factor === null || !Number.isFinite(factor)) return best; - - const bestFactor = best?.cell.verdict.factor; - - return bestFactor === undefined || bestFactor === null || factor > bestFactor ? candidate : best; - }, undefined); - /** Distinct arm names in a set of cells, alphabetically. */ const armsIn = (cells: readonly FlatCell[]): readonly string[] => [...new Set(cells.map(entry => entry.cell.competitor))].sort(); @@ -495,180 +471,151 @@ const armsIn = (cells: readonly FlatCell[]): readonly string[] => [...new Set(ce const listOf = (items: readonly string[]): string => (items.length < 2 ? (items[0] ?? '') : `${items.slice(0, -1).join(', ')} and ${items.at(-1) ?? ''}`); /** - * The arm whose numbers sit closest to ExoJS across the whole rendering table. - * - * Distance is the mean absolute log ratio, so a 2x lead and a 2x loss weigh the - * same. Averaging the raw ratios would let one lopsided win cancel a loss and - * name the wrong arm as the close one. + * How one comparison reads once the pooled runs are taken into account. * - * Only cells whose runs agreed enter the mean. An unstable cell has no ratio the - * measurement supports, so averaging one in would rank the arms partly on - * numbers the runs contradicted; an arm with no settled cell at all is not - * ranked rather than being placed last. + * `clear-lead` and `clear-loss` are the ladder's own "leads clearly" rungs, kept + * apart from the ordinary ones so a scoreboard can show at a glance how much of + * a mix is attributable to structure. `unstable` is not a sixth verdict but the + * absence of one: the runs reached different rungs, so the pair carries numbers + * and no conclusion. `absent` is an arm that produced no comparable cell at all. */ -const closestArm = (cells: readonly FlatCell[]): string | undefined => { - const scored = armsIn(cells) - .map(arm => { - const ratios = cells - .filter(entry => entry.cell.competitor === arm) - .map(entry => entry.cell.verdict.ratio) - .filter((ratio): ratio is number => ratio !== null && Number.isFinite(ratio) && ratio > 0); - - return { arm, ratios }; - }) - .filter(entry => entry.ratios.length > 0) - .map(entry => ({ arm: entry.arm, distance: entry.ratios.reduce((sum, ratio) => sum + Math.abs(Math.log(ratio)), 0) / entry.ratios.length })); - - return scored.sort((a, b) => a.distance - b.distance)[0]?.arm; -}; - -/** How a headline sentence reads for the engine. */ -export type FindingTone = 'scope' | 'lead' | 'level' | 'loss' | 'unstable'; +export type CellOutcome = 'clear-lead' | 'lead' | 'level' | 'loss' | 'clear-loss' | 'unstable' | 'absent'; -/** One generated headline sentence. */ -export interface HeadlineFinding { - readonly tone: FindingTone; - readonly text: string; -} +/** Outcomes in reading order: the widest lead first, the widest loss last, then the two that carry no verdict. */ +export const OUTCOME_ORDER: readonly CellOutcome[] = ['clear-lead', 'lead', 'level', 'loss', 'clear-loss', 'unstable', 'absent']; -/** `n of m` where `m` counts only the comparisons a verdict could be drawn from. */ -const outOfSettled = (count: number, total: number): string => `${String(count)} of ${String(total)}`; +/** The word a scoreboard and a legend print for each outcome. */ +export const OUTCOME_LABELS: Readonly> = { + 'clear-lead': 'clear lead', + lead: 'lead', + level: 'level', + loss: 'loss', + 'clear-loss': 'clear loss', + unstable: 'runs disagreed', + absent: 'not comparable', +}; /** - * The page's headline paragraph, generated from a profile's own verdicts. - * - * Which rows deserve a sentence is decided here - the widest lead, the widest - * loss, the arm that sits closest, the split against the pure-JS physics peers - * and against the WASM ceiling. Every number and every verdict inside a sentence - * is copied from the document, so a re-measurement rewrites the paragraph - * instead of leaving prose behind that no longer matches the tables under it. + * Which outcome a cell publishes. * - * No sentence is built on a cell whose pooled runs disagreed, and no denominator - * counts one: a sentence that quietly folded unstable cells into its totals - * would publish, in prose, the verdicts the tables above it refuse to print. How - * many comparisons were left out that way is stated as a finding of its own, - * because it is the reader's measure of how far the paragraph can be trusted. - * - * A sentence with nothing behind it is dropped rather than padded, so a profile - * carrying only one domain yields a shorter paragraph. + * This is the only place a comparison is turned into one of the seven words, so + * no table or scoreboard can invent an outcome for a cell whose runs did not + * agree on one. */ -export const headlineFindings = (document: BenchProfileDocument): readonly HeadlineFinding[] => { - const findings: HeadlineFinding[] = []; - const allRendering = renderingCells(document); - const allPhysics = physicsCells(document); - const rendering = settled(allRendering); - const physics = settled(allPhysics); - const { profile } = document; - const scopeParts: string[] = []; +export const outcomeOf = (cell: ProfileCell | null): CellOutcome => { + if (cell === null) return 'absent'; + if (!cell.aggregate.stable) return 'unstable'; + if (cell.verdict.side === 'neither') return 'level'; + if (cell.verdict.side === 'exojs') return cell.verdict.structural ? 'clear-lead' : 'lead'; - if (allRendering.length > 0) { - const backends = (document.rendering?.backends ?? []).map(backend => BACKEND_LABELS[backend.backend]); + return cell.verdict.structural ? 'clear-loss' : 'loss'; +}; - scopeParts.push(`${String(allRendering.length)} rendering comparisons on ${listOf(backends)} against ${listOf(armsIn(allRendering))}`); - } +/** The lowest and highest `exojs / competitor` ratio the pooled runs can have produced. */ +export interface RatioBand { + readonly low: number; + readonly high: number; +} - if (allPhysics.length > 0) { - scopeParts.push(`${String(allPhysics.length)} physics comparisons against ${listOf(armsIn(allPhysics))}`); - } +/** + * The ratio band behind a comparison whose runs disagreed. + * + * This is the one figure the page derives rather than reads: a cell the runs + * split on carries no published ratio, and printing nothing in its place would + * throw away measurements that exist. The band is the widest and narrowest + * ratio the observed extremes allow, so it is an envelope around what was + * measured and never a verdict - it is drawn without a side, and a band that + * straddles 1.00 is exactly the statement that the runs could not separate the + * pair. It returns `null` where an arm reported no extremes to bound. + */ +export const ratioBand = (cell: ProfileCell): RatioBand | null => { + const { reference, competitor } = cell.aggregate; - if (scopeParts.length > 0) { - findings.push({ - tone: 'scope', - text: `The reference measurement runs ExoJS ${profile.engineVersion} on ${profile.gpu} / ${profile.os} / ${profile.browser}, pooled from ${String(profile.runs)} separate runs taken on ${formatDay(profile.measuredAt)}, and publishes ${scopeParts.join('; ')}.`, - }); - } + if (reference.minMs === null || reference.maxMs === null || competitor.minMs === null || competitor.maxMs === null) return null; + if (competitor.minMs <= 0 || competitor.maxMs <= 0) return null; - const unstableRendering = allRendering.length - rendering.length; - const unstablePhysics = allPhysics.length - physics.length; + return { low: reference.minMs / competitor.maxMs, high: reference.maxMs / competitor.minMs }; +}; - if (unstableRendering > 0 || unstablePhysics > 0) { - const parts = [ - ...(unstableRendering > 0 ? [`${outOfSettled(unstableRendering, allRendering.length)} rendering comparisons`] : []), - ...(unstablePhysics > 0 ? [`${outOfSettled(unstablePhysics, allPhysics.length)} physics comparisons`] : []), - ]; +/** A ratio band as the scoreboard and the tables print it. */ +export const formatBand = (band: RatioBand): string => `${band.low.toFixed(2)}-${band.high.toFixed(2)}`; - findings.push({ - tone: 'unstable', - text: `${listOf(parts)} came out differently in the ${String(profile.runs)} runs behind this profile, so they publish no verdict and none of the sentences here rests on one. They stay in the tables with the range their runs observed: on this machine those pairs are too close, or too noisy, to be separated.`, - }); - } +/** One scoreboard line: everything measured against one arm, on one backend or in physics. */ +export interface ComparisonTally { + readonly key: string; + /** The pair, as the scoreboard names it. */ + readonly label: string; + /** What the pair is measured at, or what role the arm stands in. */ + readonly meta: string; + readonly counts: Readonly>; + readonly total: number; +} - const clearLeads = rendering.filter(entry => entry.cell.verdict.side === 'exojs' && entry.cell.verdict.structural); - const widestLead = widest(clearLeads); +const tally = (key: string, label: string, meta: string, cells: readonly (ProfileCell | null)[]): ComparisonTally => { + const counts = Object.fromEntries(OUTCOME_ORDER.map(outcome => [outcome, 0])) as Record; - if (widestLead !== undefined && widestLead.backend !== null) { - findings.push({ - tone: 'lead', - text: `ExoJS leads clearly - five times or better - in ${outOfSettled(clearLeads.length, rendering.length)} rendering comparisons whose runs agreed, the widest being ${widestLead.archetype} on ${BACKEND_LABELS[widestLead.backend]} at ${formatFactor(widestLead.cell.verdict.factor)} ahead of ${widestLead.cell.competitor} with ${String(widestLead.count)} nodes.`, - }); - } + for (const cell of cells) counts[outcomeOf(cell)] += 1; - const closest = closestArm(rendering); - - if (closest !== undefined) { - const against = rendering.filter(entry => entry.cell.competitor === closest); - const level = against.filter(entry => entry.cell.verdict.side === 'neither').length; - const ahead = against.filter(entry => entry.cell.verdict.side === 'exojs').length; - const behind = against.filter(entry => entry.cell.verdict.side === 'competitor').length; - const dropped = allRendering.filter(entry => entry.cell.competitor === closest).length - against.length; - const caveat = - dropped === 0 ? '' : ` A further ${String(dropped)} rows against that arm were not stable across the runs and are left out of this count entirely.`; - // An arm every one of whose rows was unstable is not merely absent from the - // ranking - it may well be the arm that actually sits closest, and naming - // another one "closest" without saying so would be the ranking laundering - // the instability it excluded. - const unranked = armsIn(allRendering).filter(arm => !rendering.some(entry => entry.cell.competitor === arm)); - const missing = - unranked.length === 0 - ? '' - : ` ${listOf(unranked)} cannot be placed against that at all: no comparison against ${unranked.length === 1 ? 'it' : 'them'} was stable across the runs, so how close ${unranked.length === 1 ? 'it sits' : 'they sit'} is not something this profile measured.`; - - findings.push({ - tone: 'level', - text: `Against ${closest}, the arm whose settled numbers sit closest to ExoJS, ${outOfSettled(level, against.length)} rows fall inside the 0.8-1.2 noise band and read as level, while ExoJS leads ${String(ahead)} and trails ${String(behind)}.${caveat}${missing}`, - }); - } + return { key, label, meta, counts, total: cells.length }; +}; - const widestLoss = widest(rendering.filter(entry => entry.cell.verdict.side === 'competitor')); +/** + * The scoreboard, one line per arm a domain was measured against. + * + * The split is per arm and never per backend: a backend line would pool two + * different opponents into one strip, so a reader would see a mix that belongs + * to neither of them. Nothing is summed across lines and no line is ranked + * against another, because the arms answer different questions. + */ +export const comparisonTallies = (document: BenchProfileDocument): readonly ComparisonTally[] => [ + ...(document.rendering?.backends ?? []).flatMap(backend => + backend.competitors.map(arm => + tally( + `${backend.backend}-${arm}`, + `${BACKEND_LABELS[backend.backend]} vs ${arm}`, + backend.headlineCount === null ? 'no headline count' : `${String(backend.headlineCount)} nodes`, + backend.sections.flatMap(section => section.rows.map(row => row.cells.find(cell => cell.competitor === arm) ?? null)), + ), + ), + ), + ...(document.physics === undefined + ? [] + : armsOfSection(document.physics.section).map(arm => + tally( + `physics-${arm}`, + `Physics vs ${arm}`, + isWasmReferenceArm(arm) ? 'Rust/WASM ceiling' : 'pure-JS peer', + (document.physics?.section.rows ?? []).map(row => row.cells.find(cell => cell.competitor === arm) ?? null), + ), + )), +]; - if (widestLoss !== undefined && widestLoss.backend !== null) { - const mechanism = widestLoss.cell.mechanism === null ? '' : ` The counters recorded for that row say: ${widestLoss.cell.mechanism}.`; +/** + * The sentence under the page title: which machine was measured, with which + * engine version, over how many runs, and how much the profile covers. + * + * Every value in it is copied from the document, so a re-measurement rewrites + * the sentence with the tables rather than leaving a claim behind that the + * numbers no longer support. A profile carrying only one domain yields a + * shorter sentence instead of a padded one. + */ +export const profileScope = (document: BenchProfileDocument): string => { + const rendering = renderingCells(document); + const physics = physicsCells(document); + const { profile } = document; + const parts: string[] = []; - findings.push({ - tone: 'loss', - text: `The widest rendering loss the runs agreed on is ${widestLoss.archetype} on ${BACKEND_LABELS[widestLoss.backend]}, where ${widestLoss.cell.competitor} leads by ${formatFactor(widestLoss.cell.verdict.factor)}.${mechanism}`, - }); - } + if (rendering.length > 0) { + const backends = (document.rendering?.backends ?? []).map(backend => BACKEND_LABELS[backend.backend]); - const peers = physics.filter(entry => !isWasmReferenceArm(entry.cell.competitor)); - const peerLeads = peers.filter(entry => entry.cell.verdict.side === 'exojs'); - const widestPeerLead = widest(peerLeads); - - if (widestPeerLead !== undefined) { - const behind = peers.filter(entry => entry.cell.verdict.side === 'competitor').length; - - findings.push({ - tone: 'lead', - // The body count belongs to the archetype the superlative names, not to - // the physics table: each archetype is measured on its own ladder, so a - // sentence that put one count in front of the whole paragraph would claim - // a scene size the other rows were not measured at. - text: `In physics, ExoJS leads the pure-JS peers (${listOf(armsIn(peers))}) in ${outOfSettled(peerLeads.length, peers.length)} comparisons whose runs agreed, by up to ${formatFactor(widestPeerLead.cell.verdict.factor)} against ${widestPeerLead.cell.competitor} on ${widestPeerLead.archetype} at ${String(widestPeerLead.count)} bodies, and trails in ${String(behind)}. Each archetype is measured on its own body-count ladder, so the counts differ per row.`, - }); + parts.push(`${String(rendering.length)} rendering comparisons on ${listOf(backends)} against ${listOf(armsIn(rendering))}`); } - const ceiling = physics.filter(entry => isWasmReferenceArm(entry.cell.competitor)); - const ceilingLosses = ceiling.filter(entry => entry.cell.verdict.side === 'competitor'); - const widestCeilingLoss = widest(ceilingLosses); - - if (widestCeilingLoss !== undefined) { - const level = ceiling.filter(entry => entry.cell.verdict.side === 'neither').length; - - findings.push({ - tone: 'loss', - text: `Against ${listOf(armsIn(ceiling))}, a Rust/WASM engine published here as a ceiling rather than as a peer, ExoJS trails in ${outOfSettled(ceilingLosses.length, ceiling.length)} comparisons whose runs agreed - by up to ${formatFactor(widestCeilingLoss.cell.verdict.factor)} on ${widestCeilingLoss.archetype} at ${String(widestCeilingLoss.count)} bodies - and is level in ${String(level)}.`, - }); + if (physics.length > 0) { + parts.push(`${String(physics.length)} physics comparisons against ${listOf(armsIn(physics))}`); } - return findings; + if (parts.length === 0) return ''; + + return `ExoJS ${profile.engineVersion} on ${profile.gpu} / ${profile.os} / ${profile.browser}, pooled from ${String(profile.runs)} separate runs taken on ${formatDay(profile.measuredAt)}: ${parts.join('; ')}.`; }; diff --git a/site/src/lib/bench-tables.ts b/site/src/lib/bench-tables.ts new file mode 100644 index 000000000..4ff518808 --- /dev/null +++ b/site/src/lib/bench-tables.ts @@ -0,0 +1,257 @@ +/** + * The published profiles reshaped into the tables the benchmarks page draws. + * + * A table has one column per comparison pair - a rendering backend against one + * arm, or one physics arm - and one row per archetype, so a reader follows a + * single archetype across every backend and every opponent on one line. The + * shape is derived entirely from the document: an arm that produced nothing + * anywhere gets no column, and an archetype a backend did not measure keeps its + * row and reports the gap rather than disappearing from it. + * + * ExoJS gets a column of its own in front of each backend's arms rather than + * appearing inside every cell. It is one measurement per row and backend, so + * repeating it beside each arm would print the same three numbers as often as + * the backend has opponents. + * + * Only the arrangement lives here. Timings, ratios and verdicts are read from + * the profile in `bench-profiles`, and no row is dropped, reordered by outcome + * or merged with another. + */ + +import { + armsOfSection, + BACKEND_LABELS, + type BenchProfileDocument, + isWasmReferenceArm, + type ProfileBackend, + type ProfileCell, + type ProfileRow, + type ProfileSpread, +} from './bench-profiles'; + +/** What a column carries: the ExoJS measurement itself, or a comparison against one arm. */ +export type ComparisonColumnKind = 'reference' | 'arm'; + +/** One column of a table. */ +export interface ComparisonColumn { + readonly key: string; + readonly kind: ComparisonColumnKind; + /** Heading the neighbouring columns share, such as the backend they were measured on; `null` where each column stands alone. */ + readonly group: string | null; + /** What this column alone is measured under, such as the role an arm stands in. */ + readonly overline: string; + readonly label: string; +} + +/** The ExoJS side of one row, under one backend. */ +export interface ComparisonReference { + readonly ms: number | null; + readonly p95Ms: number | null; + readonly overFrameBudget: boolean; + readonly spread: ProfileSpread | undefined; +} + +/** One column's outcome on one row. */ +export interface ComparisonEntry { + readonly key: string; + readonly kind: ComparisonColumnKind; + /** The published comparison, or `null` in a reference column and where an arm produced none. */ + readonly cell: ProfileCell | null; + /** The ExoJS measurement, in a reference column only. */ + readonly reference: ComparisonReference | null; + /** Scene size this column measured the row at; `null` where the column does not carry the row at all. */ + readonly count: number | null; +} + +/** One archetype, across every column of a table. */ +export interface ComparisonRow { + readonly key: string; + readonly archetype: string; + /** Category heading this archetype sits under, or `null` in a table without categories. */ + readonly section: string | null; + /** The size every column measured this row at, or `null` where they differ. */ + readonly count: number | null; + readonly entries: readonly ComparisonEntry[]; +} + +/** The structural evidence one comparison carries, listed away from the numbers. */ +export interface ComparisonMechanism { + readonly key: string; + readonly archetype: string; + readonly column: string; + readonly text: string; +} + +/** One published comparison table. */ +export interface ComparisonTable { + readonly columns: readonly ComparisonColumn[]; + readonly rows: readonly ComparisonRow[]; + /** What a count is counted in: `nodes` for rendering, `bodies` for physics. */ + readonly unit: string; + /** True where the rows were measured at different sizes, so the size belongs in a column of its own. */ + readonly countColumn: boolean; + readonly mechanisms: readonly ComparisonMechanism[]; +} + +/** + * The ExoJS side of a row. Every arm in a block times the same ExoJS scene, so + * the first cell that produced a number carries it for the whole block. + */ +const referenceOf = (row: ProfileRow | undefined): ComparisonReference => { + const cell = row?.cells.find(candidate => candidate.referenceMs !== null); + + return { + ms: cell?.referenceMs ?? null, + p95Ms: cell?.referenceP95Ms ?? null, + overFrameBudget: cell?.referenceOverFrameBudget ?? false, + spread: cell?.aggregate.reference, + }; +}; + +const referenceEntry = (key: string, row: ProfileRow | undefined): ComparisonEntry => ({ + key, + kind: 'reference', + cell: null, + reference: referenceOf(row), + count: row?.count ?? null, +}); + +const armEntry = (key: string, row: ProfileRow | undefined, arm: string): ComparisonEntry => ({ + key, + kind: 'arm', + cell: row?.cells.find(cell => cell.competitor === arm) ?? null, + reference: null, + count: row?.count ?? null, +}); + +const mechanismsOf = (rows: readonly ComparisonRow[], columns: readonly ComparisonColumn[]): readonly ComparisonMechanism[] => + rows.flatMap(row => + row.entries.flatMap((entry, index) => { + const text = entry.cell?.mechanism; + const column = columns[index]; + + return text === undefined || text === null || column === undefined + ? [] + : [{ key: `${row.key}-${column.key}`, archetype: row.archetype, column: column.label, text }]; + }), + ); + +/** Every row of a backend, flattened out of its categories. */ +const rowsOf = (backend: ProfileBackend): readonly ProfileRow[] => backend.sections.flatMap(section => section.rows); + +/** + * The rendering table: an ExoJS column and its arms, per backend. + * + * Archetypes are collected in the order the first backend publishes them and + * then extended by any a later backend adds, so the categories stay in the + * order the harness wrote them rather than being sorted into a ranking. + */ +export const renderingComparison = (document: BenchProfileDocument): ComparisonTable => { + const backends = document.rendering?.backends ?? []; + const columns = backends.flatMap(backend => [ + { key: `${backend.backend}-exojs`, kind: 'reference' as const, group: BACKEND_LABELS[backend.backend], overline: '', label: 'ExoJS' }, + ...backend.competitors.map(arm => ({ + key: `${backend.backend}-${arm}`, + kind: 'arm' as const, + group: BACKEND_LABELS[backend.backend], + overline: '', + label: `vs ${arm}`, + })), + ]); + const archetypes: { archetype: string; section: string }[] = []; + + for (const backend of backends) { + for (const section of backend.sections) { + for (const row of section.rows) { + if (!archetypes.some(entry => entry.archetype === row.archetype)) archetypes.push({ archetype: row.archetype, section: section.title }); + } + } + } + + const rows = archetypes.map(({ archetype, section }) => { + const entries = backends.flatMap(backend => { + const row = rowsOf(backend).find(candidate => candidate.archetype === archetype); + + return [referenceEntry(`${backend.backend}-exojs`, row), ...backend.competitors.map(arm => armEntry(`${backend.backend}-${arm}`, row, arm))]; + }); + const counts = [...new Set(entries.map(entry => entry.count).filter((count): count is number => count !== null))]; + + return { key: archetype, archetype, section, count: counts.length === 1 ? (counts[0] ?? null) : null, entries }; + }); + + return { columns, rows, unit: 'nodes', countColumn: false, mechanisms: mechanismsOf(rows, columns) }; +}; + +const singleBlockTable = ( + rows: readonly ProfileRow[], + arms: readonly string[], + columns: readonly ComparisonColumn[], + unit: string, + countColumn: boolean, +): ComparisonTable => { + const built = rows.map(row => ({ + key: row.archetype, + archetype: row.archetype, + section: null, + count: row.count, + entries: [referenceEntry('exojs', row), ...arms.map(arm => armEntry(arm, row, arm))], + })); + + return { columns, rows: built, unit, countColumn, mechanisms: mechanismsOf(built, columns) }; +}; + +/** + * The physics table. + * + * Every archetype carries its own body-count ladder, so the count is a column + * of its own here: two rows are two different scenes at two different sizes and + * are not comparable with each other, only the arms within one row are. + */ +export const physicsComparison = (document: BenchProfileDocument): ComparisonTable | null => { + const section = document.physics?.section; + + if (section === undefined) return null; + + const arms = armsOfSection(section); + + return singleBlockTable( + section.rows, + arms, + [ + { key: 'exojs', kind: 'reference', group: null, overline: 'per fixed step', label: 'ExoJS' }, + ...arms.map(arm => ({ + key: arm, + kind: 'arm' as const, + group: null, + overline: isWasmReferenceArm(arm) ? 'Rust/WASM ceiling' : 'pure-JS peer', + label: `vs ${arm}`, + })), + ], + 'bodies', + true, + ); +}; + +/** + * The WebGL1 block of one backend, as its own table. + * + * These arms render through a WebGL1 context and report no structural counters, + * so the block compares CPU time only and carries no mechanism list. + */ +export const webgl1Comparison = (backend: ProfileBackend): ComparisonTable | null => { + if (backend.webgl1.length === 0) return null; + + const arms = [...new Set(backend.webgl1.flatMap(row => row.cells.map(cell => cell.competitor)))].sort(); + const table = singleBlockTable( + backend.webgl1, + arms, + [ + { key: 'exojs', kind: 'reference', group: null, overline: 'CPU time only', label: 'ExoJS' }, + ...arms.map(arm => ({ key: arm, kind: 'arm' as const, group: null, overline: 'WebGL1', label: `vs ${arm}` })), + ], + 'nodes', + false, + ); + + return { ...table, mechanisms: [] }; +}; diff --git a/site/src/styles/tokens.scss b/site/src/styles/tokens.scss index 954ed3591..a486d6635 100644 --- a/site/src/styles/tokens.scss +++ b/site/src/styles/tokens.scss @@ -90,6 +90,15 @@ html[data-theme='dark'] { --grid-dot: oklch(40% 0.014 245 / 0.45); --grid-line: oklch(30% 0.014 245 / 0.6); + // Benchmark comparison hues. They carry the same meaning as the semantic + // colours above but have to hold their own against the page background at + // small sizes - a 6px bar and 0.66rem mono - so their lightness is tuned per + // theme instead of being shared. The two sides are equally saturated and + // equally dark on purpose: a loss must never read as quieter than a win. + --bench-lead: oklch(72% 0.14 230); + --bench-loss: oklch(70% 0.17 290); + --bench-mark: oklch(80% 0.14 70); + --bg-canvas: var(--color-bg); --bg-elevated: var(--color-surface); --bg-panel: var(--color-panel); @@ -124,6 +133,10 @@ html[data-theme='light'] { --grid-dot: oklch(75% 0.012 245 / 0.7); --grid-line: oklch(85% 0.01 245 / 0.7); + --bench-lead: oklch(50% 0.16 230); + --bench-loss: oklch(48% 0.2 290); + --bench-mark: oklch(52% 0.14 70); + --color-accent: oklch(54% 0.18 142); --color-accent-contrast: oklch(99% 0.01 142); --color-accent-soft: oklch(54% 0.18 142 / 0.1); From 3b4ddec5a4cef4b5ab76d27dc769d821349264f9 Mon Sep 17 00:00:00 2001 From: Exoridus Date: Wed, 9 Sep 2026 18:02:33 +0200 Subject: [PATCH 2/5] feat(bench): publish scoreboard and new arms --- packages/exojs-bench/competitors/package.json | 3 +- .../exojs-bench/competitors/pnpm-lock.yaml | 10 + .../rtx-5070-ti-windows-11-chromium.json | 3213 ++++++----------- packages/exojs-bench/src/comparison/build.ts | 25 +- .../exojs-bench/src/comparison/mechanism.ts | 4 +- packages/exojs-bench/src/comparison/render.ts | 4 +- .../exojs-bench/src/physics/PhysicsAdapter.ts | 2 +- .../src/physics/adapters/matter-js.ts | 5 +- .../src/physics/adapters/nape-js.ts | 143 + .../src/physics/adapters/perStepWork.ts | 4 +- .../exojs-bench/src/physics/adapters/scene.ts | 8 +- packages/exojs-bench/src/physics/driver.ts | 12 +- packages/exojs-bench/src/physics/index.ts | 6 +- .../exojs-bench/src/physics/page/harness.ts | 10 + .../src/rendering/adapters/phaser.ts | 62 +- .../exojs-bench/src/rendering/archetypes.ts | 6 +- packages/exojs-bench/src/rendering/driver.ts | 21 +- packages/exojs-bench/src/rendering/report.ts | 2 +- packages/exojs-bench/src/rendering/traits.ts | 6 +- packages/exojs-bench/src/shared/viteServer.ts | 2 +- .../components/BenchComparisonSummary.astro | 217 ++ .../src/components/BenchComparisonTable.astro | 869 +++-- site/src/components/BenchProfileReport.astro | 85 +- site/src/components/BenchScoreboard.astro | 214 -- .../src/components/pages/BenchmarksPage.astro | 110 +- site/src/lib/bench-profiles.ts | 193 +- site/src/lib/bench-tables.ts | 142 +- site/src/styles/tokens.scss | 9 +- 28 files changed, 2476 insertions(+), 2911 deletions(-) create mode 100644 packages/exojs-bench/src/physics/adapters/nape-js.ts create mode 100644 site/src/components/BenchComparisonSummary.astro delete mode 100644 site/src/components/BenchScoreboard.astro diff --git a/packages/exojs-bench/competitors/package.json b/packages/exojs-bench/competitors/package.json index 64da46291..f4e71b832 100644 --- a/packages/exojs-bench/competitors/package.json +++ b/packages/exojs-bench/competitors/package.json @@ -2,13 +2,14 @@ "name": "exojs-bench-competitors", "version": "0.0.0", "private": true, - "description": "Pinned exact-version competitor libraries for @codexo/exojs-bench (Pixi, Phaser, Excalibur, matter-js, planck, rapier2d-compat arms). Deliberately NOT a member of the repository workspace: a plain root `pnpm install` never resolves or downloads anything here, so a normal contributor pays zero weight for competitor libraries. Only `pnpm --filter @codexo/exojs-bench bench:setup` installs this folder - as its own workspace root (see pnpm-workspace.yaml beside this file), which applies the same minimumReleaseAge supply-chain quarantine as the repository workspace - and links the results into ../node_modules so the adapters' plain `import 'pixi.js'` (etc.) resolve unmodified.", + "description": "Pinned exact-version competitor libraries for @codexo/exojs-bench (Pixi, Phaser, Excalibur, matter-js, planck, nape-js, rapier2d-compat arms). Deliberately NOT a member of the repository workspace: a plain root `pnpm install` never resolves or downloads anything here, so a normal contributor pays zero weight for competitor libraries. Only `pnpm --filter @codexo/exojs-bench bench:setup` installs this folder - as its own workspace root (see pnpm-workspace.yaml beside this file), which applies the same minimumReleaseAge supply-chain quarantine as the repository workspace - and links the results into ../node_modules so the adapters' plain `import 'pixi.js'` (etc.) resolve unmodified.", "dependencies": { "pixi.js": "8.19.0", "phaser": "4.2.1", "excalibur": "0.32.0", "matter-js": "0.20.0", "planck": "1.5.0", + "@newkrok/nape-js": "3.42.0", "@types/matter-js": "0.20.2", "@dimforge/rapier2d-compat": "0.19.3" } diff --git a/packages/exojs-bench/competitors/pnpm-lock.yaml b/packages/exojs-bench/competitors/pnpm-lock.yaml index 9c4d82c56..039c1c61b 100644 --- a/packages/exojs-bench/competitors/pnpm-lock.yaml +++ b/packages/exojs-bench/competitors/pnpm-lock.yaml @@ -11,6 +11,9 @@ importers: '@dimforge/rapier2d-compat': specifier: 0.19.3 version: 0.19.3 + '@newkrok/nape-js': + specifier: 3.42.0 + version: 3.42.0 '@types/matter-js': specifier: 0.20.2 version: 0.20.2 @@ -35,6 +38,10 @@ packages: '@dimforge/rapier2d-compat@0.19.3': resolution: {integrity: sha512-5HZUgiSpu9Uba3He+4SCPVte3h2vgxcodkxWThRjd3X/zgq/Z4OyRUKpNEOsOR6AC+OfulPkkZ5asZGzRMpcwA==} + '@newkrok/nape-js@3.42.0': + resolution: {integrity: sha512-gjGp45ERUa4bI5Q72vpL3ruVUCoBouC29CoeE/pZ4hCrTRGjjfy1WQz0lQTR1tGuSHDd9XAbB6CgZ7EiazFA5Q==} + engines: {node: '>=18'} + '@pixi/colord@2.9.6': resolution: {integrity: sha512-nezytU2pw587fQstUu1AsJZDVEynjskwOL+kibwcdxsMBFqPsFFNA7xl0ii/gXuDi6M0xj3mfRJj8pBSc2jCfA==} @@ -50,6 +57,7 @@ packages: '@xmldom/xmldom@0.8.13': resolution: {integrity: sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw==} engines: {node: '>=10.0.0'} + deprecated: this version has critical issues, please update to the latest version earcut@3.2.3: resolution: {integrity: sha512-vnS4AVwp1KHAF13i1vp1/2D5evWy3k5u/iW/B81QVsUZtV8cv2tU0b2VNFlqvh4kYwrFMDdjPCfAmfyJW9y14Q==} @@ -99,6 +107,8 @@ snapshots: '@dimforge/rapier2d-compat@0.19.3': {} + '@newkrok/nape-js@3.42.0': {} + '@pixi/colord@2.9.6': {} '@types/earcut@3.0.0': {} diff --git a/packages/exojs-bench/results/rtx-5070-ti-windows-11-chromium.json b/packages/exojs-bench/results/rtx-5070-ti-windows-11-chromium.json index 7df66275b..9c8f25fd5 100644 --- a/packages/exojs-bench/results/rtx-5070-ti-windows-11-chromium.json +++ b/packages/exojs-bench/results/rtx-5070-ti-windows-11-chromium.json @@ -12,7 +12,7 @@ "prerelease": false }, "engineVersion": "0.17.0", - "measuredAt": "2026-09-06T17:02:01.221Z", + "measuredAt": "2026-09-09T15:39:44.282Z", "runs": 4 }, "rendering": { @@ -41,7 +41,7 @@ "headless": true, "software": false, "engineVersion": "0.17.0", - "timestamp": "2026-09-06T09:11:43.021Z" + "timestamp": "2026-09-09T14:29:34.661Z" }, { "backend": "webgpu", @@ -67,7 +67,7 @@ "software": false, "slotTier": 16, "engineVersion": "0.17.0", - "timestamp": "2026-09-06T09:25:27.695Z" + "timestamp": "2026-09-09T14:43:58.301Z" } ] }, @@ -95,7 +95,7 @@ "headless": true, "software": false, "engineVersion": "0.17.0", - "timestamp": "2026-09-06T09:29:12.221Z" + "timestamp": "2026-09-09T14:48:09.791Z" }, { "backend": "webgpu", @@ -121,7 +121,7 @@ "software": false, "slotTier": 16, "engineVersion": "0.17.0", - "timestamp": "2026-09-06T09:42:45.835Z" + "timestamp": "2026-09-09T15:02:35.495Z" } ] }, @@ -149,7 +149,7 @@ "headless": true, "software": false, "engineVersion": "0.17.0", - "timestamp": "2026-09-06T15:40:04.255Z" + "timestamp": "2026-09-09T15:06:47.979Z" }, { "backend": "webgpu", @@ -175,7 +175,7 @@ "software": false, "slotTier": 16, "engineVersion": "0.17.0", - "timestamp": "2026-09-06T15:53:45.041Z" + "timestamp": "2026-09-09T15:21:11.773Z" } ] }, @@ -203,7 +203,7 @@ "headless": true, "software": false, "engineVersion": "0.17.0", - "timestamp": "2026-09-06T16:15:18.189Z" + "timestamp": "2026-09-09T15:25:21.159Z" }, { "backend": "webgpu", @@ -229,7 +229,7 @@ "software": false, "slotTier": 16, "engineVersion": "0.17.0", - "timestamp": "2026-09-06T16:29:11.668Z" + "timestamp": "2026-09-09T15:39:44.282Z" } ] } @@ -251,9 +251,10 @@ "backends": [ { "backend": "webgl2", - "headlineCount": 5000, + "headlineCount": 1000, "competitors": [ "excalibur", + "phaser", "pixi" ], "sections": [ @@ -263,35 +264,35 @@ { "archetype": "static-heavy", "category": "Node scaling", - "count": 5000, + "count": 1000, "cells": [ { "competitor": "excalibur", - "referenceMs": 0.05000000596191967, - "referenceP95Ms": 0.10999999225168722, + "referenceMs": 0.1549999974668026, + "referenceP95Ms": 0.19999999925494194, "referenceOverFrameBudget": false, - "competitorMs": 15.46000000089407, - "competitorP95Ms": 16.524999996996485, + "competitorMs": 3.631249997764826, + "competitorP95Ms": 4.592499997466803, "competitorOverFrameBudget": false, "verdict": { "side": "exojs", - "ratio": 0.003234153037453306, - "factor": 309.1999631493745, - "label": "ExoJS leads clearly (309.20x)", + "ratio": 0.04268502514621991, + "factor": 23.42741972329745, + "label": "ExoJS leads clearly (23.43x)", "structural": true }, - "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 2/0/0 vs 1/2/1, none of them large enough to account for the gap); the difference is CPU-side", + "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/0/0 vs 1/2/1, none of them large enough to account for the gap); the difference is CPU-side", "aggregate": { "runs": 4, "reference": { - "minMs": 0.044999999925494194, - "maxMs": 0.06000000238418579, - "ratio": 1.3333333885228194 + "minMs": 0.14000000059604645, + "maxMs": 0.1550000011920929, + "ratio": 1.1071428609441738 }, "competitor": { - "minMs": 15.07999999995809, - "maxMs": 15.927499999990687, - "ratio": 1.0562002652543072 + "minMs": 3.5049999989569187, + "maxMs": 3.6950000002980232, + "ratio": 1.0542082742931949 }, "stable": true, "rungs": [ @@ -302,13 +303,50 @@ ] } }, + { + "competitor": "phaser", + "referenceMs": 0.1549999974668026, + "referenceP95Ms": 0.19999999925494194, + "referenceOverFrameBudget": false, + "competitorMs": 0.5700000002980232, + "competitorP95Ms": 0.7650000005960464, + "competitorOverFrameBudget": false, + "verdict": { + "side": "exojs", + "ratio": 0.27192981997501964, + "factor": 3.6774194168622745, + "label": "ExoJS leads (3.68x)", + "structural": false + }, + "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/0/0 vs 1/0/1, none of them large enough to account for the gap); the difference is CPU-side", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 0.14000000059604645, + "maxMs": 0.1550000011920929, + "ratio": 1.1071428609441738 + }, + "competitor": { + "minMs": 0.5699999965727329, + "maxMs": 0.5749999992549419, + "ratio": 1.0087719345829347 + }, + "stable": true, + "rungs": [ + "exojs-leads", + "exojs-leads", + "exojs-leads", + "exojs-leads" + ] + } + }, { "competitor": "pixi", - "referenceMs": 0.05000000596191967, - "referenceP95Ms": 0.10999999225168722, + "referenceMs": 0.1549999974668026, + "referenceP95Ms": 0.19999999925494194, "referenceOverFrameBudget": false, - "competitorMs": 0.04749999850173481, - "competitorP95Ms": 0.10249999462394044, + "competitorMs": 0.13750000298023224, + "competitorP95Ms": 0.17500000074505806, "competitorOverFrameBudget": false, "verdict": { "side": "neither", @@ -317,24 +355,24 @@ "label": "unstable across runs", "structural": false }, - "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 2/0/0 vs 1/0/0, none of them large enough to account for the gap); the difference is CPU-side", + "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/0/0 vs 1/0/0, none of them large enough to account for the gap); the difference is CPU-side", "aggregate": { "runs": 4, "reference": { - "minMs": 0.044999999925494194, - "maxMs": 0.06000000238418579, - "ratio": 1.3333333885228194 + "minMs": 0.14000000059604645, + "maxMs": 0.1550000011920929, + "ratio": 1.1071428609441738 }, "competitor": { - "minMs": 0.0400000000372529, - "maxMs": 0.054999977350234985, - "ratio": 1.3749994324753065 + "minMs": 0.125, + "maxMs": 0.14500000327825546, + "ratio": 1.1600000262260437 }, "stable": false, "rungs": [ "level", - "competitor-leads", "level", + "competitor-leads", "level" ] } @@ -344,35 +382,35 @@ { "archetype": "dynamic-heavy", "category": "Node scaling", - "count": 5000, + "count": 1000, "cells": [ { "competitor": "excalibur", - "referenceMs": 0.38999998988583684, - "referenceP95Ms": 0.6200000047683716, + "referenceMs": 0.3374999985098839, + "referenceP95Ms": 0.4299999997019768, "referenceOverFrameBudget": false, - "competitorMs": 15.237499996786937, - "competitorP95Ms": 16.18750000721775, + "competitorMs": 3.7025000024586916, + "competitorP95Ms": 4.484999995678663, "competitorOverFrameBudget": false, "verdict": { "side": "exojs", - "ratio": 0.025594749136543027, - "factor": 39.0705138255192, - "label": "ExoJS leads clearly (39.07x)", + "ratio": 0.09115462479021277, + "factor": 10.97037042609131, + "label": "ExoJS leads clearly (10.97x)", "structural": true }, - "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 2/0/1 vs 1/2/1, none of them large enough to account for the gap); the difference is CPU-side", + "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/0/1 vs 1/2/1, none of them large enough to account for the gap); the difference is CPU-side", "aggregate": { "runs": 4, "reference": { - "minMs": 0.36000001430511475, - "maxMs": 0.5750000000043656, - "ratio": 1.5972221587664426 + "minMs": 0.3049999997019768, + "maxMs": 0.3449999988079071, + "ratio": 1.1311475381803782 }, "competitor": { - "minMs": 14.96999999997206, - "maxMs": 15.572499990463257, - "ratio": 1.0402471603535284 + "minMs": 3.6149999983608723, + "maxMs": 3.7274999991059303, + "ratio": 1.03112033217042 }, "stable": true, "rungs": [ @@ -383,33 +421,70 @@ ] } }, + { + "competitor": "phaser", + "referenceMs": 0.3374999985098839, + "referenceP95Ms": 0.4299999997019768, + "referenceOverFrameBudget": false, + "competitorMs": 0.5675000008195639, + "competitorP95Ms": 0.8274999968707561, + "competitorOverFrameBudget": false, + "verdict": { + "side": "exojs", + "ratio": 0.594713652903045, + "factor": 1.681481491333827, + "label": "ExoJS leads (1.68x)", + "structural": false + }, + "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/0/1 vs 1/0/1, none of them large enough to account for the gap); the difference is CPU-side", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 0.3049999997019768, + "maxMs": 0.3449999988079071, + "ratio": 1.1311475381803782 + }, + "competitor": { + "minMs": 0.5549999997019768, + "maxMs": 0.5874999985098839, + "ratio": 1.058558556442088 + }, + "stable": true, + "rungs": [ + "exojs-leads", + "exojs-leads", + "exojs-leads", + "exojs-leads" + ] + } + }, { "competitor": "pixi", - "referenceMs": 0.38999998988583684, - "referenceP95Ms": 0.6200000047683716, + "referenceMs": 0.3374999985098839, + "referenceP95Ms": 0.4299999997019768, "referenceOverFrameBudget": false, - "competitorMs": 0.18375001102685928, - "competitorP95Ms": 0.33499999390915036, + "competitorMs": 0.24499999731779099, + "competitorP95Ms": 0.33500000461935997, "competitorOverFrameBudget": false, "verdict": { "side": "competitor", - "ratio": 2.122448797180368, - "factor": 2.122448797180368, - "label": "competitor leads (2.12x)", + "ratio": 1.3775510294071986, + "factor": 1.3775510294071986, + "label": "competitor leads (1.38x)", "structural": false }, - "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 2/0/1 vs 1/0/1, none of them large enough to account for the gap); the difference is CPU-side", + "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/0/1 vs 1/0/1, none of them large enough to account for the gap); the difference is CPU-side", "aggregate": { "runs": 4, "reference": { - "minMs": 0.36000001430511475, - "maxMs": 0.5750000000043656, - "ratio": 1.5972221587664426 + "minMs": 0.3049999997019768, + "maxMs": 0.3449999988079071, + "ratio": 1.1311475381803782 }, "competitor": { - "minMs": 0.16999999999825377, - "maxMs": 0.19499999983236194, - "ratio": 1.147058822555088 + "minMs": 0.22999999672174454, + "maxMs": 0.25999999791383743, + "ratio": 1.1304347896508324 }, "stable": true, "rungs": [ @@ -425,35 +500,35 @@ { "archetype": "deep-hierarchy", "category": "Node scaling", - "count": 5000, + "count": 1000, "cells": [ { "competitor": "excalibur", - "referenceMs": 0.11000001430511475, - "referenceP95Ms": 0.22000000975094736, + "referenceMs": 0.1837499998509884, + "referenceP95Ms": 0.23000000044703484, "referenceOverFrameBudget": false, - "competitorMs": 21.990000000048894, - "competitorP95Ms": 23.499999999417923, - "competitorOverFrameBudget": true, + "competitorMs": 4.872500000521541, + "competitorP95Ms": 6.535000003874302, + "competitorOverFrameBudget": false, "verdict": { "side": "exojs", - "ratio": 0.005002274411317425, - "factor": 199.90906491206164, - "label": "ExoJS leads clearly (199.91x)", + "ratio": 0.037711646963842015, + "factor": 26.51700682706331, + "label": "ExoJS leads clearly (26.52x)", "structural": true }, - "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 2/0/1 vs 1/2/1, none of them large enough to account for the gap); the difference is CPU-side", + "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/0/1 vs 1/2/1, none of them large enough to account for the gap); the difference is CPU-side", "aggregate": { "runs": 4, "reference": { - "minMs": 0.10000000000218279, - "maxMs": 0.125, - "ratio": 1.2499999999727152 + "minMs": 0.16999999433755875, + "maxMs": 0.1900000013411045, + "ratio": 1.1176471039395033 }, "competitor": { - "minMs": 21.66500000655651, - "maxMs": 22.469999998807907, - "ratio": 1.0371567039929739 + "minMs": 4.799999997019768, + "maxMs": 5.02250000089407, + "ratio": 1.0463541675025934 }, "stable": true, "rungs": [ @@ -465,36 +540,73 @@ } }, { - "competitor": "pixi", - "referenceMs": 0.11000001430511475, - "referenceP95Ms": 0.22000000975094736, + "competitor": "phaser", + "referenceMs": 0.1837499998509884, + "referenceP95Ms": 0.23000000044703484, "referenceOverFrameBudget": false, - "competitorMs": 0.08874999545514584, - "competitorP95Ms": 0.19999999622814357, + "competitorMs": 0.5974999982863665, + "competitorP95Ms": 0.8500000014901161, "competitorOverFrameBudget": false, "verdict": { - "side": "competitor", - "ratio": 1.2394368443737966, - "factor": 1.2394368443737966, - "label": "competitor leads (1.24x)", + "side": "exojs", + "ratio": 0.3075313813857481, + "factor": 3.2517006735831706, + "label": "ExoJS leads (3.25x)", "structural": false }, - "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 2/0/1 vs 1/0/1, none of them large enough to account for the gap); the difference is CPU-side", + "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/0/1 vs 1/0/1, none of them large enough to account for the gap); the difference is CPU-side", "aggregate": { "runs": 4, "reference": { - "minMs": 0.10000000000218279, - "maxMs": 0.125, - "ratio": 1.2499999999727152 + "minMs": 0.16999999433755875, + "maxMs": 0.1900000013411045, + "ratio": 1.1176471039395033 }, "competitor": { - "minMs": 0.08000000001629815, - "maxMs": 0.09000000357627869, - "ratio": 1.125000044474291 + "minMs": 0.5899999961256981, + "maxMs": 0.7874999977648258, + "ratio": 1.334745767688193 }, "stable": true, "rungs": [ - "competitor-leads", + "exojs-leads", + "exojs-leads", + "exojs-leads", + "exojs-leads" + ] + } + }, + { + "competitor": "pixi", + "referenceMs": 0.1837499998509884, + "referenceP95Ms": 0.23000000044703484, + "referenceOverFrameBudget": false, + "competitorMs": 0.14999999850988388, + "competitorP95Ms": 0.2150000035762787, + "competitorOverFrameBudget": false, + "verdict": { + "side": "neither", + "ratio": null, + "factor": null, + "label": "unstable across runs", + "structural": false + }, + "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/0/1 vs 1/0/1, none of them large enough to account for the gap); the difference is CPU-side", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 0.16999999433755875, + "maxMs": 0.1900000013411045, + "ratio": 1.1176471039395033 + }, + "competitor": { + "minMs": 0.13500000536441803, + "maxMs": 0.15000000223517418, + "ratio": 1.1111110835163693 + }, + "stable": false, + "rungs": [ + "level", "competitor-leads", "competitor-leads", "competitor-leads" @@ -506,35 +618,35 @@ { "archetype": "lifecycle-churn", "category": "Node scaling", - "count": 5000, + "count": 1000, "cells": [ { "competitor": "excalibur", - "referenceMs": 4.823749991657678, - "referenceP95Ms": 5.097500002382731, + "referenceMs": 1.1087500024586916, + "referenceP95Ms": 1.4750000014901161, "referenceOverFrameBudget": false, - "competitorMs": 45.1499999910593, - "competitorP95Ms": 59.43749999650754, - "competitorOverFrameBudget": true, + "competitorMs": 8.625, + "competitorP95Ms": 14.514999996870756, + "competitorOverFrameBudget": false, "verdict": { "side": "exojs", - "ratio": 0.10683831655842496, - "factor": 9.35993782205606, - "label": "ExoJS leads clearly (9.36x)", + "ratio": 0.12855072492274686, + "factor": 7.779030422434059, + "label": "ExoJS leads clearly (7.78x)", "structural": true }, - "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 2/0/2 vs 1/2/1, none of them large enough to account for the gap); the difference is CPU-side", + "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/0/4 vs 1/2/1, none of them large enough to account for the gap); the difference is CPU-side", "aggregate": { "runs": 4, "reference": { - "minMs": 4.46750000002794, - "maxMs": 5.327500015497208, - "ratio": 1.1925014024541443 + "minMs": 1.012499999254942, + "maxMs": 1.1549999974668026, + "ratio": 1.1407407390782427 }, "competitor": { - "minMs": 41.61499999999069, - "maxMs": 55.800000000046566, - "ratio": 1.34086266971186 + "minMs": 8.337500002235174, + "maxMs": 9.379999998956919, + "ratio": 1.1250374808326553 }, "stable": true, "rungs": [ @@ -545,33 +657,70 @@ ] } }, + { + "competitor": "phaser", + "referenceMs": 1.1087500024586916, + "referenceP95Ms": 1.4750000014901161, + "referenceOverFrameBudget": false, + "competitorMs": 1.4262499958276749, + "competitorP95Ms": 1.9100000001490116, + "competitorOverFrameBudget": false, + "verdict": { + "side": "neither", + "ratio": null, + "factor": null, + "label": "unstable across runs", + "structural": false + }, + "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/0/4 vs 1/0/1, none of them large enough to account for the gap); the difference is CPU-side", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 1.012499999254942, + "maxMs": 1.1549999974668026, + "ratio": 1.1407407390782427 + }, + "competitor": { + "minMs": 1.385000005364418, + "maxMs": 1.4499999955296516, + "ratio": 1.046931400659548 + }, + "stable": false, + "rungs": [ + "level", + "exojs-leads", + "exojs-leads", + "exojs-leads" + ] + } + }, { "competitor": "pixi", - "referenceMs": 4.823749991657678, - "referenceP95Ms": 5.097500002382731, + "referenceMs": 1.1087500024586916, + "referenceP95Ms": 1.4750000014901161, "referenceOverFrameBudget": false, - "competitorMs": 2.1437499999592546, - "competitorP95Ms": 2.6375000001571607, + "competitorMs": 0.7225000001490116, + "competitorP95Ms": 0.864999994635582, "competitorOverFrameBudget": false, "verdict": { "side": "competitor", - "ratio": 2.2501457687460578, - "factor": 2.2501457687460578, - "label": "competitor leads (2.25x)", + "ratio": 1.5346020792110977, + "factor": 1.5346020792110977, + "label": "competitor leads (1.53x)", "structural": false }, - "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 2/0/2 vs 1/0/2, none of them large enough to account for the gap); the difference is CPU-side", + "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/0/4 vs 1/0/2, none of them large enough to account for the gap); the difference is CPU-side", "aggregate": { "runs": 4, "reference": { - "minMs": 4.46750000002794, - "maxMs": 5.327500015497208, - "ratio": 1.1925014024541443 + "minMs": 1.012499999254942, + "maxMs": 1.1549999974668026, + "ratio": 1.1407407390782427 }, "competitor": { - "minMs": 1.989999994635582, - "maxMs": 2.415000006556511, - "ratio": 1.2135678457621086 + "minMs": 0.6675000041723251, + "maxMs": 0.794999998062849, + "ratio": 1.191011225608334 }, "stable": true, "rungs": [ @@ -592,35 +741,72 @@ { "archetype": "overdraw", "category": "Fill and GPU state", - "count": 5000, + "count": 1000, "cells": [ { "competitor": "excalibur", - "referenceMs": 0.04250000836327672, - "referenceP95Ms": 0.0799999893642962, + "referenceMs": 0.0925000049173832, + "referenceP95Ms": 0.12999999895691872, + "referenceOverFrameBudget": false, + "competitorMs": 4.763749999925494, + "competitorP95Ms": 56.24249999970198, + "competitorOverFrameBudget": false, + "verdict": { + "side": "exojs", + "ratio": 0.019417476760709505, + "factor": 51.49999726140836, + "label": "ExoJS leads clearly (51.50x)", + "structural": true + }, + "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/0/0 vs 1/2/1, none of them large enough to account for the gap); the difference is CPU-side", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 0.07750000059604645, + "maxMs": 0.09749999642372131, + "ratio": 1.2580644603078253 + }, + "competitor": { + "minMs": 3.339999996125698, + "maxMs": 5.114999998360872, + "ratio": 1.5314371270341682 + }, + "stable": true, + "rungs": [ + "exojs-leads-clearly", + "exojs-leads-clearly", + "exojs-leads-clearly", + "exojs-leads-clearly" + ] + } + }, + { + "competitor": "phaser", + "referenceMs": 0.0925000049173832, + "referenceP95Ms": 0.12999999895691872, "referenceOverFrameBudget": false, - "competitorMs": 14.863749995827675, - "competitorP95Ms": 15.484999999403954, + "competitorMs": 0.5400000009685755, + "competitorP95Ms": 0.7450000047683716, "competitorOverFrameBudget": false, "verdict": { "side": "exojs", - "ratio": 0.0028593059204579377, - "factor": 349.7352251975343, - "label": "ExoJS leads clearly (349.74x)", + "ratio": 0.17129630509531443, + "factor": 5.83783753796423, + "label": "ExoJS leads clearly (5.84x)", "structural": true }, - "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 2/0/0 vs 1/2/1, none of them large enough to account for the gap); the difference is CPU-side", + "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/0/0 vs 1/0/1, none of them large enough to account for the gap); the difference is CPU-side", "aggregate": { "runs": 4, "reference": { - "minMs": 0.040000000000873115, - "maxMs": 0.04500001668930054, - "ratio": 1.125000417207957 + "minMs": 0.07750000059604645, + "maxMs": 0.09749999642372131, + "ratio": 1.2580644603078253 }, "competitor": { - "minMs": 14.494999999995343, - "maxMs": 18.90250000008382, - "ratio": 1.304070369098992 + "minMs": 0.5325000025331974, + "maxMs": 0.5500000007450581, + "ratio": 1.032863846250911 }, "stable": true, "rungs": [ @@ -633,35 +819,35 @@ }, { "competitor": "pixi", - "referenceMs": 0.04250000836327672, - "referenceP95Ms": 0.0799999893642962, + "referenceMs": 0.0925000049173832, + "referenceP95Ms": 0.12999999895691872, "referenceOverFrameBudget": false, - "competitorMs": 0.04250000836327672, - "competitorP95Ms": 0.08250000416592229, + "competitorMs": 0.10625000298023224, + "competitorP95Ms": 0.14000000059604645, "competitorOverFrameBudget": false, "verdict": { "side": "neither", - "ratio": 1, - "factor": 1, - "label": "level", + "ratio": null, + "factor": null, + "label": "unstable across runs", "structural": false }, - "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 2/0/0 vs 1/0/0, none of them large enough to account for the gap); the difference is CPU-side", + "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/0/0 vs 1/0/0, none of them large enough to account for the gap); the difference is CPU-side", "aggregate": { "runs": 4, "reference": { - "minMs": 0.040000000000873115, - "maxMs": 0.04500001668930054, - "ratio": 1.125000417207957 + "minMs": 0.07750000059604645, + "maxMs": 0.09749999642372131, + "ratio": 1.2580644603078253 }, "competitor": { - "minMs": 0.03500000000349246, - "maxMs": 0.050000011920928955, - "ratio": 1.4285717690268493 + "minMs": 0.08999999612569809, + "maxMs": 0.10999999940395355, + "ratio": 1.2222222682134625 }, - "stable": true, + "stable": false, "rungs": [ - "level", + "exojs-leads", "level", "level", "level" @@ -673,35 +859,35 @@ { "archetype": "batch-breaking", "category": "Fill and GPU state", - "count": 5000, + "count": 1000, "cells": [ { "competitor": "excalibur", - "referenceMs": 1.0937499925494194, - "referenceP95Ms": 1.8125000083819032, + "referenceMs": 0.4100000001490116, + "referenceP95Ms": 0.5099999979138374, "referenceOverFrameBudget": false, - "competitorMs": 21.14249999076128, - "competitorP95Ms": 27.11500000063097, - "competitorOverFrameBudget": true, + "competitorMs": 4.668749997392297, + "competitorP95Ms": 6.33500000089407, + "competitorOverFrameBudget": false, "verdict": { "side": "exojs", - "ratio": 0.05173229244542318, - "factor": 19.33028583751601, - "label": "ExoJS leads clearly (19.33x)", + "ratio": 0.08781793850131507, + "factor": 11.387195111452371, + "label": "ExoJS leads clearly (11.39x)", "structural": true }, - "mechanism": "ExoJS issues fewer buffer uploads (0 vs 313 per frame)", + "mechanism": "ExoJS issues fewer buffer uploads (0 vs 63 per frame)", "aggregate": { "runs": 4, "reference": { - "minMs": 1.0649999999950523, - "maxMs": 1.272499999962747, - "ratio": 1.1948356807217453 + "minMs": 0.3825000002980232, + "maxMs": 0.42499999701976776, + "ratio": 1.1111111024539368 }, "competitor": { - "minMs": 20.09000000002561, - "maxMs": 22.28249999997206, - "ratio": 1.109133897458619 + "minMs": 4.357500001788139, + "maxMs": 4.887500002980232, + "ratio": 1.1216293748650839 }, "stable": true, "rungs": [ @@ -712,33 +898,70 @@ ] } }, + { + "competitor": "phaser", + "referenceMs": 0.4100000001490116, + "referenceP95Ms": 0.5099999979138374, + "referenceOverFrameBudget": false, + "competitorMs": 0.9825000017881393, + "competitorP95Ms": 1.3224999979138374, + "competitorOverFrameBudget": false, + "verdict": { + "side": "exojs", + "ratio": 0.4173027983743675, + "factor": 2.396341466905016, + "label": "ExoJS leads (2.40x)", + "structural": false + }, + "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 63/1000/0 vs 63/1000/1, none of them large enough to account for the gap); the difference is CPU-side", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 0.3825000002980232, + "maxMs": 0.42499999701976776, + "ratio": 1.1111111024539368 + }, + "competitor": { + "minMs": 0.9424999989569187, + "maxMs": 1.0249999947845936, + "ratio": 1.087533152168678 + }, + "stable": true, + "rungs": [ + "exojs-leads", + "exojs-leads", + "exojs-leads", + "exojs-leads" + ] + } + }, { "competitor": "pixi", - "referenceMs": 1.0937499925494194, - "referenceP95Ms": 1.8125000083819032, + "referenceMs": 0.4100000001490116, + "referenceP95Ms": 0.5099999979138374, "referenceOverFrameBudget": false, - "competitorMs": 0.7437500059604645, - "competitorP95Ms": 1.0750000029802322, + "competitorMs": 0.31250000186264515, + "competitorP95Ms": 0.38500000536441803, "competitorOverFrameBudget": false, "verdict": { "side": "competitor", - "ratio": 1.4705882134911337, - "factor": 1.4705882134911337, - "label": "competitor leads (1.47x)", + "ratio": 1.3119999926567079, + "factor": 1.3119999926567079, + "label": "competitor leads (1.31x)", "structural": false }, - "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 313/5000/0 vs 313/5000/0, none of them large enough to account for the gap); the difference is CPU-side", + "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 63/1000/0 vs 63/1000/0, none of them large enough to account for the gap); the difference is CPU-side", "aggregate": { "runs": 4, "reference": { - "minMs": 1.0649999999950523, - "maxMs": 1.272499999962747, - "ratio": 1.1948356807217453 + "minMs": 0.3825000002980232, + "maxMs": 0.42499999701976776, + "ratio": 1.1111111024539368 }, "competitor": { - "minMs": 0.7299999999959255, - "maxMs": 0.8700000001117587, - "ratio": 1.1917808220775543 + "minMs": 0.3049999997019768, + "maxMs": 0.3200000002980232, + "ratio": 1.0491803298711584 }, "stable": true, "rungs": [ @@ -754,35 +977,35 @@ { "archetype": "mixed-blend", "category": "Fill and GPU state", - "count": 5000, + "count": 1000, "cells": [ { "competitor": "excalibur", - "referenceMs": 0.19749999642226612, - "referenceP95Ms": 0.2649999928471516, + "referenceMs": 0.16500000096857548, + "referenceP95Ms": 0.19999999925494194, "referenceOverFrameBudget": false, - "competitorMs": 13.91125001013279, - "competitorP95Ms": 14.850000008940697, + "competitorMs": 3.4412500001490116, + "competitorP95Ms": 4.595000002533197, "competitorOverFrameBudget": false, "verdict": { "side": "exojs", - "ratio": 0.01419714233288953, - "factor": 70.43671018803339, - "label": "ExoJS leads clearly (70.44x)", + "ratio": 0.04794769370473831, + "factor": 20.856060484535412, + "label": "ExoJS leads clearly (20.86x)", "structural": true }, - "mechanism": "competitor issues fewer draw calls (120 vs 1 per frame)", + "mechanism": "competitor issues fewer draw calls (26 vs 1 per frame)", "aggregate": { "runs": 4, "reference": { - "minMs": 0.1899999976158142, - "maxMs": 0.22999999998137355, - "ratio": 1.2105263308815433 + "minMs": 0.14750000089406967, + "maxMs": 0.17499999701976776, + "ratio": 1.18644065056954 }, "competitor": { - "minMs": 13.692500000004657, - "maxMs": 14.399999999674037, - "ratio": 1.0516706225794514 + "minMs": 3.369999997317791, + "maxMs": 3.5075000002980232, + "ratio": 1.040801187860436 }, "stable": true, "rungs": [ @@ -794,37 +1017,74 @@ } }, { - "competitor": "pixi", - "referenceMs": 0.19749999642226612, - "referenceP95Ms": 0.2649999928471516, + "competitor": "phaser", + "referenceMs": 0.16500000096857548, + "referenceP95Ms": 0.19999999925494194, "referenceOverFrameBudget": false, - "competitorMs": 0.07750000059604645, - "competitorP95Ms": 0.1299999998882413, + "competitorMs": 0.5525000002235174, + "competitorP95Ms": 0.767500001937151, "competitorOverFrameBudget": false, "verdict": { - "side": "competitor", - "ratio": 2.54838703101044, - "factor": 2.54838703101044, - "label": "competitor leads (2.55x)", + "side": "exojs", + "ratio": 0.2986425355689116, + "factor": 3.3484848301833763, + "label": "ExoJS leads (3.35x)", "structural": false }, - "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 120/0/0 vs 158/0/0, none of them large enough to account for the gap); the difference is CPU-side", + "mechanism": "competitor issues fewer draw calls (26 vs 1 per frame)", "aggregate": { "runs": 4, "reference": { - "minMs": 0.1899999976158142, - "maxMs": 0.22999999998137355, - "ratio": 1.2105263308815433 + "minMs": 0.14750000089406967, + "maxMs": 0.17499999701976776, + "ratio": 1.18644065056954 }, "competitor": { - "minMs": 0.07499999999708962, - "maxMs": 0.09500000020489097, - "ratio": 1.2666666694476993 + "minMs": 0.494999997317791, + "maxMs": 0.6124999970197678, + "ratio": 1.2373737380579046 }, "stable": true, + "rungs": [ + "exojs-leads", + "exojs-leads", + "exojs-leads", + "exojs-leads" + ] + } + }, + { + "competitor": "pixi", + "referenceMs": 0.16500000096857548, + "referenceP95Ms": 0.19999999925494194, + "referenceOverFrameBudget": false, + "competitorMs": 0.11750000156462193, + "competitorP95Ms": 0.1550000011920929, + "competitorOverFrameBudget": false, + "verdict": { + "side": "neither", + "ratio": null, + "factor": null, + "label": "unstable across runs", + "structural": false + }, + "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 26/0/0 vs 32/0/0, none of them large enough to account for the gap); the difference is CPU-side", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 0.14750000089406967, + "maxMs": 0.17499999701976776, + "ratio": 1.18644065056954 + }, + "competitor": { + "minMs": 0.11500000208616257, + "maxMs": 0.125, + "ratio": 1.086956502021148 + }, + "stable": false, "rungs": [ "competitor-leads", - "competitor-leads", + "level", "competitor-leads", "competitor-leads" ] @@ -840,35 +1100,72 @@ { "archetype": "batch-breaking-atlased", "category": "Material variety", - "count": 5000, + "count": 1000, "cells": [ { "competitor": "excalibur", - "referenceMs": 0.025000002977321856, - "referenceP95Ms": 0.05499998852610588, + "referenceMs": 0.07249999791383743, + "referenceP95Ms": 0.0950000062584877, "referenceOverFrameBudget": false, - "competitorMs": 13.768750004819594, - "competitorP95Ms": 15.059999995399266, + "competitorMs": 3.3762500006705523, + "competitorP95Ms": 4.394999999552965, "competitorOverFrameBudget": false, "verdict": { "side": "exojs", - "ratio": 0.0018157060712534463, - "factor": 550.749934602391, - "label": "ExoJS leads clearly (550.75x)", + "ratio": 0.021473527700685172, + "factor": 46.56896686649638, + "label": "ExoJS leads clearly (46.57x)", "structural": true }, - "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 2/0/0 vs 1/2/1, none of them large enough to account for the gap); the difference is CPU-side", + "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/0/0 vs 1/2/1, none of them large enough to account for the gap); the difference is CPU-side", "aggregate": { "runs": 4, "reference": { - "minMs": 0.024999999906867743, - "maxMs": 0.025000005960464478, - "ratio": 1.0000002421438703 + "minMs": 0.07000000029802322, + "maxMs": 0.07499999552965164, + "ratio": 1.0714285030048725 }, "competitor": { - "minMs": 13.229999999981374, - "maxMs": 14.069999992847443, - "ratio": 1.063492062952929 + "minMs": 3.342500001192093, + "maxMs": 3.4049999974668026, + "ratio": 1.018698577786812 + }, + "stable": true, + "rungs": [ + "exojs-leads-clearly", + "exojs-leads-clearly", + "exojs-leads-clearly", + "exojs-leads-clearly" + ] + } + }, + { + "competitor": "phaser", + "referenceMs": 0.07249999791383743, + "referenceP95Ms": 0.0950000062584877, + "referenceOverFrameBudget": false, + "competitorMs": 0.6287500001490116, + "competitorP95Ms": 0.7800000011920929, + "competitorOverFrameBudget": false, + "verdict": { + "side": "exojs", + "ratio": 0.11530814774815933, + "factor": 8.672414044704514, + "label": "ExoJS leads clearly (8.67x)", + "structural": true + }, + "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/0/0 vs 1/0/1, none of them large enough to account for the gap); the difference is CPU-side", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 0.07000000029802322, + "maxMs": 0.07499999552965164, + "ratio": 1.0714285030048725 + }, + "competitor": { + "minMs": 0.505000002682209, + "maxMs": 0.7149999998509884, + "ratio": 1.415841576343377 }, "stable": true, "rungs": [ @@ -881,11 +1178,11 @@ }, { "competitor": "pixi", - "referenceMs": 0.025000002977321856, - "referenceP95Ms": 0.05499998852610588, + "referenceMs": 0.07249999791383743, + "referenceP95Ms": 0.0950000062584877, "referenceOverFrameBudget": false, - "competitorMs": 0.025000003166496754, - "competitorP95Ms": 0.05500000715255737, + "competitorMs": 0.08125000260770321, + "competitorP95Ms": 0.11249999701976776, "competitorOverFrameBudget": false, "verdict": { "side": "neither", @@ -894,25 +1191,25 @@ "label": "unstable across runs", "structural": false }, - "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 2/0/0 vs 1/0/0, none of them large enough to account for the gap); the difference is CPU-side", + "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/0/0 vs 1/0/0, none of them large enough to account for the gap); the difference is CPU-side", "aggregate": { "runs": 4, "reference": { - "minMs": 0.024999999906867743, - "maxMs": 0.025000005960464478, - "ratio": 1.0000002421438703 + "minMs": 0.07000000029802322, + "maxMs": 0.07499999552965164, + "ratio": 1.0714285030048725 }, "competitor": { - "minMs": 0.02000000001862645, - "maxMs": 0.025000005960464478, - "ratio": 1.2500002968590704 + "minMs": 0.07999999821186066, + "maxMs": 0.08999999612569809, + "ratio": 1.1249999767169352 }, "stable": false, "rungs": [ "level", "level", - "competitor-leads", - "level" + "level", + "exojs-leads" ] } } @@ -926,35 +1223,72 @@ { "archetype": "text-static", "category": "Text", - "count": 5000, + "count": 1000, "cells": [ { "competitor": "excalibur", - "referenceMs": 0.035000005504116416, - "referenceP95Ms": 0.08750000176951289, + "referenceMs": 0.11750000156462193, + "referenceP95Ms": 0.1549999974668026, "referenceOverFrameBudget": false, - "competitorMs": 61.69999999401625, - "competitorP95Ms": 65.81000000424683, - "competitorOverFrameBudget": true, + "competitorMs": 13.134999999776483, + "competitorP95Ms": 16.672500003129244, + "competitorOverFrameBudget": false, "verdict": { "side": "exojs", - "ratio": 0.0005672610292951501, - "factor": 1762.8568654584813, - "label": "ExoJS leads clearly (1762.86x)", + "ratio": 0.008945565402864212, + "factor": 111.78723255209981, + "label": "ExoJS leads clearly (111.79x)", "structural": true }, - "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/0/0 vs 3/4/3, none of them large enough to account for the gap); the difference is CPU-side", + "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/0/0 vs 1/2/1, none of them large enough to account for the gap); the difference is CPU-side", "aggregate": { "runs": 4, "reference": { - "minMs": 0.029999999998835847, - "maxMs": 0.03999999165534973, - "ratio": 1.3333330552300646 + "minMs": 0.11499999463558197, + "maxMs": 0.13499999791383743, + "ratio": 1.1739130800973732 }, "competitor": { - "minMs": 58.76999999996042, - "maxMs": 63.56499998271465, - "ratio": 1.0815892459206646 + "minMs": 12.792500000447035, + "maxMs": 13.327499996870756, + "ratio": 1.0418213794336546 + }, + "stable": true, + "rungs": [ + "exojs-leads-clearly", + "exojs-leads-clearly", + "exojs-leads-clearly", + "exojs-leads-clearly" + ] + } + }, + { + "competitor": "phaser", + "referenceMs": 0.11750000156462193, + "referenceP95Ms": 0.1549999974668026, + "referenceOverFrameBudget": false, + "competitorMs": 2.280000001192093, + "competitorP95Ms": 3.405000001192093, + "competitorOverFrameBudget": false, + "verdict": { + "side": "exojs", + "ratio": 0.05153508837859093, + "factor": 19.40425507090868, + "label": "ExoJS leads clearly (19.40x)", + "structural": true + }, + "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/0/0 vs 1/0/1, none of them large enough to account for the gap); the difference is CPU-side", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 0.11499999463558197, + "maxMs": 0.13499999791383743, + "ratio": 1.1739130800973732 + }, + "competitor": { + "minMs": 2.1424999982118607, + "maxMs": 2.3975000008940697, + "ratio": 1.1190198379906806 }, "stable": true, "rungs": [ @@ -967,11 +1301,11 @@ }, { "competitor": "pixi", - "referenceMs": 0.035000005504116416, - "referenceP95Ms": 0.08750000176951289, + "referenceMs": 0.11750000156462193, + "referenceP95Ms": 0.1549999974668026, "referenceOverFrameBudget": false, - "competitorMs": 0.030000000493600965, - "competitorP95Ms": 0.06499999645166099, + "competitorMs": 0.09999999776482582, + "competitorP95Ms": 0.13000000268220901, "competitorOverFrameBudget": false, "verdict": { "side": "neither", @@ -984,21 +1318,21 @@ "aggregate": { "runs": 4, "reference": { - "minMs": 0.029999999998835847, - "maxMs": 0.03999999165534973, - "ratio": 1.3333330552300646 + "minMs": 0.11499999463558197, + "maxMs": 0.13499999791383743, + "ratio": 1.1739130800973732 }, "competitor": { - "minMs": 0.025000000023283064, - "maxMs": 0.032499998807907104, - "ratio": 1.2999999511055649 + "minMs": 0.0925000011920929, + "maxMs": 0.10499999672174454, + "ratio": 1.1351350850654927 }, "stable": false, "rungs": [ "competitor-leads", "level", - "level", - "level" + "competitor-leads", + "competitor-leads" ] } } @@ -1007,35 +1341,35 @@ { "archetype": "text-dynamic", "category": "Text", - "count": 5000, + "count": 1000, "cells": [ { "competitor": "excalibur", - "referenceMs": 15.301249997613922, - "referenceP95Ms": 18.275000008339703, + "referenceMs": 3.724999997764826, + "referenceP95Ms": 5.092499997466803, "referenceOverFrameBudget": false, - "competitorMs": 63.270000003278255, - "competitorP95Ms": 66.95499999821186, - "competitorOverFrameBudget": true, + "competitorMs": 13.507500002160668, + "competitorP95Ms": 17.554999999701977, + "competitorOverFrameBudget": false, "verdict": { "side": "exojs", - "ratio": 0.2418405246850183, - "factor": 4.134956295279443, - "label": "ExoJS leads (4.13x)", + "ratio": 0.27577271865030334, + "factor": 3.6261744994002147, + "label": "ExoJS leads (3.63x)", "structural": false }, - "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/0/3 vs 3/4/3, none of them large enough to account for the gap); the difference is CPU-side", + "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/0/3 vs 1/2/1, none of them large enough to account for the gap); the difference is CPU-side", "aggregate": { "runs": 4, "reference": { - "minMs": 15.075000017881393, - "maxMs": 15.745000000111759, - "ratio": 1.0444444432129776 + "minMs": 3.557499997317791, + "maxMs": 3.7825000025331974, + "ratio": 1.0632466634954454 }, "competitor": { - "minMs": 60.8574999999837, - "maxMs": 64.75249999994412, - "ratio": 1.064001971818781 + "minMs": 13.269999999552965, + "maxMs": 13.697499997913837, + "ratio": 1.0322155236153183 }, "stable": true, "rungs": [ @@ -1046,33 +1380,70 @@ ] } }, + { + "competitor": "phaser", + "referenceMs": 3.724999997764826, + "referenceP95Ms": 5.092499997466803, + "referenceOverFrameBudget": false, + "competitorMs": 2.642500001937151, + "competitorP95Ms": 3.814999997615814, + "competitorOverFrameBudget": false, + "verdict": { + "side": "competitor", + "ratio": 1.4096499508170752, + "factor": 1.4096499508170752, + "label": "competitor leads (1.41x)", + "structural": false + }, + "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/0/3 vs 1/0/1, none of them large enough to account for the gap); the difference is CPU-side", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 3.557499997317791, + "maxMs": 3.7825000025331974, + "ratio": 1.0632466634954454 + }, + "competitor": { + "minMs": 2.425000000745058, + "maxMs": 2.905000001192093, + "ratio": 1.1979381444534265 + }, + "stable": true, + "rungs": [ + "competitor-leads", + "competitor-leads", + "competitor-leads", + "competitor-leads" + ] + } + }, { "competitor": "pixi", - "referenceMs": 15.301249997613922, - "referenceP95Ms": 18.275000008339703, + "referenceMs": 3.724999997764826, + "referenceP95Ms": 5.092499997466803, "referenceOverFrameBudget": false, - "competitorMs": 33.903750000892614, - "competitorP95Ms": 35.63499999279156, - "competitorOverFrameBudget": true, + "competitorMs": 7.403750000521541, + "competitorP95Ms": 10.432500001043081, + "competitorOverFrameBudget": false, "verdict": { "side": "exojs", - "ratio": 0.45131438254503037, - "factor": 2.2157503475977167, - "label": "ExoJS leads (2.22x)", + "ratio": 0.5031234168498973, + "factor": 1.9875838939501036, + "label": "ExoJS leads (1.99x)", "structural": false }, "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/0/3 vs 1/0/2, none of them large enough to account for the gap); the difference is CPU-side", "aggregate": { "runs": 4, "reference": { - "minMs": 15.075000017881393, - "maxMs": 15.745000000111759, - "ratio": 1.0444444432129776 + "minMs": 3.557499997317791, + "maxMs": 3.7825000025331974, + "ratio": 1.0632466634954454 }, "competitor": { - "minMs": 33.605000004172325, - "maxMs": 34.142500000074506, - "ratio": 1.0159946435302916 + "minMs": 7.285000000149012, + "maxMs": 7.767500001937151, + "ratio": 1.066231983771897 }, "stable": true, "rungs": [ @@ -1093,42 +1464,42 @@ { "archetype": "filter-chain-1", "category": "Render targets", - "count": 5000, + "count": 1000, "cells": [ { "competitor": "pixi", - "referenceMs": 0.09624999761581421, - "referenceP95Ms": 0.16250000894069672, + "referenceMs": 0.3200000002980232, + "referenceP95Ms": 0.4049999974668026, "referenceOverFrameBudget": false, - "competitorMs": 0.48375000059604645, - "competitorP95Ms": 0.7424999934155494, + "competitorMs": 0.45749999955296516, + "competitorP95Ms": 0.5399999991059303, "competitorOverFrameBudget": false, "verdict": { - "side": "neither", - "ratio": null, - "factor": null, - "label": "unstable across runs", + "side": "exojs", + "ratio": 0.699453553247439, + "factor": 1.429687497271516, + "label": "ExoJS leads (1.43x)", "structural": false }, "mechanism": "competitor issues fewer texture binds (9 vs 3 per frame)", "aggregate": { "runs": 4, "reference": { - "minMs": 0.08999999999650754, - "maxMs": 0.11000000033527613, - "ratio": 1.222222225994941 + "minMs": 0.30000000447034836, + "maxMs": 0.33500000089406967, + "ratio": 1.116666653007269 }, "competitor": { - "minMs": 0.3600000000005821, - "maxMs": 0.5825000000186265, - "ratio": 1.6180555556046794 + "minMs": 0.4399999976158142, + "maxMs": 0.4699999988079071, + "ratio": 1.0681818212605705 }, - "stable": false, + "stable": true, "rungs": [ "exojs-leads", - "exojs-leads-clearly", "exojs-leads", - "exojs-leads-clearly" + "exojs-leads", + "exojs-leads" ] } } @@ -1137,42 +1508,42 @@ { "archetype": "filter-chain-2", "category": "Render targets", - "count": 5000, + "count": 1000, "cells": [ { "competitor": "pixi", - "referenceMs": 0.09999999939464033, - "referenceP95Ms": 0.19749999884516, + "referenceMs": 0.2875000014901161, + "referenceP95Ms": 0.3499999977648258, "referenceOverFrameBudget": false, - "competitorMs": 0.39124999940395355, - "competitorP95Ms": 0.6275000125169754, + "competitorMs": 0.44749999791383743, + "competitorP95Ms": 0.5350000001490116, "competitorOverFrameBudget": false, "verdict": { - "side": "neither", - "ratio": null, - "factor": null, - "label": "unstable across runs", + "side": "exojs", + "ratio": 0.642458106883549, + "factor": 1.5565217238067455, + "label": "ExoJS leads (1.56x)", "structural": false }, "mechanism": "competitor issues fewer texture binds (11 vs 3 per frame)", "aggregate": { "runs": 4, "reference": { - "minMs": 0.09000000001105946, - "maxMs": 0.10999998450279236, - "ratio": 1.222222049880836 + "minMs": 0.25999999791383743, + "maxMs": 0.29499999806284904, + "ratio": 1.13461538626862 }, "competitor": { - "minMs": 0.27749999999650754, - "maxMs": 0.5775000001303852, - "ratio": 2.081081081577129 + "minMs": 0.4350000023841858, + "maxMs": 0.4624999985098839, + "ratio": 1.0632183815516638 }, - "stable": false, + "stable": true, "rungs": [ "exojs-leads", "exojs-leads", "exojs-leads", - "exojs-leads-clearly" + "exojs-leads" ] } } @@ -1181,40 +1552,40 @@ { "archetype": "filter-chain-4", "category": "Render targets", - "count": 5000, + "count": 1000, "cells": [ { "competitor": "pixi", - "referenceMs": 0.08999998867511749, - "referenceP95Ms": 0.1649999909568578, + "referenceMs": 0.24500000104308128, + "referenceP95Ms": 0.29250000417232513, "referenceOverFrameBudget": false, - "competitorMs": 0.4212499923305586, - "competitorP95Ms": 0.6199999968521297, + "competitorMs": 0.3974999990314245, + "competitorP95Ms": 0.4649999998509884, "competitorOverFrameBudget": false, "verdict": { - "side": "neither", - "ratio": null, - "factor": null, - "label": "unstable across runs", + "side": "exojs", + "ratio": 0.6163522053838111, + "factor": 1.6224489687309323, + "label": "ExoJS leads (1.62x)", "structural": false }, "mechanism": "competitor issues fewer texture binds (15 vs 7 per frame)", "aggregate": { "runs": 4, "reference": { - "minMs": 0.08000000000174623, - "maxMs": 0.10000000009313226, - "ratio": 1.2500000011368684 + "minMs": 0.22499999403953552, + "maxMs": 0.25, + "ratio": 1.1111111405455043 }, "competitor": { - "minMs": 0.2575000000069849, - "maxMs": 0.5324999988079071, - "ratio": 2.0679611603629615 + "minMs": 0.3725000023841858, + "maxMs": 0.4100000038743019, + "ratio": 1.1006711442955635 }, - "stable": false, + "stable": true, "rungs": [ "exojs-leads", - "exojs-leads-clearly", + "exojs-leads", "exojs-leads", "exojs-leads" ] @@ -1225,35 +1596,79 @@ { "archetype": "composite", "category": "Render targets", - "count": 5000, + "count": 1000, "cells": [ { "competitor": "pixi", - "referenceMs": 0.13874999433755875, - "referenceP95Ms": 0.22499999403953552, + "referenceMs": 0.36499999836087227, + "referenceP95Ms": 0.45749999955296516, "referenceOverFrameBudget": false, - "competitorMs": 0.09624999761581421, - "competitorP95Ms": 0.18500000357744284, + "competitorMs": 0.36249999701976776, + "competitorP95Ms": 0.4750000052154064, "competitorOverFrameBudget": false, "verdict": { - "side": "competitor", - "ratio": 1.4415584184363828, - "factor": 1.4415584184363828, - "label": "competitor leads (1.44x)", + "side": "neither", + "ratio": 1.0068965554804354, + "factor": 1, + "label": "level", "structural": false }, "mechanism": "ExoJS issues fewer draw calls (5 vs 8 per frame)", "aggregate": { "runs": 4, "reference": { - "minMs": 0.1299999999901047, - "maxMs": 0.16749999998137355, - "ratio": 1.2884615384163327 + "minMs": 0.33500000089406967, + "maxMs": 0.375, + "ratio": 1.1194029820870919 }, "competitor": { - "minMs": 0.09000000002561137, - "maxMs": 0.13000000035390258, - "ratio": 1.4444444479656486 + "minMs": 0.35750000178813934, + "maxMs": 0.3724999986588955, + "ratio": 1.041958032995047 + }, + "stable": true, + "rungs": [ + "level", + "level", + "level", + "level" + ] + } + } + ] + }, + { + "archetype": "mask-clip", + "category": "Render targets", + "count": 1000, + "cells": [ + { + "competitor": "pixi", + "referenceMs": 0.24249999970197678, + "referenceP95Ms": 0.29500000178813934, + "referenceOverFrameBudget": false, + "competitorMs": 0.14000000059604645, + "competitorP95Ms": 0.1875, + "competitorOverFrameBudget": false, + "verdict": { + "side": "competitor", + "ratio": 1.7321428476395655, + "factor": 1.7321428476395655, + "label": "competitor leads (1.73x)", + "structural": false + }, + "mechanism": "ExoJS issues fewer draw calls (4 vs 10 per frame)", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 0.2199999988079071, + "maxMs": 0.24500000476837158, + "ratio": 1.1136363913451346 + }, + "competitor": { + "minMs": 0.13500000536441803, + "maxMs": 0.14249999821186066, + "ratio": 1.055555500366072 }, "stable": true, "rungs": [ @@ -1267,44 +1682,44 @@ ] }, { - "archetype": "mask-clip", + "archetype": "mask-clip-animated", "category": "Render targets", - "count": 5000, + "count": 1000, "cells": [ { "competitor": "pixi", - "referenceMs": 1.2824999934528023, - "referenceP95Ms": 1.6300000101327896, + "referenceMs": 0.2587500009685755, + "referenceP95Ms": 0.32750000059604645, "referenceOverFrameBudget": false, - "competitorMs": 0.05000000013387762, - "competitorP95Ms": 0.08500000179628842, + "competitorMs": 0.19500000029802322, + "competitorP95Ms": 0.24249999970197678, "competitorOverFrameBudget": false, "verdict": { "side": "competitor", - "ratio": 25.64999980037683, - "factor": 25.64999980037683, - "label": "competitor leads clearly (25.65x)", - "structural": true + "ratio": 1.326923079862162, + "factor": 1.326923079862162, + "label": "competitor leads (1.33x)", + "structural": false }, "mechanism": "ExoJS issues fewer draw calls (4 vs 10 per frame)", "aggregate": { "runs": 4, "reference": { - "minMs": 1.110000000000582, - "maxMs": 1.3500000089406967, - "ratio": 1.21621622427026 + "minMs": 0.23499999940395355, + "maxMs": 0.2850000038743019, + "ratio": 1.2127659770092203 }, "competitor": { - "minMs": 0.04500001668930054, - "maxMs": 0.054999977350234985, - "ratio": 1.2222212656048212 + "minMs": 0.1899999976158142, + "maxMs": 0.20000000298023224, + "ratio": 1.0526316078415872 }, "stable": true, "rungs": [ - "competitor-leads-clearly", - "competitor-leads-clearly", - "competitor-leads-clearly", - "competitor-leads-clearly" + "competitor-leads", + "competitor-leads", + "competitor-leads", + "competitor-leads" ] } } @@ -1318,35 +1733,35 @@ { "archetype": "scrolling-world", "category": "Camera and world", - "count": 5000, + "count": 1000, "cells": [ { "competitor": "pixi", - "referenceMs": 0.04749999940395355, - "referenceP95Ms": 0.22749999165534973, + "referenceMs": 0.16499999910593033, + "referenceP95Ms": 0.39499999582767487, "referenceOverFrameBudget": false, - "competitorMs": 0.027500002877786756, - "competitorP95Ms": 0.05500000008032657, + "competitorMs": 0.09999999776482582, + "competitorP95Ms": 0.1275000013411045, "competitorOverFrameBudget": false, "verdict": { "side": "competitor", - "ratio": 1.727272524844784, - "factor": 1.727272524844784, - "label": "competitor leads (1.73x)", + "ratio": 1.6500000279396778, + "factor": 1.6500000279396778, + "label": "competitor leads (1.65x)", "structural": false }, - "mechanism": "competitor issues fewer draw calls (90 vs 1 per frame)", + "mechanism": "competitor issues fewer draw calls (120 vs 1 per frame)", "aggregate": { "runs": 4, "reference": { - "minMs": 0.04000000000814907, - "maxMs": 0.049999999813735485, - "ratio": 1.2499999950887286 + "minMs": 0.14249999821186066, + "maxMs": 0.17249999940395355, + "ratio": 1.2105263267967952 }, "competitor": { - "minMs": 0.024999999994179234, - "maxMs": 0.030000001192092896, - "ratio": 1.2000000479631125 + "minMs": 0.0949999988079071, + "maxMs": 0.10999999940395355, + "ratio": 1.1578947450975963 }, "stable": true, "rungs": [ @@ -1368,468 +1783,27 @@ "reason": "ExoJS-internal structural probe: a competitor arm renders a different scene here, so a wall-clock comparison would not be like for like" }, { - "archetype": "mixed-material", - "reason": "ExoJS-internal structural probe: a competitor arm renders a different scene here, so a wall-clock comparison would not be like for like" - }, - { - "archetype": "instanced-batch", - "reason": "ExoJS-internal structural probe: a competitor arm renders a different scene here, so a wall-clock comparison would not be like for like" - }, - { - "archetype": "mixed-sprite-mesh-static", - "reason": "ExoJS-internal structural probe: a competitor arm renders a different scene here, so a wall-clock comparison would not be like for like" - }, - { - "archetype": "mixed-sprite-mesh-array", - "reason": "ExoJS-internal structural probe: a competitor arm renders a different scene here, so a wall-clock comparison would not be like for like" - }, - { - "archetype": "mixed-material-atlased", - "reason": "ExoJS-internal structural probe: a competitor arm renders a different scene here, so a wall-clock comparison would not be like for like" - } - ], - "webgl1": [ - { - "archetype": "static-heavy", - "category": "Node scaling", - "count": 5000, - "cells": [ - { - "competitor": "phaser", - "referenceMs": 0.05000000596191967, - "referenceP95Ms": 0.10999999225168722, - "referenceOverFrameBudget": false, - "competitorMs": 1.9737500044284388, - "competitorP95Ms": 2.3874999922700226, - "competitorOverFrameBudget": false, - "verdict": { - "side": "exojs", - "ratio": 0.02533249188080369, - "factor": 39.47499538163375, - "label": "ExoJS leads clearly (39.47x)", - "structural": true - }, - "mechanism": null, - "aggregate": { - "runs": 4, - "reference": { - "minMs": 0.044999999925494194, - "maxMs": 0.06000000238418579, - "ratio": 1.3333333885228194 - }, - "competitor": { - "minMs": 1.8875000000116415, - "maxMs": 2.0674999952316284, - "ratio": 1.0953642358775506 - }, - "stable": true, - "rungs": [ - "exojs-leads-clearly", - "exojs-leads-clearly", - "exojs-leads-clearly", - "exojs-leads-clearly" - ] - } - } - ] - }, - { - "archetype": "dynamic-heavy", - "category": "Node scaling", - "count": 5000, - "cells": [ - { - "competitor": "phaser", - "referenceMs": 0.38999998988583684, - "referenceP95Ms": 0.6200000047683716, - "referenceOverFrameBudget": false, - "competitorMs": 2.3937500113388523, - "competitorP95Ms": 2.732500003417954, - "competitorOverFrameBudget": false, - "verdict": { - "side": "neither", - "ratio": null, - "factor": null, - "label": "unstable across runs", - "structural": false - }, - "mechanism": null, - "aggregate": { - "runs": 4, - "reference": { - "minMs": 0.36000001430511475, - "maxMs": 0.5750000000043656, - "ratio": 1.5972221587664426 - }, - "competitor": { - "minMs": 2.279999999969732, - "maxMs": 2.427499994635582, - "ratio": 1.0646929801174596 - }, - "stable": false, - "rungs": [ - "exojs-leads-clearly", - "exojs-leads-clearly", - "exojs-leads", - "exojs-leads-clearly" - ] - } - } - ] - }, - { - "archetype": "deep-hierarchy", - "category": "Node scaling", - "count": 5000, - "cells": [ - { - "competitor": "phaser", - "referenceMs": 0.11000001430511475, - "referenceP95Ms": 0.22000000975094736, - "referenceOverFrameBudget": false, - "competitorMs": 2.477500000037253, - "competitorP95Ms": 2.825000000098953, - "competitorOverFrameBudget": false, - "verdict": { - "side": "exojs", - "ratio": 0.04439960214065014, - "factor": 22.522724344064518, - "label": "ExoJS leads clearly (22.52x)", - "structural": true - }, - "mechanism": null, - "aggregate": { - "runs": 4, - "reference": { - "minMs": 0.10000000000218279, - "maxMs": 0.125, - "ratio": 1.2499999999727152 - }, - "competitor": { - "minMs": 1.5250000059604645, - "maxMs": 2.5850000083446503, - "ratio": 1.6950819660597864 - }, - "stable": true, - "rungs": [ - "exojs-leads-clearly", - "exojs-leads-clearly", - "exojs-leads-clearly", - "exojs-leads-clearly" - ] - } - } - ] - }, - { - "archetype": "overdraw", - "category": "Fill and GPU state", - "count": 5000, - "cells": [ - { - "competitor": "phaser", - "referenceMs": 0.04250000836327672, - "referenceP95Ms": 0.0799999893642962, - "referenceOverFrameBudget": false, - "competitorMs": 2.05124999769032, - "competitorP95Ms": 2.6175000013317913, - "competitorOverFrameBudget": false, - "verdict": { - "side": "exojs", - "ratio": 0.02071907783601763, - "factor": 48.264696330336676, - "label": "ExoJS leads clearly (48.26x)", - "structural": true - }, - "mechanism": null, - "aggregate": { - "runs": 4, - "reference": { - "minMs": 0.040000000000873115, - "maxMs": 0.04500001668930054, - "ratio": 1.125000417207957 - }, - "competitor": { - "minMs": 1.9149999999790452, - "maxMs": 2.7450000047683716, - "ratio": 1.4334203680409445 - }, - "stable": true, - "rungs": [ - "exojs-leads-clearly", - "exojs-leads-clearly", - "exojs-leads-clearly", - "exojs-leads-clearly" - ] - } - } - ] - }, - { - "archetype": "batch-breaking", - "category": "Fill and GPU state", - "count": 5000, - "cells": [ - { - "competitor": "phaser", - "referenceMs": 1.0937499925494194, - "referenceP95Ms": 1.8125000083819032, - "referenceOverFrameBudget": false, - "competitorMs": 3.8137499913573265, - "competitorP95Ms": 4.170000008336501, - "competitorOverFrameBudget": false, - "verdict": { - "side": "exojs", - "ratio": 0.28679121469106844, - "factor": 3.486857158707599, - "label": "ExoJS leads (3.49x)", - "structural": false - }, - "mechanism": null, - "aggregate": { - "runs": 4, - "reference": { - "minMs": 1.0649999999950523, - "maxMs": 1.272499999962747, - "ratio": 1.1948356807217453 - }, - "competitor": { - "minMs": 3.6049999999813735, - "maxMs": 4.3600000001024455, - "ratio": 1.209431345388342 - }, - "stable": true, - "rungs": [ - "exojs-leads", - "exojs-leads", - "exojs-leads", - "exojs-leads" - ] - } - } - ] - }, - { - "archetype": "mixed-blend", - "category": "Fill and GPU state", - "count": 5000, - "cells": [ - { - "competitor": "phaser", - "referenceMs": 0.19749999642226612, - "referenceP95Ms": 0.2649999928471516, - "referenceOverFrameBudget": false, - "competitorMs": 2.163749999104766, - "competitorP95Ms": 2.364999994635582, - "competitorOverFrameBudget": false, - "verdict": { - "side": "exojs", - "ratio": 0.09127671704401161, - "factor": 10.955696396462443, - "label": "ExoJS leads clearly (10.96x)", - "structural": true - }, - "mechanism": null, - "aggregate": { - "runs": 4, - "reference": { - "minMs": 0.1899999976158142, - "maxMs": 0.22999999998137355, - "ratio": 1.2105263308815433 - }, - "competitor": { - "minMs": 2.084999993443489, - "maxMs": 2.349999999627471, - "ratio": 1.1270983247085389 - }, - "stable": true, - "rungs": [ - "exojs-leads-clearly", - "exojs-leads-clearly", - "exojs-leads-clearly", - "exojs-leads-clearly" - ] - } - } - ] - }, - { - "archetype": "batch-breaking-atlased", - "category": "Material variety", - "count": 5000, - "cells": [ - { - "competitor": "phaser", - "referenceMs": 0.025000002977321856, - "referenceP95Ms": 0.05499998852610588, - "referenceOverFrameBudget": false, - "competitorMs": 1.8950000083714258, - "competitorP95Ms": 2.250000005995389, - "competitorOverFrameBudget": false, - "verdict": { - "side": "exojs", - "ratio": 0.01319261365006906, - "factor": 75.7999913076182, - "label": "ExoJS leads clearly (75.80x)", - "structural": true - }, - "mechanism": null, - "aggregate": { - "runs": 4, - "reference": { - "minMs": 0.024999999906867743, - "maxMs": 0.025000005960464478, - "ratio": 1.0000002421438703 - }, - "competitor": { - "minMs": 1.4524999856948853, - "maxMs": 2.334999999962747, - "ratio": 1.607573165548548 - }, - "stable": true, - "rungs": [ - "exojs-leads-clearly", - "exojs-leads-clearly", - "exojs-leads-clearly", - "exojs-leads-clearly" - ] - } - } - ] + "archetype": "mixed-material", + "reason": "ExoJS-internal structural probe: a competitor arm renders a different scene here, so a wall-clock comparison would not be like for like" }, { - "archetype": "text-static", - "category": "Text", - "count": 5000, - "cells": [ - { - "competitor": "phaser", - "referenceMs": 0.035000005504116416, - "referenceP95Ms": 0.08750000176951289, - "referenceOverFrameBudget": false, - "competitorMs": 9.974999998812564, - "competitorP95Ms": 10.5625, - "competitorOverFrameBudget": false, - "verdict": { - "side": "exojs", - "ratio": 0.0035087724820333692, - "factor": 284.9999551468466, - "label": "ExoJS leads clearly (285.00x)", - "structural": true - }, - "mechanism": null, - "aggregate": { - "runs": 4, - "reference": { - "minMs": 0.029999999998835847, - "maxMs": 0.03999999165534973, - "ratio": 1.3333330552300646 - }, - "competitor": { - "minMs": 8.909999999974389, - "maxMs": 10.822500005364418, - "ratio": 1.2146464652520232 - }, - "stable": true, - "rungs": [ - "exojs-leads-clearly", - "exojs-leads-clearly", - "exojs-leads-clearly", - "exojs-leads-clearly" - ] - } - } - ] + "archetype": "instanced-batch", + "reason": "ExoJS-internal structural probe: a competitor arm renders a different scene here, so a wall-clock comparison would not be like for like" }, { - "archetype": "text-dynamic", - "category": "Text", - "count": 5000, - "cells": [ - { - "competitor": "phaser", - "referenceMs": 15.301249997613922, - "referenceP95Ms": 18.275000008339703, - "referenceOverFrameBudget": false, - "competitorMs": 11.643749997019768, - "competitorP95Ms": 12.357499986886978, - "competitorOverFrameBudget": false, - "verdict": { - "side": "competitor", - "ratio": 1.3141170156977173, - "factor": 1.3141170156977173, - "label": "competitor leads (1.31x)", - "structural": false - }, - "mechanism": null, - "aggregate": { - "runs": 4, - "reference": { - "minMs": 15.075000017881393, - "maxMs": 15.745000000111759, - "ratio": 1.0444444432129776 - }, - "competitor": { - "minMs": 10.372499999997672, - "maxMs": 12.270000000018626, - "ratio": 1.1829356471459513 - }, - "stable": true, - "rungs": [ - "competitor-leads", - "competitor-leads", - "competitor-leads", - "competitor-leads" - ] - } - } - ] + "archetype": "mixed-sprite-mesh-static", + "reason": "ExoJS-internal structural probe: a competitor arm renders a different scene here, so a wall-clock comparison would not be like for like" }, { - "archetype": "lifecycle-churn", - "category": "Node scaling", - "count": 5000, - "cells": [ - { - "competitor": "phaser", - "referenceMs": 4.823749991657678, - "referenceP95Ms": 5.097500002382731, - "referenceOverFrameBudget": false, - "competitorMs": 15.714999994350364, - "competitorP95Ms": 16.26000000355998, - "competitorOverFrameBudget": false, - "verdict": { - "side": "exojs", - "ratio": 0.3069519563087397, - "factor": 3.257838822809703, - "label": "ExoJS leads (3.26x)", - "structural": false - }, - "mechanism": null, - "aggregate": { - "runs": 4, - "reference": { - "minMs": 4.46750000002794, - "maxMs": 5.327500015497208, - "ratio": 1.1925014024541443 - }, - "competitor": { - "minMs": 14.945000000298023, - "maxMs": 16.419999986886978, - "ratio": 1.0986952148919065 - }, - "stable": true, - "rungs": [ - "exojs-leads", - "exojs-leads", - "exojs-leads", - "exojs-leads" - ] - } - } - ] + "archetype": "mixed-sprite-mesh-array", + "reason": "ExoJS-internal structural probe: a competitor arm renders a different scene here, so a wall-clock comparison would not be like for like" + }, + { + "archetype": "mixed-material-atlased", + "reason": "ExoJS-internal structural probe: a competitor arm renders a different scene here, so a wall-clock comparison would not be like for like" } - ] + ], + "webgl1": [] }, { "backend": "webgpu", @@ -1848,11 +1822,11 @@ "cells": [ { "competitor": "pixi", - "referenceMs": 0.1524999999674037, - "referenceP95Ms": 0.2050000000745058, + "referenceMs": 0.24625000171363354, + "referenceP95Ms": 0.32500000298023224, "referenceOverFrameBudget": false, - "competitorMs": 0.14374999911524355, - "competitorP95Ms": 0.22499999462161213, + "competitorMs": 0.28500000201165676, + "competitorP95Ms": 0.3500000014901161, "competitorOverFrameBudget": false, "verdict": { "side": "neither", @@ -1865,21 +1839,21 @@ "aggregate": { "runs": 4, "reference": { - "minMs": 0.11499997973442078, - "maxMs": 0.16250000894069672, - "ratio": 1.4130438050160687 + "minMs": 0.22999999672174454, + "maxMs": 0.29250000044703484, + "ratio": 1.2717391505048723 }, "competitor": { - "minMs": 0.12250000005587935, - "maxMs": 0.23499998450279236, - "ratio": 1.9183672195558794 + "minMs": 0.2849999964237213, + "maxMs": 0.29999999701976776, + "ratio": 1.0526315816991987 }, "stable": false, "rungs": [ "level", - "exojs-leads", "level", - "competitor-leads" + "level", + "exojs-leads" ] } } @@ -1892,35 +1866,35 @@ "cells": [ { "competitor": "pixi", - "referenceMs": 0.42750000359956175, - "referenceP95Ms": 0.6174999922513962, + "referenceMs": 0.6887500025331974, + "referenceP95Ms": 0.8749999962747097, "referenceOverFrameBudget": false, - "competitorMs": 0.28249999543186277, - "competitorP95Ms": 0.4050000023562461, + "competitorMs": 0.5300000011920929, + "competitorP95Ms": 0.642500001937151, "competitorOverFrameBudget": false, "verdict": { - "side": "competitor", - "ratio": 1.5132743734952452, - "factor": 1.5132743734952452, - "label": "competitor leads (1.51x)", + "side": "neither", + "ratio": null, + "factor": null, + "label": "unstable across runs", "structural": false }, "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/2/1 vs 1/2/2, none of them large enough to account for the gap); the difference is CPU-side", "aggregate": { "runs": 4, "reference": { - "minMs": 0.4050000000279397, - "maxMs": 0.5499999821186066, - "ratio": 1.3580246471127502 + "minMs": 0.6225000023841858, + "maxMs": 0.7124999985098839, + "ratio": 1.1445783064754964 }, "competitor": { - "minMs": 0.2550000000046566, - "maxMs": 0.3474999964237213, - "ratio": 1.362745083989708 + "minMs": 0.5199999958276749, + "maxMs": 0.5449999943375587, + "ratio": 1.0480769205970701 }, - "stable": true, + "stable": false, "rungs": [ - "competitor-leads", + "level", "competitor-leads", "competitor-leads", "competitor-leads" @@ -1936,38 +1910,38 @@ "cells": [ { "competitor": "pixi", - "referenceMs": 0.14875000179745257, - "referenceP95Ms": 0.26750000519677997, + "referenceMs": 0.3049999978393316, + "referenceP95Ms": 0.375, "referenceOverFrameBudget": false, - "competitorMs": 0.1537500098347664, - "competitorP95Ms": 0.23999999463558197, + "competitorMs": 0.2999999988824129, + "competitorP95Ms": 0.35999999940395355, "competitorOverFrameBudget": false, "verdict": { "side": "neither", - "ratio": null, - "factor": null, - "label": "unstable across runs", + "ratio": 1.0166666632518173, + "factor": 1, + "label": "level", "structural": false }, "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/2/1 vs 1/2/2, none of them large enough to account for the gap); the difference is CPU-side", "aggregate": { "runs": 4, "reference": { - "minMs": 0.14000000001396984, - "maxMs": 0.23000001907348633, - "ratio": 1.6428572789323992 + "minMs": 0.29500000178813934, + "maxMs": 0.3100000023841858, + "ratio": 1.0508474593394037 }, "competitor": { - "minMs": 0.12749999994412065, - "maxMs": 0.18500000005587935, - "ratio": 1.4509803932310525 + "minMs": 0.2875000014901161, + "maxMs": 0.3099999986588955, + "ratio": 1.0782608593118665 }, - "stable": false, + "stable": true, "rungs": [ "level", - "competitor-leads", "level", - "exojs-leads" + "level", + "level" ] } } @@ -1980,31 +1954,31 @@ "cells": [ { "competitor": "pixi", - "referenceMs": 4.812500000902219, - "referenceP95Ms": 5.1775000024354085, + "referenceMs": 3.555000001564622, + "referenceP95Ms": 4.704999998211861, "referenceOverFrameBudget": false, - "competitorMs": 2.1162499859929085, - "competitorP95Ms": 2.4899999976623803, + "competitorMs": 2.551249999552965, + "competitorP95Ms": 2.9200000017881393, "competitorOverFrameBudget": false, "verdict": { "side": "competitor", - "ratio": 2.2740697142376, - "factor": 2.2740697142376, - "label": "competitor leads (2.27x)", + "ratio": 1.393434591744257, + "factor": 1.393434591744257, + "label": "competitor leads (1.39x)", "structural": false }, - "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/2/1 vs 1/2/3, none of them large enough to account for the gap); the difference is CPU-side", + "mechanism": "competitor issues fewer buffer uploads (13 vs 3 per frame)", "aggregate": { "runs": 4, "reference": { - "minMs": 4.5999999940395355, - "maxMs": 7.394999999785796, - "ratio": 1.6076086976886719 + "minMs": 3.4024999998509884, + "maxMs": 3.662499997764826, + "ratio": 1.0764144005658263 }, "competitor": { - "minMs": 2.080000000074506, - "maxMs": 2.147499999962747, - "ratio": 1.0324519230220306 + "minMs": 2.544999998062849, + "maxMs": 2.6150000020861626, + "ratio": 1.027504913193161 }, "stable": true, "rungs": [ @@ -2029,37 +2003,37 @@ "cells": [ { "competitor": "pixi", - "referenceMs": 0.11999998986721039, - "referenceP95Ms": 0.16500000655651093, + "referenceMs": 0.16375000402331352, + "referenceP95Ms": 0.2199999988079071, "referenceOverFrameBudget": false, - "competitorMs": 0.12000000008265488, - "competitorP95Ms": 0.17749999999068677, + "competitorMs": 0.1849999986588955, + "competitorP95Ms": 0.2524999976158142, "competitorOverFrameBudget": false, "verdict": { "side": "neither", - "ratio": null, - "factor": null, - "label": "unstable across runs", + "ratio": 0.8851351632993096, + "factor": 1, + "label": "level", "structural": false }, "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/2/0 vs 1/2/1, none of them large enough to account for the gap); the difference is CPU-side", "aggregate": { "runs": 4, "reference": { - "minMs": 0.060000000055879354, - "maxMs": 0.1474999999627471, - "ratio": 2.4583333304229504 + "minMs": 0.16250000149011612, + "maxMs": 0.17249999940395355, + "ratio": 1.0615384481362338 }, "competitor": { - "minMs": 0.10000000894069672, - "maxMs": 0.1525000035762787, - "ratio": 1.524999899417171 + "minMs": 0.17999999970197678, + "maxMs": 0.19500000029802322, + "ratio": 1.0833333367826763 }, - "stable": false, + "stable": true, "rungs": [ "level", "level", - "exojs-leads", + "level", "level" ] } @@ -2073,31 +2047,31 @@ "cells": [ { "competitor": "pixi", - "referenceMs": 0.7700000011827797, - "referenceP95Ms": 0.9350000023841858, + "referenceMs": 0.8775000013411045, + "referenceP95Ms": 1.2575000002980232, "referenceOverFrameBudget": false, - "competitorMs": 0.4725000000325963, - "competitorP95Ms": 0.6425000000745058, + "competitorMs": 0.5937500018626451, + "competitorP95Ms": 0.7925000004470348, "competitorOverFrameBudget": false, "verdict": { "side": "competitor", - "ratio": 1.629629632020444, - "factor": 1.629629632020444, - "label": "competitor leads (1.63x)", + "ratio": 1.477894734464524, + "factor": 1.477894734464524, + "label": "competitor leads (1.48x)", "structural": false }, "mechanism": "competitor issues fewer texture binds (626 vs 314 per frame)", "aggregate": { "runs": 4, "reference": { - "minMs": 0.690000000060536, - "maxMs": 0.8125, - "ratio": 1.177536231780749 + "minMs": 0.8075000010430813, + "maxMs": 0.8900000005960464, + "ratio": 1.1021671819769616 }, "competitor": { - "minMs": 0.4325000047683716, - "maxMs": 0.5149999856948853, - "ratio": 1.1907513988830984 + "minMs": 0.5824999995529652, + "maxMs": 0.5975000038743019, + "ratio": 1.0257510803997396 }, "stable": true, "rungs": [ @@ -2117,38 +2091,38 @@ "cells": [ { "competitor": "pixi", - "referenceMs": 0.22749999462394044, - "referenceP95Ms": 0.30749999987892807, + "referenceMs": 0.3374999985098839, + "referenceP95Ms": 0.4675000011920929, "referenceOverFrameBudget": false, - "competitorMs": 0.29625000001396984, - "competitorP95Ms": 0.37500000244472176, + "competitorMs": 0.37124999798834324, + "competitorP95Ms": 0.5324999988079071, "competitorOverFrameBudget": false, "verdict": { "side": "neither", - "ratio": null, - "factor": null, - "label": "unstable across runs", + "ratio": 0.909090910003132, + "factor": 1, + "label": "level", "structural": false }, "mechanism": "competitor issues fewer texture binds (240 vs 159 per frame)", "aggregate": { "runs": 4, "reference": { - "minMs": 0.22000000020489097, - "maxMs": 0.26500000059604645, - "ratio": 1.2045454561329363 + "minMs": 0.3150000050663948, + "maxMs": 0.3449999988079071, + "ratio": 1.095238073838091 }, "competitor": { - "minMs": 0.26999998092651367, - "maxMs": 0.3199999928474426, - "ratio": 1.1851852424187301 + "minMs": 0.36500000208616257, + "maxMs": 0.39000000059604645, + "ratio": 1.0684931462109482 }, - "stable": false, + "stable": true, "rungs": [ "level", "level", - "exojs-leads", - "exojs-leads" + "level", + "level" ] } } @@ -2166,37 +2140,37 @@ "cells": [ { "competitor": "pixi", - "referenceMs": 0.08750000596046448, - "referenceP95Ms": 0.12750000250525773, + "referenceMs": 0.16250000149011612, + "referenceP95Ms": 0.20749999955296516, "referenceOverFrameBudget": false, - "competitorMs": 0.09250000398606062, - "competitorP95Ms": 0.13999999989755452, + "competitorMs": 0.17999999970197678, + "competitorP95Ms": 0.23000000044703484, "competitorOverFrameBudget": false, "verdict": { "side": "neither", - "ratio": null, - "factor": null, - "label": "unstable across runs", + "ratio": 0.902777787550916, + "factor": 1, + "label": "level", "structural": false }, "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/2/0 vs 1/2/1, none of them large enough to account for the gap); the difference is CPU-side", "aggregate": { "runs": 4, "reference": { - "minMs": 0.05999999993946403, - "maxMs": 0.10999999986961484, - "ratio": 1.8333333330099575 + "minMs": 0.1599999964237213, + "maxMs": 0.16499999910593033, + "ratio": 1.0312500174622987 }, "competitor": { - "minMs": 0.0800000000745058, - "maxMs": 0.12000000476837158, - "ratio": 1.500000058207661 + "minMs": 0.17999999970197678, + "maxMs": 0.1850000023841858, + "ratio": 1.0277777927249303 }, - "stable": false, + "stable": true, "rungs": [ "level", "level", - "exojs-leads", + "level", "level" ] } @@ -2215,38 +2189,38 @@ "cells": [ { "competitor": "pixi", - "referenceMs": 0.10750000504776835, - "referenceP95Ms": 0.15000000596046448, + "referenceMs": 0.2374999988824129, + "referenceP95Ms": 0.29500000178813934, "referenceOverFrameBudget": false, - "competitorMs": 0.11749999993480742, - "competitorP95Ms": 0.15500000026077032, + "competitorMs": 0.21749999932944775, + "competitorP95Ms": 0.26999999955296516, "competitorOverFrameBudget": false, "verdict": { "side": "neither", - "ratio": null, - "factor": null, - "label": "unstable across runs", + "ratio": 1.0919540212166674, + "factor": 1, + "label": "level", "structural": false }, "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/2/0 vs 1/2/1, none of them large enough to account for the gap); the difference is CPU-side", "aggregate": { "runs": 4, "reference": { - "minMs": 0.08499999344348907, - "maxMs": 0.1699999999254942, - "ratio": 2.000000153394318 + "minMs": 0.22500000149011612, + "maxMs": 0.24500000104308128, + "ratio": 1.0888888863133797 }, "competitor": { - "minMs": 0.0949999988079071, - "maxMs": 0.13999998569488525, - "ratio": 1.4736840784384586 + "minMs": 0.19999999552965164, + "maxMs": 0.23499999940395355, + "ratio": 1.1750000232830649 }, - "stable": false, + "stable": true, "rungs": [ "level", "level", "level", - "competitor-leads" + "level" ] } } @@ -2259,31 +2233,31 @@ "cells": [ { "competitor": "pixi", - "referenceMs": 16.501249998807907, - "referenceP95Ms": 19.192500004777685, - "referenceOverFrameBudget": false, - "competitorMs": 34.44374999520369, - "competitorP95Ms": 35.6449999952456, + "referenceMs": 18.540000000968575, + "referenceP95Ms": 22.15249999985099, + "referenceOverFrameBudget": true, + "competitorMs": 37.40625, + "competitorP95Ms": 40.60249999910593, "competitorOverFrameBudget": true, "verdict": { "side": "exojs", - "ratio": 0.47907820725402184, - "factor": 2.0873418679004314, - "label": "ExoJS leads (2.09x)", + "ratio": 0.49563909777025433, + "factor": 2.0175970872732365, + "label": "ExoJS leads (2.02x)", "structural": false }, "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 1/2/3 vs 1/2/3, none of them large enough to account for the gap); the difference is CPU-side", "aggregate": { "runs": 4, "reference": { - "minMs": 15.977499999979045, - "maxMs": 22.47000000020489, - "ratio": 1.4063526834757853 + "minMs": 18.277499996125698, + "maxMs": 18.87000000104308, + "ratio": 1.0324169063079183 }, "competitor": { - "minMs": 34.31750000012107, - "maxMs": 34.855000004172325, - "ratio": 1.015662562950371 + "minMs": 36.75499999523163, + "maxMs": 39.872499998658895, + "ratio": 1.0848183921597527 }, "stable": true, "rungs": [ @@ -2308,31 +2282,31 @@ "cells": [ { "competitor": "pixi", - "referenceMs": 0.23749999701976776, - "referenceP95Ms": 0.3199999928474426, + "referenceMs": 0.3925000000745058, + "referenceP95Ms": 0.47749999910593033, "referenceOverFrameBudget": false, - "competitorMs": 0.6874999964493327, - "competitorP95Ms": 0.819999998784624, + "competitorMs": 1.0974999982863665, + "competitorP95Ms": 1.3250000029802322, "competitorOverFrameBudget": false, "verdict": { "side": "exojs", - "ratio": 0.3454545429037992, - "factor": 2.8947368634792454, - "label": "ExoJS leads (2.89x)", + "ratio": 0.35763098012515193, + "factor": 2.7961783390523185, + "label": "ExoJS leads (2.80x)", "structural": false }, "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 3/6/3 vs 2/4/2, none of them large enough to account for the gap); the difference is CPU-side", "aggregate": { "runs": 4, "reference": { - "minMs": 0.17999999999301508, - "maxMs": 0.3600000003352761, - "ratio": 2.0000000019402555 + "minMs": 0.3674999997019768, + "maxMs": 0.3999999985098839, + "ratio": 1.088435370977586 }, "competitor": { - "minMs": 0.5049999998882413, - "maxMs": 0.8550000190734863, - "ratio": 1.6930693450746566 + "minMs": 1.0649999976158142, + "maxMs": 1.1099999994039536, + "ratio": 1.0422535229003564 }, "stable": true, "rungs": [ @@ -2352,31 +2326,31 @@ "cells": [ { "competitor": "pixi", - "referenceMs": 0.2150000035762787, - "referenceP95Ms": 0.279999993450474, + "referenceMs": 0.35999999940395355, + "referenceP95Ms": 0.4700000025331974, "referenceOverFrameBudget": false, - "competitorMs": 0.6199999980162829, - "competitorP95Ms": 0.8000000054016709, + "competitorMs": 0.8799999989569187, + "competitorP95Ms": 1.0700000040233135, "competitorOverFrameBudget": false, "verdict": { "side": "exojs", - "ratio": 0.34677420042609775, - "factor": 2.8837208730385737, - "label": "ExoJS leads (2.88x)", + "ratio": 0.4090909088984871, + "factor": 2.4444444455942254, + "label": "ExoJS leads (2.44x)", "structural": false }, "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 4/8/3 vs 3/6/3, none of them large enough to account for the gap); the difference is CPU-side", "aggregate": { "runs": 4, "reference": { - "minMs": 0.17749999999068677, - "maxMs": 0.34000000008381903, - "ratio": 1.9154929583192026 + "minMs": 0.3425000011920929, + "maxMs": 0.380000002682209, + "ratio": 1.109489055064511 }, "competitor": { - "minMs": 0.5600000000558794, - "maxMs": 0.695000022649765, - "ratio": 1.2410714688935978 + "minMs": 0.7100000008940697, + "maxMs": 0.8974999971687794, + "ratio": 1.2640845014628166 }, "stable": true, "rungs": [ @@ -2396,31 +2370,31 @@ "cells": [ { "competitor": "pixi", - "referenceMs": 0.24125000089406967, - "referenceP95Ms": 0.32999998331069946, + "referenceMs": 0.4087500013411045, + "referenceP95Ms": 0.5075000002980232, "referenceOverFrameBudget": false, - "competitorMs": 0.7175000041024759, - "competitorP95Ms": 0.9000000001396984, + "competitorMs": 1.0762499999254942, + "competitorP95Ms": 1.2525000013411045, "competitorOverFrameBudget": false, "verdict": { "side": "exojs", - "ratio": 0.33623693312148534, - "factor": 2.9740932702318315, - "label": "ExoJS leads (2.97x)", + "ratio": 0.37979094203893254, + "factor": 2.6330275141145667, + "label": "ExoJS leads (2.63x)", "structural": false }, "mechanism": "structurally equivalent at this scale (draw/bind/upload per frame: 6/12/3 vs 5/10/5, none of them large enough to account for the gap); the difference is CPU-side", "aggregate": { "runs": 4, "reference": { - "minMs": 0.19500000006519258, - "maxMs": 0.3850000000093132, - "ratio": 1.974358973746665 + "minMs": 0.3674999997019768, + "maxMs": 0.41499999910593033, + "ratio": 1.1292516991631933 }, "competitor": { - "minMs": 0.6899999999441206, - "maxMs": 0.8549999892711639, - "ratio": 1.2391304193339214 + "minMs": 0.8125, + "maxMs": 1.094999998807907, + "ratio": 1.3476923062251165 }, "stable": true, "rungs": [ @@ -2440,11 +2414,11 @@ "cells": [ { "competitor": "pixi", - "referenceMs": 0.27250000089406967, - "referenceP95Ms": 0.44749999046325684, + "referenceMs": 0.39375000074505806, + "referenceP95Ms": 0.494999997317791, "referenceOverFrameBudget": false, - "competitorMs": 0.34750000946223736, - "competitorP95Ms": 0.4424999987240881, + "competitorMs": 0.4874999988824129, + "competitorP95Ms": 0.6150000020861626, "competitorOverFrameBudget": false, "verdict": { "side": "neither", @@ -2457,21 +2431,21 @@ "aggregate": { "runs": 4, "reference": { - "minMs": 0.19999999995343387, - "maxMs": 0.3925000000745058, - "ratio": 1.962500000829459 + "minMs": 0.3799999952316284, + "maxMs": 0.4049999974668026, + "ratio": 1.065789480391797 }, "competitor": { - "minMs": 0.29499998688697815, - "maxMs": 0.3674999999348074, - "ratio": 1.245762767018718 + "minMs": 0.47999999672174454, + "maxMs": 0.5225000008940697, + "ratio": 1.0885416759637236 }, "stable": false, "rungs": [ "level", "level", "exojs-leads", - "level" + "exojs-leads" ] } } @@ -2484,38 +2458,82 @@ "cells": [ { "competitor": "pixi", - "referenceMs": 1.6575000062584877, - "referenceP95Ms": 1.967500001192093, + "referenceMs": 0.35249999910593033, + "referenceP95Ms": 0.4299999997019768, "referenceOverFrameBudget": false, - "competitorMs": 0.12999999523162842, - "competitorP95Ms": 0.2125000017695129, + "competitorMs": 0.2512499988079071, + "competitorP95Ms": 0.3199999965727329, "competitorOverFrameBudget": false, "verdict": { - "side": "competitor", - "ratio": 12.750000515809445, - "factor": 12.750000515809445, - "label": "competitor leads clearly (12.75x)", - "structural": true + "side": "neither", + "ratio": null, + "factor": null, + "label": "unstable across runs", + "structural": false }, "mechanism": "ExoJS issues fewer draw calls (4 vs 10 per frame)", "aggregate": { "runs": 4, "reference": { - "minMs": 1.4449999999487773, - "maxMs": 2.6625000000931323, - "ratio": 1.842560553762985 + "minMs": 0.32999999821186066, + "maxMs": 0.369999997317791, + "ratio": 1.1212121191596196 }, "competitor": { - "minMs": 0.12250000005587935, - "maxMs": 0.17999999225139618, - "ratio": 1.4693876911778592 + "minMs": 0.24500000476837158, + "maxMs": 0.3049999997019768, + "ratio": 1.2448979337381259 }, - "stable": true, + "stable": false, + "rungs": [ + "level", + "competitor-leads", + "competitor-leads", + "competitor-leads" + ] + } + } + ] + }, + { + "archetype": "mask-clip-animated", + "category": "Render targets", + "count": 5000, + "cells": [ + { + "competitor": "pixi", + "referenceMs": 0.35624999925494194, + "referenceP95Ms": 0.4424999989569187, + "referenceOverFrameBudget": false, + "competitorMs": 0.32124999910593033, + "competitorP95Ms": 0.4050000049173832, + "competitorOverFrameBudget": false, + "verdict": { + "side": "neither", + "ratio": null, + "factor": null, + "label": "unstable across runs", + "structural": false + }, + "mechanism": "ExoJS issues fewer draw calls (4 vs 10 per frame)", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 0.32750000059604645, + "maxMs": 0.41499999910593033, + "ratio": 1.2671755674828544 + }, + "competitor": { + "minMs": 0.3199999965727329, + "maxMs": 0.32999999821186066, + "ratio": 1.0312500054569682 + }, + "stable": false, "rungs": [ - "competitor-leads-clearly", - "competitor-leads-clearly", - "competitor-leads-clearly", - "competitor-leads-clearly" + "competitor-leads", + "level", + "level", + "level" ] } } @@ -2533,11 +2551,11 @@ "cells": [ { "competitor": "pixi", - "referenceMs": 0.13249999238178134, - "referenceP95Ms": 0.3124999850988388, + "referenceMs": 0.23000000044703484, + "referenceP95Ms": 0.4299999997019768, "referenceOverFrameBudget": false, - "competitorMs": 0.09249999932944775, - "competitorP95Ms": 0.15000000759027898, + "competitorMs": 0.19250000081956387, + "competitorP95Ms": 0.24000000208616257, "competitorOverFrameBudget": false, "verdict": { "side": "neither", @@ -2550,21 +2568,21 @@ "aggregate": { "runs": 4, "reference": { - "minMs": 0.09499999997206032, - "maxMs": 0.16499999165534973, - "ratio": 1.7368420179355426 + "minMs": 0.2150000035762787, + "maxMs": 0.23499999940395355, + "ratio": 1.0930232348604552 }, "competitor": { - "minMs": 0.08500000019557774, - "maxMs": 0.13499999046325684, - "ratio": 1.5882351782662751 + "minMs": 0.1899999976158142, + "maxMs": 0.19500000029802322, + "ratio": 1.0263158039207936 }, "stable": false, "rungs": [ "level", "competitor-leads", "level", - "competitor-leads" + "level" ] } } @@ -2603,1017 +2621,8 @@ } ] }, - "physics": { - "runs": [ - { - "browser": "chromium", - "browserVersion": "151.0.7922.34", - "host": { - "cpu": "AMD Ryzen 7 3700X 8-Core Processor", - "cpuCount": 16, - "os": "win32 10.0.26200", - "platformVersion": { - "major": 11, - "source": "detected", - "evidence": "os.release() reported '10.0.26200'; build 26200 is Windows 11 (Windows 11 starts at build 22000)" - }, - "arch": "x64" - }, - "prerelease": { - "value": false, - "source": "assumed-stable", - "evidence": "browser version '151.0.7922.34' carries no pre-release marker and none was declared; the operating system's own release status cannot be read at runtime" - }, - "fixedDelta": 0.016666666666666666, - "clock": { - "resolutionMs": 0.004999995231628418, - "crossOriginIsolated": true - }, - "caveats": [ - "Step time is CPU wall-clock per step() over the timed window (median/p95), measured in one chromium page (same-run discipline). No number here was taken in Node.", - "The page's performance.now() resolves to 5.0us (cross-origin isolated: true). A cell whose step is too fast to time individually at that resolution batches steps per timing sample and records the batch as stepsPerSample; median and p95 are then per-step averages over that batch.", - "Scenes are warmed to steady state before timing; the per-cell warmupSteps/timedSteps counts are recorded for honesty.", - "All arms build the byte-identical scene (bodies, positions, shapes, sizes, static/dynamic split, gravity, perturbed-body set) from the shared deterministic RNG, and the perturbed-body selection is asserted equal across arms before each cell is timed.", - "Each arm runs at its own engine defaults for solver iterations, contact model and sleeping - those engine differences are the measured quantity in a native-vs-adapter comparison, disclosed per arm below.", - "Arm roles: matter-js and planck are the PURE-JS PEERS exojs-physics is compared against; rapier is a Rust/WASM engine and stands as the REFERENCE CEILING for what leaving JavaScript buys, not as a peer a JS solver is expected to match.", - "exojs-physics arm (native runtime, pure JS): TGS-Soft solver, 4 sub-steps per fixed step, sleeping ON by default (resting bodies deactivate). Contact count = solid contacts in the world contact graph.", - "matter-js arm (pure-JS peer): constraint solver at matter defaults (6 position / 4 velocity / 2 constraint iterations), sleeping OFF by default (a settled stack keeps paying full solve cost); matter's default per-step air drag (frictionAir) is zeroed so all arms integrate the same pure-gravity field; gravity (px/s^2) and perturbation velocity (px/s) are mapped into matter's px-per-step unit model. Contact count = active colliding pairs (engine.pairs.collisionActive), a pair-level proxy, not identical in semantics to the exojs solid-contact count.", - "planck arm (pure-JS peer): Box2D port at planck defaults (8 velocity / 3 position iterations per step), sleeping ON by default; Settings.lengthUnitsPerMeter is set to 30 so planck's absolute MKS tolerances are interpreted at the scene's pixel scale, which is the knob planck gives a pixel-coordinate game - positions, gravity (px/s^2) and velocity (px/s) then carry over unconverted. Contact count = the world contact list filtered by isTouching(), a touching collider-pair count. Rays are answered from planck's dynamic tree, but World.rayCast is Box2D's non-solid ray (an origin inside a fixture is not a hit). Continuous collision runs for every body (planck's default), where exojs and rapier restrict it to bullets and matter has none.", - "rapier arm (WASM reference ceiling, not a pure-JS peer): TGS-Soft solver at rapier defaults (4 solver / 1 internal PGS iterations), auto-sleeping ON; default lengthUnit=1 is fed a px-scale world (tuned for ~1-unit objects), exactly what attaching rapier with pixel coordinates yields. Contact count = collider pairs with a solid narrow-phase manifold (numContacts > 0), deduped." - ], - "engineVersion": "0.17.0", - "timestamp": "2026-09-06T10:03:08.002Z" - }, - { - "browser": "chromium", - "browserVersion": "151.0.7922.34", - "host": { - "cpu": "AMD Ryzen 7 3700X 8-Core Processor", - "cpuCount": 16, - "os": "win32 10.0.26200", - "platformVersion": { - "major": 11, - "source": "detected", - "evidence": "os.release() reported '10.0.26200'; build 26200 is Windows 11 (Windows 11 starts at build 22000)" - }, - "arch": "x64" - }, - "prerelease": { - "value": false, - "source": "assumed-stable", - "evidence": "browser version '151.0.7922.34' carries no pre-release marker and none was declared; the operating system's own release status cannot be read at runtime" - }, - "fixedDelta": 0.016666666666666666, - "clock": { - "resolutionMs": 0.004999995231628418, - "crossOriginIsolated": true - }, - "caveats": [ - "Step time is CPU wall-clock per step() over the timed window (median/p95), measured in one chromium page (same-run discipline). No number here was taken in Node.", - "The page's performance.now() resolves to 5.0us (cross-origin isolated: true). A cell whose step is too fast to time individually at that resolution batches steps per timing sample and records the batch as stepsPerSample; median and p95 are then per-step averages over that batch.", - "Scenes are warmed to steady state before timing; the per-cell warmupSteps/timedSteps counts are recorded for honesty.", - "All arms build the byte-identical scene (bodies, positions, shapes, sizes, static/dynamic split, gravity, perturbed-body set) from the shared deterministic RNG, and the perturbed-body selection is asserted equal across arms before each cell is timed.", - "Each arm runs at its own engine defaults for solver iterations, contact model and sleeping - those engine differences are the measured quantity in a native-vs-adapter comparison, disclosed per arm below.", - "Arm roles: matter-js and planck are the PURE-JS PEERS exojs-physics is compared against; rapier is a Rust/WASM engine and stands as the REFERENCE CEILING for what leaving JavaScript buys, not as a peer a JS solver is expected to match.", - "exojs-physics arm (native runtime, pure JS): TGS-Soft solver, 4 sub-steps per fixed step, sleeping ON by default (resting bodies deactivate). Contact count = solid contacts in the world contact graph.", - "matter-js arm (pure-JS peer): constraint solver at matter defaults (6 position / 4 velocity / 2 constraint iterations), sleeping OFF by default (a settled stack keeps paying full solve cost); matter's default per-step air drag (frictionAir) is zeroed so all arms integrate the same pure-gravity field; gravity (px/s^2) and perturbation velocity (px/s) are mapped into matter's px-per-step unit model. Contact count = active colliding pairs (engine.pairs.collisionActive), a pair-level proxy, not identical in semantics to the exojs solid-contact count.", - "planck arm (pure-JS peer): Box2D port at planck defaults (8 velocity / 3 position iterations per step), sleeping ON by default; Settings.lengthUnitsPerMeter is set to 30 so planck's absolute MKS tolerances are interpreted at the scene's pixel scale, which is the knob planck gives a pixel-coordinate game - positions, gravity (px/s^2) and velocity (px/s) then carry over unconverted. Contact count = the world contact list filtered by isTouching(), a touching collider-pair count. Rays are answered from planck's dynamic tree, but World.rayCast is Box2D's non-solid ray (an origin inside a fixture is not a hit). Continuous collision runs for every body (planck's default), where exojs and rapier restrict it to bullets and matter has none.", - "rapier arm (WASM reference ceiling, not a pure-JS peer): TGS-Soft solver at rapier defaults (4 solver / 1 internal PGS iterations), auto-sleeping ON; default lengthUnit=1 is fed a px-scale world (tuned for ~1-unit objects), exactly what attaching rapier with pixel coordinates yields. Contact count = collider pairs with a solid narrow-phase manifold (numContacts > 0), deduped." - ], - "engineVersion": "0.17.0", - "timestamp": "2026-09-06T10:17:54.524Z" - }, - { - "browser": "chromium", - "browserVersion": "151.0.7922.34", - "host": { - "cpu": "AMD Ryzen 7 3700X 8-Core Processor", - "cpuCount": 16, - "os": "win32 10.0.26200", - "platformVersion": { - "major": 11, - "source": "detected", - "evidence": "os.release() reported '10.0.26200'; build 26200 is Windows 11 (Windows 11 starts at build 22000)" - }, - "arch": "x64" - }, - "prerelease": { - "value": false, - "source": "assumed-stable", - "evidence": "browser version '151.0.7922.34' carries no pre-release marker and none was declared; the operating system's own release status cannot be read at runtime" - }, - "fixedDelta": 0.016666666666666666, - "clock": { - "resolutionMs": 0.004999999888241291, - "crossOriginIsolated": true - }, - "caveats": [ - "Step time is CPU wall-clock per step() over the timed window (median/p95), measured in one chromium page (same-run discipline). No number here was taken in Node.", - "The page's performance.now() resolves to 5.0us (cross-origin isolated: true). A cell whose step is too fast to time individually at that resolution batches steps per timing sample and records the batch as stepsPerSample; median and p95 are then per-step averages over that batch.", - "Scenes are warmed to steady state before timing; the per-cell warmupSteps/timedSteps counts are recorded for honesty.", - "All arms build the byte-identical scene (bodies, positions, shapes, sizes, static/dynamic split, gravity, perturbed-body set) from the shared deterministic RNG, and the perturbed-body selection is asserted equal across arms before each cell is timed.", - "Each arm runs at its own engine defaults for solver iterations, contact model and sleeping - those engine differences are the measured quantity in a native-vs-adapter comparison, disclosed per arm below.", - "Arm roles: matter-js and planck are the PURE-JS PEERS exojs-physics is compared against; rapier is a Rust/WASM engine and stands as the REFERENCE CEILING for what leaving JavaScript buys, not as a peer a JS solver is expected to match.", - "exojs-physics arm (native runtime, pure JS): TGS-Soft solver, 4 sub-steps per fixed step, sleeping ON by default (resting bodies deactivate). Contact count = solid contacts in the world contact graph.", - "matter-js arm (pure-JS peer): constraint solver at matter defaults (6 position / 4 velocity / 2 constraint iterations), sleeping OFF by default (a settled stack keeps paying full solve cost); matter's default per-step air drag (frictionAir) is zeroed so all arms integrate the same pure-gravity field; gravity (px/s^2) and perturbation velocity (px/s) are mapped into matter's px-per-step unit model. Contact count = active colliding pairs (engine.pairs.collisionActive), a pair-level proxy, not identical in semantics to the exojs solid-contact count.", - "planck arm (pure-JS peer): Box2D port at planck defaults (8 velocity / 3 position iterations per step), sleeping ON by default; Settings.lengthUnitsPerMeter is set to 30 so planck's absolute MKS tolerances are interpreted at the scene's pixel scale, which is the knob planck gives a pixel-coordinate game - positions, gravity (px/s^2) and velocity (px/s) then carry over unconverted. Contact count = the world contact list filtered by isTouching(), a touching collider-pair count. Rays are answered from planck's dynamic tree, but World.rayCast is Box2D's non-solid ray (an origin inside a fixture is not a hit). Continuous collision runs for every body (planck's default), where exojs and rapier restrict it to bullets and matter has none.", - "rapier arm (WASM reference ceiling, not a pure-JS peer): TGS-Soft solver at rapier defaults (4 solver / 1 internal PGS iterations), auto-sleeping ON; default lengthUnit=1 is fed a px-scale world (tuned for ~1-unit objects), exactly what attaching rapier with pixel coordinates yields. Contact count = collider pairs with a solid narrow-phase manifold (numContacts > 0), deduped." - ], - "engineVersion": "0.17.0", - "timestamp": "2026-09-06T16:33:07.319Z" - }, - { - "browser": "chromium", - "browserVersion": "151.0.7922.34", - "host": { - "cpu": "AMD Ryzen 7 3700X 8-Core Processor", - "cpuCount": 16, - "os": "win32 10.0.26200", - "platformVersion": { - "major": 11, - "source": "detected", - "evidence": "os.release() reported '10.0.26200'; build 26200 is Windows 11 (Windows 11 starts at build 22000)" - }, - "arch": "x64" - }, - "prerelease": { - "value": false, - "source": "assumed-stable", - "evidence": "browser version '151.0.7922.34' carries no pre-release marker and none was declared; the operating system's own release status cannot be read at runtime" - }, - "fixedDelta": 0.016666666666666666, - "clock": { - "resolutionMs": 0.004999999888241291, - "crossOriginIsolated": true - }, - "caveats": [ - "Step time is CPU wall-clock per step() over the timed window (median/p95), measured in one chromium page (same-run discipline). No number here was taken in Node.", - "The page's performance.now() resolves to 5.0us (cross-origin isolated: true). A cell whose step is too fast to time individually at that resolution batches steps per timing sample and records the batch as stepsPerSample; median and p95 are then per-step averages over that batch.", - "Scenes are warmed to steady state before timing; the per-cell warmupSteps/timedSteps counts are recorded for honesty.", - "All arms build the byte-identical scene (bodies, positions, shapes, sizes, static/dynamic split, gravity, perturbed-body set) from the shared deterministic RNG, and the perturbed-body selection is asserted equal across arms before each cell is timed.", - "Each arm runs at its own engine defaults for solver iterations, contact model and sleeping - those engine differences are the measured quantity in a native-vs-adapter comparison, disclosed per arm below.", - "Arm roles: matter-js and planck are the PURE-JS PEERS exojs-physics is compared against; rapier is a Rust/WASM engine and stands as the REFERENCE CEILING for what leaving JavaScript buys, not as a peer a JS solver is expected to match.", - "exojs-physics arm (native runtime, pure JS): TGS-Soft solver, 4 sub-steps per fixed step, sleeping ON by default (resting bodies deactivate). Contact count = solid contacts in the world contact graph.", - "matter-js arm (pure-JS peer): constraint solver at matter defaults (6 position / 4 velocity / 2 constraint iterations), sleeping OFF by default (a settled stack keeps paying full solve cost); matter's default per-step air drag (frictionAir) is zeroed so all arms integrate the same pure-gravity field; gravity (px/s^2) and perturbation velocity (px/s) are mapped into matter's px-per-step unit model. Contact count = active colliding pairs (engine.pairs.collisionActive), a pair-level proxy, not identical in semantics to the exojs solid-contact count.", - "planck arm (pure-JS peer): Box2D port at planck defaults (8 velocity / 3 position iterations per step), sleeping ON by default; Settings.lengthUnitsPerMeter is set to 30 so planck's absolute MKS tolerances are interpreted at the scene's pixel scale, which is the knob planck gives a pixel-coordinate game - positions, gravity (px/s^2) and velocity (px/s) then carry over unconverted. Contact count = the world contact list filtered by isTouching(), a touching collider-pair count. Rays are answered from planck's dynamic tree, but World.rayCast is Box2D's non-solid ray (an origin inside a fixture is not a hit). Continuous collision runs for every body (planck's default), where exojs and rapier restrict it to bullets and matter has none.", - "rapier arm (WASM reference ceiling, not a pure-JS peer): TGS-Soft solver at rapier defaults (4 solver / 1 internal PGS iterations), auto-sleeping ON; default lengthUnit=1 is fed a px-scale world (tuned for ~1-unit objects), exactly what attaching rapier with pixel coordinates yields. Contact count = collider pairs with a solid narrow-phase manifold (numContacts > 0), deduped." - ], - "engineVersion": "0.17.0", - "timestamp": "2026-09-06T17:02:01.221Z" - } - ], - "libraries": [ - { - "name": "@codexo/exojs-physics", - "version": "0.17.0" - }, - { - "name": "matter-js", - "version": "0.20.0" - }, - { - "name": "planck", - "version": "1.5.0" - }, - { - "name": "@dimforge/rapier2d-compat", - "version": "0.19.3" - } - ], - "section": { - "title": "Physics", - "rows": [ - { - "archetype": "box-stack", - "category": "Physics", - "count": 10000, - "cells": [ - { - "competitor": "matter-js", - "referenceMs": 17.857500004116446, - "referenceP95Ms": 18.54750000545755, - "referenceOverFrameBudget": true, - "competitorMs": 68.2212499936577, - "competitorP95Ms": 70.55750000104308, - "competitorOverFrameBudget": true, - "verdict": { - "side": "exojs", - "ratio": 0.26175861635160014, - "factor": 3.8203135924923184, - "label": "ExoJS leads (3.82x)", - "structural": false - }, - "mechanism": "ExoJS resolves fewer solved contacts (10002 vs 22463); the counters are not semantically identical across arms (see the run's caveats)", - "aggregate": { - "runs": 4, - "reference": { - "minMs": 17.09249999979511, - "maxMs": 19.375, - "ratio": 1.1335381015200967 - }, - "competitor": { - "minMs": 67.44750000000931, - "maxMs": 71.04250000417233, - "ratio": 1.0533007154329295 - }, - "stable": true, - "rungs": [ - "exojs-leads", - "exojs-leads", - "exojs-leads", - "exojs-leads" - ] - } - }, - { - "competitor": "planck", - "referenceMs": 17.857500004116446, - "referenceP95Ms": 18.54750000545755, - "referenceOverFrameBudget": true, - "competitorMs": 134.02000000327826, - "competitorP95Ms": 210.6250000030268, - "competitorOverFrameBudget": true, - "verdict": { - "side": "exojs", - "ratio": 0.13324503808147764, - "factor": 7.504969899055549, - "label": "ExoJS leads clearly (7.50x)", - "structural": true - }, - "mechanism": "ExoJS resolves fewer solved contacts (10002 vs 19865); the counters are not semantically identical across arms (see the run's caveats)", - "aggregate": { - "runs": 4, - "reference": { - "minMs": 17.09249999979511, - "maxMs": 19.375, - "ratio": 1.1335381015200967 - }, - "competitor": { - "minMs": 133.01750000030734, - "maxMs": 134.8974999994971, - "ratio": 1.0141334786714937 - }, - "stable": true, - "rungs": [ - "exojs-leads-clearly", - "exojs-leads-clearly", - "exojs-leads-clearly", - "exojs-leads-clearly" - ] - } - }, - { - "competitor": "rapier", - "referenceMs": 17.857500004116446, - "referenceP95Ms": 18.54750000545755, - "referenceOverFrameBudget": true, - "competitorMs": 16.467499998863786, - "competitorP95Ms": 23.14250000147149, - "competitorOverFrameBudget": false, - "verdict": { - "side": "neither", - "ratio": 1.0844086840958596, - "factor": 1, - "label": "level", - "structural": false - }, - "mechanism": "ExoJS resolves fewer solved contacts (10002 vs 17880); the counters are not semantically identical across arms (see the run's caveats)", - "aggregate": { - "runs": 4, - "reference": { - "minMs": 17.09249999979511, - "maxMs": 19.375, - "ratio": 1.1335381015200967 - }, - "competitor": { - "minMs": 16.18749999953434, - "maxMs": 16.777499988675117, - "ratio": 1.036447875778085 - }, - "stable": true, - "rungs": [ - "level", - "level", - "level", - "level" - ] - } - } - ] - }, - { - "archetype": "many-dynamic", - "category": "Physics", - "count": 2200, - "cells": [ - { - "competitor": "matter-js", - "referenceMs": 17.383750001201406, - "referenceP95Ms": 18.43250000406988, - "referenceOverFrameBudget": true, - "competitorMs": 15.19125000294298, - "competitorP95Ms": 16.00250000366941, - "competitorOverFrameBudget": false, - "verdict": { - "side": "neither", - "ratio": null, - "factor": null, - "label": "unstable across runs", - "structural": false - }, - "mechanism": "identical scene (2204 bodies, 6190 contacts, 0 joints on both arms); the difference is solver cost", - "aggregate": { - "runs": 4, - "reference": { - "minMs": 17.149999999906868, - "maxMs": 17.689999997615814, - "ratio": 1.0314868803330541 - }, - "competitor": { - "minMs": 14.135000000242144, - "maxMs": 16.12999999523163, - "ratio": 1.1411390162685042 - }, - "stable": false, - "rungs": [ - "level", - "level", - "competitor-leads", - "level" - ] - } - }, - { - "competitor": "planck", - "referenceMs": 17.383750001201406, - "referenceP95Ms": 18.43250000406988, - "referenceOverFrameBudget": true, - "competitorMs": 27.416249997215346, - "competitorP95Ms": 29.06500000297092, - "competitorOverFrameBudget": true, - "verdict": { - "side": "exojs", - "ratio": 0.6340673871505791, - "factor": 1.5771194359859402, - "label": "ExoJS leads (1.58x)", - "structural": false - }, - "mechanism": "identical scene (2204 bodies, 6190 contacts, 0 joints on both arms); the difference is solver cost", - "aggregate": { - "runs": 4, - "reference": { - "minMs": 17.149999999906868, - "maxMs": 17.689999997615814, - "ratio": 1.0314868803330541 - }, - "competitor": { - "minMs": 25.53250000020489, - "maxMs": 29.80000001192093, - "ratio": 1.1671399201676997 - }, - "stable": true, - "rungs": [ - "exojs-leads", - "exojs-leads", - "exojs-leads", - "exojs-leads" - ] - } - }, - { - "competitor": "rapier", - "referenceMs": 17.383750001201406, - "referenceP95Ms": 18.43250000406988, - "referenceOverFrameBudget": true, - "competitorMs": 3.908750013099052, - "competitorP95Ms": 4.112500008894131, - "competitorOverFrameBudget": false, - "verdict": { - "side": "competitor", - "ratio": 4.447393653455649, - "factor": 4.447393653455649, - "label": "competitor leads (4.45x)", - "structural": false - }, - "mechanism": "identical scene (2204 bodies, 6190 contacts, 0 joints on both arms); the difference is solver cost", - "aggregate": { - "runs": 4, - "reference": { - "minMs": 17.149999999906868, - "maxMs": 17.689999997615814, - "ratio": 1.0314868803330541 - }, - "competitor": { - "minMs": 3.8699999996460974, - "maxMs": 3.9249999970197678, - "ratio": 1.014211885627571 - }, - "stable": true, - "rungs": [ - "competitor-leads", - "competitor-leads", - "competitor-leads", - "competitor-leads" - ] - } - } - ] - }, - { - "archetype": "mixed-static-dynamic", - "category": "Physics", - "count": 3200, - "cells": [ - { - "competitor": "matter-js", - "referenceMs": 17.4987500035204, - "referenceP95Ms": 18.545000007841736, - "referenceOverFrameBudget": true, - "competitorMs": 11.001249989843927, - "competitorP95Ms": 12.917499995790422, - "competitorOverFrameBudget": false, - "verdict": { - "side": "competitor", - "ratio": 1.590614704663088, - "factor": 1.590614704663088, - "label": "competitor leads (1.59x)", - "structural": false - }, - "mechanism": "identical scene (3313 bodies, 3863 contacts, 0 joints on both arms); the difference is solver cost", - "aggregate": { - "runs": 4, - "reference": { - "minMs": 16.050000000046566, - "maxMs": 17.64249999821186, - "ratio": 1.0992211836860233 - }, - "competitor": { - "minMs": 10.819999999832362, - "maxMs": 11.390000000596046, - "ratio": 1.0526802218828573 - }, - "stable": true, - "rungs": [ - "competitor-leads", - "competitor-leads", - "competitor-leads", - "competitor-leads" - ] - } - }, - { - "competitor": "planck", - "referenceMs": 17.4987500035204, - "referenceP95Ms": 18.545000007841736, - "referenceOverFrameBudget": true, - "competitorMs": 89.73499999195337, - "competitorP95Ms": 110.02250000834465, - "competitorOverFrameBudget": true, - "verdict": { - "side": "exojs", - "ratio": 0.19500473622432193, - "factor": 5.128080575692577, - "label": "ExoJS leads clearly (5.13x)", - "structural": true - }, - "mechanism": "identical scene (3313 bodies, 3863 contacts, 0 joints on both arms); the difference is solver cost", - "aggregate": { - "runs": 4, - "reference": { - "minMs": 16.050000000046566, - "maxMs": 17.64249999821186, - "ratio": 1.0992211836860233 - }, - "competitor": { - "minMs": 89.4550000000745, - "maxMs": 128.59250000002794, - "ratio": 1.4375104801287892 - }, - "stable": true, - "rungs": [ - "exojs-leads-clearly", - "exojs-leads-clearly", - "exojs-leads-clearly", - "exojs-leads-clearly" - ] - } - }, - { - "competitor": "rapier", - "referenceMs": 17.4987500035204, - "referenceP95Ms": 18.545000007841736, - "referenceOverFrameBudget": true, - "competitorMs": 7.757500006817281, - "competitorP95Ms": 8.884999997913837, - "competitorOverFrameBudget": false, - "verdict": { - "side": "competitor", - "ratio": 2.2557202691772504, - "factor": 2.2557202691772504, - "label": "competitor leads (2.26x)", - "structural": false - }, - "mechanism": "ExoJS resolves fewer solved contacts (3863 vs 6105); the counters are not semantically identical across arms (see the run's caveats)", - "aggregate": { - "runs": 4, - "reference": { - "minMs": 16.050000000046566, - "maxMs": 17.64249999821186, - "ratio": 1.0992211836860233 - }, - "competitor": { - "minMs": 7.647500000195578, - "maxMs": 7.892499998211861, - "ratio": 1.0320366130120977 - }, - "stable": true, - "rungs": [ - "competitor-leads", - "competitor-leads", - "competitor-leads", - "competitor-leads" - ] - } - } - ] - }, - { - "archetype": "raycast", - "category": "Physics", - "count": 3200, - "cells": [ - { - "competitor": "matter-js", - "referenceMs": 24.08500001579523, - "referenceP95Ms": 25.60749999433756, - "referenceOverFrameBudget": true, - "competitorMs": 21.662499994738027, - "competitorP95Ms": 26.87250000052154, - "competitorOverFrameBudget": true, - "verdict": { - "side": "neither", - "ratio": 1.1118291989218994, - "factor": 1, - "label": "level", - "structural": false - }, - "mechanism": "identical scene (3313 bodies, 3863 contacts, 0 joints on both arms) with 38 vs 52 ray hits per step; the difference is query and solver cost", - "aggregate": { - "runs": 4, - "reference": { - "minMs": 19.442500000353903, - "maxMs": 24.40250000031665, - "ratio": 1.2551112254016954 - }, - "competitor": { - "minMs": 21.240000009536743, - "maxMs": 21.994999999878928, - "ratio": 1.035546138889039 - }, - "stable": true, - "rungs": [ - "level", - "level", - "level", - "level" - ] - } - }, - { - "competitor": "planck", - "referenceMs": 24.08500001579523, - "referenceP95Ms": 25.60749999433756, - "referenceOverFrameBudget": true, - "competitorMs": 102.21500001102686, - "competitorP95Ms": 124.2175000011921, - "competitorOverFrameBudget": true, - "verdict": { - "side": "neither", - "ratio": null, - "factor": null, - "label": "unstable across runs", - "structural": false - }, - "mechanism": "identical scene (3313 bodies, 3863 contacts, 0 joints on both arms) with 38 vs 38 ray hits per step; the difference is query and solver cost", - "aggregate": { - "runs": 4, - "reference": { - "minMs": 19.442500000353903, - "maxMs": 24.40250000031665, - "ratio": 1.2551112254016954 - }, - "competitor": { - "minMs": 95.49999999953434, - "maxMs": 112.4199999999255, - "ratio": 1.1771727748740697 - }, - "stable": false, - "rungs": [ - "exojs-leads", - "exojs-leads", - "exojs-leads-clearly", - "exojs-leads" - ] - } - }, - { - "competitor": "rapier", - "referenceMs": 24.08500001579523, - "referenceP95Ms": 25.60749999433756, - "referenceOverFrameBudget": true, - "competitorMs": 7.9787500004749745, - "competitorP95Ms": 9.055000006686896, - "competitorOverFrameBudget": false, - "verdict": { - "side": "competitor", - "ratio": 3.018643272989059, - "factor": 3.018643272989059, - "label": "competitor leads (3.02x)", - "structural": false - }, - "mechanism": "ExoJS resolves fewer solved contacts (3863 vs 6105); the counters are not semantically identical across arms (see the run's caveats)", - "aggregate": { - "runs": 4, - "reference": { - "minMs": 19.442500000353903, - "maxMs": 24.40250000031665, - "ratio": 1.2551112254016954 - }, - "competitor": { - "minMs": 7.875, - "maxMs": 8.064999997615814, - "ratio": 1.0241269838242304 - }, - "stable": true, - "rungs": [ - "competitor-leads", - "competitor-leads", - "competitor-leads", - "competitor-leads" - ] - } - } - ] - }, - { - "archetype": "body-churn", - "category": "Physics", - "count": 2400, - "cells": [ - { - "competitor": "matter-js", - "referenceMs": 18.999999999883585, - "referenceP95Ms": 19.68500000028871, - "referenceOverFrameBudget": true, - "competitorMs": 20.967499998863786, - "competitorP95Ms": 22.18749999627471, - "competitorOverFrameBudget": true, - "verdict": { - "side": "neither", - "ratio": 0.9061643019393433, - "factor": 1, - "label": "level", - "structural": false - }, - "mechanism": "ExoJS resolves fewer solved contacts (2282 vs 6244); the counters are not semantically identical across arms (see the run's caveats)", - "aggregate": { - "runs": 4, - "reference": { - "minMs": 18.490000009536743, - "maxMs": 19.52750000357628, - "ratio": 1.0561114112225212 - }, - "competitor": { - "minMs": 19.102500000037253, - "maxMs": 22.439999997615814, - "ratio": 1.1747153512666957 - }, - "stable": true, - "rungs": [ - "level", - "level", - "level", - "level" - ] - } - }, - { - "competitor": "planck", - "referenceMs": 18.999999999883585, - "referenceP95Ms": 19.68500000028871, - "referenceOverFrameBudget": true, - "competitorMs": 60.34249999804888, - "competitorP95Ms": 74.5774999926798, - "competitorOverFrameBudget": true, - "verdict": { - "side": "exojs", - "ratio": 0.31486928782363893, - "factor": 3.1759210525483477, - "label": "ExoJS leads (3.18x)", - "structural": false - }, - "mechanism": "ExoJS resolves fewer solved contacts (2282 vs 5242); the counters are not semantically identical across arms (see the run's caveats)", - "aggregate": { - "runs": 4, - "reference": { - "minMs": 18.490000009536743, - "maxMs": 19.52750000357628, - "ratio": 1.0561114112225212 - }, - "competitor": { - "minMs": 54.72750000655651, - "maxMs": 65.47750000003725, - "ratio": 1.196427755556034 - }, - "stable": true, - "rungs": [ - "exojs-leads", - "exojs-leads", - "exojs-leads", - "exojs-leads" - ] - } - }, - { - "competitor": "rapier", - "referenceMs": 18.999999999883585, - "referenceP95Ms": 19.68500000028871, - "referenceOverFrameBudget": true, - "competitorMs": 2.010000005364418, - "competitorP95Ms": 2.072499994188547, - "competitorOverFrameBudget": false, - "verdict": { - "side": "competitor", - "ratio": 9.452736293121967, - "factor": 9.452736293121967, - "label": "competitor leads clearly (9.45x)", - "structural": true - }, - "mechanism": "identical scene (2404 bodies, 2282 contacts, 0 joints on both arms); the difference is solver cost", - "aggregate": { - "runs": 4, - "reference": { - "minMs": 18.490000009536743, - "maxMs": 19.52750000357628, - "ratio": 1.0561114112225212 - }, - "competitor": { - "minMs": 1.9950000001117587, - "maxMs": 2.0249999999068677, - "ratio": 1.0150375938814178 - }, - "stable": true, - "rungs": [ - "competitor-leads-clearly", - "competitor-leads-clearly", - "competitor-leads-clearly", - "competitor-leads-clearly" - ] - } - } - ] - }, - { - "archetype": "joints", - "category": "Physics", - "count": 15000, - "cells": [ - { - "competitor": "matter-js", - "referenceMs": 18.786250002216548, - "referenceP95Ms": 19.200000000186265, - "referenceOverFrameBudget": true, - "competitorMs": 21.28500000119675, - "competitorP95Ms": 22.977499997708946, - "competitorOverFrameBudget": true, - "verdict": { - "side": "neither", - "ratio": null, - "factor": null, - "label": "unstable across runs", - "structural": false - }, - "mechanism": "competitor resolves fewer solved contacts (13125 vs 0); the counters are not semantically identical across arms (see the run's caveats)", - "aggregate": { - "runs": 4, - "reference": { - "minMs": 16.9549999833107, - "maxMs": 22.009999999776483, - "ratio": 1.2981421422259845 - }, - "competitor": { - "minMs": 20.07249999931082, - "maxMs": 23.239999994635582, - "ratio": 1.1578029640270775 - }, - "stable": false, - "rungs": [ - "exojs-leads", - "exojs-leads", - "level", - "level" - ] - } - }, - { - "competitor": "planck", - "referenceMs": 18.786250002216548, - "referenceP95Ms": 19.200000000186265, - "referenceOverFrameBudget": true, - "competitorMs": 1.554999997606501, - "competitorP95Ms": 1.7799999942071736, - "competitorOverFrameBudget": false, - "verdict": { - "side": "competitor", - "ratio": 12.081189730632067, - "factor": 12.081189730632067, - "label": "competitor leads clearly (12.08x)", - "structural": true - }, - "mechanism": "competitor resolves fewer solved contacts (13125 vs 0); the counters are not semantically identical across arms (see the run's caveats)", - "aggregate": { - "runs": 4, - "reference": { - "minMs": 16.9549999833107, - "maxMs": 22.009999999776483, - "ratio": 1.2981421422259845 - }, - "competitor": { - "minMs": 1.449999988079071, - "maxMs": 1.7850000001490116, - "ratio": 1.2310344929821284 - }, - "stable": true, - "rungs": [ - "competitor-leads-clearly", - "competitor-leads-clearly", - "competitor-leads-clearly", - "competitor-leads-clearly" - ] - } - }, - { - "competitor": "rapier", - "referenceMs": 18.786250002216548, - "referenceP95Ms": 19.200000000186265, - "referenceOverFrameBudget": true, - "competitorMs": 0.6599999981699511, - "competitorP95Ms": 0.8049999887589365, - "competitorOverFrameBudget": false, - "verdict": { - "side": "competitor", - "ratio": 28.464015233798616, - "factor": 28.464015233798616, - "label": "competitor leads clearly (28.46x)", - "structural": true - }, - "mechanism": "competitor resolves fewer solved contacts (13125 vs 1875); the counters are not semantically identical across arms (see the run's caveats)", - "aggregate": { - "runs": 4, - "reference": { - "minMs": 16.9549999833107, - "maxMs": 22.009999999776483, - "ratio": 1.2981421422259845 - }, - "competitor": { - "minMs": 0.6524999998509884, - "maxMs": 0.6899999976158142, - "ratio": 1.0574712609553865 - }, - "stable": true, - "rungs": [ - "competitor-leads-clearly", - "competitor-leads-clearly", - "competitor-leads-clearly", - "competitor-leads-clearly" - ] - } - } - ] - }, - { - "archetype": "settling-pile", - "category": "Physics", - "count": 5800, - "cells": [ - { - "competitor": "matter-js", - "referenceMs": 17.37999999523163, - "referenceP95Ms": 18.1850000070408, - "referenceOverFrameBudget": true, - "competitorMs": 69.99624999961816, - "competitorP95Ms": 71.24750000145286, - "competitorOverFrameBudget": true, - "verdict": { - "side": "exojs", - "ratio": 0.24829901595194656, - "factor": 4.02740218750416, - "label": "ExoJS leads (4.03x)", - "structural": false - }, - "mechanism": "identical scene (5804 bodies, 12178 contacts, 0 joints on both arms); the difference is solver cost", - "aggregate": { - "runs": 4, - "reference": { - "minMs": 16.882500000298023, - "maxMs": 17.707500000018626, - "ratio": 1.0488671701291894 - }, - "competitor": { - "minMs": 68.00499999988824, - "maxMs": 74.98250000178814, - "ratio": 1.1026027498259152 - }, - "stable": true, - "rungs": [ - "exojs-leads", - "exojs-leads", - "exojs-leads", - "exojs-leads" - ] - } - }, - { - "competitor": "planck", - "referenceMs": 17.37999999523163, - "referenceP95Ms": 18.1850000070408, - "referenceOverFrameBudget": true, - "competitorMs": 128.29124999721535, - "competitorP95Ms": 132.16749999765307, - "competitorOverFrameBudget": true, - "verdict": { - "side": "exojs", - "ratio": 0.13547299598069917, - "factor": 7.381544881036438, - "label": "ExoJS leads clearly (7.38x)", - "structural": true - }, - "mechanism": "identical scene (5804 bodies, 12178 contacts, 0 joints on both arms); the difference is solver cost", - "aggregate": { - "runs": 4, - "reference": { - "minMs": 16.882500000298023, - "maxMs": 17.707500000018626, - "ratio": 1.0488671701291894 - }, - "competitor": { - "minMs": 121.03749999962747, - "maxMs": 139.1925000101328, - "ratio": 1.1499948363983161 - }, - "stable": true, - "rungs": [ - "exojs-leads-clearly", - "exojs-leads-clearly", - "exojs-leads-clearly", - "exojs-leads-clearly" - ] - } - }, - { - "competitor": "rapier", - "referenceMs": 17.37999999523163, - "referenceP95Ms": 18.1850000070408, - "referenceOverFrameBudget": true, - "competitorMs": 11.788749997969717, - "competitorP95Ms": 12.372500006109476, - "competitorOverFrameBudget": false, - "verdict": { - "side": "competitor", - "ratio": 1.4742869259442137, - "factor": 1.4742869259442137, - "label": "competitor leads (1.47x)", - "structural": false - }, - "mechanism": "identical scene (5804 bodies, 12178 contacts, 0 joints on both arms); the difference is solver cost", - "aggregate": { - "runs": 4, - "reference": { - "minMs": 16.882500000298023, - "maxMs": 17.707500000018626, - "ratio": 1.0488671701291894 - }, - "competitor": { - "minMs": 11.677500000223517, - "maxMs": 11.900000005960464, - "ratio": 1.019053736307659 - }, - "stable": true, - "rungs": [ - "competitor-leads", - "competitor-leads", - "competitor-leads", - "competitor-leads" - ] - } - } - ] - } - ] - } - }, "signature": { "algorithm": "sha256", - "value": "d376f6126d5c8de4fa59485d8e1c2901e1253dbd293f59534d9cf3f5883c18c9" + "value": "cb202ad3f559b846de40a2326a448fade95ae78e30a01dea4866a65d29623885" } } diff --git a/packages/exojs-bench/src/comparison/build.ts b/packages/exojs-bench/src/comparison/build.ts index fb787849d..168c065b3 100644 --- a/packages/exojs-bench/src/comparison/build.ts +++ b/packages/exojs-bench/src/comparison/build.ts @@ -31,18 +31,8 @@ export const REFERENCE_ENGINE = 'exojs'; /** The reference arm's config. The retained tier is an opt-in and is reported separately, never as "the ExoJS number". */ export const REFERENCE_CONFIG = 'current'; -/** - * Arms that render through a WebGL1 context and therefore never share a row with - * the WebGL2/WebGPU arms. - * - * Phaser 4's renderer is WebGL1 (verified against the installed dist, see - * `adapters/phaser.ts`), so a gap against it can be caused by the backend - * generation as much as by the engine, and the harness's WebGL2 structural probe - * cannot attach to report which. Its rows go into their own clearly delimited - * block, comparing CPU time only and stating that they carry no structural - * mechanism - an observation rather than a finding. - */ -const WEBGL1_ENGINES: readonly string[] = ['phaser']; +/** Legacy arm identities whose old profiles measured Phaser through WebGL1. */ +const LEGACY_WEBGL1_ARM_KEYS: readonly string[] = ['phaser|default']; /** Category section order in the published table. */ const CATEGORY_ORDER: readonly ArchetypeCategory[] = [ @@ -157,8 +147,8 @@ export interface BackendComparison { /** Rows measured but excluded, with reasons - published so the omissions are auditable. */ readonly excluded: readonly ExcludedRow[]; /** - * The separate WebGL1 block: CPU-time-only rows against the arms in - * {@link WEBGL1_ENGINES}. Empty when no such arm ran on this backend. + * The separate legacy WebGL1 block: CPU-time-only rows for profiles measured + * before Phaser's WebGL2 context injection. Empty for new profiles. */ readonly webgl1: readonly ComparisonRow[]; } @@ -166,6 +156,9 @@ export interface BackendComparison { /** Key identifying one arm's cell within a backend. */ const cellKey = (engine: string, config: string, archetype: string, count: number): string => `${engine}|${config}|${archetype}|${count}`; +/** Identity used to preserve the old CPU-only Phaser block while new WebGL2 profiles migrate. */ +const armKeyOf = (result: { readonly spec: { readonly engine: string; readonly config: string } }): string => `${result.spec.engine}|${result.spec.config}`; + /** Whether a result can be compared at all: it measured, and it measured something. */ const isComparable = (result: { status: string; note?: string }): boolean => result.status === 'ok'; @@ -210,8 +203,8 @@ const buildBackend = (backend: Backend, results: readonly CellResult[]): Backend const onBackend = results.filter(result => result.spec.backend === backend); const byKey = new Map(onBackend.map(result => [cellKey(result.spec.engine, result.spec.config, result.spec.archetype, result.spec.nodeCount), result])); const armEngines = [...new Set(onBackend.map(result => result.spec.engine))].filter(engine => engine !== REFERENCE_ENGINE).sort(); - const competitors = armEngines.filter(engine => !WEBGL1_ENGINES.includes(engine)); - const webgl1Engines = armEngines.filter(engine => WEBGL1_ENGINES.includes(engine)); + const webgl1Engines = [...new Set(onBackend.filter(result => LEGACY_WEBGL1_ARM_KEYS.includes(armKeyOf(result))).map(result => result.spec.engine))]; + const competitors = armEngines.filter(engine => !webgl1Engines.includes(engine)); // Only archetypes the run actually MEASURED can constrain the count. An // archetype absent from the run says nothing about which count is valid, and // letting it veto would make every subset run produce an empty table; it is diff --git a/packages/exojs-bench/src/comparison/mechanism.ts b/packages/exojs-bench/src/comparison/mechanism.ts index 936bacd98..1487b052a 100644 --- a/packages/exojs-bench/src/comparison/mechanism.ts +++ b/packages/exojs-bench/src/comparison/mechanism.ts @@ -12,8 +12,8 @@ import type { StructuralCounters } from '../rendering/EngineAdapter'; * * `null` is therefore a meaningful result and never an inconvenience to work * around: it says this pair of cells carries no structural evidence, which is the - * case whenever an arm reported no counters at all (the Phaser arm renders a - * WebGL1 context the probe cannot attach to). + * case whenever an arm reported no counters at all (for example, a legacy + * Phaser profile measured before its WebGL2 context path was enabled). */ /** How far apart two counters must be before the difference is attributed to them rather than to CPU work. */ diff --git a/packages/exojs-bench/src/comparison/render.ts b/packages/exojs-bench/src/comparison/render.ts index c9a3ae04a..2c1297467 100644 --- a/packages/exojs-bench/src/comparison/render.ts +++ b/packages/exojs-bench/src/comparison/render.ts @@ -206,9 +206,9 @@ const renderRenderingBlocks = (input: RenderingInput): string[] => { } if (backend.webgl1.length > 0) { - lines.push('### WebGL1 arms - CPU time only', ''); + lines.push('### Legacy WebGL1 arms - CPU time only', ''); lines.push( - 'Phaser 4 renders a WebGL1 context (verified against the installed dist). A gap against it can be caused by the backend generation as much as by the engine, and the WebGL2 structural probe cannot attach to say which - so these rows carry NO mechanism and are observations, not findings. They never share a table with the WebGL2/WebGPU arms.', + 'These rows come from profiles measured before the Phaser arm received its verified WebGL2 context path. A gap against them can be caused by the backend generation as much as by the engine, and the WebGL2 structural probe could not attach to say which - so these rows carry NO mechanism and are observations, not findings.', '', ); diff --git a/packages/exojs-bench/src/physics/PhysicsAdapter.ts b/packages/exojs-bench/src/physics/PhysicsAdapter.ts index 3a5ffac2a..521a7cd91 100644 --- a/packages/exojs-bench/src/physics/PhysicsAdapter.ts +++ b/packages/exojs-bench/src/physics/PhysicsAdapter.ts @@ -174,7 +174,7 @@ export interface PhysicsArmIdentity { * {@link '../rendering/EngineAdapter'.EngineAdapter}. * * Every arm - the native `@codexo/exojs-physics` runtime and the matter.js, - * planck and rapier libraries an app would attach instead - implements this one + * planck, nape-js and rapier libraries an app would attach instead - implements this one * interface, so the stay-native vs. attach-an-adapter comparison rests on the * harness driving all of them through the identical calls. Implementations run * in the browser page, not in the driver process. diff --git a/packages/exojs-bench/src/physics/adapters/matter-js.ts b/packages/exojs-bench/src/physics/adapters/matter-js.ts index f1996ff9b..a6183be60 100644 --- a/packages/exojs-bench/src/physics/adapters/matter-js.ts +++ b/packages/exojs-bench/src/physics/adapters/matter-js.ts @@ -40,8 +40,9 @@ import { describePhysicsScene } from './scene'; * (`enableSleeping=false`), whereas exojs and rapier do - so a settled matter * stack keeps paying full solve cost. Kept at matter's default and disclosed. * - `frictionAir` is set to `0` (matter's default `0.01` applies a per-step linear - * drag that neither exojs nor rapier apply by default) so all three arms - * integrate the SAME pure-gravity force field - matching the SCENE, while the + * drag that exojs, planck, rapier and nape-js do not apply by default) so all + * four arms integrate the SAME pure-gravity force field - matching the SCENE, + * while the * solver differences above are left to be measured. * - Contact count is matter's active colliding-pair count (`engine.pairs * .collisionActive`), a pair-level proxy comparable to - but not semantically diff --git a/packages/exojs-bench/src/physics/adapters/nape-js.ts b/packages/exojs-bench/src/physics/adapters/nape-js.ts new file mode 100644 index 000000000..9ac2a8148 --- /dev/null +++ b/packages/exojs-bench/src/physics/adapters/nape-js.ts @@ -0,0 +1,143 @@ +import type * as Nape from '@newkrok/nape-js'; + +import type { PhysicsAdapter, PhysicsArchetypeSpec, PhysicsStructuralCounters } from '../PhysicsAdapter'; +import type { PerStepWork } from './perStepWork'; +import { createPerStepWork } from './perStepWork'; +import type { BodyDesc } from './scene'; +import { describePhysicsScene } from './scene'; + +/** + * `@newkrok/nape-js` arm of the physics benchmark. + * + * The adapter uses the shared neutral scene descriptor, so body counts, shapes, + * positions, materials and the perturbed-body selection are identical to the + * other competitor arms. Nape's default solver, sleeping and single-step + * configuration are intentionally left intact: changing them to resemble + * another engine would turn this into a tuned configuration rather than a + * library comparison. + * + * Nape's active collision arbiters are reported as its structural contact proxy. + * That count is pair-level and is disclosed as engine-specific evidence, just + * like the contact counters of matter.js and Rapier. + */ +export const createNapeJsAdapter = async (): Promise => { + const N = (await import('@newkrok/nape-js')) as typeof Nape; + + let space: Nape.Space | null = null; + let perturbedSignature = ''; + let perStep: PerStepWork | null = null; + let stepIndex = 0; + + return { + engine: 'nape-js', + config: 'default', + + setup(spec: PhysicsArchetypeSpec, bodyCount: number, seed: number): void { + const scene = describePhysicsScene(spec, bodyCount, seed); + const created = new N.Space(new N.Vec2(spec.gravity.x, spec.gravity.y)); + const table: Nape.Body[] = []; + + perturbedSignature = scene.perturbedSignature; + + const createBody = (desc: BodyDesc): Nape.Body => { + const body = new N.Body(desc.type === 'static' ? N.BodyType.STATIC : N.BodyType.DYNAMIC, new N.Vec2(desc.x, desc.y)); + const material = new N.Material(desc.restitution, desc.friction, desc.friction, desc.density); + const shape = + desc.shape.kind === 'box' + ? new N.Polygon(N.Polygon.box(desc.shape.width, desc.shape.height), material) + : new N.Circle(desc.shape.radius, undefined, material); + + body.shapes.add(shape); + + if (desc.perturb !== undefined) { + body.velocity = new N.Vec2(desc.perturb.vx, desc.perturb.vy); + } + + body.space = created; + + return body; + }; + + for (const body of scene.bodies) { + table.push(createBody(body)); + } + + for (const joint of scene.joints) { + const bodyA = table[joint.bodyA]!; + const bodyB = table[joint.bodyB]!; + const positionA = bodyA.position; + const positionB = bodyB.position; + const constraint = new N.PivotJoint( + bodyA, + bodyB, + new N.Vec2(joint.x - positionA.x, joint.y - positionA.y), + new N.Vec2(joint.x - positionB.x, joint.y - positionB.y), + ); + + constraint.space = created; + } + + stepIndex = 0; + perStep = createPerStepWork(spec, scene, table, { + createBody, + removeBody: body => { + body.space = null; + }, + castRay: ray => { + const result = created.rayCast( + new N.Ray(new N.Vec2(ray.x, ray.y), new N.Vec2(ray.dx, ray.dy)), + // The shared ray sweep starts outside the geometry. Keeping the + // default outer-surface query matches the published Nape API path. + false, + ); + + if (result !== null) { + result.dispose(); + } + + return result !== null; + }, + }); + space = created; + }, + + step(dt: number): void { + if (space === null || perStep === null) { + throw new Error('nape-js adapter: step() called before setup().'); + } + + perStep.run(stepIndex++); + space.step(dt); + }, + + sampleStructural(): PhysicsStructuralCounters { + if (space === null || perStep === null) { + throw new Error('nape-js adapter: sampleStructural() called before setup().'); + } + + const currentSpace = space; + + return { + bodyCount: currentSpace.bodies.length, + contactCount: Array.from({ length: currentSpace.arbiters.length }, (_, index) => currentSpace.arbiters.at(index)).filter(arbiter => + arbiter.isCollisionArbiter(), + ).length, + jointCount: currentSpace.constraints.length, + rayHits: perStep.rayHits, + }; + }, + + teardown(): void { + if (space !== null) { + space.clear(); + space = null; + } + + perStep = null; + }, + + mutationSignature(): string { + return perturbedSignature; + }, + }; +}; diff --git a/packages/exojs-bench/src/physics/adapters/perStepWork.ts b/packages/exojs-bench/src/physics/adapters/perStepWork.ts index 1d514ee16..e8d0c4384 100644 --- a/packages/exojs-bench/src/physics/adapters/perStepWork.ts +++ b/packages/exojs-bench/src/physics/adapters/perStepWork.ts @@ -6,8 +6,8 @@ import { rayForStep } from './scene'; * Shared per-step work for the archetypes whose cost is NOT the plain solver * step: the ray sweep and the body churn. * - * The three arms differ only in how they create, destroy and query a body, so - * only that is left to them ({@link ArmWorldOps}); the loop structure - which + * The competitor arms differ only in how they create, destroy and query a body, + * so only that is left to them ({@link ArmWorldOps}); the loop structure - which * indices churn, which rays are cast at which step index, how hits are counted - * lives here. Three hand-written copies of that loop would be three chances for * one arm to churn a different set or cast a different sweep than the others, diff --git a/packages/exojs-bench/src/physics/adapters/scene.ts b/packages/exojs-bench/src/physics/adapters/scene.ts index 5e9370ac3..0e7da5d10 100644 --- a/packages/exojs-bench/src/physics/adapters/scene.ts +++ b/packages/exojs-bench/src/physics/adapters/scene.ts @@ -4,15 +4,15 @@ import type { PhysicsArchetypeSpec, PhysicsSceneShape } from '../PhysicsAdapter' /** * Engine-neutral description of a physics scene - the fairness backbone the - * matter.js and rapier arms build from. + * matter.js, planck.js, nape-js and rapier arms build from. * * The native `adapters/exojs-physics.ts` arm builds its scene inline against the * `@codexo/exojs-physics` API. The competitor arms cannot share that code (they - * speak matter/rapier body APIs), so the risk is two hand-written transcriptions + * speak different body APIs), so the risk is several hand-written transcriptions * quietly drifting into different scenes. This module removes that risk for the * competitor arms: it produces one neutral list of {@link BodyDesc}s, drawn from * the SAME shared deterministic RNG in the SAME order as the exojs arm, so - * matter and rapier simulate a byte-identical body configuration to each other, + * all competitor arms simulate a byte-identical body configuration to each other, * and - because the draw order is a faithful transcription of exojs-physics.ts - * to the native arm as well. * @@ -23,7 +23,7 @@ import type { PhysicsArchetypeSpec, PhysicsSceneShape } from '../PhysicsAdapter' * * Coordinate convention matches exojs: +Y points DOWN, and a body's position is * the CENTRE of its box/circle. Both competitor arms adopt this same convention - * so the numeric positions are identical across all three arms. + * so the numeric positions are identical across all arms. */ /** Side length of a dynamic box / diameter reference for a dynamic circle, px. Mirrors `exojs-physics.ts`. */ diff --git a/packages/exojs-bench/src/physics/driver.ts b/packages/exojs-bench/src/physics/driver.ts index f48587895..24dc7405a 100644 --- a/packages/exojs-bench/src/physics/driver.ts +++ b/packages/exojs-bench/src/physics/driver.ts @@ -34,10 +34,10 @@ const PAGE_DIR = resolve(HERE, 'page'); * stamped into that run's provenance caveats. * * Each disclosure opens with the arm's ROLE, because the arms are not one flat - * field of competitors: matter.js and planck.js are the pure-JS PEERS, the - * libraries an ExoJS app would realistically attach instead of the native - * runtime, and they are what `exojs-physics` is compared against. rapier is a - * Rust engine compiled to WASM and stands as the REFERENCE CEILING - the + * field of competitors: matter.js, planck.js and nape-js are the JavaScript + * peers, the libraries an ExoJS app would realistically attach instead of the + * native runtime, and they are what `exojs-physics` is compared against. rapier + * is a Rust engine compiled to WASM and stands as the REFERENCE CEILING - the * ambient cost of leaving JavaScript - not as a peer a JS solver is expected to * match. */ @@ -48,6 +48,8 @@ const ARM_DISCLOSURES: Readonly> = { "matter-js arm (pure-JS peer): constraint solver at matter defaults (6 position / 4 velocity / 2 constraint iterations), sleeping OFF by default (a settled stack keeps paying full solve cost); matter's default per-step air drag (frictionAir) is zeroed so all arms integrate the same pure-gravity field; gravity (px/s^2) and perturbation velocity (px/s) are mapped into matter's px-per-step unit model. Contact count = active colliding pairs (engine.pairs.collisionActive), a pair-level proxy, not identical in semantics to the exojs solid-contact count.", planck: "planck arm (pure-JS peer): Box2D port at planck defaults (8 velocity / 3 position iterations per step), sleeping ON by default; Settings.lengthUnitsPerMeter is set to 30 so planck's absolute MKS tolerances are interpreted at the scene's pixel scale, which is the knob planck gives a pixel-coordinate game - positions, gravity (px/s^2) and velocity (px/s) then carry over unconverted. Contact count = the world contact list filtered by isTouching(), a touching collider-pair count. Rays are answered from planck's dynamic tree, but World.rayCast is Box2D's non-solid ray (an origin inside a fixture is not a hit). Continuous collision runs for every body (planck's default), where exojs and rapier restrict it to bullets and matter has none.", + 'nape-js': + 'nape-js arm (pure-JS peer): default single-step solver with default velocity/position iterations (10/10), sleeping and dynamic AABB broadphase at library defaults. Materials, body layouts, joints and perturbations come from the shared neutral scene descriptor. Contact count = active collision arbiters; ray queries use Space.rayCast with outer-surface semantics. These counters are engine-specific structural proxies, not a claim that every solver performs identical internal work.', rapier: 'rapier arm (WASM reference ceiling, not a pure-JS peer): TGS-Soft solver at rapier defaults (4 solver / 1 internal PGS iterations), auto-sleeping ON; default lengthUnit=1 is fed a px-scale world (tuned for ~1-unit objects), exactly what attaching rapier with pixel coordinates yields. Contact count = collider pairs with a solid narrow-phase manifold (numContacts > 0), deduped.', }; @@ -357,7 +359,7 @@ export const runPhysicsMatrix = async ( 'Scenes are warmed to steady state before timing; the per-cell warmupSteps/timedSteps counts are recorded for honesty.', 'All arms build the byte-identical scene (bodies, positions, shapes, sizes, static/dynamic split, gravity, perturbed-body set) from the shared deterministic RNG, and the perturbed-body selection is asserted equal across arms before each cell is timed.', 'Each arm runs at its own engine defaults for solver iterations, contact model and sleeping - those engine differences are the measured quantity in a native-vs-adapter comparison, disclosed per arm below.', - 'Arm roles: matter-js and planck are the PURE-JS PEERS exojs-physics is compared against; rapier is a Rust/WASM engine and stands as the REFERENCE CEILING for what leaving JavaScript buys, not as a peer a JS solver is expected to match.', + 'Arm roles: matter-js, planck and nape-js are the JAVASCRIPT PEERS exojs-physics is compared against; rapier is a Rust/WASM engine and stands as the REFERENCE CEILING for what leaving JavaScript buys, not as a peer a JS solver is expected to match.', ...(unavailableArms.length > 0 ? [ `Arms this browser could not run, recorded as unavailable cells rather than omitted: ${unavailableArms.map(arm => `${arm.engine} (${arm.reason})`).join('; ')}`, diff --git a/packages/exojs-bench/src/physics/index.ts b/packages/exojs-bench/src/physics/index.ts index 558e5f25a..c5c994c75 100644 --- a/packages/exojs-bench/src/physics/index.ts +++ b/packages/exojs-bench/src/physics/index.ts @@ -14,9 +14,9 @@ // provenance/report skeletons, the Vite server, CLI arg parsing) live under // `../shared` and are shared with the rendering domain. // -// The matter.js, planck.js and rapier adapter arms live under `adapters/` -// alongside the native arm: each implements the `PhysicsAdapter` interface and -// builds the shared deterministic scene from `adapters/scene.ts`. Their +// The matter.js, planck.js, nape-js and rapier adapter arms live under +// `adapters/` alongside the native arm: each implements the `PhysicsAdapter` +// interface and builds the shared deterministic scene from `adapters/scene.ts`. Their // competitor libraries are loaded lazily via dynamic `import()`; a factory that // rejects (an unlinked competitor, or a browser that refuses the arm) is // recorded as an unavailable arm carrying that reason, never omitted. diff --git a/packages/exojs-bench/src/physics/page/harness.ts b/packages/exojs-bench/src/physics/page/harness.ts index 69a54691a..f0c47a35e 100644 --- a/packages/exojs-bench/src/physics/page/harness.ts +++ b/packages/exojs-bench/src/physics/page/harness.ts @@ -120,6 +120,16 @@ const ARM_FACTORIES: ReadonlyArray<{ return module.createRapierAdapter(); }, }, + { + engine: 'nape-js', + config: 'default', + library: '@newkrok/nape-js', + create: async (): Promise => { + const module = await import('../adapters/nape-js'); + + return module.createNapeJsAdapter(); + }, + }, ]; /** One arm's resolution outcome, cached so a matrix of cells constructs each arm once. */ diff --git a/packages/exojs-bench/src/rendering/adapters/phaser.ts b/packages/exojs-bench/src/rendering/adapters/phaser.ts index 4b56d9e8b..c9ac89a2e 100644 --- a/packages/exojs-bench/src/rendering/adapters/phaser.ts +++ b/packages/exojs-bench/src/rendering/adapters/phaser.ts @@ -18,26 +18,16 @@ import { GRID_MARGIN, gridLayout, gridPosition, isScrolling, VIEWPORT_HEIGHT, VI * cycling, overdraw stacking, top-left anchoring) is a faithful transcription of * the other arms so the comparison rests on the same neutral archetypes. * - * WEBGL VERSION DISCLOSURE - EMPIRICAL, and the reason this arm is measured as it - * is. Phaser 4 "Caladan" is often described as a from-scratch WebGL2 renderer; - * against the installed 4.2.1 source it is NOT. Its `WebGLRenderer.init` requests - * a `'webgl'` (WebGL**1**) context by default (`canvas.getContext('webgl')`, - * `WebGLRenderer.js:709`), its shaders are GLSL ES 1.00 (`attribute`/`varying`; - * no `#version 300 es` anywhere in the dist), and it polyfills the WebGL2-core - * features it needs (instanced arrays, VAO) from WebGL1 extensions - its renderer - * is an evolution of the Phaser 3.85+ WebGL path, not a WebGL2 rewrite. + * WEBGL VERSION DISCLOSURE - EMPIRICAL. Phaser 4.2.1's default renderer asks for + * a WebGL1 context, but its public `GameConfig.context` path accepts a caller- + * created context. This adapter supplies a real WebGL2 context through that path + * and verifies after boot that Phaser retained the same object. The fallback path + * is deliberately not used: a browser without WebGL2 makes this arm unavailable + * instead of silently changing the measured backend. * - * By deliberate decision this arm renders through Phaser's OWN default context - * (WebGL, i.e. WebGL1) - exactly as a stock Phaser 4 app would - rather than - * injecting a WebGL2 context to force backend parity. That keeps the arm honest: - * its CPU-time column is measured identically to the other arms and IS cross-arm - * comparable, but its GPU/structural columns are NOT WebGL2-backend-comparable. - * The harness's WebGL2 draw-call structural probe cannot attach to a WebGL - * (WebGL1) context, so this arm reports NO structural counters - disclosed per - * cell by the harness (`page/harness.ts::attachProbes`) and in the report - * Methodology; the counts are omitted, never faked. Phaser 4 ships NO WebGPU - * renderer (`Phaser.AUTO/CANVAS/WEBGL/HEADLESS` only), so this arm supports the - * `'webgl2'` backend request only and never runs `'webgpu'`. + * Phaser 4 ships NO WebGPU renderer (`Phaser.AUTO/CANVAS/WEBGL/HEADLESS` only), + * so this arm supports the `'webgl2'` backend request only and never runs + * `'webgpu'`. * * The harness owns frame cadence, so Phaser's own `requestAnimationFrame` game * loop (`TimeStep`) is halted right after boot (`game.loop.stop()`), and one @@ -124,12 +114,11 @@ export const createPhaserAdapter = (): EngineAdapter => { return { engine: 'phaser', - config: 'default', + config: 'webgl2', supports(target: Backend): boolean { - // Phaser 4 renders WebGL (WebGL1) via its default context and ships no - // WebGPU renderer; it runs under the harness 'webgl2' request (disclosed) - // and never the 'webgpu' backend. + // Phaser 4 ships no WebGPU renderer; the adapter supplies WebGL2 through + // the public context injection path below. return target === 'webgl2'; }, @@ -139,26 +128,30 @@ export const createPhaserAdapter = (): EngineAdapter => { // one here, i.e. a row that looks comparable and is not - so the arm sits // the archetype out instead. // - // The render-target archetypes are sat out for the reason this arm's header - // comment establishes empirically: Phaser 4 renders a WebGL1 context, so a - // filter- or mask-heavy row's gap would be attributable to the backend - // generation rather than to the engine. + // Render-target archetypes remain out until their Phaser semantics are + // validated against the shared filter/mask contract. return !isScrolling(spec) && !usesRenderTargets(spec); }, async init(canvas: HTMLCanvasElement, target: Backend): Promise { if (target !== 'webgl2') { - throw new Error(`The phaser adapter only runs under the harness 'webgl2' backend request (Phaser 4 renders WebGL1); got '${target}'.`); + throw new Error(`The phaser adapter only runs under the harness 'webgl2' backend request; got '${target}'.`); + } + + const context = canvas.getContext('webgl2', { antialias: false, powerPreference: 'high-performance' }); + + if (context === null) { + throw new Error('Phaser requires a WebGL2 context for this arm.'); } await new Promise(resolve => { game = new Phaser.Game({ - // Force WebGL (never AUTO/Canvas): a Canvas fallback would silently - // measure a different renderer. Phaser 4's WebGLRenderer creates its - // own default `'webgl'` (WebGL1) context - no `context` is injected, so - // this measures a stock Phaser 4 app's renderer honestly. + // Force WebGL and pass the already-created WebGL2 context. Phaser's + // public type incorrectly calls this CanvasRenderingContext2D, but its + // WebGLRenderer reads the value as the renderer context at runtime. type: Phaser.WEBGL, canvas, + context: context as unknown as CanvasRenderingContext2D, width: VIEWPORT_WIDTH, height: VIEWPORT_HEIGHT, backgroundColor: '#000000', @@ -182,6 +175,11 @@ export const createPhaserAdapter = (): EngineAdapter => { // Halt Phaser's own requestAnimationFrame loop; the harness drives frames. game!.loop.stop(); + + if (game!.context !== context) { + throw new Error('Phaser did not retain the injected WebGL2 context.'); + } + scene = game!.scene.getScene(SCENE_KEY); }, diff --git a/packages/exojs-bench/src/rendering/archetypes.ts b/packages/exojs-bench/src/rendering/archetypes.ts index 3c4dd1d32..664fd7861 100644 --- a/packages/exojs-bench/src/rendering/archetypes.ts +++ b/packages/exojs-bench/src/rendering/archetypes.ts @@ -386,8 +386,8 @@ export const ARCHETYPES: readonly ArchetypeSpec[] = [ // binding and blit, so a heavy kernel would move the bottleneck into the // fragment shader and hide the thing under test. // - // WebGL2/WebGPU arms only. The Phaser arm renders WebGL1, so its gap here - // would be attributable to the backend generation rather than to the engine. + // Arms with validated render-target support only. Competitor gaps here are + // coverage gaps, not approximations of a different scene. { id: 'filter-chain-1', category: 'render-targets', @@ -426,7 +426,7 @@ export const ARCHETYPES: readonly ArchetypeSpec[] = [ // unrotated rect mask as GPU scissor/clip state, so the row measures the // nesting rather than one arm's intermediate-target policy. // - // Shares the render-target machinery of the filter rows and the same WebGL1 + // Shares the render-target machinery of the filter rows and the same coverage // exclusion; otherwise identical to `static-heavy` at depth 4. // // The nesting depth is one greater than the mask depth on purpose: the scene diff --git a/packages/exojs-bench/src/rendering/driver.ts b/packages/exojs-bench/src/rendering/driver.ts index 6dd0923f0..4b6b6d553 100644 --- a/packages/exojs-bench/src/rendering/driver.ts +++ b/packages/exojs-bench/src/rendering/driver.ts @@ -218,11 +218,10 @@ const ADAPTER_CAPABILITIES: readonly EngineAdapter[] = [ capabilityDescriptor('pixi', 'culled', ['webgl2', 'webgpu'], spec => spec.cullingEnabled), // Phaser 4 and Excalibur are committed competitor arms (pinned exact // devDependencies). Both are WebGL2-only in this harness and never run WebGPU - // (Phaser 4 ships no WebGPU renderer; Excalibur 0.32 has none). Phaser 4 is - // measured as a stock app: its WebGLRenderer creates a WebGL1 context by - // default (`getContext('webgl')`), so it runs under the 'webgl2' REQUEST while - // rendering WebGL1 (disclosed by the harness's structural-probe degrade path - // and the report Methodology); Excalibur 0.32 renders a real WebGL2 context. + // (Phaser 4 ships no WebGPU renderer; Excalibur 0.32 has none). Phaser's + // default renderer asks for WebGL1, so the adapter explicitly supplies and + // verifies a WebGL2 context through Phaser's public context path; Excalibur + // creates its own real WebGL2 context. // A missing (unlinked) competitor degrades gracefully: its per-cell dynamic // import fails in isolation (`runCellInPage` records that cell `unavailable` // and the run continues), and it is left out of Vite's pre-bundle set below. @@ -231,14 +230,10 @@ const ADAPTER_CAPABILITIES: readonly EngineAdapter[] = [ // promises off-screen content - a row that looks comparable and is not. // // Both also sit out the render-target archetypes (`filter-chain-*`, - // `mask-clip`), for two different reasons that land on the same exclusion. - // Phaser 4 renders a WebGL1 context, so a target-heavy row's gap would be - // attributable to the backend generation rather than to the engine, and a - // WebGL1-vs-WebGL2 factor is not a claim this matrix makes. Excalibur 0.32 has - // no per-node filter or clipping API at all: its `PostProcessor` chain is a - // full-SCREEN pass, not a filtered subtree, and it ships no mask source, so - // the cell would have to be approximated - which the fairness rule forbids. - capabilityDescriptor('phaser', 'default', ['webgl2'], spec => !isScrolling(spec) && !usesRenderTargets(spec)), + // `mask-clip`): Phaser's adapter and Excalibur 0.32 have no validated + // per-node equivalent for the shared filter/mask scenes, so approximating + // the cell would violate the fairness rule. + capabilityDescriptor('phaser', 'webgl2', ['webgl2'], spec => !isScrolling(spec) && !usesRenderTargets(spec)), capabilityDescriptor('excalibur', 'default', ['webgl2'], spec => !isScrolling(spec) && !usesRenderTargets(spec)), ]; diff --git a/packages/exojs-bench/src/rendering/report.ts b/packages/exojs-bench/src/rendering/report.ts index 90d56f9fd..e1768f444 100644 --- a/packages/exojs-bench/src/rendering/report.ts +++ b/packages/exojs-bench/src/rendering/report.ts @@ -184,7 +184,7 @@ const toMarkdown = (data: ReportData): string => { '', "- **Culling:** disabled on every archetype whose content is fully on-screen (`cullingEnabled: false`) — there a cull check never removes a node and can only add overhead, and the arms do not pay equally for it: ExoJS's `cullable` flag drives a real per-node bounds/intersection check in the render walk, while Pixi's `cullable` flag is inert unless something calls `Culler.shared.cull(...)`. The Phaser arm does no bounds culling (its default `willRender` checks only visibility/alpha flags), and the Excalibur arm never runs its off-screen culling system (only the draw path is stepped, not the update systems that tag entities off-screen). On those archetypes every arm therefore does identical visible-set work.", "- **`scrolling-world` is the one archetype with off-screen content**, and the only one with a moving camera: `nodeCount` leaves are laid out over 4x the viewport's area (`worldSpan: 2`), so roughly 25% are visible at any moment, and the camera travels the world diagonal at `cameraSpeed` units per frame, reflecting off the world edges on a path that is a closed form in the frame index (identical on every arm). Two disclosures apply to its rows. **(1) Camera mechanism differs by arm, idiomatically:** ExoJS moves its `View` centre (the engine has a real camera, and the view rect is what its culling and its retained-render-product validity key on); Pixi has no camera object, so that arm translates the world container under a fixed screen rect. Both show the identical world content per frame. **(2) Two Pixi arms:** `pixi default` is stock Pixi and does NOT cull — it draws the off-screen content too, which is Pixi's out-of-the-box behaviour and the honest upper bound; `pixi culled` adds the explicit per-frame `Culler.shared.cull(root, renderer.screen, false)` a Pixi app that wants culling has to write itself. Read the ExoJS rows against `pixi culled` for a culling-vs-culling comparison, and against `pixi default` for the out-of-the-box one. `pixi culled` is measured only on this archetype.", - "- **Phaser renders WebGL, not WebGL2.** The Phaser arm is measured as a stock Phaser 4 app: Phaser 4.2 is often described as a from-scratch WebGL2 renderer, but its `WebGLRenderer` requests a plain `webgl` (WebGL1) context by default (`canvas.getContext('webgl')`, WebGLRenderer.js:709), uses GLSL ES 1.00 shaders, and polyfills the WebGL2-core features it needs (instanced arrays, VAO) from WebGL1 extensions — its renderer is an evolution of the Phaser 3.85+ WebGL path, not a WebGL2 rewrite. The arm runs under the `webgl2` backend *request* but its rows are WebGL-rendered. Its CPU-time column is measured identically to the other arms and **is** cross-arm comparable; its full-frame time comes from the rAF delta (as it does for any arm when the optional GPU-timer extension is absent). The WebGL2 draw-call structural probe cannot attach to a WebGL context, so the Phaser arm reports **no structural counters** (`drawCalls`/`textureBinds`/`bufferUploads` show 0 with an explanatory `note`) — the counts are omitted, never faked. Compare structural columns only among the WebGL2 arms (ExoJS, Pixi, Excalibur). Phaser 4 ships no WebGPU renderer, so it never runs the `webgpu` backend.", + "- **Phaser uses an injected WebGL2 context.** Phaser 4.2.1's default renderer requests WebGL1, but its public `GameConfig.context` path accepts the WebGL2 context created by this adapter. The adapter verifies that Phaser retains the same context after boot, so a browser without WebGL2 makes the arm unavailable instead of silently changing the backend. Phaser ships no WebGPU renderer, and render-target rows remain excluded because no validated per-node filter or mask equivalent exists in this adapter.", '- **`hitching` marks a periodic spike, and the median alone will not show it.** A cell is marked when its `cpuMsP95` is at least 4x its `cpuMsMedian` AND at least 8ms — i.e. most frames are cheap and a few are not. This matters because an optimisation that converts per-frame work into PERIODIC work (a cache that is rebuilt every n-th frame instead of every frame) improves the median exactly as much as one that removed the work, while the worst frame is unchanged. Read `cpuMsMedian` for the amortised cost and `cpuMsP95` for the frame the player actually feels; a `hitching` row means the two answer different questions and the median must not be quoted on its own.', "- **`frameMs*` and `queueMs*` are different measurements, and only WebGL2 has one of them.** On WebGL2 `frameMs*` is an `EXT_disjoint_timer_query_webgl2` `TIME_ELAPSED` query bracketing the frame's whole GL command stream, uploads included. On WebGPU it is a hardware `timestamp-query` pair around each of the frame's render passes, which covers pass EXECUTION only — `queue.writeBuffer` is a queue operation outside every command buffer, so its device copy cannot be bracketed by any timestamp. WebGPU rows therefore also carry `queueMs*`: the `queue.onSubmittedWorkDone` interval charged to the frame that caused it (`doneAt − max(submitAt, previous doneAt)`), which is the only signal that sees upload cost — but which is floored by when the browser OBSERVES completion (measured: 0.50ms for a canvas-clearing frame doing 2µs of GPU work, 3.18ms for the identical clear into an offscreen texture), so any `queueMs` below ~4ms is observation latency rather than GPU work. Read `frameMs*` within a backend, never across the two; `cpuMs*` is the cross-backend, cross-arm metric.", '- **Competitor render-path isolation.** Each competitor arm is driven through only its render path with its own loop suppressed: Phaser via `renderer.preRender()` + `SceneManager.render()` + `renderer.postRender()` with `game.loop.stop()`; Excalibur via its public draw sequence (`beginDrawLifecycle`/`clear`/`currentScene.draw`/`flush`/`endDrawLifecycle`) with `engine.clock.stop()`. Update/input/physics subsystems are never stepped, so only rendering is measured.', diff --git a/packages/exojs-bench/src/rendering/traits.ts b/packages/exojs-bench/src/rendering/traits.ts index 5ab5ba0c3..d1ebd1638 100644 --- a/packages/exojs-bench/src/rendering/traits.ts +++ b/packages/exojs-bench/src/rendering/traits.ts @@ -39,9 +39,9 @@ export const isComposite = (spec: ArchetypeSpec): boolean => compositeBlurRadius * Whether the archetype exercises render-target machinery - a filter chain, a * mask stack, or the bloom-shaped composite. * - * This is the WebGL1 exclusion boundary: the Phaser arm renders through a WebGL1 - * context, so a target-heavy row's gap would be attributable to the backend - * generation rather than to the engine, which is not a claim this matrix makes. + * This is the render-target coverage boundary: competitor arms without a + * validated per-node equivalent sit these rows out rather than approximating + * them and making the comparison answer a different question. */ export const usesRenderTargets = (spec: ArchetypeSpec): boolean => filterChainDepth(spec) > 0 || maskDepth(spec) > 0 || isComposite(spec); diff --git a/packages/exojs-bench/src/shared/viteServer.ts b/packages/exojs-bench/src/shared/viteServer.ts index 912c0a63a..62551007d 100644 --- a/packages/exojs-bench/src/shared/viteServer.ts +++ b/packages/exojs-bench/src/shared/viteServer.ts @@ -41,7 +41,7 @@ export const RENDERING_LIBRARY_ARMS = ['pixi.js', 'phaser', 'excalibur'] as cons * own WASM, so both only reach the browser's native ESM loader through the * optimizer. */ -export const PHYSICS_LIBRARY_ARMS = ['matter-js', 'planck', '@dimforge/rapier2d-compat'] as const; +export const PHYSICS_LIBRARY_ARMS = ['matter-js', 'planck', '@dimforge/rapier2d-compat', '@newkrok/nape-js'] as const; /** Every competitor arm this package can measure, across both domains. */ export const LIBRARY_ARMS = [...RENDERING_LIBRARY_ARMS, ...PHYSICS_LIBRARY_ARMS] as const; diff --git a/site/src/components/BenchComparisonSummary.astro b/site/src/components/BenchComparisonSummary.astro new file mode 100644 index 000000000..0533e01de --- /dev/null +++ b/site/src/components/BenchComparisonSummary.astro @@ -0,0 +1,217 @@ +--- +/** + * BenchComparisonSummary - how one profile's comparisons came out, one bar per arm. + * + * This is a distribution, not a score. Nothing is summed across bars and no bar + * is ranked against another, because the arms answer different questions - and + * the block deliberately does not read as a league table. + * + * Coverage is not an outcome. A pair with no shared cell leaves the bar's track + * empty rather than taking a segment of its own: an arm that sits an archetype + * out is a gap in the matrix, and counting it beside the measured results would + * report missing coverage as a doubt about the measurement. The bar is + * therefore always the full row count wide, and how much of it is filled is the + * coverage figure printed in front of it. + * + * Ahead and behind get the same segment height and the same type, so a loss + * never reads as quieter than a lead. `unclear` is a measured outcome - the + * runs produced numbers and disagreed on the rung - so it is hatched rather + * than left empty. + * + * The bars are grouped by what kind of opponent they are, because a Rust/WASM + * solver and a JavaScript one answer different questions and a single flat list + * invites a reader to weigh them against each other. + */ + +import { OUTCOME_LABELS, OUTCOME_ORDER, SUMMARY_OF, SUMMARY_ORDER, type ComparisonTally, type SummaryState } from '../lib/bench-profiles'; + +interface Props { + tallies: readonly ComparisonTally[]; +} + +const { tallies } = Astro.props; + +/** What each state is called under a bar, with the arm's own name where it has a side. */ +const phraseOf = (state: SummaryState, arm: string, count: number): string => { + if (state === 'ahead') return `${String(count)} ExoJS ahead`; + if (state === 'behind') return `${String(count)} ${arm} ahead`; + if (state === 'level') return `${String(count)} level`; + + return `${String(count)} no clear lead`; +}; + +/** The exact outcomes behind one summary state, for the segment's own tooltip. */ +const breakdownOf = (tally: ComparisonTally, state: SummaryState): string => + OUTCOME_ORDER.filter(outcome => SUMMARY_OF[outcome] === state && tally.counts[outcome] > 0) + .map(outcome => `${String(tally.counts[outcome])} ${OUTCOME_LABELS[outcome]}`) + .join(', '); + +/** The bars, in the order they were built, split into the groups they announce. */ +const groups = tallies.reduce<{ label: string; lines: ComparisonTally[] }[]>((built, tally) => { + const last = built.at(-1); + + if (last !== undefined && last.label === tally.group) { + last.lines.push(tally); + + return built; + } + + return [...built, { label: tally.group, lines: [tally] }]; +}, []); +--- + +{ + tallies.length > 0 && ( +
+ {groups.map(group => ( +
+

{group.label}

+ {group.lines.map(tally => ( +
+ + {tally.label} + {tally.meta} + + + {SUMMARY_ORDER.filter(state => tally.summary[state] > 0).map(state => ( + + ))} + {tally.total > tally.measured && ( + + )} + + + + {tally.measured}/{tally.total} measured + + {SUMMARY_ORDER.filter(state => tally.summary[state] > 0).map(state => ( + {phraseOf(state, tally.arm, tally.summary[state])} + ))} + +
+ ))} +
+ ))} +
+ ) +} + + diff --git a/site/src/components/BenchComparisonTable.astro b/site/src/components/BenchComparisonTable.astro index ded1934b4..b55efdf0d 100644 --- a/site/src/components/BenchComparisonTable.astro +++ b/site/src/components/BenchComparisonTable.astro @@ -1,37 +1,54 @@ --- /** - * BenchComparisonTable - one archetype per row, one comparison pair per column. + * BenchComparisonTable - the comparisons of one domain, as a matrix or as cards. * - * The ratio is the statement, so it is drawn: a bar on a log axis that runs left - * from the centre when ExoJS leads and right when the arm does, with the - * published factor beside it. The milliseconds, the p95 of the same window and - * the range the pooled runs observed sit under it as the evidence, never behind - * a toggle. Both sides of a comparison get the same bar, the same type and the - * same weight; only the hue differs. + * Wide, the block is a matrix: one archetype per row, one comparison pair per + * column, so a reader follows a single archetype across every backend and every + * opponent on one line. Narrow, the same markup becomes one card per archetype + * with its comparisons stacked inside it. The switch is a container query on the + * block's own width, not on the viewport: a docs page can hand this component + * 700px of room inside a 1400px window, and a matrix squeezed into that is a + * matrix nobody can read. Measurements never scroll sideways at any width. * - * A pair whose runs reached different verdicts publishes none: it keeps its - * numbers and shows the ratio band those runs spanned, drawn without a side, so - * a cell the measurement could not settle still says what was measured. A pair - * that produced no comparable cell says so in place of the bar rather than - * leaving the column blank. + * A cell's first glance carries three things and no more: which way the pair + * went and by how much, the two medians, and a disclosure. Everything a reader + * needs only when verifying a row - p95, the range the pooled runs observed, + * the spread factor, the structural evidence, what each run concluded - sits + * behind that disclosure. It is not hidden, it is ranked: printing all of it in + * primary type is what made the previous table unreadable. * - * The axis is scaled to the widest ratio the table itself contains, up to a cap, - * and printed with it: a table whose axis had to hold a thousand-fold outlier - * would draw every ordinary row as a few pixels. A bar past the cap is drawn - * cut, so it still reads as an outlier, and its exact factor is printed beside - * it. Because each table states its own axis, two tables are never read against - * each other by bar length alone. + * The bar is a diverging one on a fixed log axis shared by every column and + * every table on the page, so a length means the same thing wherever it is + * read. A comparison past the axis is drawn cut rather than rescaling everyone + * else into a few pixels, and its exact factor is printed beside it either way. + * The bar keeps a track of its own, apart from the text: a mark that grows + * outward from a centre, laid over text that reads from the left, gives a + * column two competing origins and no continuous line to scan down. + * + * A pair whose runs landed on different rungs publishes no verdict. Its bar is + * the interval those runs spanned, drawn hollow and in neither side's hue, and + * its figure is the factor of the two pooled medians with a tilde - the pair's + * central observation, not a decision. An arm that produced no shared cell at + * all prints a dash: that is a gap in the matrix, not a result, and spelling it + * out in every affected row only crowds the ones that carry numbers. */ import { + armLabel, describeRungs, formatBand, + formatApproximate, formatFactor, formatMs, formatRange, + formatSpread, + frameShare, + FRAME_BUDGET_MS, isWideSpread, outcomeOf, + pooledFactor, ratioBand, + rungSide, type ProfileCell, type ProfileSpread, } from '../lib/bench-profiles'; @@ -46,96 +63,41 @@ interface Props { const { table, label } = Astro.props; const { columns, rows, unit, countColumn } = table; -/** Every ratio the table draws, including both ends of a disagreeing pair's band. */ -const ratios = rows - .flatMap(row => - row.entries.flatMap(entry => { - const cell = entry.cell; - - if (cell === null) return []; - - if (!cell.aggregate.stable) { - const band = ratioBand(cell); - - return band === null ? [] : [band.low, band.high]; - } - - return cell.verdict.ratio === null ? [] : [cell.verdict.ratio]; - }), - ) - .filter(ratio => Number.isFinite(ratio) && ratio > 0); - /** - * Widest half-axis the table will draw, in decades. + * Half-width of the axis in decades, fixed for every column and every table. * - * A single outlying comparison can be three orders of magnitude wide, and an - * axis stretched to hold it would compress every ordinary row into a few - * pixels. Past this point a bar is drawn cut rather than scaled down, so the - * rows a reader is comparing stay legible and the outlier still reads as one. - * Its exact factor is printed beside it either way. + * A per-column axis makes a given bar length mean something different in each + * column, which is the one thing a shared visual scale exists to prevent. Two + * decades holds every comparison a reader is weighing against another; the + * handful of order-of-magnitude outliers saturate at the edge, marked as cut, + * and are read from their printed factor instead. */ -const AXIS_CAP_DECADES = 2; - -/** Half-width of the axis in decades: the widest ratio present, up to the cap. */ -const decades = Math.min(AXIS_CAP_DECADES, Math.max(0.5, ...ratios.map(ratio => Math.abs(Math.log10(ratio))))); - -/** True where at least one bar runs past the axis and is drawn cut. */ -const clips = ratios.some(ratio => Math.abs(Math.log10(ratio)) > decades); +const AXIS_DECADES = 2; /** Where a ratio sits on the axis, as a percentage from its left edge. */ -const positionOf = (ratio: number): number => { - const clamped = Math.max(-decades, Math.min(decades, Math.log10(ratio))); - - return 50 + (clamped / decades) * 50; -}; - -interface BarShape { - readonly outcome: string; - readonly style: string; - readonly band: boolean; - /** Which end of the axis the bar was cut at, if it ran past it. */ - readonly clipped: 'left' | 'right' | null; -} - -const shapeOf = (cell: ProfileCell): BarShape | null => { - const outcome = outcomeOf(cell); - - if (outcome === 'unstable') { - const band = ratioBand(cell); +const positionOf = (ratio: number): number => 50 + (Math.max(-AXIS_DECADES, Math.min(AXIS_DECADES, Math.log10(ratio))) / AXIS_DECADES) * 50; - if (band === null) return null; - - const from = positionOf(band.low); - const to = positionOf(band.high); - - return { - outcome, - style: `left:${String(Math.min(from, to))}%;width:${String(Math.max(1.5, Math.abs(to - from)))}%`, - band: true, - clipped: Math.abs(Math.log10(band.low)) > decades ? 'left' : Math.abs(Math.log10(band.high)) > decades ? 'right' : null, - }; - } - - const ratio = cell.verdict.ratio; - - if (ratio === null || !Number.isFinite(ratio) || ratio <= 0) return null; +/** True where a ratio runs past the axis and its bar is drawn cut. */ +const saturates = (ratio: number): boolean => Math.abs(Math.log10(ratio)) > AXIS_DECADES; +/** A bar from the centre out to one ratio. */ +const barOf = (ratio: number): string => { const at = positionOf(ratio); - const style = at < 50 ? `right:50%;width:${String(50 - at)}%` : `left:50%;width:${String(at - 50)}%`; - const past = Math.abs(Math.log10(ratio)) > decades; - return { outcome, style, band: false, clipped: past ? (ratio < 1 ? 'left' : 'right') : null }; + return at < 50 ? `right:50%;width:${String(50 - at)}%` : `left:50%;width:${String(at - 50)}%`; }; -/** The figure printed beside the bar: the published factor, or the band a disagreeing pair spanned. */ -const figureOf = (cell: ProfileCell): string => { - if (outcomeOf(cell) !== 'unstable') return formatFactor(cell.verdict.factor); - - const band = ratioBand(cell); +/** A bar spanning the interval two ratios bound. */ +const intervalOf = (low: number, high: number): string => { + const from = positionOf(low); + const to = positionOf(high); - return band === null ? 'runs split' : formatBand(band); + return `left:${String(Math.min(from, to))}%;width:${String(Math.max(2, Math.abs(to - from)))}%`; }; +/** The range a pooled measurement observed, in milliseconds, or a dash where it reported none. */ +const rangeOf = (spread: ProfileSpread | undefined): string => (spread === undefined || formatRange(spread) === '' ? '-' : formatRange(spread)); + /** Consecutive columns that share a heading, such as the backend they were measured on. */ const groups = columns.reduce<{ label: string; span: number }[]>((built, column) => { if (column.group === null) return built; @@ -151,147 +113,236 @@ const groups = columns.reduce<{ label: string; span: number }[]>((built, column) return [...built, { label: column.group, span: 1 }]; }, []); -const rangeOf = (spread: ProfileSpread | undefined): { text: string; wide: boolean } => ({ - text: spread === undefined || formatRange(spread) === '' ? '-' : formatRange(spread), - wide: spread !== undefined && isWideSpread(spread), -}); +/** What a cell announces itself as once the block is stacked into cards. */ +const labelOf = (index: number): string => { + const column = columns[index]; + + if (column === undefined) return ''; + + return column.group === null ? `vs ${column.label}` : `${column.group} · vs ${column.label}`; +}; + +/** The word beside a factor: which arm the comparison favours, or nothing where it favours neither. */ +const winnerOf = (cell: ProfileCell): string => { + const side = cell.verdict.side; + + return side === 'exojs' ? 'ExoJS' : side === 'competitor' ? armLabel(cell.competitor) : ''; +}; --- -
-
- - - {groups.length > 0 && ( - - - ))} - - )} - - - {countColumn && } - {columns.map(column => ( - + ); + })} + + + ))} + +
- {groups.map(group => ( - - {group.label} -
Archetype{unit === 'bodies' ? 'Bodies' : 'Nodes'} - {column.overline !== '' && {column.overline}} - {column.label} +
+ + + {groups.length > 0 && ( + + ))} - - - {rows.map((row, index) => ( - <> - {row.section !== null && row.section !== rows[index - 1]?.section && ( - - - + )} + + + {countColumn && } + {columns.map(column => ( + + ))} + + + + {rows.map((row, index) => ( + <> + {row.section !== null && row.section !== rows[index - 1]?.section && ( + + + + )} + + + {countColumn && ( + )} - - - {countColumn && } - {row.entries.map(entry => { - if (entry.kind === 'reference') { - const reference = entry.reference; - const spread = rangeOf(reference?.spread); - - return ( - - ); - } - - const cell = entry.cell; - - if (cell === null) { - return ( - - ); - } + {row.entries.map((entry, column) => { + const cell = entry.cell; - const shape = shapeOf(cell); - const outcome = outcomeOf(cell); - const spread = rangeOf(cell.aggregate.competitor); + if (cell === null) { + const reason = + entry.count === null + ? 'This archetype was not measured on this backend.' + : `No shared cell at ${String(entry.count)} ${unit}: the arm sits this archetype out, or exceeded the run budget there.`; return ( - ); - })} - - - ))} - -
+ {groups.map(group => ( + + {group.label}
{row.section}
Archetype{unit === 'bodies' ? 'Bodies' : 'Nodes'} + {column.overline !== '' && {column.overline}} + + vs + {column.label} + +
{row.section}
+ {row.archetype} + {row.count !== null && !countColumn && ( + + {row.count} {unit} + + )} + + {row.count} +
{row.archetype}{row.count} - - - {formatMs(reference?.ms ?? null)} ms - - - p95 {formatMs(reference?.p95Ms ?? null)} · {spread.text} - - - - - - - - - {entry.count === null ? 'not measured here' : `not comparable at ${String(entry.count)} ${unit}`} - - - - - - {shape !== null && ( - - )} - - {figureOf(cell)} - - - {formatMs(cell.competitorMs)} ms - - - p95 {formatMs(cell.competitorP95Ms)} · {spread.text} + + + - +
+ Details +

{reason}

+
- + } + + const outcome = outcomeOf(cell); + const mixed = outcome === 'unstable'; + const band = mixed ? ratioBand(cell) : null; + const factor = mixed ? pooledFactor(cell) : cell.verdict.factor; + const ratio = cell.verdict.ratio; + const plotted = !mixed && ratio !== null && Number.isFinite(ratio) && ratio > 0; + + return ( +
+ + + + {band !== null && ( + + )} + {plotted && ratio !== null && ( + + )} + + + + {mixed ? (factor === null ? '-' : formatApproximate(factor)) : formatFactor(factor)} + {mixed ? 'no clear lead' : winnerOf(cell)} + + + {formatMs(cell.referenceMs)} + vs + {formatMs(cell.competitorMs)} ms + + +
+ Details +
+
+
p95
+
+ {formatMs(cell.referenceP95Ms)} vs {formatMs(cell.competitorP95Ms)} ms +
+
+
+
run range
+
+ + {rangeOf(cell.aggregate.reference)} + {' '} + vs{' '} + + {rangeOf(cell.aggregate.competitor)} + {' '} + ms +
+
+
+
spread
+
+ {formatSpread(cell.aggregate.reference)} vs {formatSpread(cell.aggregate.competitor)} +
+
+ {band !== null && ( +
+
run ratios
+
{formatBand(band)}
+
+ )} +
+
60 fps frame
+
+ {[ + { name: 'ExoJS', share: frameShare(cell.referenceMs) }, + { name: armLabel(cell.competitor), share: frameShare(cell.competitorMs) }, + ].map(entry => ( + + {entry.name} + + 100 && 'over']} + style={`width:${String(Math.min(100, entry.share ?? 0))}%`} + /> + + 100 && 'warn']}> + {entry.share === null ? '-' : `${entry.share.toFixed(1)}%`} + + + ))} +
+
+
+
runs
+
+ + {cell.aggregate.rungs.map(rung => ( + + ))} + + {describeRungs(cell.aggregate.rungs)} +
+
+ {cell.mechanism !== null && ( +
+
mechanism
+
{cell.mechanism}
+
+ )} +
+
+ +
- The bar is log-scaled and reaches {formatFactor(Math.pow(10, decades))} to each side of the centre, which is 1.00x - {clips ? '; a bar past that is drawn cut and keeps its exact factor beside it' : ''}. Under every time: the p95 of the same window, and the range the - pooled runs observed for the median. + Bars share one log axis across the whole page: the centre is 1.00x and each side reaches 100x, with anything past that drawn cut. A hollow bar is a + pair whose pooled runs landed on different rungs; it spans the interval they produced, publishes no verdict, and prints the two medians' own factor + with a tilde. A dash is a pair with no shared cell. Open a comparison for its p95, what share of a {FRAME_BUDGET_MS} ms frame each time takes, and + which way each pooled run fell.
diff --git a/site/src/components/BenchProfileReport.astro b/site/src/components/BenchProfileReport.astro index 06defa9f4..7c38076d4 100644 --- a/site/src/components/BenchProfileReport.astro +++ b/site/src/components/BenchProfileReport.astro @@ -17,7 +17,7 @@ */ import BenchComparisonTable from './BenchComparisonTable.astro'; -import BenchScoreboard from './BenchScoreboard.astro'; +import BenchComparisonSummary from './BenchComparisonSummary.astro'; import { BACKEND_LABELS, comparisonTallies, @@ -35,9 +35,11 @@ interface Props { headingLevel?: 2 | 3; /** Set false where the surrounding markup already names the machine, such as a collapsed further machine. */ showTitle?: boolean; + /** The reference profile carries the page's section anchors; a further machine must not duplicate them. */ + primary?: boolean; } -const { document: profileDocument, headingLevel = 2, showTitle = true } = Astro.props; +const { document: profileDocument, headingLevel = 2, showTitle = true, primary = false } = Astro.props; const { profile, rendering, physics } = profileDocument; const Heading = `h${String(headingLevel)}` as 'h2' | 'h3'; const SubHeading = `h${String(headingLevel + 1)}` as 'h3' | 'h4'; @@ -67,7 +69,6 @@ const renderingCountNote = .map(backend => `${String(backend.headlineCount ?? 0)} nodes on ${BACKEND_LABELS[backend.backend]}`) .join(' and ')}`; -const mechanisms = [...renderingTable.mechanisms, ...(physicsTable?.mechanisms ?? [])]; const excluded = backends.flatMap(backend => backend.excluded.map(entry => ({ backend: backend.backend, ...entry }))); /** The machine is one across the pooled runs, so the first run's stamps name it for all of them. */ @@ -96,11 +97,25 @@ const libraries = [...(rendering?.libraries ?? []), ...(physics?.libraries ?? []
{showTitle && {profile.gpu} / {profile.os} / {profile.browser}} - +
+ +
+ +

+ ExoJS ahead + the arm ahead + level, inside the noise band + no clear lead: the pooled runs landed on different verdict bands + past the {FRAME_BUDGET_MS} ms frame, or a spread of {WIDE_SPREAD_RATIO}x and up +

+

+ A bar is as wide as the arm's row count, so the unfilled part is coverage the matrix does not have rather than a result. Nothing is summed across + bars and no bar is ranked against another: the arms answer different questions, and this page publishes no score and no overall winner. +

{renderingTable.rows.length > 0 && (
- Rendering + Rendering

{renderingCountNote}, chosen from the archetype ladders before any timing was read. CPU milliseconds per frame, pooled over {profile.runs} {' '}separate runs as the median of their per-run medians; lower is better. @@ -111,7 +126,7 @@ const libraries = [...(rendering?.libraries ?? []), ...(physics?.libraries ?? [] {physicsTable !== null && physicsTable.rows.length > 0 && (

- Physics + Physics

Each archetype sits on its own body-count ladder, so every row states the count it was measured at and rows are not comparable with one another - only the arms within one row are. CPU milliseconds per fixed step, pooled the same way. @@ -120,29 +135,6 @@ const libraries = [...(rendering?.libraries ?? []), ...(physics?.libraries ?? []

)} -

- ExoJS leads - the arm leads - level, inside the noise band - the runs disagreed: the band is the ratio they spanned - not comparable - a time past the {FRAME_BUDGET_MS} ms frame - a range spanning {WIDE_SPREAD_RATIO}x or more -

- - {mechanisms.length > 0 && ( -
- Structural evidence behind each comparison ({mechanisms.length}) -
    - {mechanisms.map(entry => ( -
  • - {entry.archetype} {entry.column} - {entry.text} -
    • - ))} -
-
- )} - {webgl1Blocks.map(block => (
WebGL1 arms, CPU time only ({block.table.rows.length} rows) @@ -236,7 +228,9 @@ const libraries = [...(rendering?.libraries ?? []), ...(physics?.libraries ?? []
diff --git a/site/src/components/pages/BenchmarksPage.astro b/site/src/components/pages/BenchmarksPage.astro index d88f0d34c..33d435486 100644 --- a/site/src/components/pages/BenchmarksPage.astro +++ b/site/src/components/pages/BenchmarksPage.astro @@ -37,6 +37,19 @@ const scope = referenceProfile === undefined ? '' : profileScope(referenceProfil const REPRODUCTION_RUNS = 3; const pooledRuns = referenceProfile?.profile.runs ?? REPRODUCTION_RUNS; +const machine = referenceProfile?.profile; + +/** + * The sections the on-page navigation offers. Only the ones that exist are + * listed: a link to an empty anchor is worse than a shorter bar. + */ +const sections = [ + ...(referenceProfile === undefined ? [] : [{ id: 'summary', label: 'Overview' }]), + ...(referenceProfile?.rendering === undefined ? [] : [{ id: 'rendering', label: 'Rendering' }]), + ...(referenceProfile?.physics === undefined ? [] : [{ id: 'physics', label: 'Physics' }]), + ...(furtherProfiles.length === 0 ? [] : [{ id: 'further', label: 'Other machines' }]), + { id: 'practices', label: 'Methodology' }, +]; ---

Benchmarks

-

- ExoJS against Pixi, Phaser and Excalibur for rendering, and against matter.js, planck and Rapier for physics. Every number is written by the - harness into a signed profile in the repository, pooled over {pooledRuns} separate runs; a comparison the runs did not agree on publishes no - verdict, and every cell where ExoJS trails is published like the ones where it leads. +

+ Cross-library measurements for rendering and physics. Lower is better. Every published value is pooled from {pooledRuns} independent runs.

- {scope !== '' &&

{scope}

} + {machine !== undefined && ( +

+ + {machine.gpu} · {machine.os} · {machine.browser} + + + ExoJS {machine.engineVersion} · measured {formatDay(machine.measuredAt)} · {machine.runs} pooled runs + + {scope !== '' && {scope}} +

+ )} + + {referenceProfile === undefined ? (

No measurements are published yet. The reference measurement is {REPRODUCTION_RUNS} separate runs on one machine after a release is tagged, pooled into a single profile file in the repository; until then there is nothing here to read. The practices below already apply.

) : ( - + )} {furtherProfiles.length > 0 && ( @@ -118,9 +145,10 @@ const pooledRuns = referenceProfile?.profile.runs ?? REPRODUCTION_RUNS; median across the runs - so one unlucky run cannot set a number. Nothing is ever a mean, and nothing is aggregated across archetypes.
  • - Every number carries the range its runs observed, printed under it. A range spanning a factor of 1.2 or more is - marked: the measurement's own noise then reaches as wide as the band the ladder treats as no difference at all, which is something to - know before reading the number, not after. + Every number carries the spread its runs observed, printed under it as the factor between the slowest and the + fastest run; the two millisecond values behind it are on the cell itself. A spread of 1.2 or more is marked: the measurement's own + noise then reaches as wide as the band the ladder treats as no difference at all, which is something to know before reading the + number, not after.
  • Every median is published with the p95 of the same timed window beside it. The median is the amortised cost and the @@ -138,10 +166,11 @@ const pooledRuns = referenceProfile?.profile.runs ?? REPRODUCTION_RUNS;
  • A verdict is published only when all {pooledRuns} runs independently reach it. Each run is placed on the ladder on - its own, and the pooled medians have to land on the same rung as well. A cell whose runs disagreed publishes no verdict: it - keeps its numbers and shows the ratio band those runs spanned, and it is counted nowhere - not as a win, not as a loss, and not as a - level row. That the pair cannot be separated on this machine is the finding. The band is the one figure the page computes rather than - reads, from the published extremes of both arms, and it is drawn without a side. + its own, and the pooled medians have to land on the same rung as well. A cell whose runs disagreed publishes no verdict, and + it is counted nowhere - not as a win, not as a loss, and not as a level row. That the pair cannot be separated on this machine is the + finding. It still shows what was measured: the factor its two pooled medians work out to, printed with a tilde and plotted without a + side. That factor is the one figure the page derives rather than reads, and it is never a verdict - the tilde, the hollow mark and + the scoreboard all keep it out of the outcome counts.
  • A ratio between 0.8 and 1.2 is reported as level: inside that band the difference is not @@ -288,17 +317,64 @@ pnpm bench:compare -- \ margin: 0 0 var(--s-3); } - .bench-head p { + .lede { margin: 0; color: var(--fg-muted); line-height: 1.6; } - .scope { - margin-top: var(--s-2) !important; + .stamp { + display: flex; + flex-wrap: wrap; + gap: 0.15rem var(--s-4); + margin: var(--s-3) 0 0 !important; font-family: var(--f-mono); font-size: 0.74rem; - color: var(--fg-faint) !important; + color: var(--fg-faint); + } + + .stamp b { + font-weight: 600; + color: var(--fg-muted); + } + + /* + * The page is long and its sections are far apart, so the jump list stays + * reachable. Horizontal scrolling is fine for navigation - it is exactly + * what the measurements below must never need. + */ + .section-nav { + position: sticky; + top: 68px; + z-index: 2; + display: flex; + gap: var(--s-2); + overflow-x: auto; + margin: var(--s-5) 0 var(--s-6); + padding: var(--s-2) 0; + background: var(--bg-canvas); + border-bottom: 1px solid var(--line-soft); + scrollbar-width: none; + } + + .section-nav::-webkit-scrollbar { + display: none; + } + + .section-nav a { + flex: none; + padding: 0.25rem 0.7rem; + border: 1px solid var(--line-soft); + border-radius: var(--r-pill); + font-size: 0.76rem; + color: var(--fg-muted); + text-decoration: none; + white-space: nowrap; + } + + .section-nav a:hover { + color: var(--fg); + border-color: var(--line); } .empty { diff --git a/site/src/lib/bench-profiles.ts b/site/src/lib/bench-profiles.ts index 7c65b94ee..7f5ee81fd 100644 --- a/site/src/lib/bench-profiles.ts +++ b/site/src/lib/bench-profiles.ts @@ -349,6 +349,27 @@ export const furtherProfiles: readonly BenchProfileDocument[] = loaded.slice(1); /** Display name for a rendering backend. */ export const BACKEND_LABELS: Readonly> = { webgl2: 'WebGL2', webgpu: 'WebGPU' }; +/** + * How each arm is written where a reader sees it. + * + * The profile stores the harness's own package slugs, which are the right + * identifier inside the repository and the wrong one on a published page: a + * comparison against `matter-js` is a comparison against Matter.js. An arm with + * no entry keeps its slug rather than being guessed at, so adding one to the + * harness never silently renames it here. + */ +const ARM_LABELS: Readonly> = { + pixi: 'PixiJS', + excalibur: 'Excalibur', + phaser: 'Phaser', + 'matter-js': 'Matter.js', + planck: 'Planck', + rapier: 'Rapier', +}; + +/** An arm's published name, or its slug where none is known. */ +export const armLabel = (arm: string): string => ARM_LABELS[arm] ?? arm; + /** * Spread factor at which a measurement's own noise is called out. * @@ -420,6 +441,34 @@ const RUNG_LABELS: Readonly> = { export const describeRungs = (rungs: readonly string[]): string => rungs.map((rung, index) => `run ${String(index + 1)}: ${RUNG_LABELS[rung] ?? rung}`).join(', '); +/** Which way one run's rung fell, for showing the pooled runs as marks rather than as a sentence. */ +export type RungSide = 'exojs' | 'neither' | 'competitor'; + +/** + * The side a single run landed on. + * + * The profile records the rung each run reached but not the median behind it, + * so a run can be placed on a side and never on an axis. Four marks that all + * sit together and four that straddle the middle are different measurements + * with the same printed range, which is the whole reason to show them. + */ +export const rungSide = (rung: string): RungSide => { + if (rung.startsWith('exojs-leads')) return 'exojs'; + if (rung.startsWith('competitor-leads')) return 'competitor'; + + return 'neither'; +}; + +/** + * How much of a whole 60 fps frame one published time takes. + * + * A millisecond figure is only meaningful against the frame it has to fit in, + * and that is the comparison a reader without a benchmarking habit makes + * anyway. Nothing is derived from it: no verdict, no capacity figure, no + * ranking - it restates a published time against {@link FRAME_BUDGET_MS}. + */ +export const frameShare = (ms: number | null): number | null => (ms === null || !Number.isFinite(ms) ? null : (ms / FRAME_BUDGET_MS) * 100); + /** True when this arm stands as a reference ceiling rather than as a peer. */ export const isWasmReferenceArm = (competitor: string): boolean => WASM_REFERENCE_ARMS.includes(competitor); @@ -484,15 +533,15 @@ export type CellOutcome = 'clear-lead' | 'lead' | 'level' | 'loss' | 'clear-loss /** Outcomes in reading order: the widest lead first, the widest loss last, then the two that carry no verdict. */ export const OUTCOME_ORDER: readonly CellOutcome[] = ['clear-lead', 'lead', 'level', 'loss', 'clear-loss', 'unstable', 'absent']; -/** The word a scoreboard and a legend print for each outcome. */ +/** The word a summary and a legend print for each outcome. */ export const OUTCOME_LABELS: Readonly> = { 'clear-lead': 'clear lead', lead: 'lead', level: 'level', loss: 'loss', 'clear-loss': 'clear loss', - unstable: 'runs disagreed', - absent: 'not comparable', + unstable: 'no clear lead', + absent: 'no shared cell', }; /** @@ -520,13 +569,14 @@ export interface RatioBand { /** * The ratio band behind a comparison whose runs disagreed. * - * This is the one figure the page derives rather than reads: a cell the runs + * This is the one figure the page derives rather than reads. A cell the runs * split on carries no published ratio, and printing nothing in its place would - * throw away measurements that exist. The band is the widest and narrowest - * ratio the observed extremes allow, so it is an envelope around what was - * measured and never a verdict - it is drawn without a side, and a band that - * straddles 1.00 is exactly the statement that the runs could not separate the - * pair. It returns `null` where an arm reported no extremes to bound. + * throw away measurements that exist - but printing a single figure would + * invent the point verdict the runs failed to reach. The band is the widest and + * narrowest ratio the observed extremes allow: an envelope around what was + * measured, drawn without a side, and a band that straddles 1.00 is exactly the + * statement that the runs landed on both sides of the decision point. Returns + * `null` where an arm reported no extremes to bound. */ export const ratioBand = (cell: ProfileCell): RatioBand | null => { const { reference, competitor } = cell.aggregate; @@ -537,26 +587,99 @@ export const ratioBand = (cell: ProfileCell): RatioBand | null => { return { low: reference.minMs / competitor.maxMs, high: reference.maxMs / competitor.minMs }; }; -/** A ratio band as the scoreboard and the tables print it. */ -export const formatBand = (band: RatioBand): string => `${band.low.toFixed(2)}-${band.high.toFixed(2)}`; +/** A ratio band as the details print it. */ +export const formatBand = (band: RatioBand): string => `${band.low.toFixed(2)}x-${band.high.toFixed(2)}x`; + +/** + * The factor a pair's two pooled medians work out to. + * + * A cell whose runs landed on different rungs publishes no verdict, but its two + * medians are published and are the pair's central observation. It is printed + * with a tilde and without a side: it says how far apart the arms sit, not + * which one is faster, because that is precisely what the runs disagreed on. + * It is deliberately not the middle of the observed band - averaging two ratios + * is not a ratio anyone measured. + */ +export const pooledFactor = (cell: ProfileCell): number | null => { + const { referenceMs, competitorMs } = cell; + + if (referenceMs === null || competitorMs === null || referenceMs <= 0 || competitorMs <= 0) return null; + + const ratio = referenceMs / competitorMs; -/** One scoreboard line: everything measured against one arm, on one backend or in physics. */ + return ratio < 1 ? 1 / ratio : ratio; +}; + +/** A factor the runs did not settle, marked as such. */ +export const formatApproximate = (factor: number): string => `~${factor.toFixed(2)}x`; + +/** How far the pooled runs moved, as the single factor the profile stores. */ +export const formatSpread = (spread: ProfileSpread): string => (spread.ratio === null || !Number.isFinite(spread.ratio) ? '' : `${spread.ratio.toFixed(2)}x`); + +/** + * What a measured comparison came out as, once the ladder's five settled rungs + * are collapsed to the three directions a summary needs. + * + * `mixed` is a measured outcome, not a missing one: the runs produced numbers + * and landed on different rungs. A pair with no shared cell has no state at all + * and is counted as coverage instead - an arm that sat an archetype out is not + * a fifth performance direction, and folding the two together would report a + * gap in the matrix as a doubt about the measurement. + */ +export type SummaryState = 'ahead' | 'level' | 'behind' | 'unclear'; + +/** Which summary state an outcome falls into, or `null` where no pair was measured. */ +export const SUMMARY_OF: Readonly> = { + 'clear-lead': 'ahead', + lead: 'ahead', + level: 'level', + loss: 'behind', + 'clear-loss': 'behind', + unstable: 'unclear', + absent: null, +}; + +/** Summary states in reading order. */ +export const SUMMARY_ORDER: readonly SummaryState[] = ['ahead', 'level', 'behind', 'unclear']; + +/** One summary line: everything measured against one arm, on one backend or in physics. */ export interface ComparisonTally { readonly key: string; - /** The pair, as the scoreboard names it. */ + /** Heading this pair sits under, so peers and a reference arm are not read as the same kind of opponent. */ + readonly group: string; + /** The pair, as the summary names it. */ readonly label: string; + /** The arm on its own, for a sentence that has to name it. */ + readonly arm: string; /** What the pair is measured at, or what role the arm stands in. */ readonly meta: string; readonly counts: Readonly>; + /** The measured comparisons, rolled up to the states the summary shows. */ + readonly summary: Readonly>; + /** How many rows produced a comparison at all; the rest of `total` is coverage the matrix does not have. */ + readonly measured: number; readonly total: number; } -const tally = (key: string, label: string, meta: string, cells: readonly (ProfileCell | null)[]): ComparisonTally => { +const tally = (key: string, group: string, label: string, arm: string, meta: string, cells: readonly (ProfileCell | null)[]): ComparisonTally => { const counts = Object.fromEntries(OUTCOME_ORDER.map(outcome => [outcome, 0])) as Record; + const summary = Object.fromEntries(SUMMARY_ORDER.map(state => [state, 0])) as Record; + + let measured = 0; + + for (const cell of cells) { + const outcome = outcomeOf(cell); + const state = SUMMARY_OF[outcome]; - for (const cell of cells) counts[outcomeOf(cell)] += 1; + counts[outcome] += 1; + + if (state !== null) { + summary[state] += 1; + measured += 1; + } + } - return { key, label, meta, counts, total: cells.length }; + return { key, group, label, arm, meta, counts, summary, measured, total: cells.length }; }; /** @@ -572,7 +695,9 @@ export const comparisonTallies = (document: BenchProfileDocument): readonly Comp backend.competitors.map(arm => tally( `${backend.backend}-${arm}`, - `${BACKEND_LABELS[backend.backend]} vs ${arm}`, + `Rendering · ${BACKEND_LABELS[backend.backend]}`, + `vs ${armLabel(arm)}`, + armLabel(arm), backend.headlineCount === null ? 'no headline count' : `${String(backend.headlineCount)} nodes`, backend.sections.flatMap(section => section.rows.map(row => row.cells.find(cell => cell.competitor === arm) ?? null)), ), @@ -583,39 +708,31 @@ export const comparisonTallies = (document: BenchProfileDocument): readonly Comp : armsOfSection(document.physics.section).map(arm => tally( `physics-${arm}`, - `Physics vs ${arm}`, - isWasmReferenceArm(arm) ? 'Rust/WASM ceiling' : 'pure-JS peer', + isWasmReferenceArm(arm) ? 'Physics · WASM reference' : 'Physics · JavaScript peers', + `vs ${armLabel(arm)}`, + armLabel(arm), + isWasmReferenceArm(arm) ? 'Rust/WASM solver' : 'JavaScript solver', (document.physics?.section.rows ?? []).map(row => row.cells.find(cell => cell.competitor === arm) ?? null), ), )), ]; /** - * The sentence under the page title: which machine was measured, with which - * engine version, over how many runs, and how much the profile covers. + * The one-line stamp under the page title: which machine, which engine version, + * how many pooled runs, and what the profile covers. * - * Every value in it is copied from the document, so a re-measurement rewrites - * the sentence with the tables rather than leaving a claim behind that the - * numbers no longer support. A profile carrying only one domain yields a - * shorter sentence instead of a padded one. + * Every value is copied from the document, so a re-measurement rewrites the + * line with the tables rather than leaving a claim behind that the numbers no + * longer support. A profile carrying only one domain yields a shorter line + * instead of a padded one. */ export const profileScope = (document: BenchProfileDocument): string => { const rendering = renderingCells(document); const physics = physicsCells(document); - const { profile } = document; const parts: string[] = []; - if (rendering.length > 0) { - const backends = (document.rendering?.backends ?? []).map(backend => BACKEND_LABELS[backend.backend]); - - parts.push(`${String(rendering.length)} rendering comparisons on ${listOf(backends)} against ${listOf(armsIn(rendering))}`); - } - - if (physics.length > 0) { - parts.push(`${String(physics.length)} physics comparisons against ${listOf(armsIn(physics))}`); - } - - if (parts.length === 0) return ''; + if (rendering.length > 0) parts.push(`${String(rendering.length)} rendering comparisons against ${listOf(armsIn(rendering).map(armLabel))}`); + if (physics.length > 0) parts.push(`${String(physics.length)} physics comparisons against ${listOf(armsIn(physics).map(armLabel))}`); - return `ExoJS ${profile.engineVersion} on ${profile.gpu} / ${profile.os} / ${profile.browser}, pooled from ${String(profile.runs)} separate runs taken on ${formatDay(profile.measuredAt)}: ${parts.join('; ')}.`; + return parts.join(' · '); }; diff --git a/site/src/lib/bench-tables.ts b/site/src/lib/bench-tables.ts index 4ff518808..6546fdd01 100644 --- a/site/src/lib/bench-tables.ts +++ b/site/src/lib/bench-tables.ts @@ -8,10 +8,13 @@ * anywhere gets no column, and an archetype a backend did not measure keeps its * row and reports the gap rather than disappearing from it. * - * ExoJS gets a column of its own in front of each backend's arms rather than - * appearing inside every cell. It is one measurement per row and backend, so - * repeating it beside each arm would print the same three numbers as often as - * the backend has opponents. + * There is no column for ExoJS itself. Its median belongs to the comparison and + * is printed inside each cell beside the arm's, which repeats the value once per + * opponent - a repetition worth paying for, because it makes every cell a whole + * comparison that stands on its own. A separate reference column widened the + * table, split each backend's header across an uneven number of columns, and + * stacked into a block of its own on narrow screens, where a card should read as + * a list of comparisons and nothing else. * * Only the arrangement lives here. Timings, ratios and verdicts are read from * the profile in `bench-profiles`, and no row is dropped, reordered by outcome @@ -19,6 +22,7 @@ */ import { + armLabel, armsOfSection, BACKEND_LABELS, type BenchProfileDocument, @@ -26,16 +30,11 @@ import { type ProfileBackend, type ProfileCell, type ProfileRow, - type ProfileSpread, } from './bench-profiles'; -/** What a column carries: the ExoJS measurement itself, or a comparison against one arm. */ -export type ComparisonColumnKind = 'reference' | 'arm'; - -/** One column of a table. */ +/** One comparison pair the table has a column for. */ export interface ComparisonColumn { readonly key: string; - readonly kind: ComparisonColumnKind; /** Heading the neighbouring columns share, such as the backend they were measured on; `null` where each column stands alone. */ readonly group: string | null; /** What this column alone is measured under, such as the role an arm stands in. */ @@ -43,22 +42,11 @@ export interface ComparisonColumn { readonly label: string; } -/** The ExoJS side of one row, under one backend. */ -export interface ComparisonReference { - readonly ms: number | null; - readonly p95Ms: number | null; - readonly overFrameBudget: boolean; - readonly spread: ProfileSpread | undefined; -} - /** One column's outcome on one row. */ export interface ComparisonEntry { readonly key: string; - readonly kind: ComparisonColumnKind; - /** The published comparison, or `null` in a reference column and where an arm produced none. */ + /** The published comparison, or `null` where this column produced none. */ readonly cell: ProfileCell | null; - /** The ExoJS measurement, in a reference column only. */ - readonly reference: ComparisonReference | null; /** Scene size this column measured the row at; `null` where the column does not carry the row at all. */ readonly count: number | null; } @@ -74,14 +62,6 @@ export interface ComparisonRow { readonly entries: readonly ComparisonEntry[]; } -/** The structural evidence one comparison carries, listed away from the numbers. */ -export interface ComparisonMechanism { - readonly key: string; - readonly archetype: string; - readonly column: string; - readonly text: string; -} - /** One published comparison table. */ export interface ComparisonTable { readonly columns: readonly ComparisonColumn[]; @@ -90,57 +70,19 @@ export interface ComparisonTable { readonly unit: string; /** True where the rows were measured at different sizes, so the size belongs in a column of its own. */ readonly countColumn: boolean; - readonly mechanisms: readonly ComparisonMechanism[]; } -/** - * The ExoJS side of a row. Every arm in a block times the same ExoJS scene, so - * the first cell that produced a number carries it for the whole block. - */ -const referenceOf = (row: ProfileRow | undefined): ComparisonReference => { - const cell = row?.cells.find(candidate => candidate.referenceMs !== null); - - return { - ms: cell?.referenceMs ?? null, - p95Ms: cell?.referenceP95Ms ?? null, - overFrameBudget: cell?.referenceOverFrameBudget ?? false, - spread: cell?.aggregate.reference, - }; -}; - -const referenceEntry = (key: string, row: ProfileRow | undefined): ComparisonEntry => ({ - key, - kind: 'reference', - cell: null, - reference: referenceOf(row), - count: row?.count ?? null, -}); - -const armEntry = (key: string, row: ProfileRow | undefined, arm: string): ComparisonEntry => ({ +const entryOf = (key: string, row: ProfileRow | undefined, arm: string): ComparisonEntry => ({ key, - kind: 'arm', cell: row?.cells.find(cell => cell.competitor === arm) ?? null, - reference: null, count: row?.count ?? null, }); -const mechanismsOf = (rows: readonly ComparisonRow[], columns: readonly ComparisonColumn[]): readonly ComparisonMechanism[] => - rows.flatMap(row => - row.entries.flatMap((entry, index) => { - const text = entry.cell?.mechanism; - const column = columns[index]; - - return text === undefined || text === null || column === undefined - ? [] - : [{ key: `${row.key}-${column.key}`, archetype: row.archetype, column: column.label, text }]; - }), - ); - /** Every row of a backend, flattened out of its categories. */ const rowsOf = (backend: ProfileBackend): readonly ProfileRow[] => backend.sections.flatMap(section => section.rows); /** - * The rendering table: an ExoJS column and its arms, per backend. + * The rendering table: one column per backend-and-arm pair. * * Archetypes are collected in the order the first backend publishes them and * then extended by any a later backend adds, so the categories stay in the @@ -148,16 +90,14 @@ const rowsOf = (backend: ProfileBackend): readonly ProfileRow[] => backend.secti */ export const renderingComparison = (document: BenchProfileDocument): ComparisonTable => { const backends = document.rendering?.backends ?? []; - const columns = backends.flatMap(backend => [ - { key: `${backend.backend}-exojs`, kind: 'reference' as const, group: BACKEND_LABELS[backend.backend], overline: '', label: 'ExoJS' }, - ...backend.competitors.map(arm => ({ + const columns = backends.flatMap(backend => + backend.competitors.map(arm => ({ key: `${backend.backend}-${arm}`, - kind: 'arm' as const, group: BACKEND_LABELS[backend.backend], overline: '', - label: `vs ${arm}`, + label: armLabel(arm), })), - ]); + ); const archetypes: { archetype: string; section: string }[] = []; for (const backend of backends) { @@ -172,14 +112,14 @@ export const renderingComparison = (document: BenchProfileDocument): ComparisonT const entries = backends.flatMap(backend => { const row = rowsOf(backend).find(candidate => candidate.archetype === archetype); - return [referenceEntry(`${backend.backend}-exojs`, row), ...backend.competitors.map(arm => armEntry(`${backend.backend}-${arm}`, row, arm))]; + return backend.competitors.map(arm => entryOf(`${backend.backend}-${arm}`, row, arm)); }); const counts = [...new Set(entries.map(entry => entry.count).filter((count): count is number => count !== null))]; return { key: archetype, archetype, section, count: counts.length === 1 ? (counts[0] ?? null) : null, entries }; }); - return { columns, rows, unit: 'nodes', countColumn: false, mechanisms: mechanismsOf(rows, columns) }; + return { columns, rows, unit: 'nodes', countColumn: false }; }; const singleBlockTable = ( @@ -188,17 +128,18 @@ const singleBlockTable = ( columns: readonly ComparisonColumn[], unit: string, countColumn: boolean, -): ComparisonTable => { - const built = rows.map(row => ({ +): ComparisonTable => ({ + columns, + rows: rows.map(row => ({ key: row.archetype, archetype: row.archetype, section: null, count: row.count, - entries: [referenceEntry('exojs', row), ...arms.map(arm => armEntry(arm, row, arm))], - })); - - return { columns, rows: built, unit, countColumn, mechanisms: mechanismsOf(built, columns) }; -}; + entries: arms.map(arm => entryOf(arm, row, arm)), + })), + unit, + countColumn, +}); /** * The physics table. @@ -217,16 +158,15 @@ export const physicsComparison = (document: BenchProfileDocument): ComparisonTab return singleBlockTable( section.rows, arms, - [ - { key: 'exojs', kind: 'reference', group: null, overline: 'per fixed step', label: 'ExoJS' }, - ...arms.map(arm => ({ - key: arm, - kind: 'arm' as const, - group: null, - overline: isWasmReferenceArm(arm) ? 'Rust/WASM ceiling' : 'pure-JS peer', - label: `vs ${arm}`, - })), - ], + arms.map(arm => ({ + key: arm, + // Grouping the peers apart from the reference arm is the whole point: a + // Rust/WASM solver answers "what does leaving JavaScript buy", not "how + // does ExoJS compare to what I would otherwise reach for". + group: isWasmReferenceArm(arm) ? 'WASM reference' : 'JavaScript peers', + overline: '', + label: armLabel(arm), + })), 'bodies', true, ); @@ -236,22 +176,18 @@ export const physicsComparison = (document: BenchProfileDocument): ComparisonTab * The WebGL1 block of one backend, as its own table. * * These arms render through a WebGL1 context and report no structural counters, - * so the block compares CPU time only and carries no mechanism list. + * so the block compares CPU time only. */ export const webgl1Comparison = (backend: ProfileBackend): ComparisonTable | null => { if (backend.webgl1.length === 0) return null; const arms = [...new Set(backend.webgl1.flatMap(row => row.cells.map(cell => cell.competitor)))].sort(); - const table = singleBlockTable( + + return singleBlockTable( backend.webgl1, arms, - [ - { key: 'exojs', kind: 'reference', group: null, overline: 'CPU time only', label: 'ExoJS' }, - ...arms.map(arm => ({ key: arm, kind: 'arm' as const, group: null, overline: 'WebGL1', label: `vs ${arm}` })), - ], + arms.map(arm => ({ key: arm, group: null, overline: 'WebGL1, CPU time only', label: armLabel(arm) })), 'nodes', false, ); - - return { ...table, mechanisms: [] }; }; diff --git a/site/src/styles/tokens.scss b/site/src/styles/tokens.scss index a486d6635..a2375480a 100644 --- a/site/src/styles/tokens.scss +++ b/site/src/styles/tokens.scss @@ -92,10 +92,13 @@ html[data-theme='dark'] { // Benchmark comparison hues. They carry the same meaning as the semantic // colours above but have to hold their own against the page background at - // small sizes - a 6px bar and 0.66rem mono - so their lightness is tuned per + // small sizes - an 8px dot and 0.66rem mono - so their lightness is tuned per // theme instead of being shared. The two sides are equally saturated and // equally dark on purpose: a loss must never read as quieter than a win. - --bench-lead: oklch(72% 0.14 230); + // `--bench-mark` is a measurement warning and nothing else: a time past the + // frame budget, or runs that moved too far to trust. A comparison without a + // verdict is drawn in neutral grey, so amber keeps meaning something. + --bench-lead: oklch(76% 0.12 205); --bench-loss: oklch(70% 0.17 290); --bench-mark: oklch(80% 0.14 70); @@ -133,7 +136,7 @@ html[data-theme='light'] { --grid-dot: oklch(75% 0.012 245 / 0.7); --grid-line: oklch(85% 0.01 245 / 0.7); - --bench-lead: oklch(50% 0.16 230); + --bench-lead: oklch(48% 0.12 205); --bench-loss: oklch(48% 0.2 290); --bench-mark: oklch(52% 0.14 70); From b49803a3f6e76fbdaa5e7e3edfca656eb15bded1 Mon Sep 17 00:00:00 2001 From: Exoridus Date: Wed, 9 Sep 2026 20:06:34 +0200 Subject: [PATCH 3/5] feat(tooling): add output policy --- .github/workflows/ci.yml | 8 ++ CONTRIBUTING.md | 8 ++ scripts/ci/gates.ts | 42 ++++++--- scripts/ci/lanes.ts | 6 +- scripts/lanes.ts | 49 +++++++---- scripts/lib/output.ts | 51 +++++++++++ scripts/lib/run-command.ts | 157 ++++++++++++++++++++++++++++++++++ test/ci/output-policy.test.ts | 27 ++++++ 8 files changed, 316 insertions(+), 32 deletions(-) create mode 100644 scripts/lib/output.ts create mode 100644 scripts/lib/run-command.ts create mode 100644 test/ci/output-policy.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a7ac1e70..294022e80 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,6 +111,14 @@ jobs: - uses: actions/checkout@v6 - uses: ./.github/actions/setup - run: ${{ matrix.run }} + - name: Upload tooling logs on failure + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: tooling-logs-${{ matrix.id }} + path: .workspace/logs/ + retention-days: 1 + if-no-files-found: warn test: name: test (${{ matrix.id }}) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e110345cc..eedf8fd9c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -179,6 +179,14 @@ script of its own: `pnpm lint --fix`, `pnpm test --coverage`, through one runner, `pnpm perf ` (`pnpm perf` lists them); the cross-library benchmarks through `pnpm bench` (see `packages/exojs-bench`). +The root gate and lane runners share an output policy. `auto` is the default: +it uses `normal` for an interactive terminal and `compact` for CI or redirected +output. Override it with `--output normal|compact|silent|verbose`, or set +`EXOJS_OUTPUT` for a process tree. Compact and silent runs stream complete +output to `.workspace/logs/` without buffering it in memory; failures print the +last diagnostic lines and the log path. Benchmark and release lanes keep at +least normal output. + ## Shared configuration `@codexo/exojs-config` (private, unpublished, never a runtime dependency) centralizes diff --git a/scripts/ci/gates.ts b/scripts/ci/gates.ts index 11ba94df3..da1a6b2cf 100644 --- a/scripts/ci/gates.ts +++ b/scripts/ci/gates.ts @@ -25,12 +25,13 @@ * `dist/esm/index.d.ts`, so it can only run in a job that has the built dist. * That is why it cannot simply join the ungated typecheck job. */ -import { spawnSync } from 'node:child_process'; - import { GATE_GROUP_NAMES, GATE_GROUPS, type GateGroup } from './gate-groups.ts'; +import { readOutputOptions } from '../lib/output.ts'; +import { runCommand } from '../lib/run-command.ts'; const groupNames = GATE_GROUP_NAMES; -const requested = process.argv[2]; +const outputOptions = readOutputOptions(process.argv.slice(2)); +const requested = outputOptions.argv[0]; if (!requested) { console.error(`Usage: pnpm gates `); @@ -45,18 +46,33 @@ if (requested !== 'all' && !groupNames.includes(requested as GateGroup)) { const selected = requested === 'all' ? groupNames : [requested as GateGroup]; const scripts = selected.flatMap(group => GATE_GROUPS[group]); -console.log(`Running ${scripts.length} gate(s) from group(s): ${selected.join(', ')}\n`); +if (outputOptions.mode !== 'silent') { + console.log(`Running ${scripts.length} gate(s) from group(s): ${selected.join(', ')}\n`); +} -for (const script of scripts) { - console.log(`\n=== pnpm ${script} ===\n`); +const main = async (): Promise => { + for (const script of scripts) { + if (outputOptions.mode === 'normal' || outputOptions.mode === 'verbose') { + console.log(`\n=== pnpm ${script} ===\n`); + } - // `shell: true` so the pnpm shim resolves on Windows as well as on CI. - const result = spawnSync('pnpm', ['run', script], { stdio: 'inherit', shell: true }); + // `shell: true` so the pnpm shim resolves on Windows as well as on CI. + const result = await runCommand({ + label: `gates-${script}`, + command: 'pnpm', + args: ['run', script], + output: outputOptions.mode, + }); - if (result.status !== 0) { - console.error(`\nGate failed: pnpm ${script} (exit code ${result.status ?? 'signal'})`); - process.exit(result.status ?? 1); + if (result.status !== 0) { + if (outputOptions.mode === 'normal' || outputOptions.mode === 'verbose') { + console.error(`\nGate failed: pnpm ${script} (exit code ${result.status})`); + } + process.exit(result.status); + } } -} -console.log(`\nAll ${scripts.length} gate(s) passed.`); + if (outputOptions.mode !== 'silent') console.log(`\nAll ${scripts.length} gate(s) passed.`); +}; + +await main(); diff --git a/scripts/ci/lanes.ts b/scripts/ci/lanes.ts index 727c10da6..7d5d842c1 100644 --- a/scripts/ci/lanes.ts +++ b/scripts/ci/lanes.ts @@ -40,6 +40,8 @@ export interface Lane { local?: 'browser' | 'gate'; /** Runs on CI only: its assertions hold for the runner's software rasteriser, not a developer's GPU. */ ciOnly?: boolean; + /** Lowest output mode this lane permits; useful for reportable measurements. */ + minimumOutput?: 'compact' | 'normal' | 'silent' | 'verbose'; /** Emits `test-results/.junit.xml` for the skip budget and Codecov. */ junit?: boolean; /** Pull requests only. */ @@ -47,7 +49,7 @@ export interface Lane { timeoutMinutes?: number; } -const junit = (id: string): string => `--reporter=default --reporter=junit --outputFile.junit=./test-results/${id}.junit.xml`; +const junit = (id: string): string => `--reporter=minimal --reporter=junit --outputFile.junit=./test-results/${id}.junit.xml`; export const LANES: readonly Lane[] = [ { id: 'typecheck', stage: 'gates', when: 'typecheck', run: 'pnpm gates typecheck', local: 'gate' }, @@ -141,6 +143,7 @@ export const LANES: readonly Lane[] = [ run: 'pnpm typecheck:bench && pnpm gate:bench:structural', browser: 'chromium', local: 'browser', + minimumOutput: 'normal', timeoutMinutes: 30, }, @@ -157,6 +160,7 @@ export const LANES: readonly Lane[] = [ when: 'releaseDryRun', run: 'pnpm release:prepare --build --skip-zip', pullRequestOnly: true, + minimumOutput: 'normal', }, { id: 'create-exo-app', diff --git a/scripts/lanes.ts b/scripts/lanes.ts index a6033c154..c55953833 100644 --- a/scripts/lanes.ts +++ b/scripts/lanes.ts @@ -3,6 +3,8 @@ import { pathToFileURL } from 'node:url'; import { selectLanes, type Lane } from './ci/lanes.ts'; import { effectiveLanes, selectAreas, type LaneAreas } from './ci/select-lanes.ts'; +import { atLeastOutputMode, readOutputOptions } from './lib/output.ts'; +import { runCommand } from './lib/run-command.ts'; /** * Local lane runner - the pre-push hook's half of the lane table. @@ -72,8 +74,9 @@ const SMOKE_LANE: Lane = { local: 'browser', }; -const main = (): void => { - const argv = process.argv.slice(2); +const main = async (): Promise => { + const outputOptions = readOutputOptions(process.argv.slice(2)); + const argv = outputOptions.argv; const run = argv.includes('--run'); const quick = argv.includes('--quick'); const testsOnly = argv.includes('--tests-only'); @@ -97,16 +100,18 @@ const main = (): void => { .filter(lane => !(testsOnly && lane.local === 'gate')); const scope = all ? 'every lane' : `${files.length} changed file(s) since ${base}`; - process.stdout.write(`lanes: ${scope}\n`); - process.stdout.write( - `lanes: engine=${areas.engine} site=${areas.site} audioFx=${areas.audioFx} tilemapWorker=${areas.tilemapWorker} exampleCatalog=${areas.exampleCatalog} benchStructural=${areas.benchStructural}\n\n`, - ); - - for (const lane of selected) { - process.stdout.write(` ${lane.run}${lane.local === 'browser' ? ' (browser)' : ''}\n`); - } - if (selected.length === 0) { - process.stdout.write(' (nothing to run)\n'); + if (!run || outputOptions.mode !== 'silent') { + process.stdout.write(`lanes: ${scope}\n`); + process.stdout.write( + `lanes: engine=${areas.engine} site=${areas.site} audioFx=${areas.audioFx} tilemapWorker=${areas.tilemapWorker} exampleCatalog=${areas.exampleCatalog} benchStructural=${areas.benchStructural}\n\n`, + ); + + for (const lane of selected) { + process.stdout.write(` ${lane.run}${lane.local === 'browser' ? ' (browser)' : ''}\n`); + } + if (selected.length === 0) { + process.stdout.write(' (nothing to run)\n'); + } } if (!run) { @@ -115,18 +120,26 @@ const main = (): void => { } for (const lane of selected) { - process.stdout.write(`\n=== ${lane.id} ===\n\n`); - const result = spawnSync(lane.run, { stdio: 'inherit', shell: true }); + if (outputOptions.mode === 'normal' || outputOptions.mode === 'verbose') { + process.stdout.write(`\n=== ${lane.id} ===\n\n`); + } + const result = await runCommand({ + label: lane.id, + command: lane.run, + output: atLeastOutputMode(outputOptions.mode, lane.minimumOutput ?? 'silent'), + }); if (result.status !== 0) { - process.stderr.write(`\nlanes: ${lane.id} failed (exit ${result.status ?? 'signal'}).\n`); - process.exit(result.status ?? 1); + if (outputOptions.mode === 'normal' || outputOptions.mode === 'verbose') { + process.stderr.write(`\nlanes: ${lane.id} failed (exit ${result.status}).\n`); + } + process.exit(result.status); } } - process.stdout.write('\nlanes: all selected lanes passed.\n'); + if (outputOptions.mode !== 'silent') process.stdout.write('\nlanes: all selected lanes passed.\n'); }; const invokedPath = process.argv[1]; if (invokedPath && import.meta.url === pathToFileURL(invokedPath).href) { - main(); + await main(); } diff --git a/scripts/lib/output.ts b/scripts/lib/output.ts new file mode 100644 index 000000000..7265e6839 --- /dev/null +++ b/scripts/lib/output.ts @@ -0,0 +1,51 @@ +export const OUTPUT_MODES = ['auto', 'normal', 'compact', 'silent', 'verbose'] as const; + +export type OutputMode = (typeof OUTPUT_MODES)[number]; +export type EffectiveOutputMode = Exclude; + +const OUTPUT_RANK: Record = { + silent: 0, + compact: 1, + normal: 2, + verbose: 3, +}; + +const isOutputMode = (value: string): value is OutputMode => (OUTPUT_MODES as readonly string[]).includes(value); + +const invalidMode = (value: string): Error => new Error(`Invalid output mode '${value}'. Expected one of: ${OUTPUT_MODES.join(', ')}.`); + +/** Reads the command-line override and removes output flags from the forwarded arguments. */ +export const readOutputOptions = ( + argv: readonly string[], + env: NodeJS.ProcessEnv = process.env, + isTTY = process.stdout.isTTY === true, +): { mode: EffectiveOutputMode; argv: string[] } => { + const remaining: string[] = []; + let requested: string | undefined; + + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]!; + if (argument === '--output') { + const value = argv[index + 1]; + if (value === undefined) throw new Error('--output requires a mode.'); + requested = value; + index += 1; + } else if (argument.startsWith('--output=')) { + requested = argument.slice('--output='.length); + } else { + remaining.push(argument); + } + } + + const raw = requested ?? env['EXOJS_OUTPUT'] ?? 'auto'; + if (!isOutputMode(raw)) throw invalidMode(raw); + + if (raw !== 'auto') return { mode: raw, argv: remaining }; + + const ci = env['CI'] === 'true' || env['CI'] === '1'; + return { mode: ci || !isTTY ? 'compact' : 'normal', argv: remaining }; +}; + +/** Raises a command's mode when a lane must remain visible, such as a benchmark. */ +export const atLeastOutputMode = (mode: EffectiveOutputMode, minimum: EffectiveOutputMode): EffectiveOutputMode => + OUTPUT_RANK[mode] >= OUTPUT_RANK[minimum] ? mode : minimum; diff --git a/scripts/lib/run-command.ts b/scripts/lib/run-command.ts new file mode 100644 index 000000000..8783d6417 --- /dev/null +++ b/scripts/lib/run-command.ts @@ -0,0 +1,157 @@ +import { createWriteStream, mkdirSync } from 'node:fs'; +import { spawn } from 'node:child_process'; +import { relative, resolve } from 'node:path'; + +import type { EffectiveOutputMode } from './output.ts'; + +const DEFAULT_TAIL_LINES = 120; +const LOG_DIR = resolve(process.cwd(), '.workspace/logs'); + +export interface RunCommandOptions { + readonly label: string; + readonly command: string; + readonly args?: readonly string[]; + readonly cwd?: string; + readonly output: EffectiveOutputMode; + readonly minimumTailLines?: number; +} + +export interface RunCommandResult { + readonly status: number; + readonly signal: NodeJS.Signals | null; + readonly durationMs: number; + readonly logPath?: string; +} + +const logName = (label: string): string => { + const safe = label + .replace(/[^a-z0-9]+/gi, '-') + .replace(/^-+|-+$/g, '') + .toLowerCase(); + return `${safe || 'command'}.log`; +}; + +const displayPath = (path: string): string => relative(process.cwd(), path).replaceAll('\\', '/'); + +const duration = (durationMs: number): string => `${(durationMs / 1000).toFixed(1)}s`; + +class TailBuffer { + private readonly lines: string[] = []; + private pending = ''; + + public constructor(private readonly limit: number) {} + + public append(chunk: string): void { + const parts = `${this.pending}${chunk}`.split(/\r?\n/); + this.pending = parts.pop() ?? ''; + for (const line of parts) { + this.lines.push(line); + if (this.lines.length > this.limit) this.lines.shift(); + } + } + + public finish(): void { + if (this.pending !== '') { + this.lines.push(this.pending); + if (this.lines.length > this.limit) this.lines.shift(); + this.pending = ''; + } + } + + public text(): string { + return this.lines.join('\n'); + } +} + +const writeFailure = (options: RunCommandOptions, result: RunCommandResult, tail: string): void => { + const code = result.signal ? `signal ${result.signal}` : `exit ${result.status}`; + process.stderr.write(`FAIL ${options.label} (${code})\n`); + if (tail !== '') process.stderr.write(`\n--- last ${options.minimumTailLines ?? DEFAULT_TAIL_LINES} lines ---\n${tail}\n`); + if (result.logPath) process.stderr.write(`\nFull log: ${displayPath(result.logPath)}\n`); +}; + +const writeCompactSuccess = (label: string, durationMs: number): void => { + process.stdout.write(`PASS ${label} ${duration(durationMs)}\n`); +}; + +const hasArguments = (options: RunCommandOptions): boolean => (options.args?.length ?? 0) > 0; + +const executable = (options: RunCommandOptions): string => + hasArguments(options) && process.platform === 'win32' && options.command === 'pnpm' ? 'pnpm.cmd' : options.command; + +const windowsCommandLine = (options: RunCommandOptions): string => + [executable(options), ...(options.args ?? [])].map(argument => (/\s/.test(argument) ? JSON.stringify(argument) : argument)).join(' '); + +const spawnCommand = (options: RunCommandOptions, stdio: Parameters[2]['stdio']): ReturnType => + hasArguments(options) && process.platform === 'win32' + ? spawn(process.env['ComSpec'] ?? 'cmd.exe', ['/d', '/s', '/c', windowsCommandLine(options)], { cwd: options.cwd, stdio }) + : hasArguments(options) + ? spawn(executable(options), [...(options.args ?? [])], { cwd: options.cwd, stdio }) + : spawn(options.command, { cwd: options.cwd, shell: true, stdio }); + +const capture = (options: RunCommandOptions): Promise => + new Promise(resolveResult => { + mkdirSync(LOG_DIR, { recursive: true }); + const path = resolve(LOG_DIR, logName(options.label)); + const log = createWriteStream(path, { encoding: 'utf8' }); + const tail = new TailBuffer(options.minimumTailLines ?? DEFAULT_TAIL_LINES); + const started = Date.now(); + let settled = false; + + const finish = (status: number, signal: NodeJS.Signals | null): void => { + if (settled) return; + settled = true; + tail.finish(); + const result: RunCommandResult = { status, signal, durationMs: Date.now() - started, logPath: path }; + log.end(() => { + if (status === 0) { + if (options.output === 'compact') writeCompactSuccess(options.label, result.durationMs); + } else { + writeFailure(options, result, tail.text()); + } + resolveResult(result); + }); + }; + + log.write(`$ ${options.command}${options.args?.length ? ` ${options.args.join(' ')}` : ''}\n`); + const child = spawnCommand(options, ['inherit', 'pipe', 'pipe']); + const consume = (stream: NodeJS.ReadableStream, source: string): void => { + stream.on('data', chunk => { + log.write(chunk); + const text = typeof chunk === 'string' ? chunk : chunk.toString(); + tail.append(text); + }); + stream.on('error', error => { + const text = `[${source} error] ${String(error)}\n`; + log.write(text); + tail.append(text); + }); + }; + consume(child.stdout!, 'stdout'); + consume(child.stderr!, 'stderr'); + child.once('error', error => { + const text = `${String(error.message ?? error)}\n`; + log.write(text); + tail.append(text); + finish(1, null); + }); + child.once('close', (status, signal) => finish(status ?? 1, signal)); + }); + +/** Runs a command without buffering its complete output in memory. */ +export const runCommand = (options: RunCommandOptions): Promise => { + if (options.output === 'compact' || options.output === 'silent') return capture(options); + + return new Promise(resolveResult => { + const started = Date.now(); + let settled = false; + const finish = (status: number, signal: NodeJS.Signals | null): void => { + if (settled) return; + settled = true; + resolveResult({ status, signal, durationMs: Date.now() - started }); + }; + const child = spawnCommand(options, 'inherit'); + child.once('error', () => finish(1, null)); + child.once('close', (status, signal) => finish(status ?? 1, signal)); + }); +}; diff --git a/test/ci/output-policy.test.ts b/test/ci/output-policy.test.ts new file mode 100644 index 000000000..e36e3e4dd --- /dev/null +++ b/test/ci/output-policy.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from 'vitest'; + +import { atLeastOutputMode, readOutputOptions } from '../../scripts/lib/output.ts'; + +describe('tooling output policy', () => { + it('uses an explicit CLI mode and removes it from the remaining arguments', () => { + expect(readOutputOptions(['all', '--output', 'verbose'], { CI: 'true' }, false)).toEqual({ mode: 'verbose', argv: ['all'] }); + expect(readOutputOptions(['--output=normal', '--run'], { CI: 'true' }, false)).toEqual({ mode: 'normal', argv: ['--run'] }); + }); + + it('uses EXOJS_OUTPUT before automatic environment detection', () => { + expect(readOutputOptions([], { EXOJS_OUTPUT: 'silent', CI: 'true' }, true).mode).toBe('silent'); + expect(readOutputOptions([], { CI: 'true' }, true).mode).toBe('compact'); + expect(readOutputOptions([], {}, true).mode).toBe('normal'); + expect(readOutputOptions([], {}, false).mode).toBe('compact'); + }); + + it('rejects unknown modes', () => { + expect(() => readOutputOptions(['--output', 'chatty'])).toThrow("Invalid output mode 'chatty'"); + }); + + it('preserves a lane minimum without overriding a more verbose request', () => { + expect(atLeastOutputMode('silent', 'normal')).toBe('normal'); + expect(atLeastOutputMode('compact', 'normal')).toBe('normal'); + expect(atLeastOutputMode('verbose', 'normal')).toBe('verbose'); + }); +}); From 8440f0b104efb58ee957b450f263a5e18620b32b Mon Sep 17 00:00:00 2001 From: Dima Giesler Date: Wed, 9 Sep 2026 22:00:34 +0200 Subject: [PATCH 4/5] bench: add M3 Max macOS WebKit results --- package.json | 2 +- .../results/m3-max-macos-27-beta-webkit.json | 1298 +++++++++++++++++ 2 files changed, 1299 insertions(+), 1 deletion(-) create mode 100644 packages/exojs-bench/results/m3-max-macos-27-beta-webkit.json diff --git a/package.json b/package.json index e2206c4d9..caf32fc8c 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "description": "A TypeScript-first browser 2D runtime for games and interactive apps.", "version": "0.17.0", "type": "module", - "packageManager": "pnpm@11.4.0", + "packageManager": "pnpm@11.15.0+sha512.266f8957a30d2be6e9468e5e66bcdedd35a794175f71b067ba8504d686cce1d0c0f429b33c323c3c569ad4891e667574a49ff71d1b89a22cc66f13c65818c578", "devEngines": { "runtime": { "name": "node", diff --git a/packages/exojs-bench/results/m3-max-macos-27-beta-webkit.json b/packages/exojs-bench/results/m3-max-macos-27-beta-webkit.json new file mode 100644 index 000000000..92678330b --- /dev/null +++ b/packages/exojs-bench/results/m3-max-macos-27-beta-webkit.json @@ -0,0 +1,1298 @@ +{ + "schemaVersion": 6, + "profile": { + "slug": "m3-max-macos-27-beta-webkit", + "gpu": "m3-max", + "os": "macos-27-beta", + "browser": "webkit", + "platform": { + "name": "macos", + "version": 27, + "versionSource": "declared", + "prerelease": true + }, + "engineVersion": "0.17.0", + "measuredAt": "2026-09-09T19:14:13.252Z", + "runs": 4 + }, + "physics": { + "runs": [ + { + "browser": "webkit", + "browserVersion": "26.5", + "host": { + "cpu": "Apple M3 Max", + "cpuCount": 16, + "os": "darwin 27.0.0", + "platformVersion": { + "major": 27, + "source": "declared", + "evidence": "the runner declared '27-beta'; os.release() reported '27.0.0' on darwin, which is the kernel version and does not name the operating system's own major version; declare it with --platform=[-beta]" + }, + "arch": "arm64" + }, + "prerelease": { + "value": true, + "source": "declared", + "evidence": "the runner declared the platform as '27-beta'" + }, + "fixedDelta": 0.016666666666666666, + "clock": { + "resolutionMs": 0.01999999999999602, + "crossOriginIsolated": true + }, + "caveats": [ + "Step time is CPU wall-clock per step() over the timed window (median/p95), measured in one webkit page (same-run discipline). No number here was taken in Node.", + "The page's performance.now() resolves to 20.0us (cross-origin isolated: true). A cell whose step is too fast to time individually at that resolution batches steps per timing sample and records the batch as stepsPerSample; median and p95 are then per-step averages over that batch.", + "Scenes are warmed to steady state before timing; the per-cell warmupSteps/timedSteps counts are recorded for honesty.", + "All arms build the byte-identical scene (bodies, positions, shapes, sizes, static/dynamic split, gravity, perturbed-body set) from the shared deterministic RNG, and the perturbed-body selection is asserted equal across arms before each cell is timed.", + "Each arm runs at its own engine defaults for solver iterations, contact model and sleeping - those engine differences are the measured quantity in a native-vs-adapter comparison, disclosed per arm below.", + "Arm roles: matter-js, planck and nape-js are the JAVASCRIPT PEERS exojs-physics is compared against; rapier is a Rust/WASM engine and stands as the REFERENCE CEILING for what leaving JavaScript buys, not as a peer a JS solver is expected to match.", + "exojs-physics arm (native runtime, pure JS): TGS-Soft solver, 4 sub-steps per fixed step, sleeping ON by default (resting bodies deactivate). Contact count = solid contacts in the world contact graph.", + "matter-js arm (pure-JS peer): constraint solver at matter defaults (6 position / 4 velocity / 2 constraint iterations), sleeping OFF by default (a settled stack keeps paying full solve cost); matter's default per-step air drag (frictionAir) is zeroed so all arms integrate the same pure-gravity field; gravity (px/s^2) and perturbation velocity (px/s) are mapped into matter's px-per-step unit model. Contact count = active colliding pairs (engine.pairs.collisionActive), a pair-level proxy, not identical in semantics to the exojs solid-contact count.", + "planck arm (pure-JS peer): Box2D port at planck defaults (8 velocity / 3 position iterations per step), sleeping ON by default; Settings.lengthUnitsPerMeter is set to 30 so planck's absolute MKS tolerances are interpreted at the scene's pixel scale, which is the knob planck gives a pixel-coordinate game - positions, gravity (px/s^2) and velocity (px/s) then carry over unconverted. Contact count = the world contact list filtered by isTouching(), a touching collider-pair count. Rays are answered from planck's dynamic tree, but World.rayCast is Box2D's non-solid ray (an origin inside a fixture is not a hit). Continuous collision runs for every body (planck's default), where exojs and rapier restrict it to bullets and matter has none.", + "rapier arm (WASM reference ceiling, not a pure-JS peer): TGS-Soft solver at rapier defaults (4 solver / 1 internal PGS iterations), auto-sleeping ON; default lengthUnit=1 is fed a px-scale world (tuned for ~1-unit objects), exactly what attaching rapier with pixel coordinates yields. Contact count = collider pairs with a solid narrow-phase manifold (numContacts > 0), deduped.", + "nape-js arm (pure-JS peer): default single-step solver with default velocity/position iterations (10/10), sleeping and dynamic AABB broadphase at library defaults. Materials, body layouts, joints and perturbations come from the shared neutral scene descriptor. Contact count = active collision arbiters; ray queries use Space.rayCast with outer-surface semantics. These counters are engine-specific structural proxies, not a claim that every solver performs identical internal work." + ], + "engineVersion": "0.17.0", + "timestamp": "2026-09-09T18:50:50.763Z" + }, + { + "browser": "webkit", + "browserVersion": "26.5", + "host": { + "cpu": "Apple M3 Max", + "cpuCount": 16, + "os": "darwin 27.0.0", + "platformVersion": { + "major": 27, + "source": "declared", + "evidence": "the runner declared '27-beta'; os.release() reported '27.0.0' on darwin, which is the kernel version and does not name the operating system's own major version; declare it with --platform=[-beta]" + }, + "arch": "arm64" + }, + "prerelease": { + "value": true, + "source": "declared", + "evidence": "the runner declared the platform as '27-beta'" + }, + "fixedDelta": 0.016666666666666666, + "clock": { + "resolutionMs": 0.01999999999999602, + "crossOriginIsolated": true + }, + "caveats": [ + "Step time is CPU wall-clock per step() over the timed window (median/p95), measured in one webkit page (same-run discipline). No number here was taken in Node.", + "The page's performance.now() resolves to 20.0us (cross-origin isolated: true). A cell whose step is too fast to time individually at that resolution batches steps per timing sample and records the batch as stepsPerSample; median and p95 are then per-step averages over that batch.", + "Scenes are warmed to steady state before timing; the per-cell warmupSteps/timedSteps counts are recorded for honesty.", + "All arms build the byte-identical scene (bodies, positions, shapes, sizes, static/dynamic split, gravity, perturbed-body set) from the shared deterministic RNG, and the perturbed-body selection is asserted equal across arms before each cell is timed.", + "Each arm runs at its own engine defaults for solver iterations, contact model and sleeping - those engine differences are the measured quantity in a native-vs-adapter comparison, disclosed per arm below.", + "Arm roles: matter-js, planck and nape-js are the JAVASCRIPT PEERS exojs-physics is compared against; rapier is a Rust/WASM engine and stands as the REFERENCE CEILING for what leaving JavaScript buys, not as a peer a JS solver is expected to match.", + "exojs-physics arm (native runtime, pure JS): TGS-Soft solver, 4 sub-steps per fixed step, sleeping ON by default (resting bodies deactivate). Contact count = solid contacts in the world contact graph.", + "matter-js arm (pure-JS peer): constraint solver at matter defaults (6 position / 4 velocity / 2 constraint iterations), sleeping OFF by default (a settled stack keeps paying full solve cost); matter's default per-step air drag (frictionAir) is zeroed so all arms integrate the same pure-gravity field; gravity (px/s^2) and perturbation velocity (px/s) are mapped into matter's px-per-step unit model. Contact count = active colliding pairs (engine.pairs.collisionActive), a pair-level proxy, not identical in semantics to the exojs solid-contact count.", + "planck arm (pure-JS peer): Box2D port at planck defaults (8 velocity / 3 position iterations per step), sleeping ON by default; Settings.lengthUnitsPerMeter is set to 30 so planck's absolute MKS tolerances are interpreted at the scene's pixel scale, which is the knob planck gives a pixel-coordinate game - positions, gravity (px/s^2) and velocity (px/s) then carry over unconverted. Contact count = the world contact list filtered by isTouching(), a touching collider-pair count. Rays are answered from planck's dynamic tree, but World.rayCast is Box2D's non-solid ray (an origin inside a fixture is not a hit). Continuous collision runs for every body (planck's default), where exojs and rapier restrict it to bullets and matter has none.", + "rapier arm (WASM reference ceiling, not a pure-JS peer): TGS-Soft solver at rapier defaults (4 solver / 1 internal PGS iterations), auto-sleeping ON; default lengthUnit=1 is fed a px-scale world (tuned for ~1-unit objects), exactly what attaching rapier with pixel coordinates yields. Contact count = collider pairs with a solid narrow-phase manifold (numContacts > 0), deduped.", + "nape-js arm (pure-JS peer): default single-step solver with default velocity/position iterations (10/10), sleeping and dynamic AABB broadphase at library defaults. Materials, body layouts, joints and perturbations come from the shared neutral scene descriptor. Contact count = active collision arbiters; ray queries use Space.rayCast with outer-surface semantics. These counters are engine-specific structural proxies, not a claim that every solver performs identical internal work." + ], + "engineVersion": "0.17.0", + "timestamp": "2026-09-09T18:58:36.106Z" + }, + { + "browser": "webkit", + "browserVersion": "26.5", + "host": { + "cpu": "Apple M3 Max", + "cpuCount": 16, + "os": "darwin 27.0.0", + "platformVersion": { + "major": 27, + "source": "declared", + "evidence": "the runner declared '27-beta'; os.release() reported '27.0.0' on darwin, which is the kernel version and does not name the operating system's own major version; declare it with --platform=[-beta]" + }, + "arch": "arm64" + }, + "prerelease": { + "value": true, + "source": "declared", + "evidence": "the runner declared the platform as '27-beta'" + }, + "fixedDelta": 0.016666666666666666, + "clock": { + "resolutionMs": 0.01999999999999602, + "crossOriginIsolated": true + }, + "caveats": [ + "Step time is CPU wall-clock per step() over the timed window (median/p95), measured in one webkit page (same-run discipline). No number here was taken in Node.", + "The page's performance.now() resolves to 20.0us (cross-origin isolated: true). A cell whose step is too fast to time individually at that resolution batches steps per timing sample and records the batch as stepsPerSample; median and p95 are then per-step averages over that batch.", + "Scenes are warmed to steady state before timing; the per-cell warmupSteps/timedSteps counts are recorded for honesty.", + "All arms build the byte-identical scene (bodies, positions, shapes, sizes, static/dynamic split, gravity, perturbed-body set) from the shared deterministic RNG, and the perturbed-body selection is asserted equal across arms before each cell is timed.", + "Each arm runs at its own engine defaults for solver iterations, contact model and sleeping - those engine differences are the measured quantity in a native-vs-adapter comparison, disclosed per arm below.", + "Arm roles: matter-js, planck and nape-js are the JAVASCRIPT PEERS exojs-physics is compared against; rapier is a Rust/WASM engine and stands as the REFERENCE CEILING for what leaving JavaScript buys, not as a peer a JS solver is expected to match.", + "exojs-physics arm (native runtime, pure JS): TGS-Soft solver, 4 sub-steps per fixed step, sleeping ON by default (resting bodies deactivate). Contact count = solid contacts in the world contact graph.", + "matter-js arm (pure-JS peer): constraint solver at matter defaults (6 position / 4 velocity / 2 constraint iterations), sleeping OFF by default (a settled stack keeps paying full solve cost); matter's default per-step air drag (frictionAir) is zeroed so all arms integrate the same pure-gravity field; gravity (px/s^2) and perturbation velocity (px/s) are mapped into matter's px-per-step unit model. Contact count = active colliding pairs (engine.pairs.collisionActive), a pair-level proxy, not identical in semantics to the exojs solid-contact count.", + "planck arm (pure-JS peer): Box2D port at planck defaults (8 velocity / 3 position iterations per step), sleeping ON by default; Settings.lengthUnitsPerMeter is set to 30 so planck's absolute MKS tolerances are interpreted at the scene's pixel scale, which is the knob planck gives a pixel-coordinate game - positions, gravity (px/s^2) and velocity (px/s) then carry over unconverted. Contact count = the world contact list filtered by isTouching(), a touching collider-pair count. Rays are answered from planck's dynamic tree, but World.rayCast is Box2D's non-solid ray (an origin inside a fixture is not a hit). Continuous collision runs for every body (planck's default), where exojs and rapier restrict it to bullets and matter has none.", + "rapier arm (WASM reference ceiling, not a pure-JS peer): TGS-Soft solver at rapier defaults (4 solver / 1 internal PGS iterations), auto-sleeping ON; default lengthUnit=1 is fed a px-scale world (tuned for ~1-unit objects), exactly what attaching rapier with pixel coordinates yields. Contact count = collider pairs with a solid narrow-phase manifold (numContacts > 0), deduped.", + "nape-js arm (pure-JS peer): default single-step solver with default velocity/position iterations (10/10), sleeping and dynamic AABB broadphase at library defaults. Materials, body layouts, joints and perturbations come from the shared neutral scene descriptor. Contact count = active collision arbiters; ray queries use Space.rayCast with outer-surface semantics. These counters are engine-specific structural proxies, not a claim that every solver performs identical internal work." + ], + "engineVersion": "0.17.0", + "timestamp": "2026-09-09T19:06:25.228Z" + }, + { + "browser": "webkit", + "browserVersion": "26.5", + "host": { + "cpu": "Apple M3 Max", + "cpuCount": 16, + "os": "darwin 27.0.0", + "platformVersion": { + "major": 27, + "source": "declared", + "evidence": "the runner declared '27-beta'; os.release() reported '27.0.0' on darwin, which is the kernel version and does not name the operating system's own major version; declare it with --platform=[-beta]" + }, + "arch": "arm64" + }, + "prerelease": { + "value": true, + "source": "declared", + "evidence": "the runner declared the platform as '27-beta'" + }, + "fixedDelta": 0.016666666666666666, + "clock": { + "resolutionMs": 0.01999999999999602, + "crossOriginIsolated": true + }, + "caveats": [ + "Step time is CPU wall-clock per step() over the timed window (median/p95), measured in one webkit page (same-run discipline). No number here was taken in Node.", + "The page's performance.now() resolves to 20.0us (cross-origin isolated: true). A cell whose step is too fast to time individually at that resolution batches steps per timing sample and records the batch as stepsPerSample; median and p95 are then per-step averages over that batch.", + "Scenes are warmed to steady state before timing; the per-cell warmupSteps/timedSteps counts are recorded for honesty.", + "All arms build the byte-identical scene (bodies, positions, shapes, sizes, static/dynamic split, gravity, perturbed-body set) from the shared deterministic RNG, and the perturbed-body selection is asserted equal across arms before each cell is timed.", + "Each arm runs at its own engine defaults for solver iterations, contact model and sleeping - those engine differences are the measured quantity in a native-vs-adapter comparison, disclosed per arm below.", + "Arm roles: matter-js, planck and nape-js are the JAVASCRIPT PEERS exojs-physics is compared against; rapier is a Rust/WASM engine and stands as the REFERENCE CEILING for what leaving JavaScript buys, not as a peer a JS solver is expected to match.", + "exojs-physics arm (native runtime, pure JS): TGS-Soft solver, 4 sub-steps per fixed step, sleeping ON by default (resting bodies deactivate). Contact count = solid contacts in the world contact graph.", + "matter-js arm (pure-JS peer): constraint solver at matter defaults (6 position / 4 velocity / 2 constraint iterations), sleeping OFF by default (a settled stack keeps paying full solve cost); matter's default per-step air drag (frictionAir) is zeroed so all arms integrate the same pure-gravity field; gravity (px/s^2) and perturbation velocity (px/s) are mapped into matter's px-per-step unit model. Contact count = active colliding pairs (engine.pairs.collisionActive), a pair-level proxy, not identical in semantics to the exojs solid-contact count.", + "planck arm (pure-JS peer): Box2D port at planck defaults (8 velocity / 3 position iterations per step), sleeping ON by default; Settings.lengthUnitsPerMeter is set to 30 so planck's absolute MKS tolerances are interpreted at the scene's pixel scale, which is the knob planck gives a pixel-coordinate game - positions, gravity (px/s^2) and velocity (px/s) then carry over unconverted. Contact count = the world contact list filtered by isTouching(), a touching collider-pair count. Rays are answered from planck's dynamic tree, but World.rayCast is Box2D's non-solid ray (an origin inside a fixture is not a hit). Continuous collision runs for every body (planck's default), where exojs and rapier restrict it to bullets and matter has none.", + "rapier arm (WASM reference ceiling, not a pure-JS peer): TGS-Soft solver at rapier defaults (4 solver / 1 internal PGS iterations), auto-sleeping ON; default lengthUnit=1 is fed a px-scale world (tuned for ~1-unit objects), exactly what attaching rapier with pixel coordinates yields. Contact count = collider pairs with a solid narrow-phase manifold (numContacts > 0), deduped.", + "nape-js arm (pure-JS peer): default single-step solver with default velocity/position iterations (10/10), sleeping and dynamic AABB broadphase at library defaults. Materials, body layouts, joints and perturbations come from the shared neutral scene descriptor. Contact count = active collision arbiters; ray queries use Space.rayCast with outer-surface semantics. These counters are engine-specific structural proxies, not a claim that every solver performs identical internal work." + ], + "engineVersion": "0.17.0", + "timestamp": "2026-09-09T19:14:13.252Z" + } + ], + "libraries": [ + { + "name": "@codexo/exojs-physics", + "version": "0.17.0" + }, + { + "name": "matter-js", + "version": "0.20.0" + }, + { + "name": "planck", + "version": "1.5.0" + }, + { + "name": "@dimforge/rapier2d-compat", + "version": "0.19.3" + }, + { + "name": "@newkrok/nape-js", + "version": "3.42.0" + } + ], + "section": { + "title": "Physics", + "rows": [ + { + "archetype": "box-stack", + "category": "Physics", + "count": 10000, + "cells": [ + { + "competitor": "matter-js", + "referenceMs": 10.235000000000355, + "referenceP95Ms": 11.230000000000018, + "referenceOverFrameBudget": false, + "competitorMs": 22.664999999993597, + "competitorP95Ms": 26.17000000000553, + "competitorOverFrameBudget": true, + "verdict": { + "side": "exojs", + "ratio": 0.45157732186204486, + "factor": 2.214460185636816, + "label": "ExoJS leads (2.21x)", + "structural": false + }, + "mechanism": "ExoJS resolves fewer solved contacts (10002 vs 22396); the counters are not semantically identical across arms (see the run's caveats)", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 9.659999999999854, + "maxMs": 10.389999999999873, + "ratio": 1.0755693581780568 + }, + "competitor": { + "minMs": 21.779999999998836, + "maxMs": 23.069999999992433, + "ratio": 1.0592286501374502 + }, + "stable": true, + "rungs": [ + "exojs-leads", + "exojs-leads", + "exojs-leads", + "exojs-leads" + ] + } + }, + { + "competitor": "nape-js", + "referenceMs": 10.235000000000355, + "referenceP95Ms": 11.230000000000018, + "referenceOverFrameBudget": false, + "competitorMs": 26.220000000001164, + "competitorP95Ms": 31.570000000006985, + "competitorOverFrameBudget": true, + "verdict": { + "side": "exojs", + "ratio": 0.3903508771929787, + "factor": 2.5617977528090137, + "label": "ExoJS leads (2.56x)", + "structural": false + }, + "mechanism": "identical scene (10001 bodies, 10002 contacts, 0 joints on both arms); the difference is solver cost", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 9.659999999999854, + "maxMs": 10.389999999999873, + "ratio": 1.0755693581780568 + }, + "competitor": { + "minMs": 25.620000000024447, + "maxMs": 27.410000000003492, + "ratio": 1.069867291177882 + }, + "stable": true, + "rungs": [ + "exojs-leads", + "exojs-leads", + "exojs-leads", + "exojs-leads" + ] + } + }, + { + "competitor": "planck", + "referenceMs": 10.235000000000355, + "referenceP95Ms": 11.230000000000018, + "referenceOverFrameBudget": false, + "competitorMs": 72.32499999999709, + "competitorP95Ms": 90.47999999999593, + "competitorOverFrameBudget": true, + "verdict": { + "side": "exojs", + "ratio": 0.14151399930868672, + "factor": 7.0664386907664465, + "label": "ExoJS leads clearly (7.07x)", + "structural": true + }, + "mechanism": "ExoJS resolves fewer solved contacts (10002 vs 19865); the counters are not semantically identical across arms (see the run's caveats)", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 9.659999999999854, + "maxMs": 10.389999999999873, + "ratio": 1.0755693581780568 + }, + "competitor": { + "minMs": 71.86000000000058, + "maxMs": 73.41000000000349, + "ratio": 1.0215697188978972 + }, + "stable": true, + "rungs": [ + "exojs-leads-clearly", + "exojs-leads-clearly", + "exojs-leads-clearly", + "exojs-leads-clearly" + ] + } + }, + { + "competitor": "rapier", + "referenceMs": 10.235000000000355, + "referenceP95Ms": 11.230000000000018, + "referenceOverFrameBudget": false, + "competitorMs": 7.264999999999418, + "competitorP95Ms": 9.779999999969732, + "competitorOverFrameBudget": false, + "verdict": { + "side": "competitor", + "ratio": 1.4088093599451033, + "factor": 1.4088093599451033, + "label": "competitor leads (1.41x)", + "structural": false + }, + "mechanism": "ExoJS resolves fewer solved contacts (10002 vs 17880); the counters are not semantically identical across arms (see the run's caveats)", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 9.659999999999854, + "maxMs": 10.389999999999873, + "ratio": 1.0755693581780568 + }, + "competitor": { + "minMs": 7.079999999987194, + "maxMs": 7.5600000000267755, + "ratio": 1.0677966101752048 + }, + "stable": true, + "rungs": [ + "competitor-leads", + "competitor-leads", + "competitor-leads", + "competitor-leads" + ] + } + } + ] + }, + { + "archetype": "many-dynamic", + "category": "Physics", + "count": 2200, + "cells": [ + { + "competitor": "matter-js", + "referenceMs": 8.63000000000011, + "referenceP95Ms": 9.5, + "referenceOverFrameBudget": false, + "competitorMs": 8.080000000001746, + "competitorP95Ms": 8.989999999997963, + "competitorOverFrameBudget": false, + "verdict": { + "side": "neither", + "ratio": 1.0680693069304759, + "factor": 1, + "label": "level", + "structural": false + }, + "mechanism": "identical scene (2204 bodies, 6233 contacts, 0 joints on both arms); the difference is solver cost", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 8.159999999999854, + "maxMs": 8.679999999998472, + "ratio": 1.0637254901959101 + }, + "competitor": { + "minMs": 8.020000000004075, + "maxMs": 8.10000000000582, + "ratio": 1.0099750623443524 + }, + "stable": true, + "rungs": [ + "level", + "level", + "level", + "level" + ] + } + }, + { + "competitor": "nape-js", + "referenceMs": 8.63000000000011, + "referenceP95Ms": 9.5, + "referenceOverFrameBudget": false, + "competitorMs": 8.875, + "competitorP95Ms": 9.790000000008149, + "competitorOverFrameBudget": false, + "verdict": { + "side": "neither", + "ratio": 0.9723943661971954, + "factor": 1, + "label": "level", + "structural": false + }, + "mechanism": "identical scene (2204 bodies, 6233 contacts, 0 joints on both arms); the difference is solver cost", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 8.159999999999854, + "maxMs": 8.679999999998472, + "ratio": 1.0637254901959101 + }, + "competitor": { + "minMs": 8.820000000006985, + "maxMs": 8.939999999973224, + "ratio": 1.0136054421730323 + }, + "stable": true, + "rungs": [ + "level", + "level", + "level", + "level" + ] + } + }, + { + "competitor": "planck", + "referenceMs": 8.63000000000011, + "referenceP95Ms": 9.5, + "referenceOverFrameBudget": false, + "competitorMs": 9.35000000000582, + "competitorP95Ms": 10.910000000003492, + "competitorOverFrameBudget": false, + "verdict": { + "side": "neither", + "ratio": 0.9229946524058542, + "factor": 1, + "label": "level", + "structural": false + }, + "mechanism": "identical scene (2204 bodies, 6233 contacts, 0 joints on both arms); the difference is solver cost", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 8.159999999999854, + "maxMs": 8.679999999998472, + "ratio": 1.0637254901959101 + }, + "competitor": { + "minMs": 9.320000000006985, + "maxMs": 9.380000000004657, + "ratio": 1.0064377682400887 + }, + "stable": true, + "rungs": [ + "level", + "level", + "level", + "level" + ] + } + }, + { + "competitor": "rapier", + "referenceMs": 8.63000000000011, + "referenceP95Ms": 9.5, + "referenceOverFrameBudget": false, + "competitorMs": 1.9199999999837019, + "competitorP95Ms": 2.0599999999976717, + "competitorOverFrameBudget": false, + "verdict": { + "side": "competitor", + "ratio": 4.494791666704878, + "factor": 4.494791666704878, + "label": "competitor leads (4.49x)", + "structural": false + }, + "mechanism": "identical scene (2204 bodies, 6233 contacts, 0 joints on both arms); the difference is solver cost", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 8.159999999999854, + "maxMs": 8.679999999998472, + "ratio": 1.0637254901959101 + }, + "competitor": { + "minMs": 1.8999999999941792, + "maxMs": 1.9200000000419095, + "ratio": 1.0105263158146272 + }, + "stable": true, + "rungs": [ + "competitor-leads", + "competitor-leads", + "competitor-leads", + "competitor-leads" + ] + } + } + ] + }, + { + "archetype": "mixed-static-dynamic", + "category": "Physics", + "count": 3200, + "cells": [ + { + "competitor": "matter-js", + "referenceMs": 8.514999999999418, + "referenceP95Ms": 9.719999999999345, + "referenceOverFrameBudget": false, + "competitorMs": 5.444999999992433, + "competitorP95Ms": 6.310000000004948, + "competitorOverFrameBudget": false, + "verdict": { + "side": "competitor", + "ratio": 1.5638200183675393, + "factor": 1.5638200183675393, + "label": "competitor leads (1.56x)", + "structural": false + }, + "mechanism": "identical scene (3313 bodies, 3670 contacts, 0 joints on both arms); the difference is solver cost", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 8.43999999999869, + "maxMs": 8.540000000000873, + "ratio": 1.011848341232488 + }, + "competitor": { + "minMs": 5.430000000000291, + "maxMs": 5.4799999999959255, + "ratio": 1.0092081031299507 + }, + "stable": true, + "rungs": [ + "competitor-leads", + "competitor-leads", + "competitor-leads", + "competitor-leads" + ] + } + }, + { + "competitor": "nape-js", + "referenceMs": 8.514999999999418, + "referenceP95Ms": 9.719999999999345, + "referenceOverFrameBudget": false, + "competitorMs": 11.069999999992433, + "competitorP95Ms": 13.199999999982538, + "competitorOverFrameBudget": false, + "verdict": { + "side": "exojs", + "ratio": 0.7691960252940595, + "factor": 1.3000587199052482, + "label": "ExoJS leads (1.30x)", + "structural": false + }, + "mechanism": "identical scene (3313 bodies, 3670 contacts, 0 joints on both arms); the difference is solver cost", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 8.43999999999869, + "maxMs": 8.540000000000873, + "ratio": 1.011848341232488 + }, + "competitor": { + "minMs": 10.960000000020955, + "maxMs": 11.089999999996508, + "ratio": 1.01186131386636 + }, + "stable": true, + "rungs": [ + "exojs-leads", + "exojs-leads", + "exojs-leads", + "exojs-leads" + ] + } + }, + { + "competitor": "planck", + "referenceMs": 8.514999999999418, + "referenceP95Ms": 9.719999999999345, + "referenceOverFrameBudget": false, + "competitorMs": 19.3550000000032, + "competitorP95Ms": 30.039999999993597, + "competitorOverFrameBudget": true, + "verdict": { + "side": "exojs", + "ratio": 0.4399380005165595, + "factor": 2.2730475631244302, + "label": "ExoJS leads (2.27x)", + "structural": false + }, + "mechanism": "identical scene (3313 bodies, 3670 contacts, 0 joints on both arms); the difference is solver cost", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 8.43999999999869, + "maxMs": 8.540000000000873, + "ratio": 1.011848341232488 + }, + "competitor": { + "minMs": 19.29000000000815, + "maxMs": 19.610000000015134, + "ratio": 1.0165889061693545 + }, + "stable": true, + "rungs": [ + "exojs-leads", + "exojs-leads", + "exojs-leads", + "exojs-leads" + ] + } + }, + { + "competitor": "rapier", + "referenceMs": 8.514999999999418, + "referenceP95Ms": 9.719999999999345, + "referenceOverFrameBudget": false, + "competitorMs": 3.654999999984284, + "competitorP95Ms": 4.129999999975553, + "competitorOverFrameBudget": false, + "verdict": { + "side": "competitor", + "ratio": 2.329685362526958, + "factor": 2.329685362526958, + "label": "competitor leads (2.33x)", + "structural": false + }, + "mechanism": "ExoJS resolves fewer solved contacts (3670 vs 6105); the counters are not semantically identical across arms (see the run's caveats)", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 8.43999999999869, + "maxMs": 8.540000000000873, + "ratio": 1.011848341232488 + }, + "competitor": { + "minMs": 3.6499999999941792, + "maxMs": 3.6900000000023283, + "ratio": 1.0109589041118392 + }, + "stable": true, + "rungs": [ + "competitor-leads", + "competitor-leads", + "competitor-leads", + "competitor-leads" + ] + } + } + ] + }, + { + "archetype": "raycast", + "category": "Physics", + "count": 3200, + "cells": [ + { + "competitor": "matter-js", + "referenceMs": 10.55999999999949, + "referenceP95Ms": 11.710000000000946, + "referenceOverFrameBudget": false, + "competitorMs": 13.419999999998254, + "competitorP95Ms": 16.830000000009022, + "competitorOverFrameBudget": false, + "verdict": { + "side": "exojs", + "ratio": 0.7868852459017038, + "factor": 1.2708333333332291, + "label": "ExoJS leads (1.27x)", + "structural": false + }, + "mechanism": "identical scene (3313 bodies, 3670 contacts, 0 joints on both arms) with 41 vs 49 ray hits per step; the difference is query and solver cost", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 10.459999999999127, + "maxMs": 10.630000000001019, + "ratio": 1.0162523900575435 + }, + "competitor": { + "minMs": 13.369999999995343, + "maxMs": 13.570000000006985, + "ratio": 1.0149588631272783 + }, + "stable": true, + "rungs": [ + "exojs-leads", + "exojs-leads", + "exojs-leads", + "exojs-leads" + ] + } + }, + { + "competitor": "nape-js", + "referenceMs": 10.55999999999949, + "referenceP95Ms": 11.710000000000946, + "referenceOverFrameBudget": false, + "competitorMs": 28.210000000006403, + "competitorP95Ms": 33.079999999987194, + "competitorOverFrameBudget": true, + "verdict": { + "side": "exojs", + "ratio": 0.37433534207717456, + "factor": 2.67140151515225, + "label": "ExoJS leads (2.67x)", + "structural": false + }, + "mechanism": "identical scene (3313 bodies, 3670 contacts, 0 joints on both arms) with 41 vs 41 ray hits per step; the difference is query and solver cost", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 10.459999999999127, + "maxMs": 10.630000000001019, + "ratio": 1.0162523900575435 + }, + "competitor": { + "minMs": 27.300000000046566, + "maxMs": 28.55999999999767, + "ratio": 1.0461538461519764 + }, + "stable": true, + "rungs": [ + "exojs-leads", + "exojs-leads", + "exojs-leads", + "exojs-leads" + ] + } + }, + { + "competitor": "planck", + "referenceMs": 10.55999999999949, + "referenceP95Ms": 11.710000000000946, + "referenceOverFrameBudget": false, + "competitorMs": 19.89500000001135, + "competitorP95Ms": 30.85000000000582, + "competitorOverFrameBudget": true, + "verdict": { + "side": "exojs", + "ratio": 0.5307866298061557, + "factor": 1.8839962121223777, + "label": "ExoJS leads (1.88x)", + "structural": false + }, + "mechanism": "identical scene (3313 bodies, 3670 contacts, 0 joints on both arms) with 41 vs 37 ray hits per step; the difference is query and solver cost", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 10.459999999999127, + "maxMs": 10.630000000001019, + "ratio": 1.0162523900575435 + }, + "competitor": { + "minMs": 19.830000000001746, + "maxMs": 20.010000000009313, + "ratio": 1.0090771558248892 + }, + "stable": true, + "rungs": [ + "exojs-leads", + "exojs-leads", + "exojs-leads", + "exojs-leads" + ] + } + }, + { + "competitor": "rapier", + "referenceMs": 10.55999999999949, + "referenceP95Ms": 11.710000000000946, + "referenceOverFrameBudget": false, + "competitorMs": 3.749999999985448, + "competitorP95Ms": 4.139999999984866, + "competitorOverFrameBudget": false, + "verdict": { + "side": "competitor", + "ratio": 2.8160000000107916, + "factor": 2.8160000000107916, + "label": "competitor leads (2.82x)", + "structural": false + }, + "mechanism": "ExoJS resolves fewer solved contacts (3670 vs 6105); the counters are not semantically identical across arms (see the run's caveats)", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 10.459999999999127, + "maxMs": 10.630000000001019, + "ratio": 1.0162523900575435 + }, + "competitor": { + "minMs": 3.7000000000116415, + "maxMs": 3.779999999969732, + "ratio": 1.0216216216102267 + }, + "stable": true, + "rungs": [ + "competitor-leads", + "competitor-leads", + "competitor-leads", + "competitor-leads" + ] + } + } + ] + }, + { + "archetype": "body-churn", + "category": "Physics", + "count": 2400, + "cells": [ + { + "competitor": "matter-js", + "referenceMs": 11.340000000000146, + "referenceP95Ms": 12.890000000003056, + "referenceOverFrameBudget": false, + "competitorMs": 9.840000000003783, + "competitorP95Ms": 11.450000000004366, + "competitorOverFrameBudget": false, + "verdict": { + "side": "neither", + "ratio": 1.1524390243898155, + "factor": 1, + "label": "level", + "structural": false + }, + "mechanism": "ExoJS resolves fewer solved contacts (2282 vs 6230); the counters are not semantically identical across arms (see the run's caveats)", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 10.939999999995052, + "maxMs": 11.679999999996653, + "ratio": 1.0676416819014567 + }, + "competitor": { + "minMs": 9.740000000005239, + "maxMs": 9.89999999999418, + "ratio": 1.0164271047216482 + }, + "stable": true, + "rungs": [ + "level", + "level", + "level", + "level" + ] + } + }, + { + "competitor": "nape-js", + "referenceMs": 11.340000000000146, + "referenceP95Ms": 12.890000000003056, + "referenceOverFrameBudget": false, + "competitorMs": 5.789999999979045, + "competitorP95Ms": 12.499999999970896, + "competitorOverFrameBudget": false, + "verdict": { + "side": "competitor", + "ratio": 1.9585492228050407, + "factor": 1.9585492228050407, + "label": "competitor leads (1.96x)", + "structural": false + }, + "mechanism": "identical scene (2404 bodies, 2282 contacts, 0 joints on both arms); the difference is solver cost", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 10.939999999995052, + "maxMs": 11.679999999996653, + "ratio": 1.0676416819014567 + }, + "competitor": { + "minMs": 5.160000000032596, + "maxMs": 5.839999999967404, + "ratio": 1.1317829457229673 + }, + "stable": true, + "rungs": [ + "competitor-leads", + "competitor-leads", + "competitor-leads", + "competitor-leads" + ] + } + }, + { + "competitor": "planck", + "referenceMs": 11.340000000000146, + "referenceP95Ms": 12.890000000003056, + "referenceOverFrameBudget": false, + "competitorMs": 9.839999999996508, + "competitorP95Ms": 10.89999999999418, + "competitorOverFrameBudget": false, + "verdict": { + "side": "neither", + "ratio": 1.1524390243906677, + "factor": 1, + "label": "level", + "structural": false + }, + "mechanism": "ExoJS resolves fewer solved contacts (2282 vs 5242); the counters are not semantically identical across arms (see the run's caveats)", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 10.939999999995052, + "maxMs": 11.679999999996653, + "ratio": 1.0676416819014567 + }, + "competitor": { + "minMs": 9.739999999990687, + "maxMs": 9.929999999993015, + "ratio": 1.0195071868585739 + }, + "stable": true, + "rungs": [ + "level", + "level", + "level", + "level" + ] + } + }, + { + "competitor": "rapier", + "referenceMs": 11.340000000000146, + "referenceP95Ms": 12.890000000003056, + "referenceOverFrameBudget": false, + "competitorMs": 1.1600000000034925, + "competitorP95Ms": 1.209999999991851, + "competitorOverFrameBudget": false, + "verdict": { + "side": "competitor", + "ratio": 9.77586206893621, + "factor": 9.77586206893621, + "label": "competitor leads clearly (9.78x)", + "structural": true + }, + "mechanism": "identical scene (2404 bodies, 2282 contacts, 0 joints on both arms); the difference is solver cost", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 10.939999999995052, + "maxMs": 11.679999999996653, + "ratio": 1.0676416819014567 + }, + "competitor": { + "minMs": 1.1599999999743886, + "maxMs": 1.179999999993015, + "ratio": 1.0172413793267827 + }, + "stable": true, + "rungs": [ + "competitor-leads-clearly", + "competitor-leads-clearly", + "competitor-leads-clearly", + "competitor-leads-clearly" + ] + } + } + ] + }, + { + "archetype": "joints", + "category": "Physics", + "count": 15000, + "cells": [ + { + "competitor": "matter-js", + "referenceMs": 9.350000000000364, + "referenceP95Ms": 10.580000000001746, + "referenceOverFrameBudget": false, + "competitorMs": 9.464999999996508, + "competitorP95Ms": 10.69999999999709, + "competitorOverFrameBudget": false, + "verdict": { + "side": "neither", + "ratio": 0.9878499735873021, + "factor": 1, + "label": "level", + "structural": false + }, + "mechanism": "competitor resolves fewer solved contacts (13125 vs 0); the counters are not semantically identical across arms (see the run's caveats)", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 9.260000000002037, + "maxMs": 9.419999999998254, + "ratio": 1.0172786177101707 + }, + "competitor": { + "minMs": 9.080000000001746, + "maxMs": 9.539999999993597, + "ratio": 1.0506607929506346 + }, + "stable": true, + "rungs": [ + "level", + "level", + "level", + "level" + ] + } + }, + { + "competitor": "nape-js", + "referenceMs": 9.350000000000364, + "referenceP95Ms": 10.580000000001746, + "referenceOverFrameBudget": false, + "competitorMs": 0, + "competitorP95Ms": 0.0009999999980209394, + "competitorOverFrameBudget": false, + "verdict": { + "side": "neither", + "ratio": null, + "factor": null, + "label": "not comparable", + "structural": false + }, + "mechanism": "competitor resolves fewer solved contacts (13125 vs 0); the counters are not semantically identical across arms (see the run's caveats)", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 9.260000000002037, + "maxMs": 9.419999999998254, + "ratio": 1.0172786177101707 + }, + "competitor": { + "minMs": 0, + "maxMs": 0, + "ratio": null + }, + "stable": true, + "rungs": [ + "not-comparable", + "not-comparable", + "not-comparable", + "not-comparable" + ] + } + }, + { + "competitor": "planck", + "referenceMs": 9.350000000000364, + "referenceP95Ms": 10.580000000001746, + "referenceOverFrameBudget": false, + "competitorMs": 0.6399999999994179, + "competitorP95Ms": 0.6600000000180444, + "competitorOverFrameBudget": false, + "verdict": { + "side": "competitor", + "ratio": 14.609375000013856, + "factor": 14.609375000013856, + "label": "competitor leads clearly (14.61x)", + "structural": true + }, + "mechanism": "competitor resolves fewer solved contacts (13125 vs 0); the counters are not semantically identical across arms (see the run's caveats)", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 9.260000000002037, + "maxMs": 9.419999999998254, + "ratio": 1.0172786177101707 + }, + "competitor": { + "minMs": 0.639999999984866, + "maxMs": 0.6799999999930151, + "ratio": 1.0625000000142109 + }, + "stable": true, + "rungs": [ + "competitor-leads-clearly", + "competitor-leads-clearly", + "competitor-leads-clearly", + "competitor-leads-clearly" + ] + } + }, + { + "competitor": "rapier", + "referenceMs": 9.350000000000364, + "referenceP95Ms": 10.580000000001746, + "referenceOverFrameBudget": false, + "competitorMs": 0.29999999998835847, + "competitorP95Ms": 0.34000000002561137, + "competitorOverFrameBudget": false, + "verdict": { + "side": "competitor", + "ratio": 31.166666667877305, + "factor": 31.166666667877305, + "label": "competitor leads clearly (31.17x)", + "structural": true + }, + "mechanism": "competitor resolves fewer solved contacts (13125 vs 1875); the counters are not semantically identical across arms (see the run's caveats)", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 9.260000000002037, + "maxMs": 9.419999999998254, + "ratio": 1.0172786177101707 + }, + "competitor": { + "minMs": 0.29999999998835847, + "maxMs": 0.3200000000069849, + "ratio": 1.0666666667313418 + }, + "stable": true, + "rungs": [ + "competitor-leads-clearly", + "competitor-leads-clearly", + "competitor-leads-clearly", + "competitor-leads-clearly" + ] + } + } + ] + }, + { + "archetype": "settling-pile", + "category": "Physics", + "count": 5800, + "cells": [ + { + "competitor": "matter-js", + "referenceMs": 8.82499999999709, + "referenceP95Ms": 9.979999999995925, + "referenceOverFrameBudget": false, + "competitorMs": 25.220000000001164, + "competitorP95Ms": 29.330000000001746, + "competitorOverFrameBudget": true, + "verdict": { + "side": "exojs", + "ratio": 0.34992069785871066, + "factor": 2.857790368273029, + "label": "ExoJS leads (2.86x)", + "structural": false + }, + "mechanism": "identical scene (5804 bodies, 12187 contacts, 0 joints on both arms); the difference is solver cost", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 8.559999999997672, + "maxMs": 8.839999999996508, + "ratio": 1.0327102803737047 + }, + "competitor": { + "minMs": 25.04999999998836, + "maxMs": 25.330000000001746, + "ratio": 1.0111776447111185 + }, + "stable": true, + "rungs": [ + "exojs-leads", + "exojs-leads", + "exojs-leads", + "exojs-leads" + ] + } + }, + { + "competitor": "nape-js", + "referenceMs": 8.82499999999709, + "referenceP95Ms": 9.979999999995925, + "referenceOverFrameBudget": false, + "competitorMs": 35.81500000001688, + "competitorP95Ms": 43.20000000001164, + "competitorOverFrameBudget": true, + "verdict": { + "side": "exojs", + "ratio": 0.24640513751201815, + "factor": 4.058356940513167, + "label": "ExoJS leads (4.06x)", + "structural": false + }, + "mechanism": "identical scene (5804 bodies, 12187 contacts, 0 joints on both arms); the difference is solver cost", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 8.559999999997672, + "maxMs": 8.839999999996508, + "ratio": 1.0327102803737047 + }, + "competitor": { + "minMs": 34.85999999998603, + "maxMs": 36.84000000002561, + "ratio": 1.0567986230648414 + }, + "stable": true, + "rungs": [ + "exojs-leads", + "exojs-leads", + "exojs-leads", + "exojs-leads" + ] + } + }, + { + "competitor": "planck", + "referenceMs": 8.82499999999709, + "referenceP95Ms": 9.979999999995925, + "referenceOverFrameBudget": false, + "competitorMs": 45.02499999997963, + "competitorP95Ms": 54.179999999993015, + "competitorOverFrameBudget": true, + "verdict": { + "side": "exojs", + "ratio": 0.19600222098836387, + "factor": 5.101983002832235, + "label": "ExoJS leads clearly (5.10x)", + "structural": true + }, + "mechanism": "identical scene (5804 bodies, 12187 contacts, 0 joints on both arms); the difference is solver cost", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 8.559999999997672, + "maxMs": 8.839999999996508, + "ratio": 1.0327102803737047 + }, + "competitor": { + "minMs": 44.679999999993015, + "maxMs": 45.86999999999534, + "ratio": 1.02663384064464 + }, + "stable": true, + "rungs": [ + "exojs-leads-clearly", + "exojs-leads-clearly", + "exojs-leads-clearly", + "exojs-leads-clearly" + ] + } + }, + { + "competitor": "rapier", + "referenceMs": 8.82499999999709, + "referenceP95Ms": 9.979999999995925, + "referenceOverFrameBudget": false, + "competitorMs": 4.7999999999883585, + "competitorP95Ms": 5.279999999998836, + "competitorOverFrameBudget": false, + "verdict": { + "side": "competitor", + "ratio": 1.8385416666705194, + "factor": 1.8385416666705194, + "label": "competitor leads (1.84x)", + "structural": false + }, + "mechanism": "identical scene (5804 bodies, 12187 contacts, 0 joints on both arms); the difference is solver cost", + "aggregate": { + "runs": 4, + "reference": { + "minMs": 8.559999999997672, + "maxMs": 8.839999999996508, + "ratio": 1.0327102803737047 + }, + "competitor": { + "minMs": 4.740000000048894, + "maxMs": 4.880000000004657, + "ratio": 1.0295358649692654 + }, + "stable": true, + "rungs": [ + "competitor-leads", + "competitor-leads", + "competitor-leads", + "competitor-leads" + ] + } + } + ] + } + ] + } + }, + "signature": { + "algorithm": "sha256", + "value": "e9ab0ae10d27ef5fbe3c0417663d3738a90cd1c97f9438f967cf23d57a68016f" + } +} From a0a4982084898d0b5dd88cafd28041380ef540dc Mon Sep 17 00:00:00 2001 From: Exoridus Date: Wed, 9 Sep 2026 22:30:13 +0200 Subject: [PATCH 5/5] fix(tooling): harden output runner --- .github/workflows/ci.yml | 9 +++ scripts/lib/run-command.ts | 148 +++++++++++++++++++++++++++++------- test/ci/run-command.test.ts | 90 ++++++++++++++++++++++ 3 files changed, 219 insertions(+), 28 deletions(-) create mode 100644 test/ci/run-command.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 294022e80..fc1377d7a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -255,6 +255,15 @@ jobs: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} run: pnpm site:build + - name: Upload tooling logs on failure + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: tooling-logs-site + path: .workspace/logs/ + retention-days: 1 + if-no-files-found: warn + # Deploy Pages ships exactly this artifact; it builds nothing itself. - uses: actions/upload-artifact@v4 with: diff --git a/scripts/lib/run-command.ts b/scripts/lib/run-command.ts index 8783d6417..ba6e8df02 100644 --- a/scripts/lib/run-command.ts +++ b/scripts/lib/run-command.ts @@ -1,6 +1,7 @@ -import { createWriteStream, mkdirSync } from 'node:fs'; import { spawn } from 'node:child_process'; +import { createWriteStream, mkdirSync, type WriteStream } from 'node:fs'; import { relative, resolve } from 'node:path'; +import type { Readable } from 'node:stream'; import type { EffectiveOutputMode } from './output.ts'; @@ -21,6 +22,7 @@ export interface RunCommandResult { readonly signal: NodeJS.Signals | null; readonly durationMs: number; readonly logPath?: string; + readonly logError?: string; } const logName = (label: string): string => { @@ -66,6 +68,7 @@ class TailBuffer { const writeFailure = (options: RunCommandOptions, result: RunCommandResult, tail: string): void => { const code = result.signal ? `signal ${result.signal}` : `exit ${result.status}`; process.stderr.write(`FAIL ${options.label} (${code})\n`); + if (result.logError) process.stderr.write(`Log write failed: ${result.logError}\n`); if (tail !== '') process.stderr.write(`\n--- last ${options.minimumTailLines ?? DEFAULT_TAIL_LINES} lines ---\n${tail}\n`); if (result.logPath) process.stderr.write(`\nFull log: ${displayPath(result.logPath)}\n`); }; @@ -79,63 +82,149 @@ const hasArguments = (options: RunCommandOptions): boolean => (options.args?.len const executable = (options: RunCommandOptions): string => hasArguments(options) && process.platform === 'win32' && options.command === 'pnpm' ? 'pnpm.cmd' : options.command; +const usesWindowsShell = (options: RunCommandOptions): boolean => + process.platform === 'win32' && (options.command === 'pnpm' || options.command.endsWith('.cmd')); + const windowsCommandLine = (options: RunCommandOptions): string => [executable(options), ...(options.args ?? [])].map(argument => (/\s/.test(argument) ? JSON.stringify(argument) : argument)).join(' '); -const spawnCommand = (options: RunCommandOptions, stdio: Parameters[2]['stdio']): ReturnType => - hasArguments(options) && process.platform === 'win32' - ? spawn(process.env['ComSpec'] ?? 'cmd.exe', ['/d', '/s', '/c', windowsCommandLine(options)], { cwd: options.cwd, stdio }) - : hasArguments(options) - ? spawn(executable(options), [...(options.args ?? [])], { cwd: options.cwd, stdio }) - : spawn(options.command, { cwd: options.cwd, shell: true, stdio }); +const childEnvironment = (options: RunCommandOptions): NodeJS.ProcessEnv => ({ + ...process.env, + EXOJS_OUTPUT: options.output, +}); + +const spawnCommand = (options: RunCommandOptions, stdio: Parameters[2]['stdio']): ReturnType => { + const common = { cwd: options.cwd, env: childEnvironment(options), stdio }; + if (usesWindowsShell(options)) { + return spawn(process.env['ComSpec'] ?? 'cmd.exe', ['/d', '/s', '/c', windowsCommandLine(options)], common); + } + if (hasArguments(options)) return spawn(executable(options), [...(options.args ?? [])], common); + return spawn(options.command, { ...common, shell: true }); +}; const capture = (options: RunCommandOptions): Promise => new Promise(resolveResult => { - mkdirSync(LOG_DIR, { recursive: true }); const path = resolve(LOG_DIR, logName(options.label)); - const log = createWriteStream(path, { encoding: 'utf8' }); const tail = new TailBuffer(options.minimumTailLines ?? DEFAULT_TAIL_LINES); const started = Date.now(); + let log: WriteStream; + + try { + mkdirSync(LOG_DIR, { recursive: true }); + log = createWriteStream(path, { encoding: 'utf8' }); + } catch (error) { + const result: RunCommandResult = { + status: 1, + signal: null, + durationMs: Date.now() - started, + logPath: path, + logError: String(error), + }; + writeFailure(options, result, ''); + resolveResult(result); + return; + } + let settled = false; + let finishing = false; + let logError: string | undefined; + const paused = new Set(); - const finish = (status: number, signal: NodeJS.Signals | null): void => { - if (settled) return; - settled = true; + const resumePaused = (): void => { + for (const stream of paused) stream.resume(); + paused.clear(); + }; + + const report = (result: RunCommandResult, tailText: string): void => { + if (result.status === 0) { + if (options.output === 'compact') writeCompactSuccess(options.label, result.durationMs); + } else { + writeFailure(options, result, tailText); + } + resolveResult(result); + }; + + log.write(`$ ${options.command}${options.args?.length ? ` ${options.args.join(' ')}` : ''}\n`); + const child = spawnCommand(options, ['inherit', 'pipe', 'pipe']); + + const finishCommand = (status: number, signal: NodeJS.Signals | null): void => { + if (settled || finishing) return; + finishing = true; tail.finish(); - const result: RunCommandResult = { status, signal, durationMs: Date.now() - started, logPath: path }; + const result: RunCommandResult = { + status: logError ? 1 : status, + signal, + durationMs: Date.now() - started, + logPath: path, + ...(logError !== undefined && { logError }), + }; + if (logError) { + settled = true; + report(result, tail.text()); + return; + } log.end(() => { - if (status === 0) { - if (options.output === 'compact') writeCompactSuccess(options.label, result.durationMs); - } else { - writeFailure(options, result, tail.text()); - } - resolveResult(result); + if (settled) return; + settled = true; + report(result, tail.text()); }); }; - log.write(`$ ${options.command}${options.args?.length ? ` ${options.args.join(' ')}` : ''}\n`); - const child = spawnCommand(options, ['inherit', 'pipe', 'pipe']); - const consume = (stream: NodeJS.ReadableStream, source: string): void => { + const failLog = (error: unknown): void => { + if (logError) return; + logError = String(error instanceof Error ? error.message : error); + resumePaused(); + if (child && !child.killed) child.kill(); + if (finishing && !settled) { + settled = true; + tail.finish(); + report({ status: 1, signal: null, durationMs: Date.now() - started, logPath: path, logError }, tail.text()); + } + }; + + log.once('error', failLog); + log.on('drain', resumePaused); + const consume = (stream: Readable, source: string): void => { stream.on('data', chunk => { - log.write(chunk); + if (logError) return; const text = typeof chunk === 'string' ? chunk : chunk.toString(); tail.append(text); + try { + if (!log.write(chunk)) { + stream.pause(); + paused.add(stream); + } + } catch (error) { + failLog(error); + } }); stream.on('error', error => { const text = `[${source} error] ${String(error)}\n`; - log.write(text); tail.append(text); + if (!logError) { + try { + log.write(text); + } catch (writeError) { + failLog(writeError); + } + } }); }; consume(child.stdout!, 'stdout'); consume(child.stderr!, 'stderr'); child.once('error', error => { const text = `${String(error.message ?? error)}\n`; - log.write(text); tail.append(text); - finish(1, null); + if (!logError) { + try { + log.write(text); + } catch (writeError) { + failLog(writeError); + } + } + finishCommand(1, null); }); - child.once('close', (status, signal) => finish(status ?? 1, signal)); + child.once('close', (status, signal) => finishCommand(status ?? 1, signal)); }); /** Runs a command without buffering its complete output in memory. */ @@ -151,7 +240,10 @@ export const runCommand = (options: RunCommandOptions): Promise finish(1, null)); + child.once('error', error => { + process.stderr.write(`Failed to start ${options.label}: ${error.message}\n`); + finish(1, null); + }); child.once('close', (status, signal) => finish(status ?? 1, signal)); }); }; diff --git a/test/ci/run-command.test.ts b/test/ci/run-command.test.ts new file mode 100644 index 000000000..0e3028c9e --- /dev/null +++ b/test/ci/run-command.test.ts @@ -0,0 +1,90 @@ +import { readFileSync, rmSync } from 'node:fs'; + +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { runCommand, type RunCommandResult } from '../../scripts/lib/run-command.ts'; + +const createdLogs: string[] = []; +let sequence = 0; + +const runNode = async (script: string, output: 'compact' | 'silent' | 'normal' = 'silent'): Promise => { + const result = await runCommand({ + label: `run-command-test-${sequence++}`, + command: process.execPath, + args: ['-e', script], + output, + }); + if (result.logPath) createdLogs.push(result.logPath); + return result; +}; + +afterEach(() => { + for (const path of createdLogs.splice(0)) rmSync(path, { force: true }); +}); + +describe('runCommand', () => { + it('captures both output streams in silent mode', async () => { + const result = await runNode("process.stdout.write('stdout'); process.stderr.write('stderr');"); + + expect(result.status).toBe(0); + expect(readFileSync(result.logPath!, 'utf8')).toContain('stdout'); + expect(readFileSync(result.logPath!, 'utf8')).toContain('stderr'); + }); + + it('keeps the final unterminated line and limits the failure tail', async () => { + const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + const result = await runNode( + "for (let i = 0; i < 125; i++) process.stdout.write(`line-${i}\\n`); process.stdout.write('last-line'); process.exitCode = 2;", + 'compact', + ); + + const log = readFileSync(result.logPath!, 'utf8'); + const diagnostic = stderr.mock.calls.map(([chunk]) => String(chunk)).join(''); + stderr.mockRestore(); + expect(result.status).toBe(2); + expect(diagnostic).toContain('line-124'); + expect(diagnostic).not.toContain('line-0'); + expect(log).toContain('line-0'); + expect(log).toContain('line-124'); + expect(log).toContain('last-line'); + }); + + it('propagates the resolved output mode to child processes', async () => { + const result = await runNode("process.stdout.write(process.env.EXOJS_OUTPUT ?? 'missing');"); + + expect(readFileSync(result.logPath!, 'utf8')).toContain('silent'); + }); + + it('propagates verbose mode through the live path', async () => { + const result = await runCommand({ + label: `run-command-verbose-${sequence++}`, + command: process.execPath, + args: ['-e', "process.exit(process.env.EXOJS_OUTPUT === 'verbose' ? 0 : 1)"], + output: 'verbose', + }); + + expect(result.status).toBe(0); + }); + + it('reports a live spawn error instead of hiding it', async () => { + const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + const result = await runCommand({ + label: `run-command-missing-${sequence++}`, + command: 'exojs-command-that-does-not-exist', + args: ['--test'], + output: 'normal', + }); + const diagnostic = stderr.mock.calls.map(([chunk]) => String(chunk)).join(''); + stderr.mockRestore(); + + expect(result.status).toBe(1); + expect(diagnostic).toContain('Failed to start run-command-missing'); + }); + + it('writes a compact success result without losing the full log', async () => { + const result = await runNode("process.stdout.write('complete output');", 'compact'); + + expect(result.status).toBe(0); + expect(readFileSync(result.logPath!, 'utf8')).toContain('complete output'); + }); +});