diff --git a/packages/exojs-bench/src/suite/catalog.ts b/packages/exojs-bench/src/suite/catalog.ts index bbde2b17f..b34ed4105 100644 --- a/packages/exojs-bench/src/suite/catalog.ts +++ b/packages/exojs-bench/src/suite/catalog.ts @@ -102,11 +102,6 @@ export const loadIdFor = (value: number): string => { * scene on those, so they are development rows and never a published comparison. * They keep running under `full` through the domain ladder. */ -/** - * `fx-blur` is deliberately absent. It is measured, but its two arms do not - * produce the same picture (see its archetype), so it is an internal probe - * rather than a published comparison and has no business in a reference plan. - */ export const RENDERING_SCENARIOS: readonly ScenarioLoads[] = [ { scenarioId: 'static-heavy', diff --git a/scripts/ci/lanes.ts b/scripts/ci/lanes.ts index 027f9921b..999104a0b 100644 --- a/scripts/ci/lanes.ts +++ b/scripts/ci/lanes.ts @@ -227,6 +227,7 @@ const ALL_AREAS: LaneAreas = { benchStructural: true, release: true, guides: true, + siteData: true, createExoApp: true, }; diff --git a/scripts/ci/select-lanes.ts b/scripts/ci/select-lanes.ts index 8378e942e..93d0e6400 100644 --- a/scripts/ci/select-lanes.ts +++ b/scripts/ci/select-lanes.ts @@ -34,6 +34,7 @@ export interface LaneAreas { benchStructural: boolean; release: boolean; guides: boolean; + siteData: boolean; createExoApp: boolean; } @@ -259,6 +260,41 @@ const isBenchStructuralPath = (file: string): boolean => { */ const isGuidesPath = (file: string): boolean => file.startsWith('site/src/content/'); +/** + * Site-data area: the sources the remaining `test/site/**` suites read. Same + * reasoning as `isGuidesPath` - those suites live under `test/`, so they run on + * every engine change and would otherwise never run on the change they exist to + * check. + * + * - `site/src/lib/` every site suite that is not a guide + * suite imports from here (bench cards + * and profiles, playground navigation, + * URL state, the example catalog, source + * snippets, footgun diagnostics). + * - `examples/` the catalog sources and their generated + * twins, which `examples-sync`, + * `runtime-dts` and `assets-global-dts` + * read directly. + * - `packages/exojs-bench/results/` the committed profiles, which + * `site/src/lib/bench-profiles` globs and + * three suites validate. The bench + * package is not a runtime package and + * the structural gate covers only `src/`, + * `test/` and `baselines/`, so a + * profile-only commit reaches no other + * area that runs a test. + * + * Deliberately not `site/src/pages/` or `site/src/components/`: no suite reads + * them, and the site build already gates on the wider `site` area. + */ +const isSiteDataPath = (file: string): boolean => { + if (isDocPath(file)) return false; + if (file.startsWith('site/src/lib/')) return true; + if (file.startsWith('examples/')) return true; + if (file.startsWith('packages/exojs-bench/results/')) return true; + return false; +}; + /** * create-exo-app area: the scaffolding CLI is a standalone package with no * engine or browser impact, so it is deliberately outside `RUNTIME_PACKAGES` @@ -294,6 +330,7 @@ export const selectAreas = (changedFiles: readonly string[]): LaneAreas => { let benchStructural = false; let release = false; let guides = false; + let siteData = false; let createExoApp = false; for (const raw of changedFiles) { // Normalise Windows separators and trim stray whitespace/blank entries. @@ -307,10 +344,11 @@ export const selectAreas = (changedFiles: readonly string[]): LaneAreas => { if (!benchStructural && isBenchStructuralPath(file)) benchStructural = true; if (!release && isReleasePath(file)) release = true; if (!guides && isGuidesPath(file)) guides = true; + if (!siteData && isSiteDataPath(file)) siteData = true; if (!createExoApp && isCreateExoAppPath(file)) createExoApp = true; - if (engine && site && audioFx && tilemapWorker && exampleCatalog && benchStructural && release && guides && createExoApp) break; + if (engine && site && audioFx && tilemapWorker && exampleCatalog && benchStructural && release && guides && siteData && createExoApp) break; } - return { engine, site, audioFx, tilemapWorker, exampleCatalog, benchStructural, release, guides, createExoApp }; + return { engine, site, audioFx, tilemapWorker, exampleCatalog, benchStructural, release, guides, siteData, createExoApp }; }; /** @@ -328,16 +366,17 @@ export const selectAreas = (changedFiles: readonly string[]): LaneAreas => { * - bench-structural-gate gates on `benchStructural` (rendering source, the * bench harness, or the committed counter baseline); * - the unit lane also gates on `guides` (guide content under - * `site/src/content/**`, which the `test/site/guide-*` suites validate); + * `site/src/content/**`, which the `test/site/guide-*` suites validate) and + * on `siteData` (the sources the remaining `test/site/**` suites read); * - create-exo-app-verify gates on `createExoApp`, independent of `engine` * (the scaffolder has no engine or browser impact of its own). */ export const effectiveLanes = (areas: LaneAreas): EffectiveLanes => { - const { engine, site, audioFx, tilemapWorker, exampleCatalog, benchStructural, release, guides, createExoApp } = areas; + const { engine, site, audioFx, tilemapWorker, exampleCatalog, benchStructural, release, guides, siteData, createExoApp } = areas; return { typecheck: true, lint: true, - unit: engine || guides, + unit: engine || guides || siteData, coverage: engine, browserWebgl2: engine, browserWebgpu: engine, diff --git a/scripts/lanes.ts b/scripts/lanes.ts index c55953833..7da14b04b 100644 --- a/scripts/lanes.ts +++ b/scripts/lanes.ts @@ -62,6 +62,7 @@ const ALL_AREAS: LaneAreas = { benchStructural: true, release: true, guides: true, + siteData: true, createExoApp: true, }; diff --git a/scripts/verify-bench-results.ts b/scripts/verify-bench-results.ts index d619cf166..b602851bb 100644 --- a/scripts/verify-bench-results.ts +++ b/scripts/verify-bench-results.ts @@ -28,6 +28,7 @@ import { readdirSync, readFileSync, statSync } from 'node:fs'; import { basename, extname, join, resolve } from 'node:path'; import { isProfileSlug, SUPPORTED_BENCH_PROFILE_SCHEMA_VERSIONS } from '../packages/exojs-bench/src/profile/schema.ts'; +import { scenariosFor } from '../packages/exojs-bench/src/suite/catalog.ts'; import { computeProfileSignature, PROFILE_SIGNATURE_ALGORITHM } from '../packages/exojs-bench/src/profile/signature.ts'; const REPO_ROOT = resolve(import.meta.dirname, '..'); @@ -208,6 +209,46 @@ const CELL_FIELDS = ['competitor', 'referenceMs', 'competitorMs', 'verdict', 'ti */ const ROW_FIELDS_SINCE = 7; +/** + * Cells the `reference` plan selects for one domain, as `archetype/loadId`. + */ +const referencePlanOf = (domain: 'rendering' | 'physics'): ReadonlySet => + new Set(scenariosFor(domain).flatMap(scenario => scenario.loads.filter(load => load.reference).map(load => `${scenario.scenarioId}/${load.loadId}`))); + +/** + * A published profile has to be a `reference` run. The `full` suite adds the + * development rungs and the ExoJS-internal probes, whose competitor arms render + * some other scene - publishing them would put a comparison on the page that was + * never one. Nothing else in this gate can tell the two suites apart: both carry + * the same shape, the same provenance and a signature that recomputes, because + * both were measured rather than typed. + * + * Only the extra rows fail. A profile may hold fewer cells than the plan - an arm + * that refused a scenario leaves a gap, and that is a measurement outcome. + */ +const checkReferencePlan = (domain: 'rendering' | 'physics', section: unknown, where: string, problems: string[]): void => { + const rows = isRecord(section) ? section['rows'] : undefined; + + if (!Array.isArray(rows)) return; + + const plan = referencePlanOf(domain); + + for (const [index, row] of rows.entries()) { + if (!isRecord(row)) continue; + + const archetype = row['archetype']; + const loadId = row['loadId']; + + if (typeof archetype !== 'string' || typeof loadId !== 'string') continue; + + if (!plan.has(`${archetype}/${loadId}`)) { + problems.push( + `${where}.rows[${String(index)}] publishes '${archetype}/${loadId}', which the reference plan does not select - this looks like a 'full' run`, + ); + } + } +}; + /** Every row and cell of one published section carries the fields its schema version promises. */ const checkModelRows = (section: unknown, where: string, problems: string[]): void => { const rows = isRecord(section) ? section['rows'] : undefined; @@ -405,7 +446,10 @@ const checkProfile = (path: string): string[] => { const sections = isRecord(block) ? block['sections'] : undefined; for (const [sectionIndex, section] of (Array.isArray(sections) ? sections : []).entries()) { - checkModelRows(section, `rendering.backends[${String(index)}].sections[${String(sectionIndex)}]`, problems); + const sectionWhere = `rendering.backends[${String(index)}].sections[${String(sectionIndex)}]`; + + checkModelRows(section, sectionWhere, problems); + checkReferencePlan('rendering', section, sectionWhere, problems); } } } @@ -436,6 +480,7 @@ const checkProfile = (path: string): string[] => { problems.push('physics.section is missing'); } else if (version >= ROW_FIELDS_SINCE) { checkModelRows(physics['section'], 'physics.section', problems); + checkReferencePlan('physics', physics['section'], 'physics.section', problems); } armVersions.push(...checkLibraries(physics, 'physics', problems)); diff --git a/test/ci/lane-commands.test.ts b/test/ci/lane-commands.test.ts index eccaca754..79a435b7b 100644 --- a/test/ci/lane-commands.test.ts +++ b/test/ci/lane-commands.test.ts @@ -27,6 +27,7 @@ const allLaneKeys = Object.keys( benchStructural: true, release: true, guides: true, + siteData: true, createExoApp: true, }), ); diff --git a/test/ci/select-lanes.test.ts b/test/ci/select-lanes.test.ts index 63080a507..cfa7efc8e 100644 --- a/test/ci/select-lanes.test.ts +++ b/test/ci/select-lanes.test.ts @@ -342,8 +342,11 @@ describe('CI lane selection - example-smoke lane', () => { expect(lanes.siteBuild).toBe(true); // The catalog is not engine code. expect(areas.engine).toBe(false); - expect(lanes.unit).toBe(false); expect(lanes.browserWebgl2).toBe(false); + // It IS site data: `examples-sync` fails a `.ts` edit whose generated `.js` + // twin was not regenerated, and that suite runs in the unit lane. + expect(areas.siteData).toBe(true); + expect(lanes.unit).toBe(true); }); it('the generated `.js` twin and the catalog manifest run the lane too', () => { @@ -478,3 +481,44 @@ describe('CI lane selection - bench structural gate', () => { expect(decide('.github/workflows/ci.yml').lanes.benchStructural).toBe(true); }); }); + +describe('CI lane selection — site data gates the unit lane', () => { + it('a committed bench profile runs the unit lane that validates it', () => { + const { areas, lanes } = decide('packages/exojs-bench/results/rtx-5070-ti-windows-11-chromium.json'); + expect(areas.siteData).toBe(true); + expect(areas.engine).toBe(false); + expect(areas.benchStructural).toBe(false); + expect(lanes.unit).toBe(true); + expect(lanes.coverage).toBe(false); + expect(lanes.packageVerify).toBe(false); + expect(lanes.browserWebgl2).toBe(false); + }); + + it('a site library change runs the unit lane, without pulling in the rest of engine', () => { + const { areas, lanes } = decide('site/src/lib/bench-cards.ts'); + expect(areas.siteData).toBe(true); + expect(areas.engine).toBe(false); + expect(lanes.unit).toBe(true); + expect(lanes.browserWebgl2).toBe(false); + }); + + it('an example source change runs the unit lane alongside the smoke', () => { + const { areas, lanes } = decide('examples/rendering/sprites/basic.ts'); + expect(areas.siteData).toBe(true); + expect(lanes.unit).toBe(true); + expect(lanes.exampleSmoke).toBe(true); + }); + + it('site pages and components stay out of the area: no suite reads them', () => { + const { areas, lanes } = decide('site/src/pages/index.astro', 'site/src/components/Nav.astro'); + expect(areas.siteData).toBe(false); + expect(lanes.unit).toBe(false); + expect(lanes.siteBuild).toBe(true); + }); + + it('a bench package README stays prose: no unit lane', () => { + const { areas, lanes } = decide('packages/exojs-bench/README.md'); + expect(areas.siteData).toBe(false); + expect(lanes.unit).toBe(false); + }); +});