diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml new file mode 100644 index 000000000..287cf3c42 --- /dev/null +++ b/.github/workflows/build-docs.yml @@ -0,0 +1,48 @@ +name: Build Docs + +on: + push: + branches: [main] + paths: + - 'packages/docs/**' + - 'packages/results/**' + - 'package.json' + - 'pnpm-lock.yaml' + - 'pnpm-workspace.yaml' + - '.npmrc' + - 'tsconfig.json' + - '.github/workflows/build-docs.yml' + pull_request: + branches: [main] + paths: + - 'packages/docs/**' + - 'packages/results/**' + - 'package.json' + - 'pnpm-lock.yaml' + - 'pnpm-workspace.yaml' + - '.npmrc' + - 'tsconfig.json' + - '.github/workflows/build-docs.yml' + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup pnpm + uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5 + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '24' + cache: 'pnpm' + + - name: Install workspace dependencies + run: pnpm install --frozen-lockfile + + - name: Build docs + run: pnpm build:docs diff --git a/packages/cwv-stats/package.json b/packages/cwv-stats/package.json index 65fd9e430..4b44f3028 100644 --- a/packages/cwv-stats/package.json +++ b/packages/cwv-stats/package.json @@ -7,6 +7,6 @@ "@types/node": "^26.1.2" }, "dependencies": { - "zod": "^4.4.3" + "zod": "catalog:" } } diff --git a/packages/docs/package.json b/packages/docs/package.json index 82fa9c662..cebb51da3 100644 --- a/packages/docs/package.json +++ b/packages/docs/package.json @@ -16,7 +16,8 @@ "@astrojs/starlight": "^0.41.7", "@tanstack/charts": "^1.1.0", "astro": "^7.4.0", - "chart.js": "^4.5.1" + "chart.js": "^4.5.1", + "@framework-tracker/results": "workspace:*" }, "devDependencies": { "@astrojs/check": "^0.9.10", diff --git a/packages/docs/src/content.config.ts b/packages/docs/src/content.config.ts index 4bc575e21..2be6d7036 100644 --- a/packages/docs/src/content.config.ts +++ b/packages/docs/src/content.config.ts @@ -1,4 +1,8 @@ -import { requestThroughputContentSchema } from './lib/request-throughput' +import { + devtimeContentSchema, + runtimeContentSchema, + runtimeVersionsContentSchema, +} from './lib/result-content' import { defineCollection } from 'astro:content' import { file, glob } from 'astro/loaders' import { docsLoader } from '@astrojs/starlight/loaders' @@ -6,396 +10,9 @@ import { docsSchema } from '@astrojs/starlight/schema' import { z } from 'astro/zod' import { readdirSync } from 'node:fs' -const timeSchema = z.object({ - avgMs: z.number(), - medianMs: z.number().optional(), - standardDeviationMs: z.number().nonnegative().optional(), - minMs: z.number(), - maxMs: z.number(), - samplesMs: z.array(z.number()).nonempty().optional(), -}) - -const buildTimeSchema = z.union([ - z - .object({ - name: z.enum(['coldBuild', 'warmBuild']), - runs: z.number().int().positive(), - iterations: z.array(z.object({ timeMs: z.number().nonnegative() })), - timeMs: z.object({ - mean: z.number(), - median: z.number(), - min: z.number(), - max: z.number(), - }), - }) - .refine((value) => value.iterations.length === value.runs) - .transform((value) => ({ - avgMs: value.timeMs.mean, - medianMs: value.timeMs.median, - minMs: value.timeMs.min, - maxMs: value.timeMs.max, - samplesMs: value.iterations.map((run) => run.timeMs), - })), - timeSchema, -]) - -const devServerTimeSchema = z.union([ - z - .object({ - name: z.literal('devServer'), - runs: z.number().int().positive(), - iterations: z.array(z.object({ timeMs: z.number().nonnegative() })), - timeMs: z.object({ - mean: z.number(), - median: z.number(), - min: z.number(), - max: z.number(), - }), - }) - .refine((value) => value.iterations.length === value.runs) - .transform((value) => ({ - avgMs: value.timeMs.mean, - medianMs: value.timeMs.median, - minMs: value.timeMs.min, - maxMs: value.timeMs.max, - samplesMs: value.iterations.map((run) => run.timeMs), - })), - timeSchema, -]) - -const interactionTimingSchema = z.object({ - interactionLatencyMs: z.number().nonnegative(), - inputDelayMs: z.number().nonnegative(), - processingDurationMs: z.number().nonnegative(), - presentationDelayMs: z.number().nonnegative(), -}) - -const renderedTestSampleSchema = z.object({ - firstPaintMs: z.number().positive(), - fcpMs: z.number().positive(), - interactionTests: interactionTimingSchema, -}) - -const metricSchema = z.object({ - mean: z.number(), - median: z.number(), - min: z.number(), - max: z.number(), -}) -const ClientRenderingBenchmarkSchema = z - .object({ - name: z.literal('clientSideRendered'), - runs: z.number().int().positive(), - iterations: z.array(renderedTestSampleSchema), - firstPaintMs: metricSchema, - fcpMs: metricSchema, - interactionTests: z.object({ - scenario: z.literal('first-row-detail-navigation'), - source: z.literal('lighthouse-inp-breakdown'), - interactionLatencyMs: metricSchema, - inputDelayMs: metricSchema, - processingDurationMs: metricSchema, - presentationDelayMs: metricSchema, - }), - }) - .refine((value) => value.iterations.length === value.runs) - -const ServerRenderingBenchmarkSchema = z - .object({ - name: z.literal('serverSideRendered'), - runs: z.number().int().positive(), - iterations: z.array(renderedTestSampleSchema), - firstPaintMs: metricSchema, - fcpMs: metricSchema, - interactionTests: z.object({ - scenario: z.literal('first-row-detail-navigation'), - source: z.literal('lighthouse-inp-breakdown'), - interactionLatencyMs: metricSchema, - inputDelayMs: metricSchema, - processingDurationMs: metricSchema, - presentationDelayMs: metricSchema, - }), - }) - .refine((value) => value.iterations.length === value.runs) - -const dependencyStatsSchema = z.object({ - prodDependencies: z.number().int().nonnegative(), - devDependencies: z.number().int().nonnegative(), - allDependencies: z.number().int().nonnegative(), -}) - -const firstPartyDependencyStatsSchema = dependencyStatsSchema.extend({ - duplicateDependencies: z.number().int().nonnegative().optional(), -}) - -const devtimeSchema = z.object({ - framework: z.string(), - packagePath: z.string(), - name: z.string(), - type: z.string(), - package: z.string(), - isFocused: z.boolean(), - order: z.number(), - prodDependencies: z.number(), - devDependencies: z.number(), - allDependencies: z.number(), - firstPartyDependencies: firstPartyDependencyStatsSchema, - frameworkDependencies: dependencyStatsSchema.optional(), - install: z.object({ - name: z.literal('install'), - runs: z.number().int().positive(), - iterations: z.array(z.object({ installTimeMs: z.number().nonnegative() })), - installTimeMs: z.object({ - mean: z.number(), - median: z.number(), - min: z.number(), - max: z.number(), - }), - nodeModulesSizeBytes: z.number().nonnegative(), - }), - coldBuildTime: buildTimeSchema, - warmBuildTime: buildTimeSchema, - devServerStartTime: devServerTimeSchema.optional(), - duplicateDependencies: z.number().optional(), - depInstallSize: z.number().optional(), - vendoredCoreJsSize: z.number().optional(), - vendoredCoreJsUnnecessaryModules: z.array(z.string()).optional(), - browserBaselineTests: z - .object({ - baselineStatus: z.union([ - z.literal('high'), - z.literal('low'), - z.literal(false), - z.null(), - ]), - baselineYear: z.number().nullable(), - baselineReason: z.string().nullable(), - baselineFeatureCount: z.number(), - }) - .optional(), - minimumNodeVersion: z.string().optional(), - minimumNodeVersionImposedBy: z.array(z.string()).optional(), - timingMeasuredAt: z.string(), - runner: z.string(), - browserVersion: z.string().optional(), - frameworkVersion: z.string().optional(), - releasedAt: z.string().optional(), -}) - -const ssrLoadStageSchema = z.object({ - workers: z.number().int().positive(), - elapsedTimeMs: z.number().positive(), - requests: z.number().int().nonnegative(), - errors: z.number().int().nonnegative(), - requestsPerSec: z.number().nonnegative(), - avgLatencyMs: z.number().nonnegative(), - p50LatencyMs: z.number().nonnegative(), - p75LatencyMs: z.number().nonnegative(), - p90LatencyMs: z.number().nonnegative(), - p99LatencyMs: z.number().nonnegative(), - maxLatencyMs: z.number().nonnegative(), - bytesPerSec: z.number().nonnegative(), -}) - -const ssrLoadSweepSchema = z.array(ssrLoadStageSchema).nonempty() - -const ssrLoadStageSummarySchema = z.object({ - connections: z.number().int().positive(), - requests: metricSchema, - errors: metricSchema, - requestsPerSec: metricSchema, - avgLatencyMs: metricSchema, - p50LatencyMs: metricSchema, - p75LatencyMs: metricSchema, - p90LatencyMs: metricSchema, - p99LatencyMs: metricSchema, - maxLatencyMs: metricSchema, - bytesPerSec: metricSchema, -}) - -const ssrLoadTestsSchema = z - .object({ - name: z.enum(['ssrLoad', 'ssrRouterLinkLoad']), - runs: z.number().int().positive(), - iterations: z.array(ssrLoadSweepSchema).nonempty(), - configuration: z.object({ - workerStages: z.array(z.number().int().positive()).nonempty(), - warmupDurationMs: z.number().positive().optional(), - recoveryMs: z.number().nonnegative().optional(), - warmupAttempts: z.number().int().positive().optional(), - stageDurationMs: z.number().positive().optional(), - }), - representativeIterationIndex: z.number().int().nonnegative(), - source: z.literal('autocannon'), - path: z.enum([ - '/server-side-rendered', - '/server-side-rendered-plain-links', - ]), - totalRequests: z.number().int().nonnegative(), - totalErrors: z.number().int().nonnegative(), - stage1: ssrLoadStageSummarySchema, - stage2: ssrLoadStageSummarySchema.optional(), - stage3: ssrLoadStageSummarySchema.optional(), - stage4: ssrLoadStageSummarySchema.optional(), - stage5: ssrLoadStageSummarySchema.optional(), - stage6: ssrLoadStageSummarySchema.optional(), - stage7: ssrLoadStageSummarySchema.optional(), - }) - .refine( - (value) => - value.iterations.length === value.runs && - value.representativeIterationIndex < value.iterations.length && - value.iterations[value.representativeIterationIndex]!.some( - (stage) => stage.errors === 0, - ), - ) - .transform((value) => { - const representativeSweep = - value.iterations[value.representativeIterationIndex]! - const representativePeak = representativeSweep - .filter((stage) => stage.errors === 0) - .reduce((best, stage) => - stage.requestsPerSec > best.requestsPerSec ? stage : best, - ) - return { - ...value, - representativeSweep, - representativePeak, - stages: [ - value.stage1, - value.stage2, - value.stage3, - value.stage4, - value.stage5, - value.stage6, - value.stage7, - ].flatMap((stage) => - stage - ? [ - { - workers: stage.connections, - requests: stage.requests.mean, - errors: stage.errors.mean, - requestsPerSec: stage.requestsPerSec.mean, - avgLatencyMs: stage.avgLatencyMs.mean, - p50LatencyMs: stage.p50LatencyMs.mean, - p75LatencyMs: stage.p75LatencyMs.mean, - p90LatencyMs: stage.p90LatencyMs.mean, - p99LatencyMs: stage.p99LatencyMs.mean, - maxLatencyMs: stage.maxLatencyMs.mean, - bytesPerSec: stage.bytesPerSec.mean, - }, - ] - : [], - ), - } - }) - -const runtimeSchema = z.object({ - name: z.string(), - type: z.string(), - package: z.string(), - isFocused: z.boolean(), - order: z.number(), - ssrRequestThroughputTests: requestThroughputContentSchema, - ssrLoadTests: ssrLoadTestsSchema - .refine((value) => value.name === 'ssrLoad') - .optional(), - ssrRouterLinkLoadTests: ssrLoadTestsSchema - .refine((value) => value.name === 'ssrRouterLinkLoad') - .optional(), - clientSideRenderedTests: z - .union([ - ClientRenderingBenchmarkSchema.transform((value) => ({ - ...value, - firstPaintMs: value.firstPaintMs.mean, - fcpMs: value.fcpMs.mean, - interactionTests: { - ...value.interactionTests, - interactionLatencyMs: - value.interactionTests.interactionLatencyMs.mean, - inputDelayMs: value.interactionTests.inputDelayMs.mean, - processingDurationMs: - value.interactionTests.processingDurationMs.mean, - presentationDelayMs: value.interactionTests.presentationDelayMs.mean, - }, - })), - z.object({ - firstPaintMs: z.number(), - fcpMs: z.number(), - inpMs: z.number().optional(), - interactionTests: z - .object({ - scenario: z.literal('first-row-detail-navigation'), - source: z.literal('lighthouse-inp-breakdown'), - interactionLatencyMs: z.number().positive(), - inputDelayMs: z.number().nonnegative(), - processingDurationMs: z.number().nonnegative(), - presentationDelayMs: z.number().nonnegative(), - }) - .optional(), - runs: z.number(), - standardDeviation: interactionTimingSchema - .extend({ - firstPaintMs: z.number().nonnegative(), - fcpMs: z.number().nonnegative(), - }) - .optional(), - samples: z.array(renderedTestSampleSchema).nonempty().optional(), - }), - ]) - .optional(), - serverSideRenderedTests: z - .union([ - ServerRenderingBenchmarkSchema.transform((value) => ({ - ...value, - firstPaintMs: value.firstPaintMs.mean, - fcpMs: value.fcpMs.mean, - interactionTests: { - ...value.interactionTests, - interactionLatencyMs: - value.interactionTests.interactionLatencyMs.mean, - inputDelayMs: value.interactionTests.inputDelayMs.mean, - processingDurationMs: - value.interactionTests.processingDurationMs.mean, - presentationDelayMs: value.interactionTests.presentationDelayMs.mean, - }, - })), - z.object({ - firstPaintMs: z.number(), - fcpMs: z.number(), - inpMs: z.number().optional(), - interactionTests: z - .object({ - scenario: z.literal('first-row-detail-navigation'), - source: z.literal('lighthouse-inp-breakdown'), - interactionLatencyMs: z.number().positive(), - inputDelayMs: z.number().nonnegative(), - processingDurationMs: z.number().nonnegative(), - presentationDelayMs: z.number().nonnegative(), - }) - .optional(), - runs: z.number(), - standardDeviation: interactionTimingSchema - .extend({ - firstPaintMs: z.number().nonnegative(), - fcpMs: z.number().nonnegative(), - }) - .optional(), - samples: z.array(renderedTestSampleSchema).nonempty().optional(), - }), - ]) - .optional(), - timingMeasuredAt: z.string().optional(), - runner: z.string().optional(), - browserVersion: z.string().optional(), - frameworkVersion: z.string().optional(), - releasedAt: z.string().optional(), -}) - const devtimeCollection = defineCollection({ loader: glob({ pattern: '*.json', base: './src/content/devtime' }), - schema: devtimeSchema, + schema: devtimeContentSchema, }) const devtimeVersionsCollection = defineCollection({ @@ -403,12 +20,12 @@ const devtimeVersionsCollection = defineCollection({ pattern: '**/*.json', base: './src/content/devtime/versions', }), - schema: devtimeSchema, + schema: devtimeContentSchema, }) const runtimeCollection = defineCollection({ loader: glob({ pattern: '*.json', base: './src/content/runtime' }), - schema: runtimeSchema, + schema: runtimeContentSchema, }) const runtimeVersionsCollection = defineCollection({ @@ -416,10 +33,7 @@ const runtimeVersionsCollection = defineCollection({ pattern: '**/*.json', base: './src/content/runtime/versions', }), - schema: runtimeSchema.extend({ - ssrRequestThroughputTests: - runtimeSchema.shape.ssrRequestThroughputTests.optional(), - }), + schema: runtimeVersionsContentSchema, }) const cwvCollection = defineCollection({ diff --git a/packages/docs/src/lib/request-throughput.test.ts b/packages/docs/src/lib/request-throughput.test.ts deleted file mode 100644 index da8b853ff..000000000 --- a/packages/docs/src/lib/request-throughput.test.ts +++ /dev/null @@ -1,76 +0,0 @@ -import assert from 'node:assert/strict' -import { test } from 'node:test' -import { requestThroughputContentSchema } from './request-throughput.ts' -import { - prepareSsrRequestThroughputTestsOpsPerSecComparison, - prepareSsrRequestThroughputTestsLatencyMsComparison, - prepareSsrRequestThroughputTestsOpsPerSecHistory, -} from './prepare-chart.ts' - -const metric = (mean: number) => ({ - mean, - median: mean, - min: mean, - max: mean, -}) -const benchmark = { - name: 'ssrRequestThroughput', - runs: 1, - iterations: [ - { - opsPerSec: metric(100.125), - latencyMs: { ...metric(4), median: 2 }, - samples: 3, - }, - ], - opsPerSec: metric(100.125), - latencyMs: metric(4), - samples: 3, - bodySizeKb: 20.5, - duplicationFactor: 1.25, - durationMs: 10_000, - warmupDurationMs: 250, - minimumSamples: 64, - warmupMinimumSamples: 16, - source: 'tinybench', - path: '/ssr-throughput', -} - -test('content loader uses run means without relabeling request medians', () => { - const normalized = requestThroughputContentSchema.parse(benchmark) - assert.equal(normalized.opsPerSec, 100.125) - assert.equal(normalized.latencyMs, 4) - const records = [ - { package: 'app-astro', ssrRequestThroughputTests: normalized }, - ] - assert.equal( - prepareSsrRequestThroughputTestsOpsPerSecComparison(records).data[0]?.value, - 100.125, - ) - assert.equal( - prepareSsrRequestThroughputTestsLatencyMsComparison(records).data[0]?.value, - 4, - ) - assert.equal( - prepareSsrRequestThroughputTestsOpsPerSecHistory(records).data[0]?.value, - 100.125, - ) - assert.equal( - requestThroughputContentSchema.safeParse({ ...benchmark, iterations: [] }) - .success, - false, - ) -}) - -test('legacy mean latency remains distinct from the reported request median', () => { - const legacy = { - opsPerSec: 100, - avgLatencyMs: 4, - medianLatencyMs: 2, - samples: 64, - bodySizeKb: 20, - duplicationFactor: 1, - } - const normalized = requestThroughputContentSchema.parse(legacy) - assert.equal(normalized.latencyMs, 4) -}) diff --git a/packages/docs/src/lib/request-throughput.ts b/packages/docs/src/lib/request-throughput.ts deleted file mode 100644 index 7a1f8854b..000000000 --- a/packages/docs/src/lib/request-throughput.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { z } from 'astro/zod' - -const metricSchema = z.object({ - mean: z.number(), - median: z.number(), - min: z.number(), - max: z.number(), -}) - -const iterationSchema = z.object({ - opsPerSec: metricSchema, - latencyMs: metricSchema, - samples: z.number().int().positive(), -}) - -export const requestThroughputContentSchema = z.union([ - z - .object({ - name: z.literal('ssrRequestThroughput'), - runs: z.number().int().positive(), - iterations: z.array(iterationSchema), - opsPerSec: metricSchema, - latencyMs: metricSchema, - samples: z.number().int().positive(), - bodySizeKb: z.number().positive(), - duplicationFactor: z.number().nonnegative(), - durationMs: z.number().positive(), - warmupDurationMs: z.number().nonnegative(), - minimumSamples: z.number().int().positive(), - warmupMinimumSamples: z.number().int().nonnegative(), - source: z.literal('tinybench'), - path: z.literal('/ssr-throughput'), - }) - .refine((value) => value.runs === value.iterations.length) - .refine( - (value) => - value.samples === - value.iterations.reduce( - (count, iteration) => count + iteration.samples, - 0, - ), - ) - .transform((value) => ({ - ...value, - opsPerSec: value.opsPerSec.mean, - latencyMs: value.latencyMs.mean, - })), - // Current checked-in records have rounded summaries and no recoverable samples. - z - .object({ - opsPerSec: z.number().positive(), - avgLatencyMs: z.number().nonnegative(), - medianLatencyMs: z.number().nonnegative(), - samples: z.number().positive(), - bodySizeKb: z.number().positive(), - duplicationFactor: z.number().nonnegative(), - }) - .transform((value) => ({ ...value, latencyMs: value.avgLatencyMs })), -]) diff --git a/packages/docs/src/lib/result-content.ts b/packages/docs/src/lib/result-content.ts new file mode 100644 index 000000000..0f65cbd43 --- /dev/null +++ b/packages/docs/src/lib/result-content.ts @@ -0,0 +1,156 @@ +import { + FrameworkResultSchema, + type FrameworkResult, +} from '@framework-tracker/results' + +function formatTime( + value: NonNullable< + | FrameworkResult['coldBuildTime'] + | FrameworkResult['warmBuildTime'] + | FrameworkResult['devServerStartTime'] + >, +) { + return { + avgMs: value.timeMs.mean, + medianMs: value.timeMs.median, + minMs: value.timeMs.min, + maxMs: value.timeMs.max, + samplesMs: value.iterations.map((run) => run.timeMs), + } +} +function formatRendered( + value: NonNullable< + | FrameworkResult['clientSideRenderedTests'] + | FrameworkResult['serverSideRenderedTests'] + >, +) { + if (!('name' in value)) return value + return { + ...value, + firstPaintMs: value.firstPaintMs.mean, + fcpMs: value.fcpMs.mean, + interactionTests: { + ...value.interactionTests, + interactionLatencyMs: value.interactionTests.interactionLatencyMs.mean, + inputDelayMs: value.interactionTests.inputDelayMs.mean, + processingDurationMs: value.interactionTests.processingDurationMs.mean, + presentationDelayMs: value.interactionTests.presentationDelayMs.mean, + }, + } +} +function formatThroughput( + value: NonNullable, +) { + if ('name' in value) + return { + ...value, + opsPerSec: value.opsPerSec.mean, + latencyMs: value.latencyMs.mean, + } + return { ...value, latencyMs: value.avgLatencyMs } +} +function formatLoad(value: NonNullable) { + const representativeSweep = + value.iterations[value.representativeIterationIndex]! + const representativePeak = representativeSweep + .filter((stage) => stage.errors === 0) + .reduce((best, stage) => + stage.requestsPerSec > best.requestsPerSec ? stage : best, + ) + return { + ...value, + representativeSweep, + representativePeak, + stages: [ + value.stage1, + value.stage2, + value.stage3, + value.stage4, + value.stage5, + value.stage6, + value.stage7, + ].flatMap((stage) => + stage + ? [ + { + workers: stage.connections, + requests: stage.requests.mean, + errors: stage.errors.mean, + requestsPerSec: stage.requestsPerSec.mean, + avgLatencyMs: stage.avgLatencyMs.mean, + p50LatencyMs: stage.p50LatencyMs.mean, + p75LatencyMs: stage.p75LatencyMs.mean, + p90LatencyMs: stage.p90LatencyMs.mean, + p99LatencyMs: stage.p99LatencyMs.mean, + maxLatencyMs: stage.maxLatencyMs.mean, + bytesPerSec: stage.bytesPerSec.mean, + }, + ] + : [], + ), + } +} + +const displayMetadata = { + name: true, + type: true, + package: true, + isFocused: true, + order: true, +} as const + +export const devtimeContentSchema = FrameworkResultSchema.required({ + ...displayMetadata, + framework: true, + packagePath: true, + prodDependencies: true, + devDependencies: true, + allDependencies: true, + firstPartyDependencies: true, + install: true, + coldBuildTime: true, + warmBuildTime: true, + timingMeasuredAt: true, + runner: true, +}).transform((value) => ({ + ...formatRuntime(value), + coldBuildTime: formatTime(value.coldBuildTime), + warmBuildTime: formatTime(value.warmBuildTime), + devServerStartTime: + value.devServerStartTime && formatTime(value.devServerStartTime), +})) + +const runtimeInputSchema = FrameworkResultSchema.required(displayMetadata) +function formatRuntime>( + value: T, +) { + return { + ...value, + coldBuildTime: value.coldBuildTime && formatTime(value.coldBuildTime), + warmBuildTime: value.warmBuildTime && formatTime(value.warmBuildTime), + devServerStartTime: + value.devServerStartTime && formatTime(value.devServerStartTime), + ssrRequestThroughputTests: + value.ssrRequestThroughputTests && + formatThroughput(value.ssrRequestThroughputTests), + ssrLoadTests: value.ssrLoadTests && formatLoad(value.ssrLoadTests), + ssrRouterLinkLoadTests: + value.ssrRouterLinkLoadTests && formatLoad(value.ssrRouterLinkLoadTests), + clientSideRenderedTests: + value.clientSideRenderedTests && + formatRendered(value.clientSideRenderedTests), + serverSideRenderedTests: + value.serverSideRenderedTests && + formatRendered(value.serverSideRenderedTests), + } +} +export const runtimeContentSchema = runtimeInputSchema + .required({ ssrRequestThroughputTests: true }) + .transform((value) => ({ + ...formatRuntime(value), + ssrRequestThroughputTests: formatThroughput( + value.ssrRequestThroughputTests, + ), + })) +export const runtimeVersionsContentSchema = + runtimeInputSchema.transform(formatRuntime) diff --git a/packages/results/package.json b/packages/results/package.json new file mode 100644 index 000000000..15b0df099 --- /dev/null +++ b/packages/results/package.json @@ -0,0 +1,13 @@ +{ + "name": "@framework-tracker/results", + "version": "0.0.1", + "private": true, + "type": "module", + "exports": { + ".": "./src/schemas.ts", + "./selection": "./src/selection.ts" + }, + "dependencies": { + "zod": "catalog:" + } +} diff --git a/packages/stats-generator/src/schemas.ts b/packages/results/src/schemas.ts similarity index 79% rename from packages/stats-generator/src/schemas.ts rename to packages/results/src/schemas.ts index 7b51e3e38..d9c64b7e4 100644 --- a/packages/stats-generator/src/schemas.ts +++ b/packages/results/src/schemas.ts @@ -1,19 +1,14 @@ -import { selectRepresentativeIteration } from './ssrLoad/selection.ts' +import { selectRepresentativeIteration } from './selection.ts' import { z } from 'zod' -/** Summary of one metric; units belong to the metric being measured. */ export const MetricBaseSchema = z.object({ - /** Arithmetic mean of the measured values. */ mean: z.number(), median: z.number(), - /** Lowest measured value. */ min: z.number(), - /** Highest measured value. */ max: z.number(), }) export type MetricBase = z.infer -/** Framework identifiers configured in packages/stats-generator/src/frameworks.ts. */ export const FrameworkSchema = z.enum([ 'baseline-html', 'astro', @@ -26,7 +21,6 @@ export const FrameworkSchema = z.enum([ 'tanstack', ]) -/** Tested package folders configured in packages/stats-generator/src/frameworks.ts. */ export const PackagePathSchema = z.enum([ 'packages/app-baseline-html', 'packages/starter-astro', @@ -47,7 +41,6 @@ export const PackagePathSchema = z.enum([ 'packages/app-tanstack-start-react', ]) -/** Shared metadata for the overall result document. */ export const BaseResultSchema = z.object({ framework: FrameworkSchema, packagePath: PackagePathSchema, @@ -55,7 +48,6 @@ export const BaseResultSchema = z.object({ runner: z.string().min(1), }) -/** Identity shared by benchmarks and exact-value scans. */ export const TestBaseSchema = z.object({ /** Stable test identifier, such as install or ssrLoad. */ name: z.string().min(1), @@ -212,22 +204,20 @@ export const SSRRequestThroughputStatsSchema = ]), }) -export const SSRLoadStageSchema = z - .object({ - workers: z.number().int().positive(), - elapsedTimeMs: z.number().positive(), - requests: z.number().int().nonnegative(), - errors: z.number().int().nonnegative(), - requestsPerSec: z.number().nonnegative(), - avgLatencyMs: z.number().nonnegative(), - p50LatencyMs: z.number().nonnegative(), - p75LatencyMs: z.number().nonnegative(), - p90LatencyMs: z.number().nonnegative(), - p99LatencyMs: z.number().nonnegative(), - maxLatencyMs: z.number().nonnegative(), - bytesPerSec: z.number().nonnegative(), - }) - .strict() +export const SSRLoadStageSchema = z.strictObject({ + workers: z.number().int().positive(), + elapsedTimeMs: z.number().positive(), + requests: z.number().int().nonnegative(), + errors: z.number().int().nonnegative(), + requestsPerSec: z.number().nonnegative(), + avgLatencyMs: z.number().nonnegative(), + p50LatencyMs: z.number().nonnegative(), + p75LatencyMs: z.number().nonnegative(), + p90LatencyMs: z.number().nonnegative(), + p99LatencyMs: z.number().nonnegative(), + maxLatencyMs: z.number().nonnegative(), + bytesPerSec: z.number().nonnegative(), +}) export const SSRLoadSweepSchema = z.array(SSRLoadStageSchema).nonempty() @@ -342,14 +332,14 @@ const InteractionTestsSchema = z.object({ presentationDelayMs: z.number().nonnegative(), }) -const InteractionTimingSchema = z.object({ +export const InteractionTimingSchema = z.object({ interactionLatencyMs: z.number().nonnegative(), inputDelayMs: z.number().nonnegative(), processingDurationMs: z.number().nonnegative(), presentationDelayMs: z.number().nonnegative(), }) -const RenderedTestSampleSchema = z.object({ +export const RenderedTestSampleSchema = z.object({ firstPaintMs: z.number().positive(), fcpMs: z.number().positive(), interactionTests: InteractionTimingSchema, @@ -459,3 +449,68 @@ export const CoreJsStatsSchema = TestBaseSchema.extend({ totalVendoredBytes: z.number().int().nonnegative(), unnecessaryModules: z.array(z.string()), }) + +export const FrameworkResultSchema = z.object({ + framework: FrameworkSchema.optional(), + packagePath: PackagePathSchema.optional(), + name: z.string().optional(), + type: z.enum(['starter-kit', 'runtime-app']).optional(), + package: z.string().optional(), + isFocused: z.boolean().optional(), + order: z.number().optional(), + timingMeasuredAt: z.string().optional(), + runner: z.string().optional(), + browserVersion: z.string().optional(), + frameworkVersion: z.string().optional(), + releasedAt: z.string().optional(), + install: InstallBenchmarkSchema.optional(), + coldBuildTime: BuildStatsSchema.shape.coldBuildTime.optional(), + warmBuildTime: BuildStatsSchema.shape.warmBuildTime.optional(), + devServerStartTime: DevServerBenchmarkSchema.optional(), + ssrRequestThroughputTests: z + .union([RequestThroughputBenchmarkSchema, LegacySSRRequestThroughputSchema]) + .optional(), + ssrLoadTests: SSRLoadTestsSchema.refine( + (value) => value.name === 'ssrLoad', + ).optional(), + ssrRouterLinkLoadTests: SSRLoadTestsSchema.refine( + (value) => value.name === 'ssrRouterLinkLoad', + ).optional(), + clientSideRenderedTests: z + .union([ClientRenderingBenchmarkSchema, LegacyRenderedTestsSchema]) + .optional(), + serverSideRenderedTests: z + .union([ServerRenderingBenchmarkSchema, LegacyRenderedTestsSchema]) + .optional(), + corejs: CoreJsStatsSchema.optional(), + nodeEngines: NodeEnginesStatsSchema.partial() + .required({ name: true, minimumNodeVersion: true, imposedBy: true }) + .optional(), + dependencyScan: DependencyScanSchema.optional(), + prodDependencies: z.number().nonnegative().optional(), + devDependencies: z.number().nonnegative().optional(), + allDependencies: z.number().nonnegative().optional(), + frameworkDependencies: DependencyCountsSchema.extend({ + name: z.literal('frameworkDependencies').optional(), + }).optional(), + firstPartyDependencies: FirstPartyDependencyScanSchema.extend({ + name: z.literal('firstPartyDependencies').optional(), + }).optional(), + duplicateDependencies: z.number().nonnegative().optional(), + depInstallSize: z.number().nonnegative().optional(), + vendoredCoreJsSize: z.number().nonnegative().optional(), + vendoredCoreJsUnnecessaryModules: z.array(z.string()).optional(), + browserBaselineTests: BrowserBaselineStatsSchema.extend({ + name: z.literal('browserBaseline').optional(), + }).optional(), + minimumNodeVersion: z.string().optional(), + minimumNodeVersionImposedBy: z.array(z.string()).optional(), + testTimeMs: z.number().optional(), + packageJson: z + .object({ + dependencies: z.record(z.string(), z.string()).optional(), + devDependencies: z.record(z.string(), z.string()).optional(), + }) + .optional(), +}) +export type FrameworkResult = z.infer diff --git a/packages/stats-generator/src/ssrLoad/selection.ts b/packages/results/src/selection.ts similarity index 100% rename from packages/stats-generator/src/ssrLoad/selection.ts rename to packages/results/src/selection.ts diff --git a/packages/stats-generator/package.json b/packages/stats-generator/package.json index 0c62f7572..9a6647043 100644 --- a/packages/stats-generator/package.json +++ b/packages/stats-generator/package.json @@ -42,7 +42,8 @@ "tinyglobby": "^0.2.17", "verkit": "^0.4.0", "yaml": "2.9.0", - "zod": "^4.4.3" + "zod": "catalog:", + "@framework-tracker/results": "workspace:*" }, "devDependencies": { "@types/autocannon": "^7.12.7", diff --git a/packages/stats-generator/src/build-statistics.test.ts b/packages/stats-generator/src/build-statistics.test.ts index 691aa38c2..c9fab9c27 100644 --- a/packages/stats-generator/src/build-statistics.test.ts +++ b/packages/stats-generator/src/build-statistics.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict' import test from 'node:test' import { createBuildStats } from './build-statistics.ts' -import { BuildStatsSchema } from './schemas.ts' +import { BuildStatsSchema } from '@framework-tracker/results' test('build schema requires identity and complete iterations', () => { const result = createBuildStats([10, 20, 30], [5]) diff --git a/packages/stats-generator/src/build-statistics.ts b/packages/stats-generator/src/build-statistics.ts index a830a70a3..2f1c3e1e1 100644 --- a/packages/stats-generator/src/build-statistics.ts +++ b/packages/stats-generator/src/build-statistics.ts @@ -1,5 +1,5 @@ import { summarizeMetric } from './sample-statistics.ts' -import { BuildStatsSchema } from './schemas.ts' +import { BuildStatsSchema } from '@framework-tracker/results' export function createBuildStats(cold: number[], warm: number[]) { function benchmark(name: 'coldBuild' | 'warmBuild', samples: number[]) { diff --git a/packages/stats-generator/src/client-rendering.test.ts b/packages/stats-generator/src/client-rendering.test.ts index 0b3268c7e..d963fed6c 100644 --- a/packages/stats-generator/src/client-rendering.test.ts +++ b/packages/stats-generator/src/client-rendering.test.ts @@ -4,7 +4,7 @@ import { assembleClientRendering } from './client-rendering.ts' import { ClientRenderingBenchmarkSchema, ClientSideRenderedStatsSchema, -} from './schemas.ts' +} from '@framework-tracker/results' test('assembles client rendering from exact raw browser measurements', () => { const iterations = [1, 3].map((value) => ({ diff --git a/packages/stats-generator/src/client-rendering.ts b/packages/stats-generator/src/client-rendering.ts index 2558901b9..d2d4dfeeb 100644 --- a/packages/stats-generator/src/client-rendering.ts +++ b/packages/stats-generator/src/client-rendering.ts @@ -1,4 +1,4 @@ -import { ClientRenderingBenchmarkSchema } from './schemas.ts' +import { ClientRenderingBenchmarkSchema } from '@framework-tracker/results' import { summarizeMetric } from './sample-statistics.ts' import type { RenderedTestSample } from './types.ts' diff --git a/packages/stats-generator/src/clientSideRendered/types.ts b/packages/stats-generator/src/clientSideRendered/types.ts index dc06ad556..58c103405 100644 --- a/packages/stats-generator/src/clientSideRendered/types.ts +++ b/packages/stats-generator/src/clientSideRendered/types.ts @@ -1,5 +1,5 @@ import type { z } from 'zod' -import type { ClientRenderingBenchmarkSchema } from '../schemas.ts' +import type { ClientRenderingBenchmarkSchema } from '@framework-tracker/results' import type { InteractionTiming } from '../types.ts' export interface ClientSideRenderedRunResult { diff --git a/packages/stats-generator/src/dev-server-statistics.test.ts b/packages/stats-generator/src/dev-server-statistics.test.ts index 115697f70..1bde11b49 100644 --- a/packages/stats-generator/src/dev-server-statistics.test.ts +++ b/packages/stats-generator/src/dev-server-statistics.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict' import test from 'node:test' import { createDevServerStats } from './dev-server-statistics.ts' -import { DevServerStatsSchema } from './schemas.ts' +import { DevServerStatsSchema } from '@framework-tracker/results' test('startup results retain iterations and summarize measurements', () => { assert.deepEqual(createDevServerStats([10, 20, 30]), { diff --git a/packages/stats-generator/src/dev-server-statistics.ts b/packages/stats-generator/src/dev-server-statistics.ts index ed6424caa..991f9a964 100644 --- a/packages/stats-generator/src/dev-server-statistics.ts +++ b/packages/stats-generator/src/dev-server-statistics.ts @@ -1,5 +1,5 @@ import { summarizeMetric } from './sample-statistics.ts' -import { DevServerStatsSchema } from './schemas.ts' +import { DevServerStatsSchema } from '@framework-tracker/results' export function createDevServerStats(samples: number[]) { return DevServerStatsSchema.parse({ diff --git a/packages/stats-generator/src/final-result.test.ts b/packages/stats-generator/src/final-result.test.ts new file mode 100644 index 000000000..4b67c4353 --- /dev/null +++ b/packages/stats-generator/src/final-result.test.ts @@ -0,0 +1,48 @@ +import assert from 'node:assert/strict' +import { globSync, readFileSync } from 'node:fs' +import { join } from 'node:path' +import { test } from 'node:test' +import { FrameworkResultSchema } from '@framework-tracker/results' +import { packagesDir } from './constants.ts' + +test('shared final contract accepts every persisted result and history', () => { + const files = [ + ...globSync( + [ + '*/ci-stats.json', + '*/stats/*.json', + 'docs/src/content/devtime/**/*.json', + 'docs/src/content/runtime/**/*.json', + ], + { cwd: packagesDir }, + ), + ] + assert.ok(files.length > 0) + for (const file of files) { + const result = FrameworkResultSchema.safeParse( + JSON.parse(readFileSync(join(packagesDir, file), 'utf8')), + ) + assert.ok(result.success, `${file}: ${result.error?.message}`) + } +}) + +test('final contract validates nested benchmarks and test identity', () => { + assert.equal( + FrameworkResultSchema.safeParse({ + install: { + name: 'install', + runs: 2, + iterations: [{ installTimeMs: 10 }], + installTimeMs: { mean: 10, median: 10, min: 10, max: 10 }, + nodeModulesSizeBytes: 100, + }, + }).success, + false, + ) + assert.equal( + FrameworkResultSchema.safeParse({ + ssrRouterLinkLoadTests: { name: 'ssrLoad' }, + }).success, + false, + ) +}) diff --git a/packages/stats-generator/src/request-throughput.ts b/packages/stats-generator/src/request-throughput.ts index af8c9df70..918b2bfa6 100644 --- a/packages/stats-generator/src/request-throughput.ts +++ b/packages/stats-generator/src/request-throughput.ts @@ -4,7 +4,7 @@ import { RequestThroughputIterationSchema, RequestThroughputResultSchema, SSRRequestThroughputStatsSchema, -} from './schemas.ts' +} from '@framework-tracker/results' import { summarizeMetric } from './sample-statistics.ts' import type { CIStats } from './types.ts' diff --git a/packages/stats-generator/src/run-browser-baseline-scan.ts b/packages/stats-generator/src/run-browser-baseline-scan.ts index 8840747e4..ce2c7f104 100644 --- a/packages/stats-generator/src/run-browser-baseline-scan.ts +++ b/packages/stats-generator/src/run-browser-baseline-scan.ts @@ -1,4 +1,4 @@ -import { BrowserBaselineStatsSchema } from './schemas.ts' +import { BrowserBaselineStatsSchema } from '@framework-tracker/results' import { access, copyFile, mkdir, mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { dirname, join } from 'node:path' diff --git a/packages/stats-generator/src/run-client-side-rendered-benchmark.ts b/packages/stats-generator/src/run-client-side-rendered-benchmark.ts index bdfdd6aa7..f81ca7500 100644 --- a/packages/stats-generator/src/run-client-side-rendered-benchmark.ts +++ b/packages/stats-generator/src/run-client-side-rendered-benchmark.ts @@ -1,6 +1,6 @@ import { parseRunFrequency } from './benchmark-utils.ts' import { measurements } from './measurements.ts' -import { ClientSideRenderedStatsSchema } from './schemas.ts' +import { ClientSideRenderedStatsSchema } from '@framework-tracker/results' import { join } from 'node:path' import { runClientSideRenderedBenchmark } from './clientSideRendered/index.ts' import { packagesDir } from './constants.ts' diff --git a/packages/stats-generator/src/run-corejs-scan.ts b/packages/stats-generator/src/run-corejs-scan.ts index dee327def..6f87d8c52 100644 --- a/packages/stats-generator/src/run-corejs-scan.ts +++ b/packages/stats-generator/src/run-corejs-scan.ts @@ -1,4 +1,4 @@ -import { CoreJsStatsSchema } from './schemas.ts' +import { CoreJsStatsSchema } from '@framework-tracker/results' import { readFileSync, statSync } from 'node:fs' import { join, relative } from 'node:path' import { glob } from 'tinyglobby' diff --git a/packages/stats-generator/src/run-first-party-dependency-scan.ts b/packages/stats-generator/src/run-first-party-dependency-scan.ts index 935d7f745..66329c304 100644 --- a/packages/stats-generator/src/run-first-party-dependency-scan.ts +++ b/packages/stats-generator/src/run-first-party-dependency-scan.ts @@ -1,4 +1,4 @@ -import { FirstPartyDependencyScanSchema } from './schemas.ts' +import { FirstPartyDependencyScanSchema } from '@framework-tracker/results' import { readFile } from 'node:fs/promises' import { join } from 'node:path' import { diff --git a/packages/stats-generator/src/run-framework-dependency-scan.ts b/packages/stats-generator/src/run-framework-dependency-scan.ts index 4a4d182a5..a6f571c5b 100644 --- a/packages/stats-generator/src/run-framework-dependency-scan.ts +++ b/packages/stats-generator/src/run-framework-dependency-scan.ts @@ -1,4 +1,4 @@ -import { FrameworkDependencyScanSchema } from './schemas.ts' +import { FrameworkDependencyScanSchema } from '@framework-tracker/results' import { join } from 'node:path' import { packagesDir } from './constants.ts' import { diff --git a/packages/stats-generator/src/run-install-benchmark.ts b/packages/stats-generator/src/run-install-benchmark.ts index f5d62296e..6f2582c6f 100644 --- a/packages/stats-generator/src/run-install-benchmark.ts +++ b/packages/stats-generator/src/run-install-benchmark.ts @@ -12,7 +12,10 @@ import { getFrameworkByPackage, parseArgs, } from './utils.ts' -import { InstallBenchmarkSchema, InstallResultSchema } from './schemas.ts' +import { + InstallBenchmarkSchema, + InstallResultSchema, +} from '@framework-tracker/results' function execCommand(command: string, cwd: string): string { return execSync(command, { diff --git a/packages/stats-generator/src/run-node-engines-scan.ts b/packages/stats-generator/src/run-node-engines-scan.ts index 457eb672f..19271f3f8 100644 --- a/packages/stats-generator/src/run-node-engines-scan.ts +++ b/packages/stats-generator/src/run-node-engines-scan.ts @@ -1,4 +1,4 @@ -import { NodeEnginesStatsSchema } from './schemas.ts' +import { NodeEnginesStatsSchema } from '@framework-tracker/results' import { lstatSync, readdirSync, readFileSync } from 'node:fs' import { join, relative } from 'node:path' import { packagesDir } from './constants.ts' diff --git a/packages/stats-generator/src/run-server-side-rendered-benchmark.ts b/packages/stats-generator/src/run-server-side-rendered-benchmark.ts index 62db9d167..345091e09 100644 --- a/packages/stats-generator/src/run-server-side-rendered-benchmark.ts +++ b/packages/stats-generator/src/run-server-side-rendered-benchmark.ts @@ -1,6 +1,6 @@ import { parseRunFrequency } from './benchmark-utils.ts' import { measurements } from './measurements.ts' -import { ServerSideRenderedStatsSchema } from './schemas.ts' +import { ServerSideRenderedStatsSchema } from '@framework-tracker/results' import { join } from 'node:path' import { runServerSideRenderedBenchmark } from './serverSideRendered/index.ts' import { packagesDir } from './constants.ts' diff --git a/packages/stats-generator/src/run-ssr-load-benchmark.ts b/packages/stats-generator/src/run-ssr-load-benchmark.ts index 21967c23b..499312acc 100644 --- a/packages/stats-generator/src/run-ssr-load-benchmark.ts +++ b/packages/stats-generator/src/run-ssr-load-benchmark.ts @@ -1,4 +1,4 @@ -import { getLoadPeakStage } from './ssrLoad/selection.ts' +import { getLoadPeakStage } from '@framework-tracker/results/selection' import { validateSSRLoadForPersistence } from './ssrLoad/persistence.ts' import { getSSRLoadKind, getSSRLoadStatsKey } from './ssrLoad/config.ts' import { join } from 'node:path' diff --git a/packages/stats-generator/src/run-ssr-request-throughput-benchmark.ts b/packages/stats-generator/src/run-ssr-request-throughput-benchmark.ts index bdf645cf6..c197b3807 100644 --- a/packages/stats-generator/src/run-ssr-request-throughput-benchmark.ts +++ b/packages/stats-generator/src/run-ssr-request-throughput-benchmark.ts @@ -1,6 +1,6 @@ import { measurements } from './measurements.ts' import { parseRunFrequency } from './benchmark-utils.ts' -import { RequestThroughputResultSchema } from './schemas.ts' +import { RequestThroughputResultSchema } from '@framework-tracker/results' import { join } from 'node:path' import { runSSRRequestThroughputBenchmark, diff --git a/packages/stats-generator/src/sample-statistics.ts b/packages/stats-generator/src/sample-statistics.ts index ed2b25c0a..29e0dfac9 100644 --- a/packages/stats-generator/src/sample-statistics.ts +++ b/packages/stats-generator/src/sample-statistics.ts @@ -1,4 +1,4 @@ -import type { MetricBase } from './schemas.ts' +import type { MetricBase } from '@framework-tracker/results' import type { TimeStat } from './types.ts' function round(value: number, decimalPlaces: number): number { diff --git a/packages/stats-generator/src/save-ci-stats.ts b/packages/stats-generator/src/save-ci-stats.ts index 3dce72a46..17b9397bb 100644 --- a/packages/stats-generator/src/save-ci-stats.ts +++ b/packages/stats-generator/src/save-ci-stats.ts @@ -1,22 +1,23 @@ -import { validateSSRLoadForPersistence } from './ssrLoad/persistence.ts' import { + FrameworkResultSchema, DependencyScanSchema, CoreJsStatsSchema, NodeEnginesStatsSchema, BrowserBaselineStatsSchema, FrameworkDependencyScanSchema, FirstPartyDependencyScanSchema, -} from './schemas.ts' + ServerSideRenderedStatsSchema, + ClientSideRenderedStatsSchema, + BuildStatsSchema, + DevServerStatsSchema, + BaseResultSchema, + InstallResultSchema, +} from '@framework-tracker/results' +import { validateSSRLoadForPersistence } from './ssrLoad/persistence.ts' import { mergeRequestThroughputArtifact, validateRequestThroughputForPersistence, } from './request-throughput.ts' -import { - ServerSideRenderedStatsSchema, - ClientSideRenderedStatsSchema, -} from './schemas.ts' -import { BuildStatsSchema, DevServerStatsSchema } from './schemas.ts' -import { BaseResultSchema, InstallResultSchema } from './schemas.ts' import { mergeSSRLoadArtifact } from './ssrLoad/merge.ts' import { join } from 'node:path' import { getFrameworks } from './get-frameworks.ts' @@ -332,6 +333,7 @@ async function main() { const ciStatsPath = join(packagesDir, packageName, 'ci-stats.json') validateSSRLoadForPersistence(stats) validateRequestThroughputForPersistence(stats) + FrameworkResultSchema.parse(stats) writeJsonFile(ciStatsPath, stats) console.info(` ✓ Saved ${ciStatsPath}`) @@ -522,6 +524,7 @@ async function main() { const ciStatsPath = join(packagesDir, packageName, 'ci-stats.json') validateSSRLoadForPersistence(stats) validateRequestThroughputForPersistence(stats) + FrameworkResultSchema.parse(stats) writeJsonFile(ciStatsPath, stats) console.info(` ✓ Saved ${ciStatsPath}`) diff --git a/packages/stats-generator/src/save-stats.ts b/packages/stats-generator/src/save-stats.ts index c8738187a..76c3fdefa 100644 --- a/packages/stats-generator/src/save-stats.ts +++ b/packages/stats-generator/src/save-stats.ts @@ -1,3 +1,4 @@ +import { FrameworkResultSchema } from '@framework-tracker/results' import { validateSSRLoadForPersistence } from './ssrLoad/persistence.ts' import { validateRequestThroughputForPersistence } from './request-throughput.ts' import { access, mkdir, readFile, readdir, writeFile } from 'node:fs/promises' @@ -64,6 +65,7 @@ async function saveVersionedStats( ) validateSSRLoadForPersistence(fileStats) validateRequestThroughputForPersistence(fileStats) + FrameworkResultSchema.parse(fileStats) const outputPath = join(outputDir, fileName) const versionedStats = { ...metadata, @@ -122,6 +124,7 @@ export async function saveStats( validateRequestThroughputForPersistence(mergedStats) const docsStats = getDocsStats(mergedStats) + FrameworkResultSchema.parse(docsStats) await writeFile(filePath, `${JSON.stringify(docsStats, null, 2)}\n`, 'utf-8') await saveVersionedStats(packageName, docsStats, collection) diff --git a/packages/stats-generator/src/schemas.test.ts b/packages/stats-generator/src/schemas.test.ts index 3648af701..0e9786df6 100644 --- a/packages/stats-generator/src/schemas.test.ts +++ b/packages/stats-generator/src/schemas.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict' import { test } from 'node:test' import { z } from 'zod' -import { createBenchmarkBaseSchema } from './schemas.ts' +import { createBenchmarkBaseSchema } from '@framework-tracker/results' test('common benchmark schema requires identity and one iteration per measured run', () => { const schema = createBenchmarkBaseSchema(z.object({ value: z.number() })) diff --git a/packages/stats-generator/src/server-rendering.test.ts b/packages/stats-generator/src/server-rendering.test.ts index 97e83ba17..76ab4958c 100644 --- a/packages/stats-generator/src/server-rendering.test.ts +++ b/packages/stats-generator/src/server-rendering.test.ts @@ -4,7 +4,7 @@ import { assembleServerRendering } from './server-rendering.ts' import { ServerRenderingBenchmarkSchema, ServerSideRenderedStatsSchema, -} from './schemas.ts' +} from '@framework-tracker/results' test('assembles server rendering from exact raw browser measurements', () => { const iterations = [1, 3].map((value) => ({ diff --git a/packages/stats-generator/src/server-rendering.ts b/packages/stats-generator/src/server-rendering.ts index 6626c3eb9..484b11b11 100644 --- a/packages/stats-generator/src/server-rendering.ts +++ b/packages/stats-generator/src/server-rendering.ts @@ -1,4 +1,4 @@ -import { ServerRenderingBenchmarkSchema } from './schemas.ts' +import { ServerRenderingBenchmarkSchema } from '@framework-tracker/results' import { summarizeMetric } from './sample-statistics.ts' import type { RenderedTestSample } from './types.ts' diff --git a/packages/stats-generator/src/serverSideRendered/types.ts b/packages/stats-generator/src/serverSideRendered/types.ts index 586105f9b..7993e8e37 100644 --- a/packages/stats-generator/src/serverSideRendered/types.ts +++ b/packages/stats-generator/src/serverSideRendered/types.ts @@ -1,5 +1,5 @@ import type { z } from 'zod' -import type { ServerRenderingBenchmarkSchema } from '../schemas.ts' +import type { ServerRenderingBenchmarkSchema } from '@framework-tracker/results' import type { InteractionTiming } from '../types.ts' export interface ServerSideRenderedRunResult { firstPaintMs: number | null diff --git a/packages/stats-generator/src/ssr-load.test.ts b/packages/stats-generator/src/ssr-load.test.ts index d9a3f9228..84e668b2c 100644 --- a/packages/stats-generator/src/ssr-load.test.ts +++ b/packages/stats-generator/src/ssr-load.test.ts @@ -1,7 +1,7 @@ -import { getLoadPeakStage } from './ssrLoad/selection.ts' +import { getLoadPeakStage } from '@framework-tracker/results/selection' import { normalizeCIStats } from './utils.ts' import { validateSSRLoadForPersistence } from './ssrLoad/persistence.ts' -import { SSRLoadTestsSchema } from './schemas.ts' +import { SSRLoadTestsSchema } from '@framework-tracker/results' import assert from 'node:assert/strict' import { readFileSync } from 'node:fs' import { test } from 'node:test' diff --git a/packages/stats-generator/src/ssrLoad/merge.ts b/packages/stats-generator/src/ssrLoad/merge.ts index 486645ee7..8bdc3555d 100644 --- a/packages/stats-generator/src/ssrLoad/merge.ts +++ b/packages/stats-generator/src/ssrLoad/merge.ts @@ -1,6 +1,9 @@ import { validateSSRLoadForPersistence } from './persistence.ts' import type { CIStats } from '../types.ts' -import { SSRLoadStatsSchema, SSRRouterLinkLoadStatsSchema } from '../schemas.ts' +import { + SSRLoadStatsSchema, + SSRRouterLinkLoadStatsSchema, +} from '@framework-tracker/results' import { getSSRLoadStatsKey, type SSRLoadKind } from './config.ts' /** Jobs upload a full ci-stats file, including possibly stale sibling results. diff --git a/packages/stats-generator/src/ssrLoad/persistence.ts b/packages/stats-generator/src/ssrLoad/persistence.ts index 391a2626e..d30b1a4c2 100644 --- a/packages/stats-generator/src/ssrLoad/persistence.ts +++ b/packages/stats-generator/src/ssrLoad/persistence.ts @@ -1,5 +1,8 @@ import { isDeepStrictEqual } from 'node:util' -import { SSRLoadStatsSchema, SSRRouterLinkLoadStatsSchema } from '../schemas.ts' +import { + SSRLoadStatsSchema, + SSRRouterLinkLoadStatsSchema, +} from '@framework-tracker/results' import type { CIStats } from '../types.ts' import { summarizeLoadSweeps } from './run-load-test.ts' diff --git a/packages/stats-generator/src/ssrLoad/run-load-test.ts b/packages/stats-generator/src/ssrLoad/run-load-test.ts index f7d1e9bc2..101c7ad9e 100644 --- a/packages/stats-generator/src/ssrLoad/run-load-test.ts +++ b/packages/stats-generator/src/ssrLoad/run-load-test.ts @@ -1,6 +1,9 @@ import { measurements } from '../measurements.ts' -import { getLoadPeakStage, selectRepresentativeIteration } from './selection.ts' -import { SSRLoadTestsSchema } from '../schemas.ts' +import { + getLoadPeakStage, + selectRepresentativeIteration, +} from '@framework-tracker/results/selection' +import { SSRLoadTestsSchema } from '@framework-tracker/results' import { summarizeMetric } from '../sample-statistics.ts' import { getSSRLoadKind } from './config.ts' import { setTimeout as delay } from 'node:timers/promises' diff --git a/packages/stats-generator/src/ssrLoad/types.ts b/packages/stats-generator/src/ssrLoad/types.ts index 12fa38af0..a45471fdd 100644 --- a/packages/stats-generator/src/ssrLoad/types.ts +++ b/packages/stats-generator/src/ssrLoad/types.ts @@ -1,5 +1,8 @@ import type { z } from 'zod' -import type { SSRLoadTestsSchema, SSRLoadSweepSchema } from '../schemas.ts' +import type { + SSRLoadTestsSchema, + SSRLoadSweepSchema, +} from '@framework-tracker/results' export type SSRLoadStageStats = z.infer[number] export type SSRLoadSweep = z.infer diff --git a/packages/stats-generator/src/ssrRequestThroughput/types.ts b/packages/stats-generator/src/ssrRequestThroughput/types.ts index e88f02817..f8591d8d3 100644 --- a/packages/stats-generator/src/ssrRequestThroughput/types.ts +++ b/packages/stats-generator/src/ssrRequestThroughput/types.ts @@ -1,5 +1,5 @@ import type { z } from 'zod' -import type { RequestThroughputBenchmarkSchema } from '../schemas.ts' +import type { RequestThroughputBenchmarkSchema } from '@framework-tracker/results' import type { IncomingMessage, ServerResponse } from './mock-http.ts' export type WebServerRenderHandler = (request: Request) => Promise diff --git a/packages/stats-generator/src/types.ts b/packages/stats-generator/src/types.ts index 4b1e7e1f9..90a05208b 100644 --- a/packages/stats-generator/src/types.ts +++ b/packages/stats-generator/src/types.ts @@ -14,7 +14,7 @@ import type { DevServerStatsSchema, InstallBenchmarkSchema, InstallResultSchema, -} from './schemas.ts' +} from '@framework-tracker/results' import type { SSRLoadTests } from './ssrLoad/types.ts' import type { MeasurementName } from './measurements.ts' diff --git a/packages/stats-generator/src/validate-stats.ts b/packages/stats-generator/src/validate-stats.ts index 60a475801..b73aa7685 100644 --- a/packages/stats-generator/src/validate-stats.ts +++ b/packages/stats-generator/src/validate-stats.ts @@ -19,7 +19,7 @@ import { SSRRouterLinkLoadStatsSchema, ClientSideRenderedStatsSchema, ServerSideRenderedStatsSchema, -} from './schemas.ts' +} from '@framework-tracker/results' type BenchmarkType = | 'dependencies' @@ -37,7 +37,7 @@ type BenchmarkType = interface BenchmarkConfig { type: BenchmarkType file: string - schema: z.ZodSchema + schema: z.ZodType } const STARTER_BENCHMARKS: BenchmarkConfig[] = [ @@ -104,7 +104,7 @@ const APP_BENCHMARKS: BenchmarkConfig[] = [ }, ] -function validateFile(filePath: string, schema: z.ZodSchema): string[] { +function validateFile(filePath: string, schema: z.ZodType): string[] { const data = readJsonFile(filePath) if (!data) { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index df90ce1fc..9c16217c8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4,6 +4,12 @@ settings: autoInstallPeers: true excludeLinksFromLockfile: false +catalogs: + default: + zod: + specifier: 4.6.7 + version: 4.6.7 + importers: .: @@ -39,8 +45,8 @@ importers: packages/cwv-stats: dependencies: zod: - specifier: ^4.4.3 - version: 4.4.3 + specifier: 'catalog:' + version: 4.6.7 devDependencies: '@types/node': specifier: ^26.1.2 @@ -54,6 +60,9 @@ importers: '@astrojs/starlight': specifier: ^0.41.7 version: 0.41.7(astro@7.4.0(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.1))(typescript@6.0.3) + '@framework-tracker/results': + specifier: workspace:* + version: link:../results '@tanstack/charts': specifier: ^1.1.0 version: 1.1.0 @@ -77,8 +86,17 @@ importers: specifier: ^2.1.1 version: 2.1.1(@typescript-eslint/parser@8.62.1(eslint@10.6.0(jiti@2.7.0))(typescript@6.0.3))(eslint@10.6.0(jiti@2.7.0)) + packages/results: + dependencies: + zod: + specifier: 'catalog:' + version: 4.6.7 + packages/stats-generator: dependencies: + '@framework-tracker/results': + specifier: workspace:* + version: link:../results autocannon: specifier: ^8.0.0 version: 8.0.0 @@ -107,8 +125,8 @@ importers: specifier: 2.9.0 version: 2.9.0 zod: - specifier: ^4.4.3 - version: 4.4.3 + specifier: 'catalog:' + version: 4.6.7 devDependencies: '@types/autocannon': specifier: ^7.12.7 @@ -4419,9 +4437,6 @@ packages: zod@3.25.76: resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} - zod@4.4.3: - resolution: {integrity: sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==} - zod@4.6.7: resolution: {integrity: sha512-g8W2D2kU6a8untc053fX1uVBKzM7yqAxW/3zwk83+6RyF2+SKOuV+c0wBDhUwzV1PaGapmZH+rmcUzPf+3CGwg==} @@ -4680,7 +4695,7 @@ snapshots: dependencies: sitemap: 9.0.1 stream-replace-string: 2.0.0 - zod: 4.4.3 + zod: 4.6.7 '@astrojs/starlight@0.41.7(astro@7.4.0(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.1))(typescript@6.0.3)': dependencies: @@ -7323,7 +7338,7 @@ snapshots: tinyglobby: 0.2.17 unbash: 4.0.11 yaml: 2.9.0 - zod: 4.4.3 + zod: 4.6.7 legacy-javascript@0.0.3: {} @@ -9124,8 +9139,6 @@ snapshots: zod@3.25.76: {} - zod@4.4.3: {} - zod@4.6.7: {} zwitch@2.0.4: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index dc6a2e90e..668c78fbd 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,4 +1,5 @@ packages: + - 'packages/results' - 'packages/docs' - 'packages/stats-generator' - 'packages/cwv-stats' @@ -10,3 +11,6 @@ allowBuilds: minimumReleaseAgeExclude: - baseline-detector@0.4.1 + +catalog: + zod: 4.6.7