diff --git a/.github/scripts/measurement-control.ts b/.github/scripts/measurement-control.ts new file mode 100644 index 00000000..cfc87b23 --- /dev/null +++ b/.github/scripts/measurement-control.ts @@ -0,0 +1,330 @@ +import { appendFileSync, readFileSync } from 'node:fs' +import { setTimeout } from 'node:timers/promises' +import { fileURLToPath } from 'node:url' + +interface Run { + id: number + run_attempt: number + name: string + path: string + event: string + status: string + conclusion: string | null + display_title: string + head_sha: string +} +interface PullRequest { + state: string + base: { ref: string; repo: { full_name: string } } + head: { sha: string } +} +interface RelayEvent { + action: string + repository: { default_branch: string } + workflow_run: Run +} +interface RequestOptions { + method?: 'GET' | 'POST' + body?: unknown +} +export type GitHubApi = ( + path: string, + options?: RequestOptions, +) => Promise + +class ApiError extends Error { + status: number + constructor(status: number, message: string) { + super(message) + this.status = status + } +} + +export function githubApi( + token: string, + apiUrl = 'https://api.github.com', + request: typeof fetch = fetch, +): GitHubApi { + return async (path: string, options: RequestOptions = {}): Promise => { + const response = await request(`${apiUrl}${path}`, { + method: options.method ?? 'GET', + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + 'User-Agent': 'framework-tracker-measurements', + 'X-GitHub-Api-Version': '2026-03-10', + }, + body: + options.body === undefined ? undefined : JSON.stringify(options.body), + signal: AbortSignal.timeout(30_000), + }) + const text = await response.text() + if (!response.ok) + throw new ApiError( + response.status, + `GitHub API ${response.status}: ${text.slice(0, 500)}`, + ) + return (text ? JSON.parse(text) : undefined) as T + } +} + +export function workerTitle(run: Pick) { + return `PR measurements ${run.id}-${run.run_attempt}` +} + +async function matchingRuns( + api: GitHubApi, + repository: string, + workflow: string, + event: string, + title: string, +): Promise { + const matches: Run[] = [] + for (let page = 1; page <= 10; page++) { + let data: { workflow_runs: Run[] } + try { + data = await api( + `/repos/${repository}/actions/workflows/${workflow}/runs?event=${event}&per_page=100&page=${page}`, + ) + } catch (error) { + if (error instanceof ApiError && error.status === 404) + throw new Error( + 'The measurement worker and relay must be installed on main before this PR check can run', + { cause: error }, + ) + throw error + } + matches.push( + ...data.workflow_runs.filter((run) => run.display_title === title), + ) + if (data.workflow_runs.length < 100) return matches + } + throw new Error( + 'Too many recent workflow runs to identify the measurement worker safely', + ) +} + +async function cancel(api: GitHubApi, repository: string, id: number) { + try { + await api(`/repos/${repository}/actions/runs/${id}/cancel`, { + method: 'POST', + }) + } catch (error) { + // A worker may finish between checking its status and requesting cancellation. + if (!(error instanceof ApiError) || error.status !== 409) throw error + } +} + +export async function relay( + api: GitHubApi, + repository: string, + event: RelayEvent, +) { + const parent = event.workflow_run + if ( + parent.event !== 'pull_request' || + parent.name !== 'Measure PR' || + parent.path !== '.github/workflows/measure-pr.yml' + ) + throw new Error('Unexpected PR controller workflow') + const title = workerTitle(parent) + const workers = await matchingRuns( + api, + repository, + 'measure.yml', + 'workflow_dispatch', + title, + ) + if (event.action === 'completed') { + if (parent.conclusion !== 'success') + for (const worker of workers) + if (worker.status !== 'completed') + await cancel(api, repository, worker.id) + return + } + if (event.action !== 'in_progress') + throw new Error('Unexpected controller event') + if (workers.length) return // A repeated delivery must not start duplicate benchmarks. + + const current = await api( + `/repos/${repository}/actions/runs/${parent.id}`, + ) + if ( + current.run_attempt !== parent.run_attempt || + current.status === 'completed' + ) + return + const match = /^Measurements for PR #([1-9]\d*) at ([a-f0-9]{40})$/.exec( + parent.display_title, + ) + if (!match) + throw new Error( + 'Missing PR number or exact merge commit in the controller run name', + ) + const [, number, mergeSha] = match + const pr = await api(`/repos/${repository}/pulls/${number}`) + const commit = await api<{ parents: { sha: string }[] }>( + `/repos/${repository}/commits/${mergeSha}`, + ) + if ( + pr.state !== 'open' || + pr.base.ref !== 'main' || + pr.base.repo.full_name !== repository || + commit.parents.length !== 2 || + commit.parents[1]?.sha !== pr.head.sha || + (parent.head_sha !== pr.head.sha && parent.head_sha !== mergeSha) + ) + throw new Error( + 'The PR changed or closed before its measurements could start; rerun the current PR check', + ) + // Other PRs may advance main while this run is queued. Keep measuring this + // run's merge snapshot, provided its base remains part of the target branch. + const comparison = await api<{ merge_base_commit: { sha: string } }>( + `/repos/${repository}/compare/${commit.parents[0]!.sha}...${pr.base.ref}`, + ) + if (comparison.merge_base_commit.sha !== commit.parents[0]!.sha) + throw new Error( + 'The measurement merge commit is not based on the PR target branch', + ) + + // The workflow definition always comes from the trusted default branch. Only + // read-only benchmark jobs check out the pinned PR merge commit, including forks. + const dispatched = await api<{ workflow_run_id: number }>( + `/repos/${repository}/actions/workflows/measure.yml/dispatches`, + { + method: 'POST', + body: { + ref: event.repository.default_branch, + inputs: { + 'source-sha': mergeSha, + 'base-sha': commit.parents[0]!.sha, + 'pr-number': number, + 'controller-run-id': String(parent.id), + 'controller-run-attempt': String(parent.run_attempt), + }, + }, + }, + ) + if (!Number.isSafeInteger(dispatched?.workflow_run_id)) + throw new Error('GitHub did not return the dispatched measurement run ID') + // Close the race where the controller is cancelled while dispatch is in flight. + const after = await api(`/repos/${repository}/actions/runs/${parent.id}`) + if (after.run_attempt !== parent.run_attempt || after.status === 'completed') + await cancel(api, repository, dispatched.workflow_run_id) +} + +interface WaitOptions { + sleep?: (milliseconds: number) => Promise + now?: () => number + startupTimeoutMs?: number + timeoutMs?: number + report?: (message: string) => void +} + +export async function waitForMeasurements( + api: GitHubApi, + repository: string, + parent: Pick, + options: WaitOptions = {}, +) { + const sleep = options.sleep ?? setTimeout + const now = options.now ?? Date.now + const report = options.report ?? console.info + const started = now() + const title = workerTitle(parent) + let worker: Run | undefined + while (now() - started < (options.timeoutMs ?? 355 * 60_000)) { + if (!worker) { + const matches = await matchingRuns( + api, + repository, + 'measure.yml', + 'workflow_dispatch', + title, + ) + if (matches.length > 1) + throw new Error(`Multiple measurement workers found for ${title}`) + worker = matches[0] + if (worker) + report( + `Measurement details: https://github.com/${repository}/actions/runs/${worker.id}`, + ) + else { + const relays = await matchingRuns( + api, + repository, + 'measure-pr-dispatch.yml', + 'workflow_run', + `Measurements dispatch ${parent.id}-${parent.run_attempt}`, + ) + const failed = relays.find( + (run) => run.status === 'completed' && run.conclusion !== 'success', + ) + if (failed) + throw new Error( + `Measurement dispatch failed: https://github.com/${repository}/actions/runs/${failed.id}`, + ) + if (now() - started >= (options.startupTimeoutMs ?? 10 * 60_000)) + throw new Error( + 'Measurement worker did not start; check that the relay and worker workflows are enabled on main', + ) + } + } + if (worker) { + worker = await api(`/repos/${repository}/actions/runs/${worker.id}`) + if (worker.status === 'completed') { + if (worker.conclusion !== 'success') + throw new Error( + `Measurements or validation ${worker.conclusion}: https://github.com/${repository}/actions/runs/${worker.id}`, + ) + report( + 'All selected measurements and complete-result validation passed.', + ) + return worker + } + } + await sleep(15_000) + } + throw new Error( + 'Timed out waiting for measurements; the relay will cancel the worker when this check finishes', + ) +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + try { + const repository = process.env.GITHUB_REPOSITORY! + if (!/^[\w.-]+\/[\w.-]+$/.test(repository)) + throw new Error('Invalid GitHub repository') + const token = process.env.GITHUB_TOKEN + if (!token) throw new Error('Missing GitHub token') + const api = githubApi(token, process.env.GITHUB_API_URL) + if (process.argv[2] === 'relay') + await relay( + api, + repository, + JSON.parse(readFileSync(process.env.GITHUB_EVENT_PATH!, 'utf8')), + ) + else if (process.argv[2] === 'wait') { + const parent = { + id: Number(process.env.GITHUB_RUN_ID), + run_attempt: Number(process.env.GITHUB_RUN_ATTEMPT), + } + if ( + !Number.isSafeInteger(parent.id) || + !Number.isSafeInteger(parent.run_attempt) + ) + throw new Error('Missing controller run ID or attempt') + await waitForMeasurements(api, repository, parent, { + report(message) { + console.info(message) + if (process.env.GITHUB_STEP_SUMMARY) + appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${message}\n\n`) + }, + }) + } else throw new Error('Expected wait or relay') + } catch (error) { + console.error(error) + process.exitCode = 1 + } +} diff --git a/.github/workflows/generate-stats.yml b/.github/workflows/generate-stats.yml deleted file mode 100644 index 20483a31..00000000 --- a/.github/workflows/generate-stats.yml +++ /dev/null @@ -1,156 +0,0 @@ -name: Generate Stats - -env: - RUNNER_LABEL: 'Depot depot-ubuntu-24.04 GitHub Actions runner (2 CPUs, 8 GB RAM, 100 GB disk, 2 GB disk accelerator)' - -on: - workflow_dispatch: - push: - branches: [main] - paths: - - 'packages/starter-*/**' - - '!packages/starter-*/ci-stats.json' - - '!packages/starter-*/stats/**' - - '!packages/starter-*/install-stats.json' - - '!packages/starter-*/build-stats.json' - - '!packages/starter-*/dev-server-stats.json' - - '!packages/starter-*/corejs-stats.json' - - '!packages/starter-*/browser-baseline-stats.json' - - '!packages/starter-*/e18e-stats.json' - - '!packages/starter-*/node-engines-stats.json' - - 'packages/app-*/**' - - '!packages/app-*/ci-stats.json' - - '!packages/app-*/stats/**' - - 'packages/stats-generator/**' - - '.github/workflows/generate-stats.yml' - - '.github/workflows/measure-framework.yml' - - 'packages/results/**' - - 'packages/stats-generator/src/frameworks.ts' - - 'package.json' - -jobs: - setup: - runs-on: depot-ubuntu-24.04 - outputs: - install-matrix: ${{ steps.set-matrix.outputs.install }} - build-matrix: ${{ steps.set-matrix.outputs.build }} - dev-server-matrix: ${{ steps.set-matrix.outputs.dev-server }} - ssr-request-throughput-matrix: ${{ steps.set-matrix.outputs.ssr-request-throughput }} - ssr-load-matrix: ${{ steps.set-matrix.outputs.ssr-load }} - ssr-router-link-load-matrix: ${{ steps.set-matrix.outputs.ssr-router-link-load }} - deps-matrix: ${{ steps.set-matrix.outputs.deps }} - client-side-rendered-matrix: ${{ steps.set-matrix.outputs.client-side-rendered }} - server-side-rendered-matrix: ${{ steps.set-matrix.outputs.server-side-rendered }} - steps: - - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Setup pnpm - uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5 - - - name: Setup Node.js - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: '24' - - - name: Install workspace dependencies - run: pnpm install --frozen-lockfile - - - name: Set matrices from TypeScript config - id: set-matrix - run: | - FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) - echo "install=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | contains(["install"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT - echo "build=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | contains(["build"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT - echo "dev-server=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | contains(["devServer"])) | {name, displayName, package: .starter.package, devServerPort: .starter.devServerPort, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT - echo "ssr-request-throughput=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | contains(["ssrRequestThroughput"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT - echo "ssr-load=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | contains(["ssrLoad"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT - echo "ssr-router-link-load=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | contains(["ssrRouterLinkLoad"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT - echo "deps=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | contains(["dependencies"])) | {name, displayName, package: .starter.package}]')" >> $GITHUB_OUTPUT - echo "client-side-rendered=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | contains(["clientSideRendered"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT - echo "server-side-rendered=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | contains(["serverSideRendered"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT - - measure: - needs: setup - uses: ./.github/workflows/measure-framework.yml - with: - install-matrix: ${{ needs.setup.outputs.install-matrix }} - build-matrix: ${{ needs.setup.outputs.build-matrix }} - dev-server-matrix: ${{ needs.setup.outputs.dev-server-matrix }} - ssr-request-throughput-matrix: ${{ needs.setup.outputs.ssr-request-throughput-matrix }} - ssr-load-matrix: ${{ needs.setup.outputs.ssr-load-matrix }} - ssr-router-link-load-matrix: ${{ needs.setup.outputs.ssr-router-link-load-matrix }} - deps-matrix: ${{ needs.setup.outputs.deps-matrix }} - client-side-rendered-matrix: ${{ needs.setup.outputs.client-side-rendered-matrix }} - server-side-rendered-matrix: ${{ needs.setup.outputs.server-side-rendered-matrix }} - - generate-stats: - needs: [setup, measure] - if: always() && needs.setup.result == 'success' && needs.measure.result == 'success' - runs-on: depot-ubuntu-24.04 - permissions: - contents: write - pull-requests: write - steps: - - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Setup pnpm - uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5 - - - name: Setup Node.js - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: '24' - cache: 'pnpm' - - - name: Install dependencies - run: pnpm install --frozen-lockfile - - - name: Download artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - path: artifacts - - - name: List downloaded artifacts - run: find artifacts -type f -name "*.json" 2>/dev/null || echo "No artifacts found" - - - name: Save CI stats - run: pnpm --filter @framework-tracker/stats-generator save:ci-stats $GITHUB_WORKSPACE/artifacts - - - name: Format - run: pnpm format - - - name: Create Pull Request - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - if [[ -z $(git status -s) ]]; then - echo "No changes to commit" - exit 0 - fi - - git checkout -B automated-stats-update - git add packages/ results/ - git commit -m "Update CI stats and generated stats after starter changes" - git push --force-with-lease origin automated-stats-update - - if ! gh pr list --head automated-stats-update --state open | grep -q .; then - gh pr create \ - --title "Update CI stats and generated stats after starter changes" \ - --body "Automated stats update triggered by changes to starter packages. - - Includes: - - Combined framework measurements (results//latest.json) - - Versioned stats history (results//versions/.json) - - Starter and app test results with project attribution - - This PR was automatically created and will auto-merge if all checks pass." \ - --base main \ - --head automated-stats-update - else - echo "PR already exists, updated with new commits" - fi - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/lint-and-type-check.yml b/.github/workflows/lint-and-type-check.yml index cc2152f8..0bacd65d 100644 --- a/.github/workflows/lint-and-type-check.yml +++ b/.github/workflows/lint-and-type-check.yml @@ -31,3 +31,6 @@ jobs: - name: Run stats generator tests run: pnpm --filter @framework-tracker/stats-generator test + + - name: Validate first-party dependency manifests + run: pnpm --filter @framework-tracker/stats-generator validate:first-party-dependencies diff --git a/.github/workflows/measure-framework.yml b/.github/workflows/measure-framework.yml index 83f2ea84..09a602e8 100644 --- a/.github/workflows/measure-framework.yml +++ b/.github/workflows/measure-framework.yml @@ -1,463 +1,124 @@ -name: Measure Framework - -env: - RUNNER_LABEL: 'Depot depot-ubuntu-24.04 GitHub Actions runner (2 CPUs, 8 GB RAM, 100 GB disk, 2 GB disk accelerator)' +name: Measure Frameworks on: workflow_call: inputs: - install-matrix: - description: 'JSON array of frameworks to measure install time' - type: string - required: true - build-matrix: - description: 'JSON array of frameworks to measure build time' + source-sha: type: string - required: true - dev-server-matrix: - description: 'JSON array of frameworks to measure dev server startup time' - type: string - required: false - default: '[]' - ssr-request-throughput-matrix: - description: 'JSON array of frameworks to measure SSR request throughput performance' + default: '' + measurements: + description: 'JSON object mapping each measurement type to its frameworks' type: string required: true - ssr-load-matrix: - description: 'JSON array of frameworks to measure SSR load performance' - type: string - required: false - default: '[]' - ssr-router-link-load-matrix: - description: 'JSON array of frameworks to measure SSR router link load performance' - type: string - required: false - default: '[]' - deps-matrix: - description: 'JSON array of frameworks to run e18e dependency analysis' - type: string - required: false - default: '[]' - client-side-rendered-matrix: - description: 'JSON array of frameworks to measure client-side rendered paint and interaction performance' - type: string - required: false - default: '[]' - server-side-rendered-matrix: - description: 'JSON array of frameworks to measure server-side rendered paint and interaction performance' - type: string - required: false - default: '[]' + +permissions: + contents: read jobs: measure-install: - if: inputs.install-matrix != '[]' - runs-on: depot-ubuntu-24.04 + if: toJson(fromJson(inputs.measurements)['install']) != '[]' strategy: fail-fast: false matrix: - framework: ${{ fromJson(inputs.install-matrix) }} - steps: - - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Setup pnpm - uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5 - - - name: Setup Node.js - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: '24' - - - name: Install stats-generator dependencies - run: pnpm install --frozen-lockfile --filter @framework-tracker/stats-generator... - - - name: Run install benchmark - run: | - pnpm --filter @framework-tracker/stats-generator run:install ${{ matrix.framework.package }} - - - name: Upload install stats - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: install-stats-${{ matrix.framework.name }} - path: packages/${{ matrix.framework.package }}/install-stats.json - retention-days: 1 - if-no-files-found: error + framework: ${{ fromJson(inputs.measurements)['install'] }} + uses: ./.github/workflows/measure-job.yml + with: + framework: ${{ matrix.framework }} + source-sha: ${{ inputs.source-sha }} + task: install measure-build: - if: inputs.build-matrix != '[]' - runs-on: depot-ubuntu-24.04 + if: toJson(fromJson(inputs.measurements)['build']) != '[]' strategy: fail-fast: false matrix: - framework: ${{ fromJson(inputs.build-matrix) }} - steps: - - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Setup pnpm - uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5 - - - name: Setup Node.js - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: '24' - cache: 'pnpm' - - - name: Install workspace dependencies - run: pnpm install --frozen-lockfile - - - name: Run build benchmark - run: | - pnpm --filter @framework-tracker/stats-generator run:build ${{ matrix.framework.package }} - - - name: Scan build output for vendored core-js - run: pnpm --filter @framework-tracker/stats-generator run:corejs ${{ matrix.framework.package }} - - - name: Scan build output for browser baseline requirements - if: contains(toJson(matrix.framework.measurements), 'browserBaseline') - run: pnpm --filter @framework-tracker/stats-generator run:browser-baseline ${{ matrix.framework.package }} - - - name: Upload build stats - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: build-stats-${{ matrix.framework.name }} - path: packages/${{ matrix.framework.package }}/build-stats.json - retention-days: 1 - if-no-files-found: error - - - name: Upload core-js stats - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: corejs-stats-${{ matrix.framework.name }} - path: packages/${{ matrix.framework.package }}/corejs-stats.json - retention-days: 1 - if-no-files-found: warn - - - name: Upload browser baseline stats - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: contains(toJson(matrix.framework.measurements), 'browserBaseline') - with: - name: browser-baseline-stats-${{ matrix.framework.name }} - path: packages/${{ matrix.framework.package }}/browser-baseline-stats.json - retention-days: 1 - if-no-files-found: error + framework: ${{ fromJson(inputs.measurements)['build'] }} + uses: ./.github/workflows/measure-job.yml + with: + framework: ${{ matrix.framework }} + source-sha: ${{ inputs.source-sha }} + task: build measure-dev-server: - if: inputs.dev-server-matrix != '[]' - runs-on: depot-ubuntu-24.04 - timeout-minutes: 30 + if: toJson(fromJson(inputs.measurements)['dev-server']) != '[]' strategy: fail-fast: false matrix: - framework: ${{ fromJson(inputs.dev-server-matrix) }} - steps: - - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Setup pnpm - uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5 - - - name: Setup Node.js - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: '24' - cache: 'pnpm' - - - name: Install workspace dependencies - run: pnpm install --frozen-lockfile - - - name: Run dev server benchmark - run: | - pnpm --filter @framework-tracker/stats-generator run:dev-server ${{ matrix.framework.package }} - - - name: Upload dev server stats - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: dev-server-stats-${{ matrix.framework.name }} - path: packages/${{ matrix.framework.package }}/dev-server-stats.json - retention-days: 1 - if-no-files-found: error + framework: ${{ fromJson(inputs.measurements)['dev-server'] }} + uses: ./.github/workflows/measure-job.yml + with: + framework: ${{ matrix.framework }} + source-sha: ${{ inputs.source-sha }} + task: dev-server measure-ssr-request-throughput: - if: inputs.ssr-request-throughput-matrix != '[]' - runs-on: depot-ubuntu-24.04 + if: toJson(fromJson(inputs.measurements)['ssr-request-throughput']) != '[]' strategy: fail-fast: false matrix: - framework: ${{ fromJson(inputs.ssr-request-throughput-matrix) }} - steps: - - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Setup pnpm - uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5 - - - name: Setup Node.js - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: '24' - cache: 'pnpm' + framework: ${{ fromJson(inputs.measurements)['ssr-request-throughput'] }} + uses: ./.github/workflows/measure-job.yml + with: + framework: ${{ matrix.framework }} + source-sha: ${{ inputs.source-sha }} + task: ssr-request-throughput - - name: Install workspace dependencies - run: pnpm install --frozen-lockfile - - - name: Install package dependencies - working-directory: ./packages/${{ matrix.framework.package }} - run: pnpm install --frozen-lockfile - - - name: Build app - working-directory: ./packages/${{ matrix.framework.package }} - run: pnpm build - - - name: Run SSR request throughput benchmark - run: pnpm --filter @framework-tracker/stats-generator run:ssr-request-throughput ${{ matrix.framework.package }} - - - name: Upload SSR request throughput stats - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: ssr-request-throughput-stats-${{ matrix.framework.name }} - path: packages/${{ matrix.framework.package }}/ci-stats.json - retention-days: 1 - if-no-files-found: error - - measure-dependencies: - if: inputs.deps-matrix != '[]' - runs-on: depot-ubuntu-24.04 + measure-ssr-load: + if: toJson(fromJson(inputs.measurements)['ssr-load']) != '[]' strategy: fail-fast: false matrix: - framework: ${{ fromJson(inputs.deps-matrix) }} - steps: - - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Setup pnpm - uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5 - - - name: Setup Node.js - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: '24' - cache: 'pnpm' - - - name: Install stats-generator dependencies - run: pnpm install --frozen-lockfile --filter @framework-tracker/stats-generator... + framework: ${{ fromJson(inputs.measurements)['ssr-load'] }} + uses: ./.github/workflows/measure-job.yml + with: + framework: ${{ matrix.framework }} + source-sha: ${{ inputs.source-sha }} + task: ssr-load - - name: Install package dependencies - working-directory: ./packages/${{ matrix.framework.package }} - run: pnpm install --frozen-lockfile - - - name: Run e18e analysis - working-directory: ./packages/${{ matrix.framework.package }} - run: npx @e18e/cli@0.5.0 analyze --json > e18e-stats.json || true - - - name: Collect framework dependency stats - env: - STARTER_PACKAGE: ${{ matrix.framework.package }} - run: node packages/stats-generator/src/run-framework-dependency-scan.ts "$STARTER_PACKAGE" - - - name: Collect first-party dependency stats - env: - STARTER_PACKAGE: ${{ matrix.framework.package }} - run: node packages/stats-generator/src/run-first-party-dependency-scan.ts "$STARTER_PACKAGE" - - - name: Scan installed dependencies for Node engine constraints - run: pnpm --filter @framework-tracker/stats-generator run:node-engines ${{ matrix.framework.package }} - - - name: Upload dependency stats - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: e18e-stats-${{ matrix.framework.name }} - path: | - packages/${{ matrix.framework.package }}/e18e-stats.json - packages/${{ matrix.framework.package }}/framework-dependency-stats.json - packages/${{ matrix.framework.package }}/first-party-dependency-stats.json - retention-days: 1 - if-no-files-found: error - - - name: Upload node engines stats - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: node-engines-stats-${{ matrix.framework.name }} - path: packages/${{ matrix.framework.package }}/node-engines-stats.json - retention-days: 1 - if-no-files-found: error - - measure-ssr-load: - if: inputs.ssr-load-matrix != '[]' - runs-on: depot-ubuntu-24.04-16 - env: - RUNNER_LABEL: 'Depot depot-ubuntu-24.04-16 GitHub Actions runner (16 CPUs, 64 GB RAM, 180 GB disk, 8 GB disk accelerator; server CPUs 0-11; Autocannon CPUs 12-15)' + measure-ssr-router-link-load: + if: toJson(fromJson(inputs.measurements)['ssr-router-link-load']) != '[]' strategy: fail-fast: false matrix: - framework: ${{ fromJson(inputs.ssr-load-matrix) }} - steps: - - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Setup pnpm - uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5 - - - name: Setup Node.js - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: '24' - cache: 'pnpm' + framework: ${{ fromJson(inputs.measurements)['ssr-router-link-load'] }} + uses: ./.github/workflows/measure-job.yml + with: + framework: ${{ matrix.framework }} + source-sha: ${{ inputs.source-sha }} + task: ssr-router-link-load - - name: Install workspace dependencies - run: pnpm install --frozen-lockfile - - - name: Install package dependencies - working-directory: ./packages/${{ matrix.framework.package }} - run: pnpm install --frozen-lockfile - - - name: Build app - working-directory: ./packages/${{ matrix.framework.package }} - run: pnpm build - - - name: Run SSR load benchmark - run: pnpm --filter @framework-tracker/stats-generator run:ssr-load:containerized ${{ matrix.framework.package }} - - - name: Upload SSR load stats - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: ssr-load-stats-${{ matrix.framework.name }} - path: packages/${{ matrix.framework.package }}/ci-stats.json - retention-days: 1 - if-no-files-found: error - - measure-ssr-router-link-load: - if: inputs.ssr-router-link-load-matrix != '[]' - runs-on: depot-ubuntu-24.04-16 - env: - RUNNER_LABEL: 'Depot depot-ubuntu-24.04-16 GitHub Actions runner (16 CPUs, 64 GB RAM, 180 GB disk, 8 GB disk accelerator; server CPUs 0-11; Autocannon CPUs 12-15)' + measure-dependencies: + if: toJson(fromJson(inputs.measurements)['deps']) != '[]' strategy: fail-fast: false matrix: - framework: ${{ fromJson(inputs.ssr-router-link-load-matrix) }} - steps: - - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Setup pnpm - uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5 - - - name: Setup Node.js - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: '24' - cache: 'pnpm' - - - name: Install workspace dependencies - run: pnpm install --frozen-lockfile - - - name: Install package dependencies - working-directory: ./packages/${{ matrix.framework.package }} - run: pnpm install --frozen-lockfile - - - name: Build app - working-directory: ./packages/${{ matrix.framework.package }} - run: pnpm build - - - name: Run SSR router link load benchmark - run: pnpm --filter @framework-tracker/stats-generator run:ssr-router-link-load:containerized ${{ matrix.framework.package }} - - - name: Upload SSR router link load stats - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: ssr-router-link-load-stats-${{ matrix.framework.name }} - path: packages/${{ matrix.framework.package }}/ci-stats.json - retention-days: 1 - if-no-files-found: error + framework: ${{ fromJson(inputs.measurements)['deps'] }} + uses: ./.github/workflows/measure-job.yml + with: + framework: ${{ matrix.framework }} + source-sha: ${{ inputs.source-sha }} + task: deps measure-client-side-rendered: - if: inputs.client-side-rendered-matrix != '[]' - runs-on: depot-ubuntu-24.04 + if: toJson(fromJson(inputs.measurements)['client-side-rendered']) != '[]' strategy: fail-fast: false matrix: - framework: ${{ fromJson(inputs.client-side-rendered-matrix) }} - steps: - - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Setup pnpm - uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5 - - - name: Setup Node.js - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: '24' - cache: 'pnpm' - - - name: Install workspace dependencies - run: pnpm install --frozen-lockfile - - - name: Install package dependencies - working-directory: ./packages/${{ matrix.framework.package }} - run: pnpm install --frozen-lockfile - - - name: Build app - working-directory: ./packages/${{ matrix.framework.package }} - run: pnpm build - - - name: Verify runner Chrome - run: google-chrome --version - - - name: Run client-side rendered benchmark - run: pnpm --filter @framework-tracker/stats-generator run:client-side-rendered ${{ matrix.framework.package }} - - - name: Upload client-side rendered stats - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: client-side-rendered-stats-${{ matrix.framework.name }} - path: packages/${{ matrix.framework.package }}/ci-stats.json - retention-days: 1 - if-no-files-found: error + framework: ${{ fromJson(inputs.measurements)['client-side-rendered'] }} + uses: ./.github/workflows/measure-job.yml + with: + framework: ${{ matrix.framework }} + source-sha: ${{ inputs.source-sha }} + task: client-side-rendered measure-server-side-rendered: - if: inputs.server-side-rendered-matrix != '[]' - runs-on: depot-ubuntu-24.04 + if: toJson(fromJson(inputs.measurements)['server-side-rendered']) != '[]' strategy: fail-fast: false matrix: - framework: ${{ fromJson(inputs.server-side-rendered-matrix) }} - steps: - - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Setup pnpm - uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5 - - - name: Setup Node.js - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: '24' - cache: 'pnpm' - - - name: Install workspace dependencies - run: pnpm install --frozen-lockfile - - - name: Install package dependencies - working-directory: ./packages/${{ matrix.framework.package }} - run: pnpm install --frozen-lockfile - - - name: Build app - working-directory: ./packages/${{ matrix.framework.package }} - run: pnpm build - - - name: Verify runner Chrome - run: google-chrome --version - - - name: Run server-side rendered benchmark - run: pnpm --filter @framework-tracker/stats-generator run:server-side-rendered ${{ matrix.framework.package }} - - - name: Upload server-side rendered stats - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: server-side-rendered-stats-${{ matrix.framework.name }} - path: packages/${{ matrix.framework.package }}/ci-stats.json - retention-days: 1 - if-no-files-found: error + framework: ${{ fromJson(inputs.measurements)['server-side-rendered'] }} + uses: ./.github/workflows/measure-job.yml + with: + framework: ${{ matrix.framework }} + source-sha: ${{ inputs.source-sha }} + task: server-side-rendered diff --git a/.github/workflows/measure-job.yml b/.github/workflows/measure-job.yml new file mode 100644 index 00000000..a15292e0 --- /dev/null +++ b/.github/workflows/measure-job.yml @@ -0,0 +1,47 @@ +name: Measure Job + +on: + workflow_call: + inputs: + source-sha: + type: string + default: '' + framework: + type: string + required: true + task: + type: string + required: true + +permissions: + contents: read + +jobs: + measure: + runs-on: ${{ (inputs.task == 'ssr-load' || inputs.task == 'ssr-router-link-load') && 'depot-ubuntu-24.04-16' || 'depot-ubuntu-24.04' }} + timeout-minutes: ${{ inputs.task == 'dev-server' && 30 || 360 }} + env: + RUNNER_LABEL: ${{ (inputs.task == 'ssr-load' || inputs.task == 'ssr-router-link-load') && 'Depot depot-ubuntu-24.04-16 GitHub Actions runner (16 CPUs, 64 GB RAM, 180 GB disk, 8 GB disk accelerator; server CPUs 0-11; Autocannon CPUs 12-15)' || 'Depot depot-ubuntu-24.04 GitHub Actions runner (2 CPUs, 8 GB RAM, 100 GB disk, 2 GB disk accelerator)' }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.source-sha || github.sha }} + persist-credentials: false + - uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '24' + cache: ${{ inputs.source-sha == '' && 'pnpm' || '' }} + package-manager-cache: false + - run: pnpm install --frozen-lockfile + - name: Run and validate measurement + env: + FRAMEWORK: ${{ inputs.framework }} + TASK: ${{ inputs.task }} + run: node packages/stats-generator/src/run-measurement.ts "$FRAMEWORK" "$TASK" "$GITHUB_WORKSPACE/artifacts" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ inputs.task }}-stats-${{ inputs.framework }} + path: artifacts/${{ inputs.task }}-stats-${{ inputs.framework }}/ + retention-days: 1 + if-no-files-found: error diff --git a/.github/workflows/measure-pr-dispatch.yml b/.github/workflows/measure-pr-dispatch.yml new file mode 100644 index 00000000..4beb37be --- /dev/null +++ b/.github/workflows/measure-pr-dispatch.yml @@ -0,0 +1,39 @@ +name: Dispatch PR Measurements +run-name: Measurements dispatch ${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }} + +on: + workflow_run: + workflows: [Measure PR] + types: [in_progress, completed] + +permissions: + contents: read + pull-requests: read + actions: write + +concurrency: + group: measure-dispatch-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }} + cancel-in-progress: false + +jobs: + dispatch: + if: >- + github.event.workflow_run.event == 'pull_request' && + (github.event.action == 'in_progress' || github.event.workflow_run.conclusion != 'success') + runs-on: depot-ubuntu-24.04 + timeout-minutes: 15 + steps: + # workflow_run uses the trusted default-branch revision. Never check out PR code here. + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.sha }} + persist-credentials: false + sparse-checkout: .github/scripts + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '24' + package-manager-cache: false + - name: Start or cancel the separate worker + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: node .github/scripts/measurement-control.ts relay diff --git a/.github/workflows/measure-pr.yml b/.github/workflows/measure-pr.yml new file mode 100644 index 00000000..0ed44146 --- /dev/null +++ b/.github/workflows/measure-pr.yml @@ -0,0 +1,35 @@ +name: Measure PR +run-name: 'Measurements for PR #${{ github.event.pull_request.number }} at ${{ github.sha }}' + +on: + pull_request: + branches: [main] + types: [opened, synchronize, reopened, closed] + +permissions: + contents: read + actions: read + +concurrency: + group: measure-pr-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + check: + name: Measure and validate + if: github.event.action != 'closed' + runs-on: depot-ubuntu-24.04 + timeout-minutes: 360 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + sparse-checkout: .github/scripts + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '24' + package-manager-cache: false + - name: Wait for isolated measurements and validation + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: node .github/scripts/measurement-control.ts wait diff --git a/.github/workflows/measure.yml b/.github/workflows/measure.yml new file mode 100644 index 00000000..e735192a --- /dev/null +++ b/.github/workflows/measure.yml @@ -0,0 +1,162 @@ +name: Measure +run-name: ${{ inputs.controller-run-id != '' && format('PR measurements {0}-{1}', inputs.controller-run-id, inputs.controller-run-attempt) || format('Measurements ({0})', github.event_name) }} + +on: + push: + branches: [main] + schedule: + # Monday and Thursday at 04:00 UTC. + - cron: '0 4 * * 1,4' + workflow_dispatch: + inputs: + source-sha: + description: 'Exact PR merge commit to measure; empty for a normal refresh' + type: string + default: '' + base-sha: + description: 'Base commit for the PR change selection' + type: string + default: '' + pr-number: + type: string + default: '' + controller-run-id: + type: string + default: '' + controller-run-attempt: + type: string + default: '' + +permissions: + contents: read + +concurrency: + group: measure-${{ inputs.controller-run-id != '' && format('pr-{0}-{1}', inputs.controller-run-id, inputs.controller-run-attempt) || 'results' }} + cancel-in-progress: ${{ inputs.controller-run-id != '' }} + +jobs: + select: + runs-on: depot-ubuntu-24.04 + outputs: + measurements: ${{ steps.select.outputs.measurements }} + has-changes: ${{ steps.select.outputs.has-changes }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.source-sha || github.sha }} + persist-credentials: false + fetch-depth: 0 + - uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '24' + cache: ${{ inputs.source-sha == '' && 'pnpm' || '' }} + package-manager-cache: false + - run: pnpm install --frozen-lockfile + - id: select + env: + MEASUREMENT_BASE_SHA: ${{ inputs.base-sha }} + MEASUREMENT_HEAD_SHA: ${{ inputs.source-sha }} + run: node packages/stats-generator/src/measurement-ci.ts select + + measure: + needs: select + if: needs.select.outputs.has-changes == 'true' + uses: ./.github/workflows/measure-framework.yml + with: + measurements: ${{ needs.select.outputs.measurements }} + source-sha: ${{ inputs.source-sha }} + + assemble: + name: Validate results + needs: [select, measure] + if: always() && needs.select.outputs.has-changes == 'true' && needs.measure.result == 'success' + runs-on: depot-ubuntu-24.04 + env: + RUNNER_LABEL: 'Depot depot-ubuntu-24.04 GitHub Actions runner (2 CPUs, 8 GB RAM, 100 GB disk, 2 GB disk accelerator)' + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.source-sha || github.sha }} + persist-credentials: false + - uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '24' + cache: ${{ inputs.source-sha == '' && 'pnpm' || '' }} + package-manager-cache: false + - run: pnpm install --frozen-lockfile + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: artifacts + - name: Assemble and validate temporary results + run: node packages/stats-generator/src/measurement-ci.ts assemble artifacts artifacts/results + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: measured-results + path: artifacts/results/ + retention-days: 7 + if-no-files-found: error + + results-pr: + name: Save results + needs: [select, assemble] + if: >- + (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && + github.ref == 'refs/heads/main' && inputs.source-sha == '' && + inputs.pr-number == '' && inputs.controller-run-id == '' && + needs.select.outputs.has-changes == 'true' && needs.assemble.result == 'success' + runs-on: depot-ubuntu-24.04 + permissions: + contents: write + pull-requests: write + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + - uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '24' + cache: pnpm + - run: pnpm install --frozen-lockfile + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: measured-results + path: artifacts/results + - name: Preserve pending results PR updates + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + OPEN_PRS=$(gh pr list --head automated-stats-update --state open --json number --jq 'length') + if [[ "$OPEN_PRS" != "0" ]]; then + git fetch origin automated-stats-update + git diff --binary HEAD...FETCH_HEAD -- results/ > "$RUNNER_TEMP/pending-results.patch" + if [[ -s "$RUNNER_TEMP/pending-results.patch" ]]; then + git apply --3way "$RUNNER_TEMP/pending-results.patch" + fi + fi + - name: Update canonical results + run: node packages/stats-generator/src/measurement-ci.ts promote artifacts/results + - run: pnpm exec prettier --write results/ + - name: Create or update results pull request + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + if [[ -z $(git status --short -- results/) ]]; then + echo "No changes to commit" + exit 0 + fi + git checkout -B automated-stats-update + git add results/ + git commit -m "Update framework measurements" + git push --force-with-lease origin automated-stats-update + if ! gh pr list --head automated-stats-update --state open | grep -q .; then + gh pr create --title "Update framework measurements" \ + --body "Automated framework measurements with validated current results and version history under results/." \ + --base main --head automated-stats-update + else + echo "PR already exists, updated with new commits" + fi diff --git a/.github/workflows/preview-stats.yml b/.github/workflows/preview-stats.yml deleted file mode 100644 index 571c5d92..00000000 --- a/.github/workflows/preview-stats.yml +++ /dev/null @@ -1,151 +0,0 @@ -name: Preview Stats - -env: - RUNNER_LABEL: 'Depot depot-ubuntu-24.04 GitHub Actions runner (2 CPUs, 8 GB RAM, 100 GB disk, 2 GB disk accelerator)' - -on: - workflow_dispatch: - -jobs: - setup: - runs-on: ubuntu-latest - outputs: - install-matrix: ${{ steps.detect.outputs.install }} - build-matrix: ${{ steps.detect.outputs.build }} - dev-server-matrix: ${{ steps.detect.outputs.dev-server }} - ssr-request-throughput-matrix: ${{ steps.detect.outputs.ssr-request-throughput }} - ssr-load-matrix: ${{ steps.detect.outputs.ssr-load }} - ssr-router-link-load-matrix: ${{ steps.detect.outputs.ssr-router-link-load }} - deps-matrix: ${{ steps.detect.outputs.deps }} - framework-names: ${{ steps.detect.outputs.framework-names }} - has-changes: ${{ steps.detect.outputs.has-changes }} - pr-number: ${{ steps.detect.outputs.pr-number }} - steps: - - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - fetch-depth: 0 - - - name: Setup pnpm - uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5 - - - name: Setup Node.js - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: '24' - - - name: Install workspace dependencies - run: pnpm install --frozen-lockfile - - - name: Detect PR, changed frameworks, and build matrices - id: detect - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - PR_INFO=$(gh pr view --json number,baseRefName 2>/dev/null) - if [[ -z "$PR_INFO" ]]; then - echo "::error::No open PR found for branch ${{ github.ref_name }}" - exit 1 - fi - PR_NUMBER=$(echo "$PR_INFO" | jq -r '.number') - BASE_REF=$(echo "$PR_INFO" | jq -r '.baseRefName') - echo "pr-number=$PR_NUMBER" >> $GITHUB_OUTPUT - echo "Detected PR #$PR_NUMBER targeting $BASE_REF" - - FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) - - CHANGED_FILES=$(git diff --name-only origin/$BASE_REF...HEAD -- 'packages/starter-*/package.json' 'packages/starter-*/pnpm-lock.yaml' 'packages/starter-*/first-party-dependencies.json' 'packages/app-*/package.json') - - if [[ -z "$CHANGED_FILES" ]]; then - echo "No framework dependency files changed, skipping" - echo "has-changes=false" >> $GITHUB_OUTPUT - echo "framework-names=[]" >> $GITHUB_OUTPUT - echo "install=[]" >> $GITHUB_OUTPUT - echo "build=[]" >> $GITHUB_OUTPUT - echo "dev-server=[]" >> $GITHUB_OUTPUT - echo "ssr-request-throughput=[]" >> $GITHUB_OUTPUT - echo "ssr-load=[]" >> $GITHUB_OUTPUT - echo "ssr-router-link-load=[]" >> $GITHUB_OUTPUT - echo "deps=[]" >> $GITHUB_OUTPUT - exit 0 - fi - - echo "Changed files:" - echo "$CHANGED_FILES" - - CHANGED_PACKAGES=$(echo "$CHANGED_FILES" | grep -oE 'packages/[^/]+' | sed 's|packages/||' | sort -u) - - echo "Changed packages:" - echo "$CHANGED_PACKAGES" - - NAMES_JSON=$(echo "$CHANGED_PACKAGES" | while read -r PKG; do - echo "$FRAMEWORKS" | jq -r --arg pkg "$PKG" '.[] | select(.starter.package == $pkg or .app.package == $pkg) | .name' - done | sort -u | jq -R . | jq -s -c .) - - echo "Detected frameworks: $NAMES_JSON" - echo "has-changes=true" >> $GITHUB_OUTPUT - echo "framework-names=$NAMES_JSON" >> $GITHUB_OUTPUT - - echo "install=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | contains(["install"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT - echo "build=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | contains(["build"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT - echo "dev-server=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | contains(["devServer"])) | {name, displayName, package: .starter.package, devServerPort: .starter.devServerPort, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT - echo "ssr-request-throughput=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.app) | select(.app.measurements | contains(["ssrRequestThroughput"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT - echo "ssr-load=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.app) | select(.app.measurements | contains(["ssrLoad"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT - echo "ssr-router-link-load=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.app) | select(.app.measurements | contains(["ssrRouterLinkLoad"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT - echo "deps=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | contains(["dependencies"])) | {name, displayName, package: .starter.package}]')" >> $GITHUB_OUTPUT - - measure: - needs: setup - if: needs.setup.result == 'success' && needs.setup.outputs.has-changes == 'true' - uses: ./.github/workflows/measure-framework.yml - with: - install-matrix: ${{ needs.setup.outputs.install-matrix }} - build-matrix: ${{ needs.setup.outputs.build-matrix }} - dev-server-matrix: ${{ needs.setup.outputs.dev-server-matrix }} - ssr-request-throughput-matrix: ${{ needs.setup.outputs.ssr-request-throughput-matrix }} - ssr-load-matrix: ${{ needs.setup.outputs.ssr-load-matrix }} - ssr-router-link-load-matrix: ${{ needs.setup.outputs.ssr-router-link-load-matrix }} - deps-matrix: ${{ needs.setup.outputs.deps-matrix }} - - preview-comment: - needs: [setup, measure] - if: always() && needs.setup.result == 'success' && needs.setup.outputs.has-changes == 'true' && needs.measure.result == 'success' - runs-on: ubuntu-latest - permissions: - pull-requests: write - steps: - - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Setup pnpm - uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5 - - - name: Setup Node.js - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: '24' - cache: 'pnpm' - - - name: Install dependencies - run: pnpm install --frozen-lockfile - - - name: Download artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - path: artifacts - - - name: List downloaded artifacts - run: find artifacts -type f -name "*.json" 2>/dev/null || echo "No artifacts found" - - - name: Save CI stats - run: pnpm --filter @framework-tracker/stats-generator save:ci-stats $GITHUB_WORKSPACE/artifacts - - - name: Generate comment - run: | - FRAMEWORK_ARGS=$(echo '${{ needs.setup.outputs.framework-names }}' | jq -r '.[]' | tr '\n' ' ') - pnpm --filter @framework-tracker/stats-generator generate:preview-comment $FRAMEWORK_ARGS - - - name: Post preview comment - run: gh pr comment ${{ needs.setup.outputs.pr-number }} --body-file comment.md - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/validate-stats.yml b/.github/workflows/validate-stats.yml deleted file mode 100644 index c2c395d7..00000000 --- a/.github/workflows/validate-stats.yml +++ /dev/null @@ -1,155 +0,0 @@ -name: Validate Stats Generation - -on: - pull_request: - paths: - - 'packages/stats-generator/**' - - '.github/workflows/generate-stats.yml' - - '.github/workflows/measure-framework.yml' - - '.github/workflows/validate-stats.yml' - - 'packages/results/**' - - 'packages/stats-generator/src/frameworks.ts' - - 'packages/app-*/**' - - '!packages/app-*/ci-stats.json' - - '!packages/app-*/stats/**' - - 'packages/starter-*/**' - - '!packages/starter-*/ci-stats.json' - - '!packages/starter-*/stats/**' - - '!packages/starter-*/install-stats.json' - - '!packages/starter-*/build-stats.json' - - '!packages/starter-*/dev-server-stats.json' - - '!packages/starter-*/corejs-stats.json' - - '!packages/starter-*/browser-baseline-stats.json' - - '!packages/starter-*/e18e-stats.json' - - '!packages/starter-*/node-engines-stats.json' - -jobs: - validate: - runs-on: depot-ubuntu-24.04 - steps: - - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Setup pnpm - uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5 - - - name: Setup Node.js - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: '24' - cache: 'pnpm' - - - name: Install workspace dependencies - run: pnpm install --frozen-lockfile - - - name: Validate first-party dependency manifests - run: pnpm --filter @framework-tracker/stats-generator validate:first-party-dependencies - - - name: Install all package dependencies - run: | - FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) - for PKG in $(echo "$FRAMEWORKS" | jq -r '.[].starter.package // empty, .[].app.package // empty'); do - echo "Installing $PKG..." - (cd packages/$PKG && pnpm install --frozen-lockfile) - done - - - name: Run all benchmarks and validate - run: | - set -e - - echo "=== Running SSR request throughput benchmarks ===" - FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) - echo "$FRAMEWORKS" | jq -c '.[] | select(.app) | select(.app.measurements | contains(["ssrRequestThroughput"]))' | while read -r framework; do - PKG=$(echo "$framework" | jq -r '.app.package') - echo "Building $PKG..." - (cd packages/$PKG && pnpm build) - echo "Running SSR request throughput benchmark for $PKG..." - pnpm --filter @framework-tracker/stats-generator run:ssr-request-throughput $PKG - done - - echo "" - echo "=== Running SSR load benchmarks ===" - FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) - echo "$FRAMEWORKS" | jq -c '.[] | select(.app) | select(.app.measurements | contains(["ssrLoad"]))' | while read -r framework; do - PKG=$(echo "$framework" | jq -r '.app.package') - echo "Building $PKG..." - (cd packages/$PKG && pnpm build) - echo "Running SSR load benchmark for $PKG..." - pnpm --filter @framework-tracker/stats-generator run:ssr-load $PKG - done - - echo "" - echo "=== Running SSR router link load benchmarks ===" - FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) - echo "$FRAMEWORKS" | jq -c '.[] | select(.app) | select(.app.measurements | contains(["ssrRouterLinkLoad"]))' | while read -r framework; do - PKG=$(echo "$framework" | jq -r '.app.package') - echo "Building $PKG..." - (cd packages/$PKG && pnpm build) - echo "Running SSR router link load benchmark for $PKG..." - pnpm --filter @framework-tracker/stats-generator run:ssr-router-link-load $PKG - done - - echo "" - echo "=== Running starter benchmarks ===" - FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) - echo "$FRAMEWORKS" | jq -c '.[] | select(.starter)' | while read -r framework; do - PKG=$(echo "$framework" | jq -r '.starter.package') - echo "Running install benchmark for $PKG..." - pnpm --filter @framework-tracker/stats-generator run:install $PKG - echo "Running build benchmark for $PKG..." - pnpm --filter @framework-tracker/stats-generator run:build $PKG - echo "Running dev server benchmark for $PKG..." - pnpm --filter @framework-tracker/stats-generator run:dev-server $PKG - echo "Running Node engines scan for $PKG..." - pnpm --filter @framework-tracker/stats-generator run:node-engines $PKG - done - - echo "" - echo "=== Running dependency scans ===" - FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) - echo "$FRAMEWORKS" | jq -c '.[] | select(.starter) | select(.starter.measurements | contains(["dependencies"]))' | while read -r framework; do - PKG=$(echo "$framework" | jq -r '.starter.package') - echo "Running e18e analysis for $PKG..." - (cd packages/$PKG && npx @e18e/cli@0.5.0 analyze --json > e18e-stats.json || true) - echo "Scanning core-js usage for $PKG..." - pnpm --filter @framework-tracker/stats-generator run:corejs "$PKG" - echo "Scanning framework dependencies for $PKG..." - pnpm --filter @framework-tracker/stats-generator run:framework-dependencies "$PKG" - echo "Scanning first-party dependencies for $PKG..." - pnpm --filter @framework-tracker/stats-generator run:first-party-dependencies "$PKG" - done - - echo "" - echo "=== Running browser baseline scans ===" - FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) - echo "$FRAMEWORKS" | jq -c '.[] | select(.starter) | select(.starter.measurements | contains(["browserBaseline"]))' | while read -r framework; do - PKG=$(echo "$framework" | jq -r '.starter.package') - echo "Scanning browser baseline requirements for $PKG..." - pnpm --filter @framework-tracker/stats-generator run:browser-baseline $PKG - done - - echo "" - echo "=== Running client-side rendered benchmarks ===" - FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) - echo "$FRAMEWORKS" | jq -c '.[] | select(.app) | select(.app.measurements | contains(["clientSideRendered"]))' | while read -r framework; do - PKG=$(echo "$framework" | jq -r '.app.package') - echo "Building client-side rendered app for $PKG..." - (cd packages/$PKG && pnpm build) - echo "Running client-side rendered benchmark for $PKG..." - pnpm --filter @framework-tracker/stats-generator run:client-side-rendered $PKG 1 - done - - echo "" - echo "=== Running server-side rendered benchmarks ===" - FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) - echo "$FRAMEWORKS" | jq -c '.[] | select(.app) | select(.app.measurements | contains(["serverSideRendered"]))' | while read -r framework; do - PKG=$(echo "$framework" | jq -r '.app.package') - echo "Building server-side rendered app for $PKG..." - (cd packages/$PKG && pnpm build) - echo "Running server-side rendered benchmark for $PKG..." - pnpm --filter @framework-tracker/stats-generator run:server-side-rendered $PKG 1 - done - - echo "" - echo "=== Validating all outputs ===" - pnpm --filter @framework-tracker/stats-generator validate diff --git a/.gitignore b/.gitignore index b715504c..4a8822d8 100644 --- a/.gitignore +++ b/.gitignore @@ -88,3 +88,5 @@ packages/*/node-engines-stats.json # Temporary runtime measurement artifacts; persisted results live in /results. packages/app-*/ci-stats.json packages/starter-*/ci-stats.json + +packages/*/browser-baseline-stats.json diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a4726810..9bb17dbb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,52 +43,26 @@ Having both a `starter` and `app` project for each framework allows us to get a ### How Metrics Work -Please note that metrics collection is in early stages and the process is likely to change as we iterate on it. The current process is as follows but is subject to change: - -We currently run scripts in CI using GitHub Actions or if the test is more complex we run it locally in container and push the results to the repo. - -#### CI Metrics Collection - -The current flow for collecting metrics is as follows: - -1. A PR is merged which triggers the CI Pipeline: `generate-stats` which uses functions from `packages/stats-generator` to run measurements for each framework -2. The CI Pipeline reads the framework config from `.github/frameworks.json` and runs measurements based on each framework's `app` and `starter` config -3. Repository benchmarks run on Depot; see [Default Benchmark Setup](./packages/docs/src/content/docs/methodology.md#default-benchmark-setup) for the shared environment and test-specific exceptions. -4. The collected metrics are passed into the final step which runs the scripts from `packages/stats-generator` -5. The `stats-generator` reads `frameworks.json` and generates stats only for the configured measurements -6. Stats are then saved into each package and passed into `packages/docs` - -### Framework Configuration - -All frameworks are configured in `.github/frameworks.json`. Each entry specifies what measurements to run: - -```json -{ - "name": "astro", - "displayName": "Astro", - "frameworkPackage": "astro", - "starter": { - "package": "starter-astro", - "buildScript": "build:astro", - "buildOutputDir": "dist", - "measurements": [ - { "type": "install", "runFrequency": 5 }, - { "type": "build", "runFrequency": 5 }, - { "type": "dependencies" } - ] - }, - "app": { - "package": "app-astro", - "buildScript": "build:app-astro", - "buildOutputDir": "dist", - "measurements": [{ "type": "ssrRequestThroughput" }] - } -}, -``` - -#### Container Metrics Collection - -Coming soon but will also pull from `.github/frameworks.json` +The shared [measurement pipeline](./.github/workflows/measure.yml) uses +[framework configuration](./packages/results/src/frameworks.ts) and the same +runners, assembly, and schema validation in CI. +Measurements are grouped by type (install, build, load, and so on), with a +separate Depot runner for each framework and measurement. Load measurements use +the larger Depot runners. All measurement types share the same worker workflow. + +- Source PRs measure all frameworks and their configured starter and app projects. + Docs-only and results-only changes skip benchmarks. PR results are temporary artifacts. + The Validate results job assembles and validates the complete temporary results; + the Save results job never runs for PRs. +- After source changes merge to `main`, measurements open or update the existing + `automated-stats-update` results PR. Scheduled runs measure all frameworks on + Monday and Thursday at 04:00 UTC. Manual runs also measure all frameworks; + runs on `main` update the results PR. + +Canonical data lives in `results//latest.json` and `versions/`. +Starter and app versions remain separate, and every test identifies its project. +The docs load these files directly. See [methodology](https://frameworks.e18e.dev/methodology/) +for measurement details and runner resources. ### Versioning @@ -98,18 +72,10 @@ the framework version, also update the other direct packages in its starter and app projects to the latest compatible versions available on the framework release date. -For a newer patch within a major or minor version already tracked (for example, -SolidStart `2.0.0` to `2.0.5`), update the exact framework version in both the -starter and app `package.json` files, regenerate both lockfiles, and update the -starter's `first-party-dependencies.json`. Update `frameworkVersion` and the -framework package version in both `ci-stats.json` files, then rename the -existing `stats/.json` files to the new patch version and update -their version fields. Make the same changes to the current and versioned stats -JSON files under `packages/docs/src/content/devtime` and `runtime`. Replace the -old patch entry rather than adding another point to the major or minor history. -Leave measured values and diagnostic messages unchanged; they still reflect the -previous patch until the Generate Stats workflow reruns after the change reaches -`main` and opens a stats update PR. +For a newer patch within a tracked major or minor version, update the exact +framework version, dependencies, lockfiles, and starter first-party dependency +manifest. CI measures the updated projects and opens a separate results PR after +merge. Keep starter and app version metadata separate in stored results. Framework version bumps are currently manual; CI handles the measurements after the change reaches `main`. @@ -142,7 +108,7 @@ new first-party package, remove one no longer declared, and leave third-party packages out. Confirm ownership from the package's source project rather than assuming matching names or scopes are first-party. -The validation workflow checks that every entry remains in the same dependency +The lint and type check workflow checks that every entry remains in the same dependency section with the same version as the starter's `package.json`. When an upgrade changes the build tool, adapter, starter setup, or benchmark @@ -191,7 +157,7 @@ Adding a new framework increases the maintenance burden, so please open an issue 2. **Create the app package** (optional): If runtime performance testing is planned, add an `app-*` package (e.g., `packages/app-my-framework`) with a more complex setup that includes features like dynamic routing or client-side interactivity. Make sure the framework dependency is pinned to the same exact version as the starter package. -3. **Add an entry to `.github/frameworks.json`**: Configure the framework's measurements: +3. **Add an entry to `packages/results/src/frameworks.ts`**: Configure the framework's measurements: ```json { @@ -201,13 +167,9 @@ Adding a new framework increases the maintenance burden, so please open an issue "focusedFramework": false, "starter": { "package": "starter-my-framework", - "buildScript": "build:my-framework", + "buildScript": "build", "buildOutputDir": "dist", - "measurements": [ - { "type": "install", "runFrequency": 5 }, - { "type": "build", "runFrequency": 5 }, - { "type": "dependencies" } - ] + "measurements": ["install", "build", "dependencies"] } } ``` @@ -218,7 +180,7 @@ Adding a new framework increases the maintenance burden, so please open an issue 5. **Test locally**: Make sure the framework builds successfully by running the build script from inside the package directory. -6. CI for `sync-version` and `validate-stats` will automatically run on the new framework once it's added to `frameworks.json`. +6. The shared measurement pipeline automatically includes frameworks added to the configuration. 7. **Submit a PR**: Open a pull request with the new packages, configuration, and methodology updates. Once merged, the CI will automatically pick up the new framework and raise a PR with new metrics. diff --git a/packages/results/src/storage.ts b/packages/results/src/storage.ts index 418c2eb8..49f2d0c5 100644 --- a/packages/results/src/storage.ts +++ b/packages/results/src/storage.ts @@ -92,13 +92,17 @@ export function resultPath( ? join(root, framework, 'versions', `${version}.json`) : join(root, framework, 'latest.json') } -export function readProjectResult(packageName: string, version?: string) { +export function readProjectResult( + packageName: string, + version?: string, + root = resultsRoot, +) { const location = resultLocation(packageName) try { return projectResult( CombinedResultSchema.parse( JSON.parse( - readFileSync(resultPath(location.framework, version), 'utf8'), + readFileSync(resultPath(location.framework, version, root), 'utf8'), ), ), location.project, @@ -141,6 +145,7 @@ export function mergeProjectResult( export function saveProjectResult( packageName: string, stats: Record, + root = resultsRoot, ) { const { framework } = resultLocation(packageName) const version = @@ -149,8 +154,8 @@ export function saveProjectResult( ? stats.frameworkVersion : undefined for (const path of [ - resultPath(framework), - ...(version ? [resultPath(framework, version)] : []), + resultPath(framework, undefined, root), + ...(version ? [resultPath(framework, version, root)] : []), ]) { let existing: CombinedResult | null = null try { diff --git a/packages/stats-generator/package.json b/packages/stats-generator/package.json index 4267e3f0..1e84d48e 100644 --- a/packages/stats-generator/package.json +++ b/packages/stats-generator/package.json @@ -21,9 +21,6 @@ "run:framework-dependencies": "node src/run-framework-dependency-scan.ts", "run:first-party-dependencies": "node src/run-first-party-dependency-scan.ts", "run:node-engines": "node src/run-node-engines-scan.ts", - "save:ci-stats": "node src/save-ci-stats.ts", - "generate:preview-comment": "node src/generate-preview-comment.ts", - "validate": "node src/validate-stats.ts", "validate:first-party-dependencies": "node src/validate-first-party-dependencies.ts", "sync:versions": "node src/sync-versions.ts", "lint": "eslint .", diff --git a/packages/stats-generator/src/generate-preview-comment.ts b/packages/stats-generator/src/generate-preview-comment.ts deleted file mode 100644 index 26e474a1..00000000 --- a/packages/stats-generator/src/generate-preview-comment.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { execFileSync } from 'node:child_process' -import { writeFileSync } from 'node:fs' -import { join } from 'node:path' - -const MAX_DIFF_LINES = 500 -const COMMENT_MARKER = '' - -async function main() { - const frameworkNames = process.argv.slice(2) - - if (frameworkNames.length === 0) { - console.error( - 'Usage: generate:preview-comment [framework-name...]', - ) - process.exit(1) - } - - const repoRoot = execFileSync('git', ['rev-parse', '--show-toplevel'], { - encoding: 'utf-8', - }).trim() - - const fullDiff = execFileSync('git', ['diff', '--', 'results/'], { - cwd: repoRoot, - encoding: 'utf-8', - }) - - const diffLines = fullDiff.trimEnd().split('\n').filter(Boolean) - const truncated = diffLines.length > MAX_DIFF_LINES - const diffSlice = diffLines.slice(0, MAX_DIFF_LINES).join('\n') - - const frameworkDisplay = frameworkNames.map((n) => `\`${n}\``).join(', ') - - const lines: string[] = [ - COMMENT_MARKER, - '', - `## Stats Preview: ${frameworkDisplay}`, - '', - `Stats were measured on this PR branch for ${frameworkDisplay}. Below is the diff against the base branch.`, - '', - '
', - 'View stats diff', - '', - ] - - if (!diffSlice) { - lines.push('No stats changes detected.') - } else { - lines.push('```diff') - lines.push(diffSlice) - lines.push('```') - if (truncated) { - lines.push('') - lines.push( - `_(Diff truncated at ${MAX_DIFF_LINES} lines — ${diffLines.length} total lines)_`, - ) - } - } - - lines.push('') - lines.push('
') - - const outputPath = join(repoRoot, 'comment.md') - writeFileSync(outputPath, lines.join('\n')) - console.info(`✓ Comment written to ${outputPath}`) -} - -main().catch((error) => { - console.error('Failed to generate preview comment:', error) - process.exit(1) -}) diff --git a/packages/stats-generator/src/measurement-ci.ts b/packages/stats-generator/src/measurement-ci.ts new file mode 100644 index 00000000..df6e1f4c --- /dev/null +++ b/packages/stats-generator/src/measurement-ci.ts @@ -0,0 +1,82 @@ +import { execFileSync } from 'node:child_process' +import { appendFileSync, readFileSync } from 'node:fs' +import { resolve } from 'node:path' +import { frameworks } from './frameworks.ts' +import { + measurementJobs, + shouldMeasure, + tasks, +} from './measurement-pipeline.ts' +import { assembleResults } from './save-ci-stats.ts' +import { promoteResults } from './promote-results.ts' + +try { + const [command, ...args] = process.argv.slice(2) + if (command === 'select') { + let hasChanges = true + const selectedBase = process.env.MEASUREMENT_BASE_SHA + const selectedHead = process.env.MEASUREMENT_HEAD_SHA + if (selectedBase || selectedHead) { + if ( + !/^[a-f0-9]{40}$/.test(selectedBase ?? '') || + !/^[a-f0-9]{40}$/.test(selectedHead ?? '') + ) + throw new Error( + 'PR measurements require exact base and merge commit SHAs', + ) + const files = execFileSync( + 'git', + ['diff', '--name-only', '-z', `${selectedBase}...${selectedHead}`], + { encoding: 'utf8' }, + ) + .split('\0') + .filter(Boolean) + hasChanges = shouldMeasure(files) + } else if ( + ['pull_request', 'push'].includes(process.env.GITHUB_EVENT_NAME!) + ) { + const event = JSON.parse( + readFileSync(process.env.GITHUB_EVENT_PATH!, 'utf8'), + ) + const base = event.pull_request?.base.sha ?? event.before + const head = event.pull_request?.head.sha ?? event.after + // Missing history fails clearly rather than silently skipping measurements. + const files = execFileSync( + 'git', + [ + 'diff', + '--name-only', + '-z', + ...(event.pull_request ? [`${base}...${head}`] : [base, head]), + ], + { encoding: 'utf8' }, + ) + .split('\0') + .filter(Boolean) + hasChanges = shouldMeasure(files) + } + const jobs = hasChanges ? measurementJobs(frameworks) : [] + const outputs = { + measurements: Object.fromEntries( + Object.keys(tasks).map((task) => [ + task, + jobs.filter((job) => job.task === task).map((job) => job.framework), + ]), + ), + 'has-changes': hasChanges, + } + for (const [key, value] of Object.entries(outputs)) + appendFileSync( + process.env.GITHUB_OUTPUT!, + `${key}=${JSON.stringify(value)}\n`, + ) + } else if (command === 'assemble') { + const output = resolve(args[1]!) + await assembleResults(frameworks, resolve(args[0]!), output) + } else if (command === 'promote') { + promoteResults(frameworks, resolve(args[0]!)) + } else throw new Error('Expected select, assemble, or promote') +} catch (error) { + console.error('Measurement pipeline failed:', error) + process.exitCode = 1 +} diff --git a/packages/stats-generator/src/measurement-pipeline.ts b/packages/stats-generator/src/measurement-pipeline.ts new file mode 100644 index 00000000..da54967d --- /dev/null +++ b/packages/stats-generator/src/measurement-pipeline.ts @@ -0,0 +1,116 @@ +import { frameworks } from './frameworks.ts' +import type { FrameworkConfig } from './types.ts' + +export const tasks = { + install: { + project: 'starter', + measurement: 'install', + files: ['install-stats.json'], + }, + build: { + project: 'starter', + measurement: 'build', + files: ['build-stats.json', 'corejs-stats.json'], + }, + 'dev-server': { + project: 'starter', + measurement: 'devServer', + files: ['dev-server-stats.json'], + }, + deps: { + project: 'starter', + measurement: 'dependencies', + files: [ + 'e18e-stats.json', + 'framework-dependency-stats.json', + 'first-party-dependency-stats.json', + 'node-engines-stats.json', + ], + }, + 'client-side-rendered': { + project: 'app', + measurement: 'clientSideRendered', + files: ['ci-stats.json'], + }, + 'server-side-rendered': { + project: 'app', + measurement: 'serverSideRendered', + files: ['ci-stats.json'], + }, + 'ssr-request-throughput': { + project: 'app', + measurement: 'ssrRequestThroughput', + files: ['ci-stats.json'], + }, + 'ssr-load': { + project: 'app', + measurement: 'ssrLoad', + files: ['ci-stats.json'], + }, + 'ssr-router-link-load': { + project: 'app', + measurement: 'ssrRouterLinkLoad', + files: ['ci-stats.json'], + }, +} as const +export type TaskName = keyof typeof tasks +export interface MeasurementJob { + framework: string + package: string + task: TaskName +} + +export function measurementJobs( + selected: readonly FrameworkConfig[], +): MeasurementJob[] { + return selected.flatMap((framework) => + Object.entries(tasks).flatMap(([task, config]) => { + const project = framework[config.project] + return project?.measurements.includes(config.measurement) + ? [ + { + framework: framework.name, + package: project.package, + task: task as TaskName, + }, + ] + : [] + }), + ) +} + +export function artifactFiles(job: MeasurementJob): string[] { + const files: string[] = [...tasks[job.task].files] + const framework: FrameworkConfig | undefined = frameworks.find( + (item) => item.name === job.framework, + ) + if ( + job.task === 'build' && + framework?.starter?.measurements.includes('browserBaseline') + ) + files.push('browser-baseline-stats.json') + return files +} +export function artifactName(job: MeasurementJob) { + return `${job.task}-stats-${job.framework}` +} +export function isLoadJob(job: MeasurementJob) { + return job.task === 'ssr-load' || job.task === 'ssr-router-link-load' +} + +const generated = /(?:^|\/)(?:ci-stats\.json|[^/]*-stats\.json|stats\/.*)$/ +export function shouldMeasure(files: string[]): boolean { + return files.some( + (file) => + !generated.test(file) && + !/\.(?:md|mdx)$/.test(file) && + (/^packages\/(?:starter-[^/]+|app-[^/]+|stats-generator|results|testdata)\//.test( + file, + ) || + /^(?:package\.json|pnpm-lock\.yaml|pnpm-workspace\.yaml|tsconfig\.json|Dockerfile|\.dockerignore|docker-compose\.yml|\.npmrc|\.nvmrc)$/.test( + file, + ) || + /^\.github\/workflows\/(?:measure|benchmark)/.test(file) || + /^\.github\/scripts\/measurement-/.test(file)), + ) +} diff --git a/packages/stats-generator/src/promote-results.test.ts b/packages/stats-generator/src/promote-results.test.ts new file mode 100644 index 00000000..cebb02b3 --- /dev/null +++ b/packages/stats-generator/src/promote-results.test.ts @@ -0,0 +1,114 @@ +import assert from 'node:assert/strict' +import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { test } from 'node:test' +import { + readProjectResult, + resultPath, + saveProjectResult, +} from '@framework-tracker/results/storage' +import { frameworks } from './frameworks.ts' +import { promoteResults } from './promote-results.ts' + +const selectedFrameworks = frameworks.filter((item) => item.name === 'astro') + +for (const upgradedProject of ['starter', 'app'] as const) { + test(`promotion preserves sibling history when ${upgradedProject} advances to another version`, () => { + const directory = mkdtempSync(join(tmpdir(), 'promote-results-')) + const sourceRoot = join(directory, 'incoming') + const destinationRoot = join(directory, 'canonical') + const oldStarter = { + frameworkVersion: '1.0.0', + runner: 'old-starter', + prodDependencies: 3, + devDependencies: 2, + } + const oldApp = { + frameworkVersion: '1.0.0', + runner: 'old-app', + testTimeMs: 20, + } + const newStarter = { + frameworkVersion: upgradedProject === 'starter' ? '2.0.0' : '1.0.0', + runner: 'new-starter', + prodDependencies: 4, + } + const newApp = { + frameworkVersion: upgradedProject === 'app' ? '2.0.0' : '1.0.0', + runner: 'new-app', + testTimeMs: 30, + } + + try { + saveProjectResult('starter-astro', oldStarter, destinationRoot) + saveProjectResult('app-astro', oldApp, destinationRoot) + saveProjectResult( + 'app-astro', + { frameworkVersion: '0.5.0', testTimeMs: 10 }, + destinationRoot, + ) + saveProjectResult('starter-next-js', oldStarter, destinationRoot) + const unrelatedHistoryPath = resultPath('astro', '0.5.0', destinationRoot) + const unrelatedHistory = readFileSync(unrelatedHistoryPath, 'utf8') + const otherFrameworkPath = resultPath('next', undefined, destinationRoot) + const otherFramework = readFileSync(otherFrameworkPath, 'utf8') + + saveProjectResult('starter-astro', newStarter, sourceRoot) + saveProjectResult('app-astro', newApp, sourceRoot) + const sourceLatestPath = resultPath('astro', undefined, sourceRoot) + const sourceLatest = readFileSync(sourceLatestPath, 'utf8') + + promoteResults(selectedFrameworks, sourceRoot, destinationRoot) + + assert.deepEqual( + readProjectResult('starter-astro', '1.0.0', destinationRoot), + upgradedProject === 'starter' ? oldStarter : newStarter, + ) + assert.deepEqual( + readProjectResult('app-astro', '1.0.0', destinationRoot), + upgradedProject === 'app' ? oldApp : newApp, + ) + assert.deepEqual( + readProjectResult(`${upgradedProject}-astro`, '2.0.0', destinationRoot), + upgradedProject === 'starter' ? newStarter : newApp, + ) + assert.equal( + readFileSync(resultPath('astro', undefined, destinationRoot), 'utf8'), + sourceLatest, + ) + assert.equal(readFileSync(sourceLatestPath, 'utf8'), sourceLatest) + assert.equal(readFileSync(unrelatedHistoryPath, 'utf8'), unrelatedHistory) + assert.equal(readFileSync(otherFrameworkPath, 'utf8'), otherFramework) + } finally { + rmSync(directory, { recursive: true, force: true }) + } + }) +} + +test('promotion creates results when there is no existing destination', () => { + const directory = mkdtempSync(join(tmpdir(), 'promote-results-')) + const sourceRoot = join(directory, 'incoming') + const destinationRoot = join(directory, 'canonical') + const starter = { frameworkVersion: '1.0.0', prodDependencies: 3 } + const app = { frameworkVersion: '1.0.0', testTimeMs: 20 } + try { + saveProjectResult('starter-astro', starter, sourceRoot) + saveProjectResult('app-astro', app, sourceRoot) + + promoteResults(selectedFrameworks, sourceRoot, destinationRoot) + + for (const version of [undefined, '1.0.0']) { + assert.deepEqual( + readProjectResult('starter-astro', version, destinationRoot), + starter, + ) + assert.deepEqual( + readProjectResult('app-astro', version, destinationRoot), + app, + ) + } + } finally { + rmSync(directory, { recursive: true, force: true }) + } +}) diff --git a/packages/stats-generator/src/promote-results.ts b/packages/stats-generator/src/promote-results.ts new file mode 100644 index 00000000..831d448b --- /dev/null +++ b/packages/stats-generator/src/promote-results.ts @@ -0,0 +1,72 @@ +import { copyFileSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { dirname } from 'node:path' +import { + CombinedResultSchema, + mergeProjectResult, + projectResult, + resultPath, + resultsRoot, + type CombinedResult, +} from '@framework-tracker/results/storage' +import type { FrameworkConfig } from './types.ts' +import { readRequiredJson, validateResults } from './validate-stats.ts' + +function readExistingResult(path: string): CombinedResult | null { + try { + return CombinedResultSchema.parse(JSON.parse(readFileSync(path, 'utf8'))) + } catch (error) { + if (error instanceof Error && 'code' in error && error.code === 'ENOENT') { + return null + } + throw error + } +} + +export function promoteResults( + selectedFrameworks: FrameworkConfig[], + sourceRoot: string, + destinationRoot = resultsRoot, +) { + validateResults(selectedFrameworks, sourceRoot) + for (const framework of selectedFrameworks) { + const sourceLatestPath = resultPath(framework.name, undefined, sourceRoot) + const latest = CombinedResultSchema.parse( + readRequiredJson(sourceLatestPath), + ) + + // Update each measured project while preserving its sibling's older results. + for (const project of ['starter', 'app'] as const) { + const projectConfig = framework[project] + const version = latest.projects[project]?.frameworkVersion + if (!projectConfig || !version || version === 'unknown') continue + + const incomingVersion = CombinedResultSchema.parse( + readRequiredJson(resultPath(framework.name, version, sourceRoot)), + ) + const destinationVersionPath = resultPath( + framework.name, + version, + destinationRoot, + ) + const mergedVersion = mergeProjectResult( + readExistingResult(destinationVersionPath), + projectConfig.package, + projectResult(incomingVersion, project), + ) + mkdirSync(dirname(destinationVersionPath), { recursive: true }) + writeFileSync( + destinationVersionPath, + JSON.stringify(mergedVersion, null, 2) + '\n', + ) + } + + // Current results are a complete snapshot and must not retain old measurements. + const destinationLatestPath = resultPath( + framework.name, + undefined, + destinationRoot, + ) + mkdirSync(dirname(destinationLatestPath), { recursive: true }) + copyFileSync(sourceLatestPath, destinationLatestPath) + } +} diff --git a/packages/stats-generator/src/result-storage.test.ts b/packages/stats-generator/src/result-storage.test.ts index 18578f8a..75fec671 100644 --- a/packages/stats-generator/src/result-storage.test.ts +++ b/packages/stats-generator/src/result-storage.test.ts @@ -76,3 +76,162 @@ test('stored tests always validate their result and require project metadata', ( false, ) }) + +// Source changes trigger the full pipeline independently of publishing. +import { frameworks } from './frameworks.ts' +import { + artifactFiles, + isLoadJob, + measurementJobs, + shouldMeasure, +} from './measurement-pipeline.ts' +import { validateArtifact, validateArtifacts } from './validate-stats.ts' +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +test('any framework or shared source change triggers measurements', () => { + for (const files of [ + ['packages/app-astro/src/pages/index.astro'], + ['packages/starter-astro/package.json'], + [ + 'packages/starter-astro/package.json', + 'packages/app-next-js/pnpm-lock.yaml', + ], + ['packages/stats-generator/src/run-measurement.ts'], + ['packages/results/src/frameworks.ts'], + ['pnpm-lock.yaml'], + ['pnpm-workspace.yaml'], + ['.github/workflows/measure.yml'], + ['.github/workflows/measure-pr.yml'], + ['.github/scripts/measurement-control.ts'], + ['packages/testdata/data.json'], + ]) + assert.equal(shouldMeasure(files), true) +}) + +test('docs and generated results never select benchmarks', () => { + assert.equal( + shouldMeasure([ + 'README.md', + 'CONTRIBUTING.md', + 'packages/docs/src/pages/index.astro', + 'packages/starter-astro/README.md', + 'results/astro/latest.json', + 'packages/starter-astro/ci-stats.json', + 'packages/app-next-js/stats/1.json', + 'packages/starter-next-js/build-stats.json', + ]), + false, + ) + assert.equal(shouldMeasure([]), false) +}) + +test('measurement jobs cover every configured framework and project', () => { + const jobs = measurementJobs(frameworks) + assert.deepEqual( + new Set(jobs.map((job) => job.framework)), + new Set(frameworks.map((framework) => framework.name)), + ) + assert.deepEqual( + new Set(jobs.map((job) => job.package)), + new Set( + frameworks.flatMap((framework) => + [framework.starter, framework.app].flatMap((project) => + project ? [project.package] : [], + ), + ), + ), + ) +}) + +test('job manifest keeps load jobs separate and requires companion scan artifacts', () => { + const jobs = measurementJobs( + frameworks.filter((item) => item.name === 'next'), + ) + assert.deepEqual( + jobs.filter(isLoadJob).map((job) => job.task), + ['ssr-load', 'ssr-router-link-load'], + ) + assert.deepEqual(artifactFiles(jobs.find((job) => job.task === 'build')!), [ + 'build-stats.json', + 'corejs-stats.json', + 'browser-baseline-stats.json', + ]) + assert.ok( + artifactFiles(jobs.find((job) => job.task === 'deps')!).includes( + 'first-party-dependency-stats.json', + ), + ) +}) + +test('missing and malformed artifacts fail before results can be assembled', () => { + const directory = mkdtempSync(join(tmpdir(), 'measurement-artifacts-')) + try { + const selected = frameworks.filter((item) => item.name === 'astro') + assert.throws( + () => validateArtifacts(selected, directory), + /Missing or invalid result artifact/, + ) + const job = measurementJobs(selected).find((job) => job.task === 'install')! + writeFileSync(join(directory, 'install-stats.json'), '{') + assert.throws( + () => validateArtifact(job, directory), + /Missing or invalid result artifact/, + ) + writeFileSync(join(directory, 'install-stats.json'), '{}') + assert.throws( + () => validateArtifact(job, directory), + /Invalid result artifact/, + ) + } finally { + rmSync(directory, { recursive: true, force: true }) + } +}) + +test('dependency artifacts validate e18e output with package lockfile counts', () => { + const directory = mkdtempSync(join(tmpdir(), 'measurement-dependencies-')) + try { + const job = measurementJobs( + frameworks.filter((item) => item.name === 'astro'), + ).find((job) => job.task === 'deps')! + const reports = { + 'e18e-stats.json': { + stats: { + dependencyCount: { production: 2, development: 3 }, + installSize: 1234, + }, + }, + 'framework-dependency-stats.json': { + name: 'frameworkDependencies', + prodDependencies: 1, + devDependencies: 0, + allDependencies: 10, + }, + 'first-party-dependency-stats.json': { + name: 'firstPartyDependencies', + prodDependencies: 1, + devDependencies: 0, + allDependencies: 10, + }, + 'node-engines-stats.json': { + name: 'nodeEngines', + minimumNodeVersion: '24.0.0', + imposedBy: [], + packagesScanned: 10, + packagesDeclaringNodeEngine: 5, + unsatisfiableRanges: [], + }, + } + for (const [file, report] of Object.entries(reports)) + writeFileSync(join(directory, file), JSON.stringify(report)) + assert.doesNotThrow(() => validateArtifact(job, directory)) + writeFileSync(join(directory, 'e18e-stats.json'), '{}') + assert.throws( + () => validateArtifact(job, directory), + /Invalid result artifact/, + ) + } finally { + rmSync(directory, { recursive: true, force: true }) + } +}) diff --git a/packages/stats-generator/src/run-measurement.ts b/packages/stats-generator/src/run-measurement.ts new file mode 100644 index 00000000..fb694e9a --- /dev/null +++ b/packages/stats-generator/src/run-measurement.ts @@ -0,0 +1,121 @@ +import { execFileSync, spawnSync } from 'node:child_process' +import { copyFileSync, mkdirSync, rmSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { packagesDir } from './constants.ts' +import type { FrameworkConfig } from './types.ts' +import { frameworks } from './frameworks.ts' +import { + artifactFiles, + artifactName, + tasks, + type MeasurementJob, + type TaskName, +} from './measurement-pipeline.ts' +import { validateArtifact } from './validate-stats.ts' +import { validateFirstPartyDependencyManifest } from './first-party-dependencies.ts' +import { getPackageJsonDeps, readJsonFile } from './utils.ts' + +const repository = join(packagesDir, '..') +export function runMeasurement(job: MeasurementJob, artifacts: string) { + const config = tasks[job.task] + const framework: FrameworkConfig | undefined = frameworks.find( + (item) => item.name === job.framework, + ) + if ( + !framework || + framework[config.project]?.package !== job.package || + !framework[config.project]?.measurements.includes(config.measurement) + ) + throw new Error(`Unsupported measurement: ${JSON.stringify(job)}`) + const project = join(packagesDir, job.package) + const run = (command: string, args: string[], cwd = repository) => + execFileSync(command, args, { cwd, stdio: 'inherit' }) + const script = (name: string) => + run(process.execPath, [ + join(packagesDir, 'stats-generator/src', name), + job.package, + ]) + for (const file of artifactFiles(job)) + rmSync(join(project, file), { force: true }) + // Runtime runners must start with fresh data, rather than canonical measurements. + rmSync(join(project, 'ci-stats.json'), { force: true }) + if (config.project === 'app' || job.task === 'deps') + run('pnpm', ['install', '--frozen-lockfile'], project) + if (config.project === 'app') run('pnpm', ['build'], project) + switch (job.task) { + case 'install': + script('run-install-benchmark.ts') + break + case 'build': + script('run-build-benchmark.ts') + script('run-corejs-scan.ts') + if (artifactFiles(job).includes('browser-baseline-stats.json')) + script('run-browser-baseline-scan.ts') + break + case 'dev-server': + script('run-dev-server-benchmark.ts') + break + case 'deps': { + const errors = validateFirstPartyDependencyManifest( + readJsonFile(join(project, 'first-party-dependencies.json')), + getPackageJsonDeps(job.package), + ) + if (errors.length) throw new Error(errors.join('\n')) + // e18e can return a nonzero status for findings; its JSON must still validate. + const report = spawnSync( + 'pnpm', + ['dlx', '@e18e/cli@0.5.0', 'analyze', '--json'], + { + cwd: project, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'inherit'], + }, + ) + if (report.error || report.signal || !report.stdout.trim()) + throw new Error('Dependency analysis produced no report', { + cause: report.error, + }) + writeFileSync(join(project, 'e18e-stats.json'), report.stdout) + script('run-framework-dependency-scan.ts') + script('run-first-party-dependency-scan.ts') + script('run-node-engines-scan.ts') + break + } + case 'ssr-load': + case 'ssr-router-link-load': + run('pnpm', [ + '--filter', + '@framework-tracker/stats-generator', + `run:${job.task}:containerized`, + job.package, + ]) + break + default: + script(`run-${job.task}-benchmark.ts`) + } + validateArtifact(job, project) + const output = join(artifacts, artifactName(job)) + mkdirSync(output, { recursive: true }) + for (const file of artifactFiles(job)) + copyFileSync(join(project, file), join(output, file)) +} +if (process.argv[1] === fileURLToPath(import.meta.url)) { + try { + const [framework, task, artifacts] = process.argv.slice(2) + if (!framework || !task || !(task in tasks) || !artifacts) + throw new Error( + 'Expected framework, measurement task, and artifact directory', + ) + const config = frameworks.find((item) => item.name === framework) + const packageName = config?.[tasks[task as TaskName].project]?.package + if (!packageName) throw new Error(`Unknown framework: ${framework}`) + runMeasurement( + { framework, task: task as TaskName, package: packageName }, + artifacts, + ) + } catch (error) { + console.error(error) + process.exitCode = 1 + } +} diff --git a/packages/stats-generator/src/save-ci-stats.ts b/packages/stats-generator/src/save-ci-stats.ts index 232cb65f..1a8e91c7 100644 --- a/packages/stats-generator/src/save-ci-stats.ts +++ b/packages/stats-generator/src/save-ci-stats.ts @@ -1,7 +1,4 @@ -import { - readProjectResult, - saveProjectResult, -} from '@framework-tracker/results/storage' +import { saveProjectResult } from '@framework-tracker/results/storage' import { DependencyScanSchema, CoreJsStatsSchema, @@ -23,7 +20,8 @@ import { } from './request-throughput.ts' import { mergeSSRLoadArtifact } from './ssrLoad/merge.ts' import { join } from 'node:path' -import { getFrameworks } from './get-frameworks.ts' +import type { FrameworkConfig } from './types.ts' +import { validateArtifacts, validateResults } from './validate-stats.ts' import { getPublishTimes, withReleasedAt } from './release-dates.ts' import { getDependencyCountsFromPackageMetadata, @@ -74,8 +72,12 @@ async function getReleasedAt( } } -async function main() { - const artifactsDir = process.argv[2] || 'artifacts' +export async function assembleResults( + frameworks: FrameworkConfig[], + artifactsDir: string, + outputRoot: string, +) { + validateArtifacts(frameworks, artifactsDir) const timestamp = new Date().toISOString() const runner = process.env.RUNNER_LABEL || 'local' @@ -84,8 +86,6 @@ async function main() { console.info(` Timestamp: ${timestamp}`) console.info(` Runner: ${runner}\n`) - const frameworks = await getFrameworks() - for (const framework of frameworks) { const { name, displayName } = framework @@ -94,10 +94,7 @@ async function main() { const packageName = framework.starter.package console.info(`Processing ${displayName} starter (${packageName})...`) - const existingStats = readProjectResult(packageName) as CIStats | null - let stats: CIStats = { - ...(existingStats ? normalizeCIStats(existingStats) : {}), ...BaseResultSchema.parse({ framework: framework.name, packagePath: `packages/${packageName}`, @@ -106,7 +103,7 @@ async function main() { }), } - let frameworkVersion = existingStats?.frameworkVersion + let frameworkVersion: string | undefined // Load install stats from artifact const installStatsPath = join( @@ -172,7 +169,7 @@ async function main() { // Load core-js stats from artifact const coreJsArtifactPath = join( artifactsDir, - `corejs-stats-${name}`, + `build-stats-${name}`, 'corejs-stats.json', ) const coreJsStats = readJsonFile(coreJsArtifactPath) @@ -193,7 +190,7 @@ async function main() { // Load browser baseline stats from artifact const browserBaselineArtifactPath = join( artifactsDir, - `browser-baseline-stats-${name}`, + `build-stats-${name}`, 'browser-baseline-stats.json', ) const browserBaselineStats = readJsonFile( @@ -216,7 +213,7 @@ async function main() { const nodeEnginesArtifactPath = join( artifactsDir, - `node-engines-stats-${name}`, + `deps-stats-${name}`, 'node-engines-stats.json', ) const nodeEnginesStats = readJsonFile( @@ -244,12 +241,12 @@ async function main() { // Load dependency stats from artifacts const e18eArtifactPath = join( artifactsDir, - `e18e-stats-${name}`, + `deps-stats-${name}`, 'e18e-stats.json', ) const frameworkDependencyStatsPath = join( artifactsDir, - `e18e-stats-${name}`, + `deps-stats-${name}`, 'framework-dependency-stats.json', ) const frameworkDependencies = readJsonFile( @@ -272,7 +269,7 @@ async function main() { const firstPartyDependencyStatsPath = join( artifactsDir, - `e18e-stats-${name}`, + `deps-stats-${name}`, 'first-party-dependency-stats.json', ) const firstPartyDependencies = readJsonFile( @@ -315,11 +312,10 @@ async function main() { } // check for releasedAt, if not fetch from repository - const releasedAt = - existingStats?.frameworkVersion === frameworkVersion && - existingStats?.releasedAt - ? existingStats?.releasedAt - : await getReleasedAt(framework.frameworkPackage, frameworkVersion) + const releasedAt = await getReleasedAt( + framework.frameworkPackage, + frameworkVersion, + ) stats = withReleasedAt( { @@ -331,7 +327,7 @@ async function main() { validateSSRLoadForPersistence(stats) validateRequestThroughputForPersistence(stats) - saveProjectResult(packageName, { ...stats }) + saveProjectResult(packageName, { ...stats }, outputRoot) console.info('') } @@ -341,10 +337,7 @@ async function main() { const packageName = framework.app.package console.info(`Processing ${displayName} app (${packageName})...`) - const existingStats = readProjectResult(packageName) as CIStats | null - let stats: CIStats = { - ...(existingStats ? normalizeCIStats(existingStats) : {}), ...BaseResultSchema.parse({ framework: framework.name, packagePath: `packages/${packageName}`, @@ -353,7 +346,7 @@ async function main() { }), } - let frameworkVersion = existingStats?.frameworkVersion + let frameworkVersion: string | undefined // Load SSR request throughput stats from artifact const ssrRequestThroughputStatsPath = join( @@ -489,11 +482,10 @@ async function main() { } // check for releasedAt, if not fetch from repository - const releasedAt = - existingStats?.frameworkVersion === frameworkVersion && - existingStats?.releasedAt - ? existingStats?.releasedAt - : await getReleasedAt(framework.frameworkPackage, frameworkVersion) + const releasedAt = await getReleasedAt( + framework.frameworkPackage, + frameworkVersion, + ) stats = withReleasedAt( { @@ -505,16 +497,12 @@ async function main() { validateSSRLoadForPersistence(stats) validateRequestThroughputForPersistence(stats) - saveProjectResult(packageName, { ...stats }) + saveProjectResult(packageName, { ...stats }, outputRoot) console.info('') } } + validateResults(frameworks, outputRoot) console.info('Done!') } - -main().catch((error) => { - console.error('Failed to save CI stats:', error) - process.exit(1) -}) diff --git a/packages/stats-generator/src/validate-stats.ts b/packages/stats-generator/src/validate-stats.ts index b73aa768..3c9e8392 100644 --- a/packages/stats-generator/src/validate-stats.ts +++ b/packages/stats-generator/src/validate-stats.ts @@ -1,215 +1,128 @@ -import { validateSSRLoadForPersistence } from './ssrLoad/persistence.ts' -import type { CIStats } from './types.ts' +import { readFileSync } from 'node:fs' import { join } from 'node:path' import { z } from 'zod' -import { getFrameworks } from './get-frameworks.ts' -import { packagesDir } from './constants.ts' -import { readJsonFile } from './utils.ts' import { + CombinedResultSchema, + projectResult, + resultPath, +} from '@framework-tracker/results/storage' +import type { FrameworkConfig, CIStats } from './types.ts' +import { + artifactFiles, + artifactName, + measurementJobs, + type MeasurementJob, +} from './measurement-pipeline.ts' +import { validateSSRLoadForPersistence } from './ssrLoad/persistence.ts' +import { validateRequestThroughputForPersistence } from './request-throughput.ts' +import { + getDependencyCountsFromPackageMetadata, + getDependencyStatsFromE18e, +} from './utils.ts' +import type { E18eStats } from './types.ts' +import { + BuildStatsSchema, CoreJsStatsSchema, - FrameworkDependencyScanSchema, - FirstPartyDependencyScanSchema, InstallResultSchema, - BuildStatsSchema, DevServerStatsSchema, BrowserBaselineStatsSchema, NodeEnginesStatsSchema, - SSRRequestThroughputStatsSchema, + FrameworkDependencyScanSchema, + FirstPartyDependencyScanSchema, + DependencyScanSchema, + RequestThroughputResultSchema, SSRLoadStatsSchema, SSRRouterLinkLoadStatsSchema, ClientSideRenderedStatsSchema, ServerSideRenderedStatsSchema, } from '@framework-tracker/results' -type BenchmarkType = - | 'dependencies' - | 'install' - | 'build' - | 'devServer' - | 'browserBaseline' - | 'nodeEngines' - | 'ssrRequestThroughput' - | 'ssrLoad' - | 'ssrRouterLinkLoad' - | 'clientSideRendered' - | 'serverSideRendered' - -interface BenchmarkConfig { - type: BenchmarkType - file: string - schema: z.ZodType +const fileSchemas: Record = { + 'install-stats.json': InstallResultSchema, + 'build-stats.json': BuildStatsSchema, + 'corejs-stats.json': CoreJsStatsSchema, + 'dev-server-stats.json': DevServerStatsSchema, + 'browser-baseline-stats.json': BrowserBaselineStatsSchema, + 'node-engines-stats.json': NodeEnginesStatsSchema, + 'framework-dependency-stats.json': FrameworkDependencyScanSchema, + 'first-party-dependency-stats.json': FirstPartyDependencyScanSchema, } - -const STARTER_BENCHMARKS: BenchmarkConfig[] = [ - { - type: 'dependencies', - file: 'corejs-stats.json', - schema: CoreJsStatsSchema, - }, - { - type: 'dependencies', - file: 'framework-dependency-stats.json', - schema: FrameworkDependencyScanSchema, - }, - { - type: 'dependencies', - file: 'first-party-dependency-stats.json', - schema: FirstPartyDependencyScanSchema, - }, - - { type: 'install', file: 'install-stats.json', schema: InstallResultSchema }, - { type: 'build', file: 'build-stats.json', schema: BuildStatsSchema }, - { - type: 'devServer', - file: 'dev-server-stats.json', - schema: DevServerStatsSchema, - }, - { - type: 'browserBaseline', - file: 'browser-baseline-stats.json', - schema: BrowserBaselineStatsSchema, - }, - { - type: 'nodeEngines', - file: 'node-engines-stats.json', - schema: NodeEnginesStatsSchema, - }, -] - -const APP_BENCHMARKS: BenchmarkConfig[] = [ - { - type: 'ssrRouterLinkLoad', - file: 'ci-stats.json', - schema: SSRRouterLinkLoadStatsSchema, - }, - { - type: 'clientSideRendered', - file: 'ci-stats.json', - schema: ClientSideRenderedStatsSchema, - }, - { - type: 'serverSideRendered', - file: 'ci-stats.json', - schema: ServerSideRenderedStatsSchema, - }, - { - type: 'ssrRequestThroughput', - file: 'ci-stats.json', - schema: SSRRequestThroughputStatsSchema, - }, - { - type: 'ssrLoad', - file: 'ci-stats.json', - schema: SSRLoadStatsSchema, - }, -] - -function validateFile(filePath: string, schema: z.ZodType): string[] { - const data = readJsonFile(filePath) - - if (!data) { - return [`File not found: ${filePath}`] +const appSchemas: Record = { + 'ssr-request-throughput': RequestThroughputResultSchema, + 'ssr-load': SSRLoadStatsSchema, + 'ssr-router-link-load': SSRRouterLinkLoadStatsSchema, + 'client-side-rendered': ClientSideRenderedStatsSchema, + 'server-side-rendered': ServerSideRenderedStatsSchema, +} +export function readRequiredJson(path: string): unknown { + try { + return JSON.parse(readFileSync(path, 'utf8')) + } catch (cause) { + throw new Error(`Missing or invalid result artifact: ${path}`, { cause }) } - - const result = schema.safeParse(data) - if (result.success) { - if ( - schema === SSRLoadStatsSchema || - schema === SSRRouterLinkLoadStatsSchema - ) { - try { - validateSSRLoadForPersistence(data as CIStats) - } catch (error) { - return [error instanceof Error ? error.message : String(error)] +} +export function validateArtifact(job: MeasurementJob, directory: string) { + for (const file of artifactFiles(job)) { + const path = join(directory, file) + const value = readRequiredJson(path) + try { + if (file === 'e18e-stats.json') + DependencyScanSchema.parse({ + name: 'dependencies', + ...getDependencyStatsFromE18e(value as E18eStats), + allDependencies: getDependencyCountsFromPackageMetadata(job.package) + .allDependencies, + }) + else + (file === 'ci-stats.json' + ? appSchemas[job.task]! + : fileSchemas[file]! + ).parse(value) + if (file === 'ci-stats.json') { + validateSSRLoadForPersistence(value as CIStats) + validateRequestThroughputForPersistence(value as CIStats) } + } catch (cause) { + throw new Error(`Invalid result artifact: ${path}`, { cause }) } - return [] } - - return result.error.issues.map( - (issue) => `${issue.path.join('.') || 'root'}: ${issue.message}`, - ) } - -async function main() { - const [target = 'all', filterType] = process.argv.slice(2) - - console.info('Validating stats output...\n') - - const frameworks = await getFrameworks() - let hasFailures = false - - for (const framework of frameworks) { - // Validate starter - if (framework.starter) { - const pkg = framework.starter.package - if (target !== 'all' && target !== framework.name && target !== pkg) - continue - - console.info(`Validating ${framework.displayName} starter (${pkg})...`) - - const errors: string[] = [] - for (const { type: benchType, file, schema } of STARTER_BENCHMARKS) { - if (filterType && filterType !== benchType) continue - if (!framework.starter.measurements.includes(benchType)) { - continue - } - - const filePath = join(packagesDir, pkg, file) - const fileErrors = validateFile(filePath, schema) - errors.push(...fileErrors.map((e) => `[${benchType}] ${e}`)) - } - - if (errors.length === 0) { - console.info(' ✓ Valid\n') - } else { - console.info(' ✗ Invalid') - errors.forEach((e) => console.error(` ${e}`)) - console.info('') - hasFailures = true - } - } - - // Validate app - if (framework.app) { - const pkg = framework.app.package - if (target !== 'all' && target !== framework.name && target !== pkg) - continue - - console.info(`Validating ${framework.displayName} app (${pkg})...`) - - const errors: string[] = [] - for (const { type: benchType, file, schema } of APP_BENCHMARKS) { - if (filterType && filterType !== benchType) continue - if (!framework.app.measurements.includes(benchType)) { - continue - } - - const filePath = join(packagesDir, pkg, file) - const fileErrors = validateFile(filePath, schema) - errors.push(...fileErrors.map((e) => `[${benchType}] ${e}`)) - } - - if (errors.length === 0) { - console.info(' ✓ Valid\n') - } else { - console.info(' ✗ Invalid') - errors.forEach((e) => console.error(` ${e}`)) - console.info('') - hasFailures = true +export function validateArtifacts( + selected: FrameworkConfig[], + directory: string, +) { + for (const job of measurementJobs(selected)) + validateArtifact(job, join(directory, artifactName(job))) +} +export function validateResults(selected: FrameworkConfig[], root: string) { + for (const framework of selected) { + const combined = CombinedResultSchema.parse( + readRequiredJson(resultPath(framework.name, undefined, root)), + ) + if (combined.framework !== framework.name) + throw new Error(`Wrong framework result: ${framework.name}`) + for (const project of ['starter', 'app'] as const) { + if (!framework[project]) continue + if (!combined.projects[project]) + throw new Error(`Missing ${framework.name} ${project} results`) + const result = projectResult(combined, project) as CIStats + validateSSRLoadForPersistence(result) + validateRequestThroughputForPersistence(result) + if (result.frameworkVersion && result.frameworkVersion !== 'unknown') { + const version = CombinedResultSchema.parse( + readRequiredJson( + resultPath(framework.name, result.frameworkVersion, root), + ), + ) + if ( + version.framework !== framework.name || + version.projects[project]?.frameworkVersion !== + result.frameworkVersion + ) + throw new Error( + `Missing version metadata for ${framework.name} ${project}`, + ) } } } - - if (hasFailures) { - console.error('Validation failed!') - process.exit(1) - } - - console.info('✓ All validations passed!') } - -main().catch((error) => { - console.error('Validation failed:', error) - process.exit(1) -})