diff --git a/.github/actions/bun-cache/action.yml b/.github/actions/bun-cache/action.yml new file mode 100644 index 0000000..9a43c91 --- /dev/null +++ b/.github/actions/bun-cache/action.yml @@ -0,0 +1,10 @@ +name: Cache Bun downloads +description: Reuse package downloads while keeping frozen dependency installation +runs: + using: composite + steps: + - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ~/.bun/install/cache + key: bun-v1-${{ runner.os }}-${{ runner.arch }}-1.3.14-${{ hashFiles('**/bun.lock') }} + restore-keys: bun-v1-${{ runner.os }}-${{ runner.arch }}-1.3.14- diff --git a/.github/actions/ci-tools/action.yml b/.github/actions/ci-tools/action.yml new file mode 100644 index 0000000..1664053 --- /dev/null +++ b/.github/actions/ci-tools/action.yml @@ -0,0 +1,28 @@ +name: Prepare pinned CI tools +description: Restore or install verified tool binaries independently of application builds +inputs: + group: + description: worker, browser, or notices + required: true +runs: + using: composite + steps: + - id: info + shell: bash + env: + TOOL_GROUP: ${{ inputs.group }} + run: bun tools/ci/setup.ts info "$TOOL_GROUP" + - id: cache + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ steps.info.outputs.path }} + key: ${{ steps.info.outputs.key }} + - shell: bash + env: + TOOL_GROUP: ${{ inputs.group }} + run: bun tools/ci/setup.ts install "$TOOL_GROUP" + - if: steps.cache.outputs.cache-hit != 'true' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ steps.info.outputs.path }} + key: ${{ steps.info.outputs.key }} diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 3520af0..1205b25 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -1,9 +1,33 @@ name: Verify on: push: + branches: [main] pull_request: workflow_dispatch: +permissions: + contents: read +concurrency: + group: verify-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: + changes: + runs-on: ubuntu-24.04 + outputs: + native: ${{ steps.select.outputs.native }} + tooling: ${{ steps.select.outputs.tooling }} + worker: ${{ steps.select.outputs.worker }} + configurable-chart: ${{ steps.select.outputs.configurable-chart }} + notices: ${{ steps.select.outputs.notices }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: oven-sh/setup-bun@v2 + with: + bun-version: '1.3.14' + - run: bun test tools/ci + - id: select + run: bun tools/ci/changes.ts usage-policy: runs-on: ubuntu-24.04 steps: @@ -14,6 +38,8 @@ jobs: - run: python3 tools/verification/check_prohibited_usage.py --tree HEAD - run: python3 tools/verification/test_prohibited_usage.py notices: + needs: changes + if: needs.changes.outputs.notices == 'true' runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 @@ -21,7 +47,18 @@ jobs: with: bun-version: '1.3.14' - run: rustup toolchain install 1.96.0 --profile minimal - - run: cargo install cargo-about --version 0.9.2 --features cli --locked + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + with: + cache-bin: false + workspaces: . -> target + - uses: ./.github/actions/ci-tools + with: + group: notices + - uses: ./.github/actions/bun-cache + - run: bun install --frozen-lockfile + working-directory: examples/configurable-chart + - run: bun examples/configurable-chart/node_modules/typescript/bin/tsc --project tools/notices/tsconfig.json + - run: bun examples/configurable-chart/node_modules/typescript/bin/tsc --project tools/ci/tsconfig.json - run: cargo fetch --locked - run: cargo fetch --locked --manifest-path examples/cloudflare-worker/Cargo.toml - run: cargo fetch --locked --manifest-path examples/configurable-chart/wasm/Cargo.toml @@ -31,6 +68,8 @@ jobs: - run: test -s target/distribution/dataset-builder/ASTROLOGY_ENGINE_SOURCE.tar.gz && test -s target/distribution/dataset-builder/THIRD_PARTY_NOTICES.html && test -s target/distribution/dataset-builder/RUST_LIBRARY_NOTICES.html - run: cargo package --locked --allow-dirty --list | python3 -c 'import sys; files=set(sys.stdin.read().splitlines()); assert {"LICENSE", "NOTICE", "THIRD_PARTY_NOTICES.md", "notices/runtime.txt", "notices/RUST_LIBRARY_NOTICES.html"} <= files' configurable-chart: + needs: changes + if: needs.changes.outputs.configurable-chart == 'true' runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 @@ -38,44 +77,94 @@ jobs: with: bun-version: '1.3.14' - run: rustup toolchain install 1.96.0 --profile minimal --component rustfmt --component clippy --target wasm32-unknown-unknown - - run: cargo install wasm-bindgen-cli --version 0.2.122 --locked + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + with: + cache-bin: false + workspaces: examples/configurable-chart/wasm -> target + - uses: ./.github/actions/ci-tools + with: + group: browser + - uses: ./.github/actions/bun-cache - run: cargo fmt --manifest-path examples/configurable-chart/wasm/Cargo.toml -- --check - run: cargo clippy --locked --manifest-path examples/configurable-chart/wasm/Cargo.toml --target wasm32-unknown-unknown -- -D warnings - run: bun install --frozen-lockfile working-directory: examples/configurable-chart - run: bun run build && bun run check working-directory: examples/configurable-chart - - run: bun examples/configurable-chart/node_modules/typescript/bin/tsc --project tools/notices/tsconfig.json - run: test -s examples/configurable-chart/build/ASTROLOGY_ENGINE_SOURCE.tar.gz && test -s examples/configurable-chart/build/THIRD_PARTY_NOTICES.html && test -s examples/configurable-chart/engine/THIRD_PARTY_NOTICES.txt && test -s examples/configurable-chart/build/RUST_LIBRARY_NOTICES.html - wasm-and-host: + native: + needs: changes + if: needs.changes.outputs.native == 'true' runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/setup-python@v5 with: - node-version: '22' + python-version: '3.13' + - run: rustup toolchain install 1.96.0 --profile minimal --component rustfmt --component clippy + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + with: + cache-bin: false + workspaces: . -> target + - run: cargo fmt --all -- --check + - run: cargo test --locked --workspace + - run: cargo clippy --locked --workspace --all-targets -- -D warnings + - run: cargo fetch --locked --target wasm32-unknown-unknown + - run: python tools/verification/check_boundaries.py + - run: cargo package --locked --allow-dirty + tooling: + needs: changes + if: needs.changes.outputs.tooling == 'true' + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: '3.13' + cache: pip + cache-dependency-path: tools/jpl/requirements.lock + - run: python -m pip install --require-hashes -r tools/jpl/requirements.lock + - run: python tools/verification/test_compare.py + - run: python tools/verification/test_acquisition.py + - run: python tools/verification/test_generation.py + worker: + needs: changes + if: needs.changes.outputs.worker == 'true' + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' - uses: oven-sh/setup-bun@v2 with: bun-version: '1.3.14' - run: rustup toolchain install 1.96.0 --profile minimal --component rustfmt --component clippy --target wasm32-unknown-unknown - - run: cargo fmt --all -- --check - - run: cargo test --locked --workspace + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + with: + cache-bin: false + workspaces: examples/cloudflare-worker -> target + - uses: ./.github/actions/ci-tools + with: + group: worker + - uses: ./.github/actions/bun-cache - run: cargo fmt --manifest-path examples/cloudflare-worker/Cargo.toml -- --check - - run: cargo clippy --locked --workspace --all-targets -- -D warnings - run: cargo clippy --locked --manifest-path examples/cloudflare-worker/Cargo.toml --target wasm32-unknown-unknown --features verification -- -D warnings - - run: cargo install worker-build --version 0.8.3 --locked - run: bun install --frozen-lockfile working-directory: examples/cloudflare-worker - run: bun run build:verification working-directory: examples/cloudflare-worker - run: test -s examples/cloudflare-worker/build/worker/ASTROLOGY_ENGINE_SOURCE.tar.gz && test -s examples/cloudflare-worker/build/worker/THIRD_PARTY_NOTICES.txt && test -s examples/cloudflare-worker/build/RUST_LIBRARY_NOTICES.html - run: node examples/cloudflare-worker/verify.mjs - - run: python tools/verification/test_compare.py - - run: python -m pip install --require-hashes -r tools/jpl/requirements.lock - - run: python tools/verification/test_acquisition.py - - run: python tools/verification/test_generation.py - - run: python tools/verification/check_boundaries.py - - run: cargo package --locked --allow-dirty + verify: + if: always() + needs: [changes, usage-policy, native, tooling, notices, configurable-chart, worker] + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: oven-sh/setup-bun@v2 + with: + bun-version: '1.3.14' + - run: bun tools/ci/gate.ts + env: + CI_NEEDS: ${{ toJSON(needs) }} diff --git a/tools/README.md b/tools/README.md index b1cd23e..181fd86 100644 --- a/tools/README.md +++ b/tools/README.md @@ -171,7 +171,8 @@ astronomical accuracy. See the [regular verification workflow](../.github/workflows/verify.yml) for the repository's formatting, linting, packaging, and local Wasm checks, and the [full-data workflow](../.github/workflows/full-data.yml) for explicit regeneration -and reference-dataset parity runs. +and reference-dataset parity runs. The [CI guide](ci/README.md) describes PR +selection, independent jobs, tool caching and download failure handling. ## Commit checks diff --git a/tools/ci/README.md b/tools/ci/README.md new file mode 100644 index 0000000..bb94b26 --- /dev/null +++ b/tools/ci/README.md @@ -0,0 +1,71 @@ +# Regular CI + +The Verify workflow runs once for each PR update, on pushes to `main`, and on +manual dispatch. Feature branches without a PR can use manual dispatch. New PR +updates cancel superseded runs. Main and manual runs always select every job. + +PR selection uses the merge base of the PR base/head commits. Renames include +both paths, and unavailable comparisons select everything. The `changes` job +runs the CI helper tests and records its selections in the run summary: + +| Changes | Checks | +| --- | --- | +| Prose documentation | Usage policy, helper tests, final gate | +| Worker example source | Worker, notices | +| Browser example source | Browser, notices | +| Native builder or Python preparation source | Native, preparation tooling, notices | +| Runtime, shared verification, CI, licensing, notice inputs, unknown files | Everything | + +Notice input fingerprints from both base and head take precedence over prose +and adapter-specific rules. Executable documentation is code. Native tests, +preparation tooling, and Worker integration run independently. The final +`verify` job accepts intentionally skipped jobs and rejects selected jobs that +fail, are cancelled, or unexpectedly skip. No deployment or full-data generation +occurs in regular CI. + +## Caches and tool downloads + +Rust dependency caches are separate for each job and workspace, keyed by the +Rust toolchain, manifests/lockfiles and compiler environment. They exclude +workspace build products. Bun/pip caches hold package downloads, not installed +application output. Dependency installation remains locked and hash-checked. + +The `ci-tools` composite action caches tools separately from source compilation. +Its key includes platform, tool versions, download checksums, setup code and the +Rust toolchain. Successful setup saves its cache immediately so later failures +do not discard it. Every restored executable must report the expected version. + +`worker-build` and `cargo-about` are installed with exact versions and `--locked` +on a cold cache. Their versions come from the existing notice tool constants. +`downloads.json` records the official Linux x64 helper archives and SHA-256 +hashes: wasm-bindgen 0.2.122, Binaryen 129, and esbuild 0.28.0. These match the +Worker build tool's resolver. Archives are verified before extraction; cached +binaries are version checked. Update URLs, checksums, expected version output, +and corresponding notice material together when intentionally upgrading tools. + +Transport errors, HTTP 408/429 and 5xx responses get at most three attempts, +with 2/5-second backoff and a 60-second timeout per attempt. Other HTTP errors, +checksum mismatches, wrong versions, compilation errors and test failures are +not retried. Cargo uses its bounded network retry setting during installation. +Tool setup exports the supported Worker binary overrides and the browser's +`WASM_BINDGEN` override, so builds use the validated binaries. Wasm optimization +stays enabled. Local developer build setup is unchanged. + +To invalidate a broken tool cache, delete its entry in GitHub Actions caches; +never bypass version or checksum checks. No final Wasm/site/native distribution, +notice bundle, source archive, or dataset is restored as a CI artifact cache. + +## Local checks + +```sh +bun test tools/ci +bun tools/ci/changes.ts # without a PR event, selects every job +bun tools/ci/setup.ts info worker # prints the platform/tool cache identity +bun examples/configurable-chart/node_modules/typescript/bin/tsc --project tools/ci/tsconfig.json +actionlint +``` + +TypeScript checking uses the browser example's locked development dependencies. +The notices job installs them and checks both notice and CI tooling, independent +of whether browser integration was selected. The full-data workflow retains its +explicit invocation and existing setup. diff --git a/tools/ci/changes.test.ts b/tools/ci/changes.test.ts new file mode 100644 index 0000000..843a56f --- /dev/null +++ b/tools/ci/changes.test.ts @@ -0,0 +1,60 @@ +// Copyright (c) Jack Asher +// SPDX-License-Identifier: MPL-2.0 + +import { test, expect } from 'bun:test'; +import { mkdtempSync, writeFileSync, renameSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { allJobs, changedPaths, determineSelection, jobs, selectJobs } from './changes.ts'; +import { parseSelection, verifyResults, type Result } from './gate.ts'; +const chosen = (paths: string[], inputs: string[] = []) => jobs.filter(job => selectJobs(paths, inputs)[job]); +test('prose skips expensive checks; executable docs and unknown files run all', () => { + expect(chosen(['README.md', 'docs/architecture.md'])).toEqual([]); + for (const path of ['docs/demo.ts', 'unknown.config', 'tests/fixtures/regression.txt', 'src/new.rs', '.github/workflows/verify.yml', 'LICENSE', 'NOTICE']) { + expect(chosen([path])).toEqual([...jobs]); + } +}); +test('gate rejects absent or malformed selection outputs', () => { + expect(() => parseSelection({})).toThrow('selection'); + const outputs = Object.fromEntries(jobs.map(job => [job, 'true'])); + expect(parseSelection(outputs)).toEqual(allJobs()); + expect(() => parseSelection({ ...outputs, worker: 'tru' })).toThrow('worker'); +}); +test('adapter and preparation changes select their consumers', () => { + expect(chosen(['examples/cloudflare-worker/src/lib.rs'])).toEqual(['worker', 'notices']); + expect(chosen(['examples/configurable-chart/main.ts'])).toEqual(['configurable-chart', 'notices']); + expect(chosen(['tools/jpl/horizons.py', 'tools/dataset-builder/src/main.rs'])).toEqual(['native', 'tooling', 'notices']); +}); +test('notice inputs override prose and narrow adapter selection', () => { + expect(chosen(['tools/notices/README.md'])).toEqual([...jobs]); + expect(chosen(['THIRD_PARTY_NOTICES.md'], ['THIRD_PARTY_NOTICES.md'])).toEqual([...jobs]); + expect(chosen(['examples/cloudflare-worker/Cargo.toml'], ['examples/cloudflare-worker/Cargo.toml'])).toEqual([...jobs]); +}); +test('main/manual and unreliable PR comparisons run everything', () => { + const fail = () => { throw new Error('unavailable base'); }; + expect(determineSelection('push', fail)).toEqual(allJobs()); + expect(determineSelection('workflow_dispatch', fail)).toEqual(allJobs()); + expect(determineSelection('pull_request', fail)).toEqual(allJobs()); +}); +test('renames and deletions include former paths and use the merge base', () => { + const dir = mkdtempSync(join(tmpdir(), 'ci-diff-')); + const git = (args: string[]) => execFileSync('git', args, { cwd: dir, encoding: 'utf8' }); + try { + git(['init', '-q']); git(['config', 'user.email', 'test@example.invalid']); git(['config', 'user.name', 'CI test']); + writeFileSync(join(dir, 'old.rs'), 'source'); git(['add', '.']); git(['commit', '-qm', 'base']); + const base = git(['rev-parse', 'HEAD']).trim(); + renameSync(join(dir, 'old.rs'), join(dir, 'README.md')); git(['add', '-A']); git(['commit', '-qm', 'rename']); + const head = git(['rev-parse', 'HEAD']).trim(); + expect(changedPaths(base, head, git).sort()).toEqual(['README.md', 'old.rs']); + expect(chosen(changedPaths(base, head, git))).toEqual([...jobs]); + } finally { rmSync(dir, { recursive: true, force: true }); } +}); +test('gate accepts intentional skips, rejects selected skips and every failure/cancellation', () => { + const selection = selectJobs(['README.md'], []); + const results: Record = { changes: 'success', 'usage-policy': 'success', ...Object.fromEntries(jobs.map(job => [job, 'skipped' as const])) }; + verifyResults(selection, results); + expect(() => verifyResults(allJobs(), results)).toThrow(); + for (const result of ['failure', 'cancelled'] as const) expect(() => verifyResults(selection, { ...results, worker: result })).toThrow(); + expect(() => verifyResults(selection, { ...results, changes: 'failure' })).toThrow(); +}); diff --git a/tools/ci/changes.ts b/tools/ci/changes.ts new file mode 100644 index 0000000..bfd8ca9 --- /dev/null +++ b/tools/ci/changes.ts @@ -0,0 +1,61 @@ +// Copyright (c) Jack Asher +// SPDX-License-Identifier: MPL-2.0 + +import { execFileSync } from 'node:child_process'; +import { appendFileSync, readFileSync } from 'node:fs'; + +export const jobs = ['native', 'tooling', 'worker', 'configurable-chart', 'notices'] as const; +export type Job = typeof jobs[number]; +export type Selection = Record; +export const allJobs = (): Selection => Object.fromEntries(jobs.map(job => [job, true])) as Selection; +export const selectJobs = (paths: string[], noticeInputs: string[]): Selection => { + const selected = Object.fromEntries(jobs.map(job => [job, false])) as Selection; + const inputs = new Set(noticeInputs); + for (const path of paths) { + // Legal/build inputs take precedence over documentation exemptions. + if (inputs.has(path) || /^(LICENSE|NOTICE)(\.|$)/.test(path) || + /^(src\/|\.github\/|\.cargo\/|tools\/(ci|notices|verification)\/|notices\/)/.test(path)) return allJobs(); + const prose = /^(docs\/.*\.(md|rst|txt)|[^/]+\.md)$/i.test(path) || /(^|\/)(README|AGENTS)\.md$/.test(path); + if (prose) continue; + const affected: Job[] | undefined = path.startsWith('examples/cloudflare-worker/') ? ['worker', 'notices'] + : path.startsWith('examples/configurable-chart/') ? ['configurable-chart', 'notices'] + : /^(tools\/dataset-builder\/|tools\/jpl\/|tools\/regenerate\.py$)/.test(path) ? ['native', 'tooling', 'notices'] + : undefined; + if (!affected) return allJobs(); + for (const job of affected) selected[job] = true; + } + return selected; +}; +export const changedPaths = (base: string, head: string, run = (args: string[]): string => + execFileSync('git', args, { encoding: 'utf8' })): string[] => { + if (![base, head].every(ref => /^[a-f0-9]{40}$/.test(ref))) throw new Error('Expected base/head commit IDs'); + const ancestor = run(['merge-base', base, head]).trim(); + // Disable rename collapsing so both old and new paths affect selection. + return run(['diff', '--name-only', '--no-renames', '-z', ancestor, head, '--']).split('\0').filter(Boolean); +}; +export const determineSelection = (event: string, collect: () => { paths: string[]; inputs: string[] }): Selection => { + if (event !== 'pull_request') return allJobs(); + try { + const { paths, inputs } = collect(); + return selectJobs(paths, inputs); + } catch (error) { + console.warn(`Change detection unavailable; running all checks: ${error}`); + return allJobs(); + } +}; +if (import.meta.main) { + const selected = determineSelection(process.env.GITHUB_EVENT_NAME ?? '', () => { + const event = JSON.parse(readFileSync(process.env.GITHUB_EVENT_PATH!, 'utf8')); + const base: string = event.pull_request.base.sha; + const head: string = event.pull_request.head.sha; + // Union both versions so deleting a fingerprinted input cannot exempt it. + const inputs = [base, head].flatMap(ref => Object.keys(JSON.parse(execFileSync('git', + ['show', `${ref}:notices/inventory.json`], { encoding: 'utf8' })).inputs)); + return { paths: changedPaths(base, head), inputs }; + }); + const output = jobs.map(job => `${job}=${selected[job]}`).join('\n') + '\n'; + if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, output); + if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, + `### Selected checks\n${jobs.map(job => `- ${job}: ${selected[job] ? 'run' : 'unaffected'}`).join('\n')}\n`); + console.log(output); +} diff --git a/tools/ci/download.test.ts b/tools/ci/download.test.ts new file mode 100644 index 0000000..6a5728c --- /dev/null +++ b/tools/ci/download.test.ts @@ -0,0 +1,30 @@ +// Copyright (c) Jack Asher +// SPDX-License-Identifier: MPL-2.0 + +import { test, expect } from 'bun:test'; +import { downloadVerified, hash, reuseOrInstall, validateVersion, type Download } from './download.ts'; +const artifact: Download = { url: 'https://example.invalid/tool.tar.gz', sha256: hash('archive'), version: '1', binary: 'tool', versionOutput: 'tool 1' }; +test('transient failure retries, then verifies bytes', async () => { + let calls = 0; const delays: number[] = []; + const bytes = await downloadVerified(artifact, async () => ++calls === 1 ? new Response('', { status: 502 }) : new Response('archive'), async ms => { delays.push(ms); }); + expect(new TextDecoder().decode(bytes)).toBe('archive'); expect(calls).toBe(2); expect(delays).toEqual([2000]); +}); +test('network errors exhaust exactly three attempts', async () => { + let calls = 0; const delays: number[] = []; + await expect(downloadVerified(artifact, async () => { calls++; throw new TypeError('network failure'); }, async ms => { delays.push(ms); })).rejects.toThrow('network failure'); + expect(calls).toBe(3); expect(delays).toEqual([2000, 5000]); +}); +test('checksum mismatches and permanent HTTP errors are not retried', async () => { + for (const response of [new Response('corrupt'), new Response('', { status: 404 })]) { + let calls = 0; + await expect(downloadVerified(artifact, async () => { calls++; return response; }, async () => { throw new Error('unexpected retry'); })).rejects.toThrow(); + expect(calls).toBe(1); + } +}); +test('warm installations validate without downloading; wrong versions fail', async () => { + let installs = 0; let checks = 0; + await reuseOrInstall(() => true, async () => { installs++; }, () => { checks++; validateVersion('tool 1\n', 'tool 1'); }); + expect(installs).toBe(0); expect(checks).toBe(1); + await expect(reuseOrInstall(() => true, async () => { installs++; }, () => validateVersion('tool 2', 'tool 1'))).rejects.toThrow('version mismatch'); + expect(installs).toBe(0); +}); diff --git a/tools/ci/download.ts b/tools/ci/download.ts new file mode 100644 index 0000000..31f30ee --- /dev/null +++ b/tools/ci/download.ts @@ -0,0 +1,42 @@ +// Copyright (c) Jack Asher +// SPDX-License-Identifier: MPL-2.0 + +import { createHash } from 'node:crypto'; +export type Download = { url: string; sha256: string; version: string; binary: string; versionOutput: string }; +type Fetch = (url: string, init: RequestInit) => Promise; +export const hash = (bytes: string | Uint8Array): string => createHash('sha256').update(bytes).digest('hex'); +export const downloadVerified = async (artifact: Download, fetcher: Fetch = fetch, + sleep: (ms: number) => Promise = Bun.sleep): Promise => { + for (let attempt = 0; attempt < 3; attempt++) { + let bytes: Uint8Array | undefined; + let failure: unknown; + let transient = false; + try { + const response = await fetcher(artifact.url, { signal: AbortSignal.timeout(60_000) }); + if (response.ok) bytes = new Uint8Array(await response.arrayBuffer()); + else { + failure = new Error(`HTTP ${response.status}: ${artifact.url}`); + transient = response.status === 408 || response.status === 429 || response.status >= 500; + await response.body?.cancel(); + } + } catch (error) { + failure = error; + transient = true; // Fetch/response transport failed; verification is outside this block. + } + if (bytes) { + if (hash(bytes) !== artifact.sha256) throw new Error(`Checksum mismatch: ${artifact.url}`); + return bytes; + } + if (!transient || attempt === 2) throw failure; + console.warn(`Tool download attempt ${attempt + 1} failed: ${failure}`); + await sleep([2_000, 5_000][attempt]!); + } + throw new Error('Download attempts exhausted'); +}; +export const validateVersion = (actual: string, expected: string): void => { + if (actual.trim() !== expected) throw new Error(`Tool version mismatch: expected ${expected}, got ${actual.trim()}`); +}; +export const reuseOrInstall = async (exists: () => boolean, install: () => Promise, validate: () => void): Promise => { + if (!exists()) await install(); + validate(); +}; diff --git a/tools/ci/downloads.json b/tools/ci/downloads.json new file mode 100644 index 0000000..f81a88a --- /dev/null +++ b/tools/ci/downloads.json @@ -0,0 +1,23 @@ +{ + "wasm-bindgen": { + "version": "0.2.122", + "url": "https://github.com/wasm-bindgen/wasm-bindgen/releases/download/0.2.122/wasm-bindgen-0.2.122-x86_64-unknown-linux-musl.tar.gz", + "sha256": "122a3fe2ea9c6e3e89b50e42dd1a346e499f3f65a54aae0b4eaeb658139d1e0e", + "binary": "wasm-bindgen", + "versionOutput": "wasm-bindgen 0.2.122" + }, + "wasm-opt": { + "version": "129", + "url": "https://github.com/WebAssembly/binaryen/releases/download/version_129/binaryen-version_129-x86_64-linux.tar.gz", + "sha256": "50b9fa62b9abea752da92ec57e0c555fee578760cd237c40107957715d2976ba", + "binary": "bin/wasm-opt", + "versionOutput": "wasm-opt version 129 (version_129)" + }, + "esbuild": { + "version": "0.28.0", + "url": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.0.tgz", + "sha256": "e94bf1c7f44197b22cf6a787578eca9af805aa9624104488252de2a765c6a4f0", + "binary": "bin/esbuild", + "versionOutput": "0.28.0" + } +} diff --git a/tools/ci/gate.ts b/tools/ci/gate.ts new file mode 100644 index 0000000..f0a6f01 --- /dev/null +++ b/tools/ci/gate.ts @@ -0,0 +1,24 @@ +// Copyright (c) Jack Asher +// SPDX-License-Identifier: MPL-2.0 + +import { jobs, type Selection } from './changes.ts'; +export type Result = 'success' | 'failure' | 'cancelled' | 'skipped'; +export const parseSelection = (outputs: Record): Selection => { + for (const job of jobs) { + if (outputs[job] !== 'true' && outputs[job] !== 'false') throw new Error(`Missing or invalid selection: ${job}`); + } + return Object.fromEntries(jobs.map(job => [job, outputs[job] === 'true'])) as Selection; +}; +export const verifyResults = (selection: Selection, results: Record): void => { + for (const job of ['changes', 'usage-policy', ...jobs]) { + const required = job === 'changes' || job === 'usage-policy' || selection[job as keyof Selection]; + const result = results[job]; + if (result !== 'success' && !(result === 'skipped' && !required)) throw new Error(`${job}: ${result ?? 'missing result'}`); + } +}; +if (import.meta.main) { + const needs = JSON.parse(process.env.CI_NEEDS!); + const selection = parseSelection(needs.changes?.outputs ?? {}); + verifyResults(selection, Object.fromEntries(Object.entries(needs).map(([name, job]) => [name, (job as { result: Result }).result]))); + console.log('All selected verification jobs passed.'); +} diff --git a/tools/ci/setup.ts b/tools/ci/setup.ts new file mode 100644 index 0000000..28c130f --- /dev/null +++ b/tools/ci/setup.ts @@ -0,0 +1,72 @@ +// Copyright (c) Jack Asher +// SPDX-License-Identifier: MPL-2.0 + +import { execFileSync } from 'node:child_process'; +import { appendFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { generatorVersion, wasmBindgenVersion, workerBuildVersion } from '../notices/core.ts'; +import { downloadVerified, hash, reuseOrInstall, validateVersion, type Download } from './download.ts'; +import downloads from './downloads.json'; + +const groups = ['worker', 'browser', 'notices'] as const; +type Group = typeof groups[number]; +const [command, groupArg] = process.argv.slice(2); +if (!groups.includes(groupArg as Group) || !['info', 'install'].includes(command!)) throw new Error('Usage: setup.ts info|install worker|browser|notices'); +const group = groupArg as Group; +const root = resolve(import.meta.dir, '../..'); +const cache = join(process.env.CI_TOOL_CACHE ?? join(process.env.RUNNER_TEMP ?? '/tmp', 'astrology-ci-tools'), group); +const binaries: Record = group === 'worker' ? downloads : group === 'browser' ? { 'wasm-bindgen': downloads['wasm-bindgen'] } : {}; +validateVersion(downloads['wasm-bindgen'].version, wasmBindgenVersion); +const cargoTool = group === 'worker' ? { name: 'worker-build', version: workerBuildVersion, features: [] } + : group === 'notices' ? { name: 'cargo-about', version: generatorVersion, features: ['--features', 'cli'] } : undefined; +const key = hash(JSON.stringify({ binaries, cargoTool, + rust: readFileSync(join(root, 'rust-toolchain.toml'), 'utf8'), + scripts: ['setup.ts', 'download.ts'].map(path => readFileSync(join(import.meta.dir, path), 'utf8')) })); +const output = (file: string | undefined, value: string) => { if (file) appendFileSync(file, value + '\n'); }; +if (command === 'info') { + output(process.env.GITHUB_OUTPUT, `path=${cache}\nkey=ci-tools-v1-${process.platform}-${process.arch}-${group}-${key}`); + console.log(`Tool cache: ${cache}\nIdentity: ${key}`); +} else { + if (process.platform !== 'linux' || process.arch !== 'x64') throw new Error('CI helper archives support Linux x64 only. Local builds keep their existing setup.'); + mkdirSync(cache, { recursive: true }); + const binaryPaths: Record = {}; + for (const [name, artifact] of Object.entries(binaries)) { + const destination = join(cache, `${name}-${artifact.version}`); + const binary = join(destination, artifact.binary); + await reuseOrInstall(() => existsSync(binary), async () => { + const staging = mkdtempSync(join(cache, `${name}-download-`)); + try { + const archive = join(staging, 'archive.tar.gz'); + writeFileSync(archive, await downloadVerified(artifact)); + const extracted = join(staging, 'extracted'); + mkdirSync(extracted); + execFileSync('tar', ['-xzf', archive, '--strip-components=1', '-C', extracted]); + validateVersion(execFileSync(join(extracted, artifact.binary), ['--version'], { encoding: 'utf8' }), artifact.versionOutput); + renameSync(extracted, destination); + } finally { rmSync(staging, { recursive: true, force: true }); } + }, () => validateVersion(execFileSync(binary, ['--version'], { encoding: 'utf8' }), artifact.versionOutput)); + binaryPaths[name] = binary; + console.log(`Validated ${name} ${artifact.version}`); + } + if (cargoTool) { + const destination = join(cache, `${cargoTool.name}-${cargoTool.version}`); + const binary = join(destination, 'bin', cargoTool.name); + const expected = cargoTool.name === 'worker-build' ? cargoTool.version : `${cargoTool.name} ${cargoTool.version}`; + await reuseOrInstall(() => existsSync(binary), async () => { + const staging = mkdtempSync(join(cache, 'cargo-install-')); + try { + execFileSync('cargo', ['install', cargoTool.name, '--bin', cargoTool.name, '--version', cargoTool.version, + '--locked', '--root', staging, ...cargoTool.features], { stdio: 'inherit', env: { ...process.env, CARGO_NET_RETRY: '3' } }); + validateVersion(execFileSync(join(staging, 'bin', cargoTool.name), ['--version'], { encoding: 'utf8' }), expected); + renameSync(staging, destination); + } finally { rmSync(staging, { recursive: true, force: true }); } + }, () => validateVersion(execFileSync(binary, ['--version'], { encoding: 'utf8' }), expected)); + output(process.env.GITHUB_PATH, dirname(binary)); + if (cargoTool.name === 'cargo-about') output(process.env.GITHUB_ENV, `CARGO_ABOUT=${binary}`); + console.log(`Validated ${cargoTool.name} ${cargoTool.version}`); + } + for (const [name, binary] of Object.entries(binaryPaths)) { + output(process.env.GITHUB_ENV, `${name.toUpperCase().replaceAll('-', '_')}_BIN=${binary}`); + if (name === 'wasm-bindgen') output(process.env.GITHUB_ENV, `WASM_BINDGEN=${binary}`); + } +} diff --git a/tools/ci/tsconfig.json b/tools/ci/tsconfig.json new file mode 100644 index 0000000..b988dbb --- /dev/null +++ b/tools/ci/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../notices/tsconfig.json", + "compilerOptions": { "resolveJsonModule": true }, + "include": ["*.ts"] +}