diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a4fb5c1678..528946595e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,7 +14,7 @@ name: CI # + one pending, so a burst of merges is tested at # its tip). PR tier unscoped + medium-weight jobs. # full schedule / tags / everything, incl. parity, compile-smoke, -# workflow_dispatch doc-tests, package smokes, the 8-shard +# workflow_dispatch doc-tests, package smokes, the 12-shard # / label auto-optimize gap suite. `full-suite-gate` is # what release-packages.yml waits for. # @@ -314,6 +314,7 @@ jobs: python3 scripts/ci_plan.py --self-test python3 scripts/ci_pr_files.py --self-test python3 scripts/ci_cargo_test_shard.py --self-test + python3 scripts/ci_compile_smoke_shard.py --self-test python3 scripts/ci_cargo_test_shard.py --package perry \ --total-shards "${{ fromJSON(needs.plan.outputs.plan).cargo_test_perry.total }}" \ --validate @@ -2990,7 +2991,7 @@ jobs: # --------------------------------------------------------------------------- # gap-suite (was `conformance-smoke`): the gap suite, sharded. Runs in every - # tier -- 6 fast-mode shards on a PR, 3 in the sweep, 8 auto-optimize shards + # tier -- 6 fast-mode shards on a PR, 3 in the sweep, 12 auto-optimize shards # in the full tier (scripts/ci_plan.py GAP_SUITE). The `gate` fan-in below is # what branch protection requires; a single shard's red bubbles up through it. # --------------------------------------------------------------------------- @@ -3071,7 +3072,7 @@ jobs: # protection requires. 2026-08-16: in the harness's default (auto-optimize) # mode 96% of a shard's wall time was ~10 tests at ~200 s each -- the # feature-stripped runtime rebuild per distinct feature set, redone in - # every shard. That mode is now the full tier's 8-shard arm; PR and sweep + # every shard. That mode is now the full tier's 12-shard arm; PR and sweep # tiers use `fast` mode against one prebuilt release build (~1.5 s/test). strategy: fail-fast: false @@ -3489,14 +3490,23 @@ jobs: # --------------------------------------------------------------------------- - # Compile smoke test (all 130+ test files must compile) + # Compile smoke test (all top-level test files must compile) # --------------------------------------------------------------------------- compile-smoke: + name: compile-smoke (${{ matrix.shard }}/4) # Sweep/full-tier only. Chained behind `check` so a sweep's fan-out does # not take every runner slot the moment a merge lands -- PR gates share # the same 20-slot pool. needs: [plan, check] if: fromJSON(needs.plan.outputs.plan).jobs.compile_smoke + # The unsharded current-head job ran into the hosted six-hour limit while + # still compiling (run36914319295). Every top-level test stays assigned + # exactly once; each shard retains the default auto-optimize path. + strategy: + fail-fast: false + max-parallel: 2 + matrix: + shard: [1, 2, 3, 4] # Release-publish decoupling (see `parity` above): aspirational extended # suite, informational only — does not block package publishing. # Was macos-14 — moved to ubuntu-latest in v0.5.392. The smoke @@ -3764,8 +3774,8 @@ jobs: # as a belt-and-suspenders safety net for any remaining race # corner the lock doesn't catch. NJOBS="${PERRY_SMOKE_JOBS:-6}" - printf '%s\n' test-files/*.ts \ - | xargs -P "$NJOBS" -n 1 -I{} bash -c 'compile_one "$@"' _ {} + python3 scripts/ci_compile_smoke_shard.py --shard '${{ matrix.shard }}/4' \ + | xargs -0 -r -P "$NJOBS" -I{} bash -c 'compile_one "$@"' _ {} # Count markers via shopt nullglob + bash array length. Pre-fix # we used `ls -1 "$LOGS_DIR"/*.fail | wc -l` which fails when @@ -3808,7 +3818,7 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: compile-smoke-error-logs + name: compile-smoke-error-logs-${{ matrix.shard }} path: /tmp/perry_smoke_logs/*.compile_error.log if-no-files-found: ignore diff --git a/changelog.d/11750-full-suite-compile-shards.md b/changelog.d/11750-full-suite-compile-shards.md new file mode 100644 index 0000000000..da5ed37b62 --- /dev/null +++ b/changelog.d/11750-full-suite-compile-shards.md @@ -0,0 +1,9 @@ +Distribute full-tier compile work across bounded jobs after current runs hit +their timeout limits. The auto-optimize gap suite now uses twelve shards. +Compile-smoke assigns the complete top-level TypeScript inventory exactly once +across four round-robin shards, with at most two smoke shards running at once. +The existing compiler invocation, platform exclusions, retry policy and failure +markers remain intact; every shard still contributes to the aggregate result. +Invalid or empty selections fail closed, and smoke error artifacts are keyed +by shard. Local selection checks cover all 2,067 current files; actual full-tier +CI must establish the resulting wall-time margin. diff --git a/docs/src/testing/ci-tiers.md b/docs/src/testing/ci-tiers.md index 9fd7a2aa84..8a84c16a28 100644 --- a/docs/src/testing/ci-tiers.md +++ b/docs/src/testing/ci-tiers.md @@ -14,13 +14,23 @@ python3 scripts/ci_plan.py --self-test # the policy's own invariants |---|---|---|---| | **pr** | every `pull_request` push | the required gate. Small, fast, must be green on `main`. | `pr-gate` — **the only required status context** | | **sweep** | every `push` to `main` (coalesced) **+ a two-hourly cron backstop** | post-merge truth for `main`: the PR tier unscoped plus the medium-weight jobs that do not fit the PR budget | `main-gate` | -| **full** | nightly `schedule`, `v*` tags, `workflow_dispatch`, PRs labelled `run-extended-tests` | everything, incl. parity, compile-smoke, doc-tests, package smokes, the 8-shard auto-optimize gap suite | `full-suite-gate` — what `release-packages.yml` waits for | +| **full** | nightly `schedule`, `v*` tags, `workflow_dispatch`, PRs labelled `run-extended-tests` | everything, incl. parity, compile-smoke, doc-tests, package smokes, the 12-shard auto-optimize gap suite | `full-suite-gate` — what `release-packages.yml` waits for | ## The job × tier matrix Generated by `python3 scripts/ci_plan.py --table`; the `lint` job checks that this copy is current. +The full gap tier uses twelve shards with auto-optimize enabled. Compile-smoke +partitions the complete top-level `test-files/*.ts` inventory into four stable +round-robin shards, with at most two smoke shards running concurrently. Each +file is assigned once before the existing platform exclusions are applied. +Every shard retains the default compiler invocation and failure markers; +`full-suite-gate` requires the matrix job's aggregate result. These splits +address current eight-way gap jobs reaching their 110-minute bound and the +unsharded smoke job reaching the hosted six-hour limit. Actual CI durations +must still establish the new margin. + | job | pr | sweep | full | |---|:-:|:-:|:-:| | `lint` | always | yes | yes | @@ -28,7 +38,7 @@ copy is current. | `warnings` | yes | yes | yes | | `cargo-test` | yes | yes | yes | | `cargo-test-perry` | | | yes | -| `gap-suite` | 6x fast | 3x fast | 8x full | +| `gap-suite` | 6x fast | 3x fast | 12x full | | `gc-call-effects` | yes | yes | yes | | `gc-stress` | 1x pr | 4x all | 4x all | | `e2e-scoped` | yes | | | @@ -100,7 +110,7 @@ Two specific costs dominated: rebuilding a feature-stripped runtime per distinct feature set, redundantly in every shard. The harness's `PERRY_SKIP_BUILD=1` mode runs the same tests against one prebuilt release compiler at ~1.5 s each; that is the `fast` gap mode the PR - and sweep tiers use. The 8-shard auto-optimize mode is kept in the full tier + and sweep tiers use. The auto-optimize mode is kept in the full tier because it is the only arm that sees auto-optimize-only link bugs. - Every job saved a fresh ~0.5–1.3 GB sccache tarball on every PR push (~200 GB/day into a 10 GB repo cache budget), evicting every useful entry within the hour — diff --git a/scripts/ci_compile_smoke_shard.py b/scripts/ci_compile_smoke_shard.py new file mode 100644 index 0000000000..14d632d30e --- /dev/null +++ b/scripts/ci_compile_smoke_shard.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Select a stable slice of the complete top-level TypeScript smoke inventory.""" +import argparse +from pathlib import Path +import sys +import tempfile +import unittest + + +def select(root, shard): + try: + index, total = map(int, shard.split('/')) + except ValueError as error: + raise ValueError('shard must be INDEX/TOTAL') from error + if not 1 <= index <= total: + raise ValueError('shard must satisfy 1 <= INDEX <= TOTAL') + if not root.is_dir(): + raise ValueError('test directory does not exist') + files = sorted(path for path in root.glob('*.ts') if path.is_file()) + if not files or len(files) < total: + raise ValueError('every shard must contain at least one test') + return files[index - 1::total] + + +class CoverageTests(unittest.TestCase): + def test_all_files_run_once_with_balanced_nonempty_shards(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name in ['a.ts', 'b space.ts', 'c\nnewline.ts', 'd.ts', 'e.ts']: + (root / name).touch() + (root / 'ignored.js').touch() + (root / 'directory.ts').mkdir() + (root / 'nested').mkdir() + (root / 'nested/fixture.ts').touch() + expected = {p for p in root.glob('*.ts') if p.is_file()} + slices = [select(root, f'{i}/4') for i in range(1, 5)] + assigned = [p for part in slices for p in part] + self.assertEqual(set(assigned), expected) + self.assertEqual(len(assigned), len(expected)) + self.assertLessEqual(max(map(len, slices)) - min(map(len, slices)), 1) + self.assertTrue(all(slices)) + self.assertEqual(select(root, '1/4'), slices[0]) + + def test_invalid_or_empty_scope_cannot_succeed(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'a.ts').touch() + for shard in ['0/4', '5/4', '1/0', '-1/4', '1/2', 'bad', '1/2/3']: + with self.subTest(shard=shard), self.assertRaises(ValueError): + select(root, shard) + with self.assertRaises(ValueError): + select(root / 'missing', '1/1') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--root', type=Path, default=Path('test-files')) + parser.add_argument('--shard') + parser.add_argument('--self-test', action='store_true') + args = parser.parse_args() + if args.self_test: + result = unittest.TextTestRunner().run(unittest.defaultTestLoader.loadTestsFromTestCase(CoverageTests)) + return 0 if result.wasSuccessful() else 1 + if not args.shard: + parser.error('--shard is required') + try: + paths = select(args.root, args.shard) + except ValueError as error: + parser.error(str(error)) + # Compute and validate the entire selection before emitting any scope. + sys.stdout.buffer.write(b'\0'.join(str(path).encode() for path in paths) + b'\0') + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/scripts/ci_plan.py b/scripts/ci_plan.py index 04c5ee3ec3..b93a14ea75 100755 --- a/scripts/ci_plan.py +++ b/scripts/ci_plan.py @@ -31,7 +31,7 @@ `await-tests` dispatches this and waits for the `full-suite-gate` job), and PRs carrying the `run-extended-tests` label. The sweep plus the slow/opt-in suites (parity, compile-smoke, doc-tests, package smokes, - the gap suite in its 8-shard auto-optimize mode). + the gap suite in its 12-shard auto-optimize mode). PR SCOPE -------- @@ -123,7 +123,10 @@ GAP_SUITE = { "pr": {"mode": "fast", "total": 6}, "sweep": {"mode": "fast", "total": 3}, - "full": {"mode": "full", "total": 8}, + # Current 8-way full runs hit the 110-minute bound while still compiling + # (e.g. run36914319295, shard1 reached137/152). Preserve auto-optimize + # coverage and distribute the complete corpus across more workers. + "full": {"mode": "full", "total": 12}, } # Parity: full tier only, sharded. The unsharded job was killed by GitHub's @@ -138,7 +141,7 @@ # The growing `perry` integration inventory cannot finish serially inside # cargo-test's 180-minute release bound (#8914). Eight round-robin shards put # the observed low-cache workload below 90 minutes while fitting alongside the -# existing eight gap workers in the org's 20-runner pool. The regular +# gap workers in the org's 20-runner pool. The regular # `cargo-test` job still covers the perry bin/unit target and every other # package; this matrix owns only `perry` integration targets. PERRY_INTEGRATION_SHARDS = 8 @@ -460,7 +463,7 @@ def check(name: str, cond: bool): "shards": list(range(1, PERRY_INTEGRATION_SHARDS + 1)), }, ) - check("full: 8 auto-optimize gap shards", full["gap"]["total"] == 8 and full["gap"]["mode"] == "full") + check("full: 12 auto-optimize gap shards", full["gap"] == {"mode": "full", "total": 12, "shards": list(range(1, 13)), "update_snapshot": False}) check("full: parity sharded (6h-cap kill, 2026-08-16)", full["parity"]["total"] >= 2 and full["parity"]["shards"][0] == 1) check("full: full GC matrix has four shards", full["gc_stress"] == {"mode": "all", "total": 4, "shards": [1, 2, 3, 4]})