From e495fb7cf09397f463497a5992920cd3e78834b9 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 1 Nov 2026 04:08:37 -0800 Subject: [PATCH 1/5] ci(tests): run heavy suites post-merge + per-touched-package, not every PR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The required check "Test & Coverage" gated on the full matrix — backend-tests (3 shards ~250s), service-test (Tier-1 real DBs ~250s), frontend, chart-lint — so every PR and every merge-queue commit paid the slowest leg. Heavy suites now run on push:main (post-merge safety net) + the nightly schedule, and on a pull_request only when the touched package needs them (a new `changes` job keyed off the PR diff). merge_group stays light, so the queue's required check runs ~backend-checks + tool-tests instead of the full matrix, cutting runner contention and serialization. test-gate accepts a skipped heavy leg; a failure or cancellation of any leg still fails the gate. The pass predicate was unit-tested across docs-only, backend, and failure cases. Latchkey is left intact but inert for the heavy path (it only routed heavy on merge_group); a clean retirement can follow once this change's effect is measured. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_015AKgpNr8tFMMj8aFvFPUdA --- .github/workflows/tests.yml | 83 +++++++++++++++++++++++++++++++------ 1 file changed, 70 insertions(+), 13 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1b3ff6460e..cd345d8ea3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,7 +2,10 @@ name: Tests on: push: - branches: [ v1.0.x ] + # main added (tests-slim fast lane): the heavy suites moved off the PR + + # merge_group critical path run here post-merge as the safety net, with the + # nightly schedule. v1.0.x stays for release-branch coverage. + branches: [ v1.0.x, main ] # No branches filter on pull_request: stacked PRs (base = another feature # branch) previously got ZERO test runs — every number on them was # single-sourced to the author's local sweep, and the 2026-08-22 threading @@ -61,6 +64,43 @@ concurrency: cancel-in-progress: true jobs: + # Which packages a pull_request touches — gates the heavy suites below so a PR + # runs only the matrices its changes need (tests-slim fast lane). On push and + # schedule the heavy suites run regardless (see their if:), so this is a no-op + # there and the outputs stay false. + changes: + name: Changed packages + runs-on: ubuntu-latest + outputs: + backend: ${{ steps.filter.outputs.backend }} + frontend: ${{ steps.filter.outputs.frontend }} + chart: ${{ steps.filter.outputs.chart }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - id: filter + run: | + backend=false; frontend=false; chart=false + if [ "${{ github.event_name }}" = "pull_request" ]; then + base="${{ github.event.pull_request.base.sha }}" + files=$(git diff --name-only "$base"...HEAD 2>/dev/null || git diff --name-only "$base" HEAD) + printf '%s\n' "$files" + printf '%s\n' "$files" | grep -qE '^(backend/|cli/|package(-lock)?\.json)' && backend=true + printf '%s\n' "$files" | grep -qE '^(frontend/|backend/)' && frontend=true + printf '%s\n' "$files" | grep -qE '^(k8s/|charts?/|helm/)' && chart=true + # A change to this workflow runs the full suite as a self-test, so a + # PR that edits the test pipeline proves every heavy leg still works. + if printf '%s\n' "$files" | grep -qE '^\.github/workflows/tests\.yml$'; then + backend=true; frontend=true; chart=true + fi + fi + { + echo "backend=$backend" + echo "frontend=$frontend" + echo "chart=$chart" + } >> "$GITHUB_OUTPUT" + # Heavy jobs (backend, frontend, service tests) run on Latchkey runners on # merge-queue runs when vars.LATCHKEY_HEAVY is 'on': those runners don't # count against the org's 20-concurrent-job cap that made the queue wait @@ -175,8 +215,10 @@ jobs: shard: [1, 2, 3] # Routed by latchkey-route (backend on medium, the rest on small; see # the route job). Skipped routing leaves the output empty: GitHub-hosted. - needs: latchkey-route - if: ${{ !cancelled() }} + needs: [latchkey-route, changes] + # tests-slim fast lane: off the PR + merge_group critical path. Runs post-merge + # (push:main) and nightly, and on a PR only when backend/cli/deps changed. + if: ${{ !cancelled() && (github.event_name == 'push' || github.event_name == 'schedule' || (github.event_name == 'pull_request' && needs.changes.outputs.backend == 'true')) }} runs-on: ${{ needs.latchkey-route.outputs.backend || 'ubuntu-latest' }} env: JWT_SECRET: test-jwt-secret @@ -225,8 +267,10 @@ jobs: frontend-tests: name: Frontend tests # Routed by latchkey-route, as backend-tests. - needs: latchkey-route - if: ${{ !cancelled() }} + needs: [latchkey-route, changes] + # tests-slim fast lane: post-merge + nightly, and on a PR only when frontend or + # backend changed (the frontend typecheck follows imports into backend). + if: ${{ !cancelled() && (github.event_name == 'push' || github.event_name == 'schedule' || (github.event_name == 'pull_request' && needs.changes.outputs.frontend == 'true')) }} runs-on: ${{ needs.latchkey-route.outputs.frontend || 'ubuntu-latest' }} steps: - uses: actions/checkout@v4 @@ -305,7 +349,7 @@ jobs: name: Test & Coverage if: always() # latchkey-probe is left out on purpose: see the comment above it. - needs: [latchkey-route, latchkey-backstop, backend-tests, backend-checks, frontend-tests, tool-tests, service-test, chart-lint] + needs: [changes, latchkey-route, latchkey-backstop, backend-tests, backend-checks, frontend-tests, tool-tests, service-test, chart-lint] runs-on: ubuntu-latest env: NEEDS_JSON: ${{ toJSON(needs) }} @@ -318,12 +362,20 @@ jobs: <(printf '%s\n' "$NEEDS_JSON" | jq -r 'keys[]' | sort) - name: Verify every check passed run: | - # Suite jobs run unconditionally; a skipped result is a failed gate. - # Only the Latchkey routing jobs may be skipped (off on PR runs). - # Revisit this predicate if a later PR adds path-based job skipping. + # Light legs (changes, backend-checks, tool-tests) must succeed. Heavy + # legs (backend/frontend/service/chart) may be SKIPPED — they run + # post-merge (push:main) + nightly, and on a PR only for the touched + # package (tests-slim fast lane). Latchkey routing jobs may be skipped too + # (off on PR runs). A failure or cancellation of any leg fails the gate. echo "$NEEDS_JSON" | jq -e ' - to_entries | all(.value.result == "success" - or ((.key | startswith("latchkey-")) and .value.result == "skipped")) + (["backend-tests","frontend-tests","service-test","chart-lint"]) as $heavy + | to_entries + | all( + .key as $k | .value.result as $r + | $r == "success" + or ($r == "skipped" + and (($k | startswith("latchkey-")) or ($heavy | index($k) != null))) + ) ' service-test: @@ -333,8 +385,10 @@ jobs: matrix: shard: [1, 2] # Routed by latchkey-route, as backend-tests. - needs: latchkey-route - if: ${{ !cancelled() }} + needs: [latchkey-route, changes] + # tests-slim fast lane: post-merge + nightly, and on a PR only when backend/cli + # changed (these are the real-DB Tier-1 integration tests). + if: ${{ !cancelled() && (github.event_name == 'push' || github.event_name == 'schedule' || (github.event_name == 'pull_request' && needs.changes.outputs.backend == 'true')) }} runs-on: ${{ needs.latchkey-route.outputs.service || 'ubuntu-latest' }} services: @@ -404,6 +458,9 @@ jobs: chart-lint: name: Chart Lint (Tier 1.5) + needs: changes + # tests-slim fast lane: post-merge + nightly, and on a PR only when charts/k8s changed. + if: ${{ !cancelled() && (github.event_name == 'push' || github.event_name == 'schedule' || (github.event_name == 'pull_request' && needs.changes.outputs.chart == 'true')) }} runs-on: ubuntu-latest env: From d78bec8b0e9c94979e67d536a61a3acf364aea52 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 1 Nov 2026 04:16:59 -0800 Subject: [PATCH 2/5] =?UTF-8?q?ci(tests):=20address=20review=20=E2=80=94?= =?UTF-8?q?=20heavy=20on=20workflow=5Fdispatch;=20chart=20covers=20root=20?= =?UTF-8?q?manifests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Heavy suites also run on workflow_dispatch, so a manual run tests the selected revision (and the latchkey dispatch input routes heavy as its description says). - chart-lint's touched-package filter includes root package.json/lock, since the job runs a root `npm ci` — a broken root dep is caught on the PR, not post-merge. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_015AKgpNr8tFMMj8aFvFPUdA --- .github/workflows/tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cd345d8ea3..94619e5d71 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -88,7 +88,7 @@ jobs: printf '%s\n' "$files" printf '%s\n' "$files" | grep -qE '^(backend/|cli/|package(-lock)?\.json)' && backend=true printf '%s\n' "$files" | grep -qE '^(frontend/|backend/)' && frontend=true - printf '%s\n' "$files" | grep -qE '^(k8s/|charts?/|helm/)' && chart=true + printf '%s\n' "$files" | grep -qE '^(k8s/|charts?/|helm/|package(-lock)?\.json)' && chart=true # A change to this workflow runs the full suite as a self-test, so a # PR that edits the test pipeline proves every heavy leg still works. if printf '%s\n' "$files" | grep -qE '^\.github/workflows/tests\.yml$'; then @@ -218,7 +218,7 @@ jobs: needs: [latchkey-route, changes] # tests-slim fast lane: off the PR + merge_group critical path. Runs post-merge # (push:main) and nightly, and on a PR only when backend/cli/deps changed. - if: ${{ !cancelled() && (github.event_name == 'push' || github.event_name == 'schedule' || (github.event_name == 'pull_request' && needs.changes.outputs.backend == 'true')) }} + if: ${{ !cancelled() && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && needs.changes.outputs.backend == 'true')) }} runs-on: ${{ needs.latchkey-route.outputs.backend || 'ubuntu-latest' }} env: JWT_SECRET: test-jwt-secret @@ -270,7 +270,7 @@ jobs: needs: [latchkey-route, changes] # tests-slim fast lane: post-merge + nightly, and on a PR only when frontend or # backend changed (the frontend typecheck follows imports into backend). - if: ${{ !cancelled() && (github.event_name == 'push' || github.event_name == 'schedule' || (github.event_name == 'pull_request' && needs.changes.outputs.frontend == 'true')) }} + if: ${{ !cancelled() && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && needs.changes.outputs.frontend == 'true')) }} runs-on: ${{ needs.latchkey-route.outputs.frontend || 'ubuntu-latest' }} steps: - uses: actions/checkout@v4 @@ -388,7 +388,7 @@ jobs: needs: [latchkey-route, changes] # tests-slim fast lane: post-merge + nightly, and on a PR only when backend/cli # changed (these are the real-DB Tier-1 integration tests). - if: ${{ !cancelled() && (github.event_name == 'push' || github.event_name == 'schedule' || (github.event_name == 'pull_request' && needs.changes.outputs.backend == 'true')) }} + if: ${{ !cancelled() && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && needs.changes.outputs.backend == 'true')) }} runs-on: ${{ needs.latchkey-route.outputs.service || 'ubuntu-latest' }} services: @@ -460,7 +460,7 @@ jobs: name: Chart Lint (Tier 1.5) needs: changes # tests-slim fast lane: post-merge + nightly, and on a PR only when charts/k8s changed. - if: ${{ !cancelled() && (github.event_name == 'push' || github.event_name == 'schedule' || (github.event_name == 'pull_request' && needs.changes.outputs.chart == 'true')) }} + if: ${{ !cancelled() && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && needs.changes.outputs.chart == 'true')) }} runs-on: ubuntu-latest env: From 1b13a71834cb42cb3cb080a3caf9674afcd01d08 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 1 Nov 2026 04:24:56 -0800 Subject: [PATCH 3/5] ci(tests): chart filter covers chart-lint's script and runbook inputs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit chart-lint runs scripts/verify-backend-worker-chart.sh and scripts/monitoring/apply-alerting.py --check (which reads docs/runbooks/ alerting.md). A PR editing only those would not set the `chart` output, so chart-lint would skip and test-gate would accept the skip — a broken chart verifier or alerting runbook could pass the PR gate. Add the three paths to the changes-job chart filter so they trigger the job. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_015AKgpNr8tFMMj8aFvFPUdA --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 94619e5d71..cd5460cb07 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -88,7 +88,7 @@ jobs: printf '%s\n' "$files" printf '%s\n' "$files" | grep -qE '^(backend/|cli/|package(-lock)?\.json)' && backend=true printf '%s\n' "$files" | grep -qE '^(frontend/|backend/)' && frontend=true - printf '%s\n' "$files" | grep -qE '^(k8s/|charts?/|helm/|package(-lock)?\.json)' && chart=true + printf '%s\n' "$files" | grep -qE '^(k8s/|charts?/|helm/|scripts/verify-backend-worker-chart\.sh$|scripts/monitoring/apply-alerting\.py$|docs/runbooks/alerting\.md$|package(-lock)?\.json)' && chart=true # A change to this workflow runs the full suite as a self-test, so a # PR that edits the test pipeline proves every heavy leg still works. if printf '%s\n' "$files" | grep -qE '^\.github/workflows/tests\.yml$'; then From 2d0a7b1b8496c7f45c1b16579f546fd6654a026c Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 1 Nov 2026 04:39:26 -0800 Subject: [PATCH 4/5] ci(tests): run heavy legs on merge_group; packages/ in backend filter; --no-renames MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses Sable's gate review (2 of 3 filter points + the merge_group decision): - merge_group now runs every heavy leg (presser ruling). Heavy suites skip only on an unrelated pull_request; the queue keeps full coverage so an inter-PR conflict between two concurrently-queued PRs (e.g. #3186's v2-layout-invariants failure) is caught as a dequeue, not as red-on-main. - backend filter includes packages/: backend imports packages/commonly-apps, packages/integration-sdk and maps packages/seats in jest.config.js, so a packages/ edit must run backend-tests + service-test, not just typecheck. - git diff uses --no-renames so a file moved out of backend/ or frontend/ still reports the old path and triggers the right legs. Not applied: docs/design/ and e2e/ in the frontend filter. Verified neither is a test input — signalAccent.test.ts asserts an href string in rendered preview HTML (it reads design-system/preview/, not the .md), and the e2e/ reference in v2-layout-invariants.test.ts is a comment. Adding them would run frontend-tests on edits the suite does not exercise; and with merge_group now full, any real cross-file breakage is caught in the queue regardless. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_015AKgpNr8tFMMj8aFvFPUdA --- .github/workflows/tests.yml | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cd5460cb07..89137d4843 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,9 +2,11 @@ name: Tests on: push: - # main added (tests-slim fast lane): the heavy suites moved off the PR + - # merge_group critical path run here post-merge as the safety net, with the - # nightly schedule. v1.0.x stays for release-branch coverage. + # main added (tests-slim fast lane): the heavy suites skip on an unrelated + # pull_request, but still run on merge_group (so an inter-PR conflict between + # two queued PRs is caught in the queue, not as red-on-main — #3186) and here + # post-merge as a safety net, with the nightly schedule. v1.0.x stays for + # release-branch coverage. branches: [ v1.0.x, main ] # No branches filter on pull_request: stacked PRs (base = another feature # branch) previously got ZERO test runs — every number on them was @@ -84,9 +86,9 @@ jobs: backend=false; frontend=false; chart=false if [ "${{ github.event_name }}" = "pull_request" ]; then base="${{ github.event.pull_request.base.sha }}" - files=$(git diff --name-only "$base"...HEAD 2>/dev/null || git diff --name-only "$base" HEAD) + files=$(git diff --no-renames --name-only "$base"...HEAD 2>/dev/null || git diff --no-renames --name-only "$base" HEAD) printf '%s\n' "$files" - printf '%s\n' "$files" | grep -qE '^(backend/|cli/|package(-lock)?\.json)' && backend=true + printf '%s\n' "$files" | grep -qE '^(backend/|cli/|packages/|package(-lock)?\.json)' && backend=true printf '%s\n' "$files" | grep -qE '^(frontend/|backend/)' && frontend=true printf '%s\n' "$files" | grep -qE '^(k8s/|charts?/|helm/|scripts/verify-backend-worker-chart\.sh$|scripts/monitoring/apply-alerting\.py$|docs/runbooks/alerting\.md$|package(-lock)?\.json)' && chart=true # A change to this workflow runs the full suite as a self-test, so a @@ -216,9 +218,10 @@ jobs: # Routed by latchkey-route (backend on medium, the rest on small; see # the route job). Skipped routing leaves the output empty: GitHub-hosted. needs: [latchkey-route, changes] - # tests-slim fast lane: off the PR + merge_group critical path. Runs post-merge - # (push:main) and nightly, and on a PR only when backend/cli/deps changed. - if: ${{ !cancelled() && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && needs.changes.outputs.backend == 'true')) }} + # tests-slim fast lane: skipped only on an unrelated PR. Runs on merge_group + # (queue catches inter-PR conflicts), post-merge (push:main), nightly, and on + # a PR only when backend/cli/packages/deps changed. + if: ${{ !cancelled() && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'merge_group' || (github.event_name == 'pull_request' && needs.changes.outputs.backend == 'true')) }} runs-on: ${{ needs.latchkey-route.outputs.backend || 'ubuntu-latest' }} env: JWT_SECRET: test-jwt-secret @@ -270,7 +273,7 @@ jobs: needs: [latchkey-route, changes] # tests-slim fast lane: post-merge + nightly, and on a PR only when frontend or # backend changed (the frontend typecheck follows imports into backend). - if: ${{ !cancelled() && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && needs.changes.outputs.frontend == 'true')) }} + if: ${{ !cancelled() && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'merge_group' || (github.event_name == 'pull_request' && needs.changes.outputs.frontend == 'true')) }} runs-on: ${{ needs.latchkey-route.outputs.frontend || 'ubuntu-latest' }} steps: - uses: actions/checkout@v4 @@ -388,7 +391,7 @@ jobs: needs: [latchkey-route, changes] # tests-slim fast lane: post-merge + nightly, and on a PR only when backend/cli # changed (these are the real-DB Tier-1 integration tests). - if: ${{ !cancelled() && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && needs.changes.outputs.backend == 'true')) }} + if: ${{ !cancelled() && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'merge_group' || (github.event_name == 'pull_request' && needs.changes.outputs.backend == 'true')) }} runs-on: ${{ needs.latchkey-route.outputs.service || 'ubuntu-latest' }} services: @@ -460,7 +463,7 @@ jobs: name: Chart Lint (Tier 1.5) needs: changes # tests-slim fast lane: post-merge + nightly, and on a PR only when charts/k8s changed. - if: ${{ !cancelled() && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && needs.changes.outputs.chart == 'true')) }} + if: ${{ !cancelled() && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'merge_group' || (github.event_name == 'pull_request' && needs.changes.outputs.chart == 'true')) }} runs-on: ubuntu-latest env: From 99c97b3b2fea9b7479575c8b0befbfa38c8c78ca Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 1 Nov 2026 04:42:02 -0800 Subject: [PATCH 5/5] ci(tests): frontend filter covers README.md + docs/assets (real jest inputs) readmeFrames.test.ts and landingHeroContent.test.tsx fs-read README.md, and landingFrames.test.ts reads docs/assets/landing-source (readmeFrames also reads docs/assets/readme frames). A Growth PR editing only the README or those assets would otherwise skip frontend-tests on the PR and surface only in the queue. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_015AKgpNr8tFMMj8aFvFPUdA --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 89137d4843..3dab426d24 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -89,7 +89,7 @@ jobs: files=$(git diff --no-renames --name-only "$base"...HEAD 2>/dev/null || git diff --no-renames --name-only "$base" HEAD) printf '%s\n' "$files" printf '%s\n' "$files" | grep -qE '^(backend/|cli/|packages/|package(-lock)?\.json)' && backend=true - printf '%s\n' "$files" | grep -qE '^(frontend/|backend/)' && frontend=true + printf '%s\n' "$files" | grep -qE '^(frontend/|backend/|README\.md$|docs/assets/)' && frontend=true printf '%s\n' "$files" | grep -qE '^(k8s/|charts?/|helm/|scripts/verify-backend-worker-chart\.sh$|scripts/monitoring/apply-alerting\.py$|docs/runbooks/alerting\.md$|package(-lock)?\.json)' && chart=true # A change to this workflow runs the full suite as a self-test, so a # PR that edits the test pipeline proves every heavy leg still works.