From 85fbc2e23f23366ae13251cfb70d27ebc9f0ae25 Mon Sep 17 00:00:00 2001 From: song Date: Thu, 15 Oct 2026 16:21:27 +0800 Subject: [PATCH 1/6] ci: distinguish qualification failures and remove duplicate coverage work Signed-off-by: song --- .github/workflows/ci-maintenance.yml | 95 +++++++++++++ .github/workflows/python-tests.yml | 133 +++++++------------ scripts/ci/merge-dashboard-coverage.test.mjs | 5 +- scripts/ci/qualification_summary.test.ts | 77 +++++++++++ scripts/ci/qualification_summary.ts | 90 +++++++++++++ scripts/ci/test_review_gate.py | 66 ++++++++- tests/test_ci_resource_policy.py | 36 ++++- tests/test_python_ci_workflow.py | 9 +- tests/test_sonarcloud_workflow.py | 13 +- 9 files changed, 423 insertions(+), 101 deletions(-) create mode 100644 .github/workflows/ci-maintenance.yml create mode 100644 scripts/ci/qualification_summary.test.ts create mode 100644 scripts/ci/qualification_summary.ts diff --git a/.github/workflows/ci-maintenance.yml b/.github/workflows/ci-maintenance.yml new file mode 100644 index 0000000000..87c8e0bed5 --- /dev/null +++ b/.github/workflows/ci-maintenance.yml @@ -0,0 +1,95 @@ +name: CI Maintenance + +# Trusted base-branch code only; never checks out or executes PR content. +on: + pull_request_target: + types: [closed, synchronize] + +permissions: + contents: read + +concurrency: + group: ci-maintenance-pr-${{ github.event.number }} + cancel-in-progress: true + +jobs: + cancel-obsolete-pr: + if: github.event_name == 'pull_request_target' && (github.event.action == 'closed' || github.event.action == 'synchronize') + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + actions: write + contents: read + pull-requests: read + steps: + - uses: actions/github-script@v8 + with: + script: | + const repo = context.repo; + const pull_number = context.payload.number; + const readPR = async () => (await github.rest.pulls.get({...repo, pull_number})).data; + const pr = await readPR(); + const closed = context.payload.action === 'closed'; + if (!closed && context.payload.action !== 'synchronize') return core.info('No obsolete-run cleanup event.'); + if (pr.state !== (closed ? 'closed' : 'open')) return core.info('PR state changed; retain qualification.'); + if (!closed && context.payload.pull_request?.head.sha !== pr.head.sha) { + return core.info('Synchronize head changed; stop cleanup.'); + } + const cutoff = Date.parse(pr.closed_at); + if (!pr.head.repo?.id || !pr.head.ref || !pr.head.sha || (closed && !Number.isFinite(cutoff))) { + throw new Error('PR has no verifiable source repository, branch, head or close time.'); + } + const unchanged = current => current.state === pr.state && current.head.sha === pr.head.sha && + current.head.ref === pr.head.ref && current.head.repo?.id === pr.head.repo.id && + (!closed || current.closed_at === pr.closed_at); + // Another open PR may use the same source branch against a different base. + const hasOtherOpenPR = async (head = pr.head.label) => (await github.paginate(github.rest.pulls.list, + {...repo, state: 'open', head, per_page: 100})) + .some(p => closed || p.number !== pull_number); + const cancellationScopeUnchanged = async () => { + const current = await readPR(); + return unchanged(current) && !(await hasOtherOpenPR(current.head.label)); + }; + if (await hasOtherOpenPR()) { + return core.info('Source branch has another open PR; retain its qualification.'); + } + const runs = await github.paginate(github.rest.actions.listWorkflowRuns, + {...repo, workflow_id: 'python-tests.yml', event: 'pull_request', + branch: pr.head.ref, per_page: 100}); + for (const run of runs) { + const created = Date.parse(run.created_at); + if (run.status === 'completed' || run.head_repository?.id !== pr.head.repo.id || + run.head_branch !== pr.head.ref || !run.head_sha || !Number.isFinite(created) || + (closed ? created > cutoff : run.head_sha === pr.head.sha)) continue; + // GitHub can omit PR associations after merge. Only the exact closed head + // is safe in that case; superseded heads remain in their concurrency lane. + if (run.pull_requests?.length + ? !run.pull_requests.some(p => p.number === pull_number) + : !closed || run.head_sha !== pr.head.sha) continue; + if (!(await cancellationScopeUnchanged())) return core.info('PR or source branch changed; stop cleanup.'); + const params = {...repo, run_id: run.id}; + try { + await github.rest.actions.cancelWorkflowRun(params); + await new Promise(resolve => setTimeout(resolve, 1500)); + const {data: remaining} = await github.rest.actions.getWorkflowRun(params); + if (remaining.status !== 'completed') { + if (!(await cancellationScopeUnchanged())) return core.info('PR or source branch changed; stop force cancellation.'); + await github.request('POST /repos/{owner}/{repo}/actions/runs/{run_id}/force-cancel', params); + let terminal = false; + for (let attempt = 0; attempt < 10; attempt++) { + await new Promise(resolve => setTimeout(resolve, 1000)); + if ((await github.rest.actions.getWorkflowRun(params)).data.status === 'completed') { + terminal = true; + break; + } + } + if (!terminal) throw new Error(`Cancellation ${run.id} has not completed; inspect its run.`); + } + core.info(`Retired obsolete Python qualification ${run.id}.`); + } catch (error) { + if (error.status !== 409 || + (await github.rest.actions.getWorkflowRun(params)).data.status !== 'completed') throw error; + core.info(`Qualification ${run.id} completed before cancellation.`); + } + } + diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index dfcb4926bf..170dd92384 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -3,9 +3,6 @@ name: Python Tests on: pull_request: types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled, closed] - # Trusted base-branch code only; never checks out or executes PR content. - pull_request_target: - types: [closed, synchronize] # Inert until the main ruleset enables a merge queue; queue runs are full. merge_group: workflow_dispatch: @@ -43,86 +40,6 @@ env: LOOPX_USAGE_PING: "0" jobs: - cancel-obsolete-pr: - if: github.event_name == 'pull_request_target' && (github.event.action == 'closed' || github.event.action == 'synchronize') - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - actions: write - contents: read - pull-requests: read - steps: - - uses: actions/github-script@v8 - with: - script: | - const repo = context.repo; - const pull_number = context.payload.number; - const readPR = async () => (await github.rest.pulls.get({...repo, pull_number})).data; - const pr = await readPR(); - const closed = context.payload.action === 'closed'; - if (!closed && context.payload.action !== 'synchronize') return core.info('No obsolete-run cleanup event.'); - if (pr.state !== (closed ? 'closed' : 'open')) return core.info('PR state changed; retain qualification.'); - if (!closed && context.payload.pull_request?.head.sha !== pr.head.sha) { - return core.info('Synchronize head changed; stop cleanup.'); - } - const cutoff = Date.parse(pr.closed_at); - if (!pr.head.repo?.id || !pr.head.ref || !pr.head.sha || (closed && !Number.isFinite(cutoff))) { - throw new Error('PR has no verifiable source repository, branch, head or close time.'); - } - const unchanged = current => current.state === pr.state && current.head.sha === pr.head.sha && - current.head.ref === pr.head.ref && current.head.repo?.id === pr.head.repo.id && - (!closed || current.closed_at === pr.closed_at); - // Another open PR may use the same source branch against a different base. - const hasOtherOpenPR = async (head = pr.head.label) => (await github.paginate(github.rest.pulls.list, - {...repo, state: 'open', head, per_page: 100})) - .some(p => closed || p.number !== pull_number); - const cancellationScopeUnchanged = async () => { - const current = await readPR(); - return unchanged(current) && !(await hasOtherOpenPR(current.head.label)); - }; - if (await hasOtherOpenPR()) { - return core.info('Source branch has another open PR; retain its qualification.'); - } - const runs = await github.paginate(github.rest.actions.listWorkflowRuns, - {...repo, workflow_id: 'python-tests.yml', event: 'pull_request', - branch: pr.head.ref, per_page: 100}); - for (const run of runs) { - const created = Date.parse(run.created_at); - if (run.status === 'completed' || run.head_repository?.id !== pr.head.repo.id || - run.head_branch !== pr.head.ref || !run.head_sha || !Number.isFinite(created) || - (closed ? created > cutoff : run.head_sha === pr.head.sha)) continue; - // GitHub can omit PR associations after merge. Only the exact closed head - // is safe in that case; superseded heads remain in their concurrency lane. - if (run.pull_requests?.length - ? !run.pull_requests.some(p => p.number === pull_number) - : !closed || run.head_sha !== pr.head.sha) continue; - if (!(await cancellationScopeUnchanged())) return core.info('PR or source branch changed; stop cleanup.'); - const params = {...repo, run_id: run.id}; - try { - await github.rest.actions.cancelWorkflowRun(params); - await new Promise(resolve => setTimeout(resolve, 1500)); - const {data: remaining} = await github.rest.actions.getWorkflowRun(params); - if (remaining.status !== 'completed') { - if (!(await cancellationScopeUnchanged())) return core.info('PR or source branch changed; stop force cancellation.'); - await github.request('POST /repos/{owner}/{repo}/actions/runs/{run_id}/force-cancel', params); - let terminal = false; - for (let attempt = 0; attempt < 10; attempt++) { - await new Promise(resolve => setTimeout(resolve, 1000)); - if ((await github.rest.actions.getWorkflowRun(params)).data.status === 'completed') { - terminal = true; - break; - } - } - if (!terminal) throw new Error(`Cancellation ${run.id} has not completed; inspect its run.`); - } - core.info(`Retired obsolete Python qualification ${run.id}.`); - } catch (error) { - if (error.status !== 409 || - (await github.rest.actions.getWorkflowRun(params)).data.status !== 'completed') throw error; - core.info(`Qualification ${run.id} completed before cancellation.`); - } - } - changes: # Closed events cancel the old lane without starting another qualification. if: github.event_name != 'pull_request_target' && (github.event_name != 'pull_request' || github.event.action != 'closed') @@ -148,7 +65,9 @@ jobs: - name: Install workflow contract parser run: python -m pip install --disable-pip-version-check 'pyyaml>=6,<7' - name: Validate merge gate semantics - run: python -m unittest discover -s scripts/ci -p 'test_*.py' + run: | + python -m unittest discover -s scripts/ci -p 'test_*.py' + node --test scripts/ci/qualification_summary.test.ts - name: Classify the exact pull-request change id: classify env: @@ -453,14 +372,18 @@ jobs: - name: Reject undefined CSS custom properties run: node scripts/check-css-custom-properties.mjs dashboard - - name: Measure Dashboard unit and browser interactions + # The coverage merger consumes the unit report from shard 1 only. + - name: Measure Dashboard unit coverage once + if: matrix.shard == 1 + run: npm run test:dashboard:coverage + + - name: Measure Dashboard browser interactions env: LOOPX_PERSONAL_WORKSPACE_SHARD: "${{ matrix.shard }}/3" LOOPX_DASHBOARD_COVERAGE: "1" LOOPX_PLAYWRIGHT_PACKAGE: ${{ github.workspace }}/apps/presentation/dashboard/node_modules/playwright run: | node --test examples/personal-workspace-browser/shard.test.mjs - npm run test:dashboard:coverage cd apps/presentation/dashboard npx playwright install --with-deps chromium --only-shell node ../../../examples/personal-workspace-browser-smoke.mjs @@ -942,6 +865,9 @@ jobs: run: python scripts/chat_bundle.py verify --source merge-gate: + permissions: + contents: read + actions: read # Open PRs always receive an outcome; closed PRs only cancel their old lane. if: always() && github.event_name != 'pull_request_target' && (github.event_name != 'pull_request' || github.event.action != 'closed') needs: [changes, checks, pytest, node-minimum-compatibility, stage2c-correctness-e2e, windows-powershell, presentation] @@ -953,6 +879,41 @@ jobs: with: python-version: "3.11" - name: Reject incomplete or unsuccessful qualification + id: verdict env: NEEDS_JSON: ${{ toJSON(needs) }} run: python scripts/ci/review_gate.py verify + - name: Explain qualification with current-attempt job evidence + if: always() + # Reporting failure cannot change the preceding gate's verdict. + continue-on-error: true + timeout-minutes: 1 + uses: actions/github-script@v8 + env: + NEEDS_JSON: ${{ toJSON(needs) }} + GATE_RESULT: ${{ steps.verdict.outcome }} + with: + script: | + const {pathToFileURL} = await import('node:url'); + const {renderQualificationSummary} = await import(pathToFileURL( + `${process.env.GITHUB_WORKSPACE}/scripts/ci/qualification_summary.ts`).href); + let jobs = null; + try { + jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRunAttempt, { + ...context.repo, run_id: context.runId, + attempt_number: Number(process.env.GITHUB_RUN_ATTEMPT), per_page: 100, + request: {signal: AbortSignal.timeout(10000)}, + }); + } catch { + core.warning('Job evidence unavailable; the qualification verdict is unchanged.'); + } + await core.summary.addRaw(renderQualificationSummary({ + repository: `${context.repo.owner}/${context.repo.repo}`, + serverUrl: process.env.GITHUB_SERVER_URL, + runId: context.runId, runAttempt: Number(process.env.GITHUB_RUN_ATTEMPT), + sourceSha: context.payload.pull_request?.head.sha || context.sha, + checkoutSha: context.sha, + baseSha: context.payload.pull_request?.base.sha || context.payload.merge_group?.base_sha || '', + gateResult: process.env.GATE_RESULT, + needs: JSON.parse(process.env.NEEDS_JSON), jobs, + })).write(); diff --git a/scripts/ci/merge-dashboard-coverage.test.mjs b/scripts/ci/merge-dashboard-coverage.test.mjs index 54a4df63b5..be6e78702d 100644 --- a/scripts/ci/merge-dashboard-coverage.test.mjs +++ b/scripts/ci/merge-dashboard-coverage.test.mjs @@ -17,12 +17,15 @@ test("coverage merge preserves disjoint hits and rejects missing or duplicate ac await mkdir(shard); await writeReceipt(index,`scenario-${index}`); await writeFile(resolve(shard,"browser-coverage.json"),JSON.stringify({[source]:file(index===2?1:0)})); - await writeFile(resolve(shard,"lcov.info"),"unit-coverage-preserved"); + if (index === 1) await writeFile(resolve(shard,"lcov.info"),"unit-coverage-preserved"); } let result = run(); assert.equal(result.status,0,result.stderr); assert.match(await readFile(resolve(root,"merged/browser-lcov.info"),"utf8"),/DA:1,1/); assert.equal(await readFile(resolve(root,"merged/lcov.info"),"utf8"),"unit-coverage-preserved"); + await rm(resolve(root,"dashboard-coverage-1/lcov.info")); + assert.notEqual(run().status,0,"unit coverage is still required"); + await writeFile(resolve(root,"dashboard-coverage-1/lcov.info"),"unit-coverage-preserved"); await writeReceipt(3,"scenario-2"); result=run();assert.notEqual(result.status,0);assert.match(result.stderr,/overlapping/); await rm(resolve(root,"dashboard-coverage-3"),{recursive:true}); diff --git a/scripts/ci/qualification_summary.test.ts b/scripts/ci/qualification_summary.test.ts new file mode 100644 index 0000000000..79f415e7ba --- /dev/null +++ b/scripts/ci/qualification_summary.test.ts @@ -0,0 +1,77 @@ +import assert from "node:assert/strict"; +import {test} from "node:test"; +import {renderQualificationSummary, type JobEvidence, type QualificationSummary} from "./qualification_summary.ts"; + +const job = (name: string, conclusion: string | null = "failure", overrides: Partial = {}): JobEvidence => ({ + id: 10, run_id: 42, run_attempt: 2, name, status: "completed", conclusion, + steps: [{name: "Run tests", conclusion: "failure"}], ...overrides, +}); +const input = (overrides: Partial = {}): QualificationSummary => ({ + repository: "example/project", serverUrl: "https://github.com", runId: 42, runAttempt: 2, + sourceSha: "a".repeat(40), checkoutSha: "b".repeat(40), baseSha: "c".repeat(40), + gateResult: "failure", needs: {changes: {result: "success", outputs: {change_kind: "full"}}, + pytest: {result: "failure"}}, jobs: [], ...overrides, +}); + +test("shows failed steps ahead of downstream rollups without inventing a cause", () => { + const result = renderQualificationSummary(input({jobs: [ + job("pytest", "failure", {id: 11}), job("test-shard (2)"), + job("checks", "failure", {id: 12}), job("test-shard (3)", "cancelled", {id: 13}), + ]})); + assert.match(result, /Gate step: failure/); + assert.ok(result.indexOf("test-shard") < result.indexOf("Aggregate failures")); + assert.match(result, /Run tests/); + assert.match(result, /https:\/\/github.com\/example\/project\/actions\/runs\/42\/job\/10/); + assert.match(result, /Aggregate failures[\s\S]*pytest/); + assert.match(result, /cancelled/); + for (const sha of ["a", "b", "c"]) assert.ok(result.includes(sha.repeat(40))); + assert.doesNotMatch(result, /inherited|flaky|safe to merge/i); +}); + +test("coverage or aggregation failures remain visible even without a failing test job", () => { + const result = renderQualificationSummary(input({jobs: [job("pytest", "failure", { + steps: [{name: "Combine complete coverage", conclusion: "failure"}], + })]})); + assert.match(result, /Aggregate failures[\s\S]*Combine complete coverage/); + assert.match(result, /Gate step: failure/); +}); + +test("exempt checks and unavailable evidence cannot become passing test claims", () => { + const docs = renderQualificationSummary(input({gateResult: "success", needs: { + changes: {result: "success", outputs: {change_kind: "docs"}}, pytest: {result: "skipped"}, + }, jobs: [job("test-shard", "skipped")]})); + assert.match(docs, /Gate step: success/); + assert.match(docs, /pytest \| skipped/); + assert.match(docs, /Skipped jobs are not passing test evidence/); + const missing = renderQualificationSummary(input({jobs: null})); + assert.match(missing, /Job evidence unavailable/); + assert.match(missing, /Gate step: failure/); + assert.doesNotMatch(missing, /all.*pass/i); +}); + +test("does not mix earlier attempts or other runs into the current attempt", () => { + const result = renderQualificationSummary(input({jobs: [ + job("old green", "success", {run_attempt: 1}), + job("unrelated workflow", "success", {run_id: 41}), job("current failure"), + ]})); + assert.doesNotMatch(result, /old green|unrelated workflow/); + assert.match(result, /Excluded 2 jobs from another run or attempt/); + assert.match(result, /current failure/); +}); + +test("reports unfinished jobs without calling them assertion failures", () => { + const result = renderQualificationSummary(input({jobs: [job("waiting", null, { + status: "queued", steps: [], + })]})); + assert.match(result, /Incomplete jobs[\s\S]*waiting[\s\S]*queued/); + assert.doesNotMatch(result, /Failing jobs/); +}); + +test("renders metadata as bounded text and constructs links from trusted run identity", () => { + const result = renderQualificationSummary(input({jobs: [job("\n| [click](javascript:x) " + "a".repeat(400), "failure", { + steps: [{name: "