From 68fa6cf3ebf7c1d64a122a02ea8ae2a9f2bb9a9d Mon Sep 17 00:00:00 2001 From: song Date: Wed, 14 Oct 2026 15:35:00 +0800 Subject: [PATCH 1/4] test(ci): model both PR-number fields in concurrency fixtures GitHub's pull_request and pull_request_target payloads expose the PR number as both github.event.number and github.event.pull_request.number. tests/test_ci_resource_policy.py modelled only the nested form, so the python-tests concurrency group, which reads github.event.number, failed test_main_supersession_does_not_cancel_nightly_or_manual_full_qualification on main with AttributeError. scripts/ci/test_review_gate.py modelled only the flat form with pull_request=None. Give both fixtures the real shape so the tests check the cancellation policy rather than one field spelling. No workflow change. Signed-off-by: song --- scripts/ci/test_review_gate.py | 2 +- tests/test_ci_resource_policy.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/ci/test_review_gate.py b/scripts/ci/test_review_gate.py index ff20a4854b..07f44c2c25 100644 --- a/scripts/ci/test_review_gate.py +++ b/scripts/ci/test_review_gate.py @@ -52,7 +52,7 @@ def test_closed_pr_cancels_same_lane_and_skips_qualification(self): def expression(value, action, *, number=17, event="pull_request"): ref = "refs/heads/main" if action == "closed" or event != "pull_request" else f"refs/pull/{number}/merge" github = SimpleNamespace(event_name=event, ref=ref, event=SimpleNamespace( - action=action, number=number, pull_request=None)) + action=action, number=number, pull_request=SimpleNamespace(number=number))) return eval(value.strip().removeprefix("${{").removesuffix("}}").strip() .replace("&&", "and").replace("||", "or"), {"__builtins__": {}}, {"github": github, "always": lambda: True, diff --git a/tests/test_ci_resource_policy.py b/tests/test_ci_resource_policy.py index 2f3f7a431e..ff0f158b15 100644 --- a/tests/test_ci_resource_policy.py +++ b/tests/test_ci_resource_policy.py @@ -20,7 +20,8 @@ def event(name: str, *, pr: int = 101, tag: str = "", backend: bool = True) -> d "github": SimpleNamespace( event_name=name, ref=f"refs/pull/{pr}/merge" if name == "pull_request" else "refs/heads/main", - event=SimpleNamespace(pull_request=SimpleNamespace(number=pr), release=SimpleNamespace(tag_name=tag)), + # GitHub exposes the PR number both as `event.number` and `event.pull_request.number`. + event=SimpleNamespace(number=pr, pull_request=SimpleNamespace(number=pr), release=SimpleNamespace(tag_name=tag)), ), "inputs": SimpleNamespace(tag=tag), "needs": SimpleNamespace(changes=SimpleNamespace(outputs=SimpleNamespace(backend_tests=str(backend).lower()))), From ee0833128aada01bc17de9e76c34e611e93c014a Mon Sep 17 00:00:00 2001 From: song Date: Wed, 14 Oct 2026 15:36:16 +0800 Subject: [PATCH 2/4] ci: queue superseded main pushes instead of cancelling the running qualification Main pushes to python-tests used cancel-in-progress, so each push cancelled the previous default-branch run. With pushes arriving faster than a full run completes, the newest main almost never reached a result: of the last 100 Python Tests runs on main, 95 were cancelled, 3 failed and none succeeded, and the only completed runs were the nightly schedule. Eight sampled cancelled push runs had already consumed 1,345 runner-minutes (mean 168) before cancellation, producing no signal. Drop push from cancel-in-progress. GitHub's default queue keeps the running run, holds at most one pending run per group, and replaces that pending run when a newer push arrives, so intermediate pushes cost nothing and the newest main always completes. PR heads keep cancelling their own superseded runs; scheduled, manual and merge-queue lanes are unchanged. Rename the policy test that encoded the old default and update the resource cadence doc in both languages. Signed-off-by: song --- .github/workflows/python-tests.yml | 6 +++++- docs/development/ci-impact-selection.md | 12 ++++++++---- tests/test_ci_resource_policy.py | 7 +++++-- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 4b413eb057..3cc5bb690e 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -37,7 +37,11 @@ permissions: concurrency: # A merged PR's closed event uses main as github.ref; keep its original PR lane. group: python-tests-${{ github.event_name }}-${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('pr-{0}', github.event.number) || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' || github.event_name == 'push' }} + # Only a PR's own superseded head is cancelled. Main pushes queue: the running + # default-branch qualification completes and at most one newer push waits, + # replacing any older pending push, so the newest main always reaches a result + # without spending runner time on runs that can never finish. + cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} env: LOOPX_USAGE_PING: "0" diff --git a/docs/development/ci-impact-selection.md b/docs/development/ci-impact-selection.md index eac77ede20..58cd71f964 100644 --- a/docs/development/ci-impact-selection.md +++ b/docs/development/ci-impact-selection.md @@ -193,9 +193,11 @@ to one and background shard occupancy from five runners to two; the complete sweep may take longer. The existing fleet-health readback still checks receipts. Python Tests also runs daily at `20:17 UTC`. Its event-scoped concurrency keeps -that full run separate from main pushes and PRs. New PR heads and main pushes -cancel superseded runs; scheduled, manual and merge-queue qualification is not -cancelled by that policy. Release-artifact, desktop-artifact, signed-desktop, +that full run separate from main pushes and PRs. A new PR head cancels that +PR's superseded run. Main pushes queue instead: the running default-branch +qualification completes, at most one newer push waits, and a further push +replaces that pending successor before it starts. Scheduled, manual and +merge-queue qualification is never cancelled by this policy. Release-artifact, desktop-artifact, signed-desktop, DCO and optional Ark PR checks cancel only superseded validation for the same PR. Signed-desktop PRs use separate groups from each other and from channel publication. Release publication remains serialized and is never cancelled by a PR update. @@ -203,7 +205,9 @@ Release publication remains serialized and is never cancelled by a PR update. 全量 smoke 改为每天一次与指定 ref 的手动运行，不再随普通主干提交启动。五个完整 分片共享一次源码绑定的前端构建，同一时刻最多运行两个分片；全量用例仍保留， 代价是后台 sweep 可能更久。每日完整 Python 验证与 PR／主干提交使用不同并发组， -避免新提交取消夜间基线。过时 PR 检查可取消，发布通道仍保留串行发布保护。 +避免新提交取消夜间基线。同一 PR 的过时 head 会被取消；主干提交改为排队：运行中的 +主干验证跑完，最多保留一个更新的待运行提交，再新的提交在其启动前替换它。发布通道 +仍保留串行发布保护。 To force a complete smoke sweep before release, open Actions → Full Public Smokes → Run workflow and select the candidate ref. `ci:full` forces full Python diff --git a/tests/test_ci_resource_policy.py b/tests/test_ci_resource_policy.py index ff0f158b15..06a441d9a1 100644 --- a/tests/test_ci_resource_policy.py +++ b/tests/test_ci_resource_policy.py @@ -71,13 +71,16 @@ def test_signed_desktop_prs_do_not_share_either_publication_channel() -> None: assert expression(concurrency["group"], event("schedule")) == expression(concurrency["group"], event("workflow_dispatch")) -def test_main_supersession_does_not_cancel_nightly_or_manual_full_qualification() -> None: +def test_only_pr_heads_cancel_in_progress_and_main_pushes_queue() -> None: + # A main push must never cancel the running default-branch qualification; + # it waits as the single pending successor so the newest main always reaches + # a result. Nightly, manual and merge-queue runs keep their own lanes. policy = workflow("python-tests") assert "schedule" in policy[True] concurrency = policy["concurrency"] contexts = [event(name) for name in ("push", "pull_request", "schedule", "workflow_dispatch", "merge_group")] assert len({expression(concurrency["group"], context) for context in contexts}) == 5 - assert [expression(concurrency["cancel-in-progress"], context) for context in contexts] == [True, True, False, False, False] + assert [expression(concurrency["cancel-in-progress"], context) for context in contexts] == [False, True, False, False, False] def test_required_merge_gate_workflow_is_not_suppressed_by_path_filters() -> None: From 1305980ec76c3cfb00e88f4a6346393cd8dd0dd3 Mon Sep 17 00:00:00 2001 From: song Date: Wed, 14 Oct 2026 15:37:25 +0800 Subject: [PATCH 3/4] docs(ci): record six Python test shards The test-shard matrix and --splits moved from four to six, but the job comments and the testing guide (both languages, including the local reproduce command) still described four lanes and four partitions. Align the readback with the shipped workflow. No behavior change. Signed-off-by: song --- .github/workflows/python-tests.yml | 4 ++-- docs/development/testing-and-quality.md | 28 ++++++++++++------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 3cc5bb690e..689e7bb938 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -574,14 +574,14 @@ jobs: needs: [changes, chat-bundle] if: needs.changes.outputs.python_tests == 'true' runs-on: ubuntu-latest - # Share four long-test lanes across current PRs and other workflow runs. + # Share six long-test lanes across current PRs and other workflow runs. # Do not replace another candidate's pending shard or cancel its active head. # Workflow-level concurrency still retires obsolete heads of the same PR. concurrency: group: loopx-python-test-shard-${{ matrix.shard }} queue: max cancel-in-progress: false - # All four shards were still progressing at 83-94% when the 45m job + # All shards were still progressing at 83-94% when the 45m job # deadline cancelled run 37916979857. Reserve setup/reporting headroom # without changing test-level deadlines, collection or worker count. timeout-minutes: 60 diff --git a/docs/development/testing-and-quality.md b/docs/development/testing-and-quality.md index 891d170280..6262e6dd9f 100644 --- a/docs/development/testing-and-quality.md +++ b/docs/development/testing-and-quality.md @@ -122,7 +122,7 @@ shell；不能因为过滤了 `CI` 就恢复采集。不能按系统、安装渠 `python-tests.yml` publishes `merge-gate` for every open pull request. Code, workflow, policy and unknown paths require the existing `pytest` aggregate -(including TypeScript checks and all four Python shards), Stage 2C correctness +(including TypeScript checks and all six Python shards), Stage 2C correctness aggregate, and Windows tests to succeed. Failed, cancelled, missing or unexpectedly skipped results cannot pass the gate. The client-only exception below retains common checks and substitutes packaged Dashboard qualification @@ -203,8 +203,8 @@ The [job exemption policy](ci-impact-selection.md) additionally permits pure Dashboard-client changes to skip backend Python/Windows and Stage2c, while requiring frontend type-check/build, scoped coverage and real packaged Dashboard browser smoke. Backend-only TS/lint/minimum-Node jobs are also exempt. -Mixed, prompt, dependency and unknown changes stay full. Full Python runs four -complete shards and combines all four coverage files. The `ci:full` label forces +Mixed, prompt, dependency and unknown changes stay full. Full Python runs six +complete shards and combines all six coverage files. The `ci:full` label forces full qualification; main stays full. No selected-only report impersonates full coverage. 新的前端豁免由目标分支已审阅的策略控制；CI 自身变更仍全量验证。 @@ -616,7 +616,7 @@ network latency, provider availability, or a two-hour matrix. 它刻意不包含真实模型调用和 full smoke catalog，因此普通迭代不依赖凭证、网络 时延、模型服务可用性或两小时级测试矩阵。 -The Linux suite uses four hosted runners with two xdist workers each. +The Linux suite uses six hosted runners with two xdist workers each. `pytest-split` partitions the complete collection using `least_duration`; without a timing file, tests have equal weight and alternate between shards. Lint, type checks, and the CLI budget run separately. The required `pytest` @@ -630,14 +630,14 @@ An explicitly loaded CI observer also streams pytest-owned phase durations from the xdist controller. Read the downloaded log with `node scripts/ci/pytest_timing.ts JOB_LOG`; incomplete evidence remains incomplete and does not replace JUnit, coverage or the required gate. See the -[timing observation boundary](ci-impact-selection.md#four-complete-python-shards). +[timing observation boundary](ci-impact-selection.md#six-complete-python-shards). The reusable Sonar workflow consumes that same run's XML and never reruns pytest or reads cross-run artifacts. Missing Sonar tokens still skip analysis successfully; test jobs receive no Sonar secret. The trigger is the union of the former Python and Sonar paths. Client-only PRs use the exemption above; Sonar-configuration changes remain full, including on forks without a token. -Linux 全套测试分到四台 hosted runner，每台保留两个 xdist worker。`pytest-split` +Linux 全套测试分到六台 hosted runner，每台保留两个 xdist worker。`pytest-split` 按完整 collection 分片；没有历史耗时时，等权测试交替分配。lint、类型检查和 CLI 预算独立执行。必需的 `pytest` 汇总检查会拒绝失败／跳过的分片和缺失的 coverage， 合并后再执行原有 19.6% 门槛；不要求单个分片达到全套覆盖率。coverage 使用相对路径， @@ -646,12 +646,12 @@ Linux 全套测试分到四台 hosted runner，每台保留两个 xdist worker CI 同时在 xdist controller 流式记录 pytest 原生 phase 耗时。下载日志后用 `node scripts/ci/pytest_timing.ts JOB_LOG` 回读；不完整证据保持不完整，不能替代 JUnit、coverage 或必过门禁。归属与测量边界见 -[分片耗时记录](ci-impact-selection.md#four-complete-python-shards)。 +[分片耗时记录](ci-impact-selection.md#six-complete-python-shards)。 Sonar 只复用同一次 run 的 XML，不重复测试、不跨 run 取产物。缺少 token 仍成功跳过 Sonar，测试 job 不接收 Sonar secret。触发范围取原有两套 workflow 的并集；纯前端 PR 使用前述豁免，Sonar 配置变更仍全量运行，包括没有 token 的 fork。 -Full Python shards share four repository-wide job concurrency groups, one per +Full Python shards share six repository-wide job concurrency groups, one per `matrix.shard`, across PR, main, merge-queue and manual runs carrying this workflow version. Each group admits one running job and uses `queue: max` to retain up to 100 pending jobs instead of replacing another current @@ -666,16 +666,16 @@ unchanged: cancelled, skipped or incomplete dependencies cannot become a green aggregate. This adds one short trusted cleanup job per update without checkout, PR-code execution or new permissions on qualification jobs. Older workflow versions on the base branch retain only closed-event cleanup. Queue overflow, cancellation and missing coverage remain failures of -the required gate; collection, four partitions, two workers, reports and the +the required gate; collection, six partitions, two workers, reports and the coverage floor stay unchanged. Contributors can wait longer for a shard during -overlapping full runs. This limits long-shard occupancy to four runners; it +overlapping full runs. This limits long-shard occupancy to six runners; it does not reserve release capacity or prioritize releases over other jobs or repositories in the organization's shared hosted pool. Older workflow versions do not participate. Roll back by removing this job-level concurrency block, retaining workflow-level cancellation and the required gate. See GitHub's [concurrency queue contract](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/control-workflow-concurrency). -全量 Python 分片按 `matrix.shard` 共用四个仓库级 job concurrency 组，适用于 +全量 Python 分片按 `matrix.shard` 共用六个仓库级 job concurrency 组，适用于 携带本版 workflow 的 PR、main、merge queue 和手动运行。每组允许一个运行中 job， 用 `queue: max` 保留至多 100 个等待 job，不替换其他有效候选。workflow 级取消 请求清退同 PR 的旧 head；可信 base 分支的 `synchronize` 清理还会取消遗留 @@ -685,13 +685,13 @@ PR、当前 head 和无关联 run 均被保护。必过汇总条件不变，取 不能变绿。每次更新增加一个短清理 job，不 checkout 或执行 PR 代码，不增加普通 验证 job 的权限；base 分支旧版 workflow 仍只清理 closed 事件。队列溢出、取消 和缺失 coverage 仍不能通过必需检查。 -完整 collection、四分片、双 worker、报告和覆盖率门槛保持不变；重叠的全量运行 -会增加贡献者等待分片的时间。此限制只把长分片占用约束到四台 runner，不预留发布 +完整 collection、六分片、双 worker、报告和覆盖率门槛保持不变；重叠的全量运行 +会增加贡献者等待分片的时间。此限制只把长分片占用约束到六台 runner，不预留发布 容量，也不保证发布优先于组织共享 hosted 池中的其他 job 或仓库。旧版 workflow 不参与该限制。回退时移除此 job 的 concurrency 块，保留 workflow 级取消和必需 检查；队列行为遵循上述 GitHub 官方契约。 -Reproduce one shard locally with `uv run --extra test python -m pytest -v -n 2 --splits 4 --group 1 +Reproduce one shard locally with `uv run --extra test python -m pytest -v -n 2 --splits 6 --group 1 --splitting-algorithm least_duration --cov=loopx`. Omit the split arguments to run the complete suite locally. 全量本地测试仍省略分片参数即可。 From 5c13b80fff0563b5eddd3c7f4a098ad0a58d5445 Mon Sep 17 00:00:00 2001 From: song Date: Thu, 15 Oct 2026 08:13:37 +0800 Subject: [PATCH 4/4] Revert "ci: queue superseded main pushes instead of cancelling the running qualification" The maintainer confirmed on #6416 that cancelling a superseded main push is by design, to keep CI efficient. Restore the accepted cancel-in-progress expression, the test that encodes that default, and the supersession prose in ci-impact-selection.md. The webhook-fixture and six-shard readback commits are unchanged. Signed-off-by: song --- .github/workflows/python-tests.yml | 6 +----- docs/development/ci-impact-selection.md | 12 ++++-------- tests/test_ci_resource_policy.py | 7 ++----- 3 files changed, 7 insertions(+), 18 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 689e7bb938..709645afd3 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -37,11 +37,7 @@ permissions: concurrency: # A merged PR's closed event uses main as github.ref; keep its original PR lane. group: python-tests-${{ github.event_name }}-${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('pr-{0}', github.event.number) || github.ref }} - # Only a PR's own superseded head is cancelled. Main pushes queue: the running - # default-branch qualification completes and at most one newer push waits, - # replacing any older pending push, so the newest main always reaches a result - # without spending runner time on runs that can never finish. - cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' || github.event_name == 'push' }} env: LOOPX_USAGE_PING: "0" diff --git a/docs/development/ci-impact-selection.md b/docs/development/ci-impact-selection.md index 58cd71f964..eac77ede20 100644 --- a/docs/development/ci-impact-selection.md +++ b/docs/development/ci-impact-selection.md @@ -193,11 +193,9 @@ to one and background shard occupancy from five runners to two; the complete sweep may take longer. The existing fleet-health readback still checks receipts. Python Tests also runs daily at `20:17 UTC`. Its event-scoped concurrency keeps -that full run separate from main pushes and PRs. A new PR head cancels that -PR's superseded run. Main pushes queue instead: the running default-branch -qualification completes, at most one newer push waits, and a further push -replaces that pending successor before it starts. Scheduled, manual and -merge-queue qualification is never cancelled by this policy. Release-artifact, desktop-artifact, signed-desktop, +that full run separate from main pushes and PRs. New PR heads and main pushes +cancel superseded runs; scheduled, manual and merge-queue qualification is not +cancelled by that policy. Release-artifact, desktop-artifact, signed-desktop, DCO and optional Ark PR checks cancel only superseded validation for the same PR. Signed-desktop PRs use separate groups from each other and from channel publication. Release publication remains serialized and is never cancelled by a PR update. @@ -205,9 +203,7 @@ Release publication remains serialized and is never cancelled by a PR update. 全量 smoke 改为每天一次与指定 ref 的手动运行，不再随普通主干提交启动。五个完整 分片共享一次源码绑定的前端构建，同一时刻最多运行两个分片；全量用例仍保留， 代价是后台 sweep 可能更久。每日完整 Python 验证与 PR／主干提交使用不同并发组， -避免新提交取消夜间基线。同一 PR 的过时 head 会被取消；主干提交改为排队：运行中的 -主干验证跑完，最多保留一个更新的待运行提交，再新的提交在其启动前替换它。发布通道 -仍保留串行发布保护。 +避免新提交取消夜间基线。过时 PR 检查可取消，发布通道仍保留串行发布保护。 To force a complete smoke sweep before release, open Actions → Full Public Smokes → Run workflow and select the candidate ref. `ci:full` forces full Python diff --git a/tests/test_ci_resource_policy.py b/tests/test_ci_resource_policy.py index 654eba6a02..9cd4dfacac 100644 --- a/tests/test_ci_resource_policy.py +++ b/tests/test_ci_resource_policy.py @@ -72,10 +72,7 @@ def test_signed_desktop_prs_do_not_share_either_publication_channel() -> None: assert expression(concurrency["group"], event("schedule")) == expression(concurrency["group"], event("workflow_dispatch")) -def test_only_pr_heads_cancel_in_progress_and_main_pushes_queue() -> None: - # A main push must never cancel the running default-branch qualification; - # it waits as the single pending successor so the newest main always reaches - # a result. Nightly, manual and merge-queue runs keep their own lanes. +def test_main_supersession_does_not_cancel_nightly_or_manual_full_qualification() -> None: policy = workflow("python-tests") assert "schedule" in policy[True] concurrency = policy["concurrency"] @@ -84,7 +81,7 @@ def test_only_pr_heads_cancel_in_progress_and_main_pushes_queue() -> None: if context["github"].event_name != "pull_request": del context["github"].event.number assert len({expression(concurrency["group"], context) for context in contexts}) == 5 - assert [expression(concurrency["cancel-in-progress"], context) for context in contexts] == [False, True, False, False, False] + assert [expression(concurrency["cancel-in-progress"], context) for context in contexts] == [True, True, False, False, False] @pytest.mark.parametrize("name", ["pull_request", "pull_request_target"])