From 113573bf3ba734fda79ff2de6935cd21af3515dc Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:47:53 -0600 Subject: [PATCH 1/6] Mutants: two that fell through to the whole suite name the tests that kill them "P-58 keeps the first head's status" and "the impostor workflow posts from the Lens's own file" each named a test that passes with the mutant in place, so their legs ran the whole suite after it: 10 to 21 minutes of a 27-minute step, the slowest leg of five of the last six merge-group runs. Each now names the one test the whole suite fails on. Co-Authored-By: Claude Opus 5.5 --- tools/mutants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/mutants.py b/tools/mutants.py index cb134af6..eada9849 100644 --- a/tools/mutants.py +++ b/tools/mutants.py @@ -4189,7 +4189,7 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ('a mutant a row names is killed by any test', 'tools/mutants.py', '({"only": first[0]} if name in _held_by_rows(root) ' 'else {"first": first[0]})', '{"first": first[0]}', RUN_ALONE), ("the impostor workflow posts from the Lens's own file", "sim/world.py", - '"workflow": IMPOSTOR_WORKFLOW,', '"workflow": WORKFLOWS[lens],', ("tests.test_simulator.Simulation.test_the_seeds_take_a_pull_request_out_of_the_queue_for_each_cause",)), + '"workflow": IMPOSTOR_WORKFLOW,', '"workflow": WORKFLOWS[lens],', ("tests.test_simulator.Simulation.test_the_seeds_serve_items_with_an_acceptance_contract_and_without",)), ("a neutral check is read as passing", "config/defaults.json", '"neutral": "unreadable"', '"neutral": "passing"', ("tests.test_decide.Classes",)), ("a neutral check is read as failing", "config/defaults.json", @@ -13248,7 +13248,7 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ('P-58 may write OUT inside the repo', "tools/trial_p58_edits.sh", 'case "$out/" in "$root"/*)', 'case "$out/" in /nowhere/*)', ('tests.test_tools.EditsReadTrial.test_each_refusal_comes_before_anything_is_minted',)), ("P-58's OUT is read without its links", "tools/trial_p58_edits.sh", '2>/dev/null && pwd -P); then', '2>/dev/null && pwd); then', ('tests.test_tools.EditsReadTrial.test_an_output_folder_reached_through_a_link_into_the_repo_is_refused',)), ("P-58's raw answers may be kept inside the repo", "tools/trial_p58_edits.sh", 'case "$scratch/" in "$root"/*)', 'case "$scratch/" in /nowhere/*)', ('tests.test_tools.EditsReadTrial.test_a_scratch_directory_inside_the_repo_is_refused_before_anything_is_minted',)), - ("P-58 keeps the first head's status", "tools/trial_p58_edits.sh", 'head && $0 != ""', 'NR == 1 && $0 != ""', ('tests.test_tools.EditsReadTrial.test_the_status_kept_is_the_last_heads',)), + ("P-58 keeps the first head's status", "tools/trial_p58_edits.sh", 'head && $0 != ""', 'NR == 1 && $0 != ""', ('tests.test_tools.ReachScript.test_a_head_ends_where_the_next_line_is_no_head',)), ("P-56 mints for another App's installation", "tools/trial_p56_comments.sh", '["apps"]["controller"]', '["apps"]["reader"]', ("tests.test_tools.CommentsReadTrial.test_each_read_is_made_with_each_token_kept_with_its_status_and_no_token_is_kept",)), ("P-56 keeps a read without its redaction", "tools/trial_p56_comments.sh", ' (cd "$root" && python3 -m tools.redact_trial < "$scratch/raw-$n" > "$out/clean-$n") 2> "$out/redacted-$n"\n', From 9e068f8cce49c3fe8b55ce3e8b8d199c1423636d Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:54:34 -0600 Subject: [PATCH 2/6] CI: make verify runs in three legs, each told its place and the count by the matrix On 2026-11-09 merge group pr-657-a81e6e58 was cut at the checks step's 26 minutes, and the ten runs before took 19 to 23.4. checks becomes a matrix of three legs, as the mutants are: each leg runs every check and every third test of the whole suite, told CHECKS_SHARD and CHECKS_SHARDS by strategy.job-index and job-total, never by a number written in, so across the legs every test runs once, the 300 seeds included. verify still needs checks, whose result is every leg's. tools/check_workflows.py holds the legs' job as it holds the shards' (_legs); tools/verify.py refuses a leg it cannot read before any check runs, and runs every test where neither is set, as at a desk. Each new rule has its mutant. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/verify.yml | 12 +++++- docs/standards/workflows.md | 18 ++++++-- examples/workflows/verify.yml | 12 +++++- tests/test_tools.py | 78 +++++++++++++++++++++++++++++++++++ tools/check_workflows.py | 44 +++++++++++++++++++- tools/mutants.py | 38 +++++++++++++++-- tools/verify.py | 47 ++++++++++++++++++--- 7 files changed, 233 insertions(+), 16 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 47aaec5d..733542ec 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -1,5 +1,5 @@ # The required check, `factory / verify`: it passes only when `make verify` and every shard of the mutation check pass on the commit, with the history tools/check_decisions.py compares it with. -# `checks` runs `make verify`, its simulators on 300 seeds. On 2026-10-21 the runs that finished took 11 to 14.5 minutes, and six runs in three pull requests were cut at its 15-minute timeout with the tests still running, so its timeout was raised to 25 minutes; by 2026-10-29 a run took up to 23, so the job has 28 and its step 26, two kept for the checkout. Raise both before a run nears the step's. +# `checks` runs `make verify` in three legs, its simulators on 300 seeds: each leg runs every check and every third test, told its place and the count by the matrix itself, never by a number written in, so that across the legs each test runs once. As one job, on 2026-10-21 the runs that finished took 11 to 14.5 minutes, and six runs in three pull requests were cut at its 15-minute timeout with the tests still running, so its timeout was raised to 25 minutes; by 2026-10-29 a run took up to 23, so the job was given 28 and its step 26, two kept for the checkout; and on 2026-11-09 the ten runs before took 19 to 23.4 and a merge group was cut at 26.2, so its tests were dealt into legs. Add a leg before one nears the step's limit. # About 5,200 mutants, in twelve shards of about 434, each kill confirmed by two runs of one test. With six shards of 168, of about 1,000 mutants then, the legs ran 14 to 25 minutes against a 25-minute timeout, and legs were cut at 25:00 in three pull requests, once 22 ms after the last kill had printed. With eight shards of 651 the legs mostly ran 6 to 14 minutes, and on 2026-11-06 one was cut at its step's 27 minutes on a runner that ran it 2.8 times slower than a re-run of the same tree ("CI: twelve mutant shards, so a slow runner does not cut a leg at its limit", issue 583); the first run of twelve took 4.4 to 12.3 minutes a leg, 7.3 the median. Every mutant names the tests that kill it, which run first; the whole suite, its slow modules last, runs only when they pass ("Mutation run: every mutant names its tests, the slow suites run last, and a builder may break a few", issue 217). Add a shard before a leg nears its timeout, or before a leg of the median time would take more than 22 minutes on a runner three times slower. A runner as slow may still cut the longest leg: run it again. # `verify` checks out the base, never the head, and runs the base's tools/needs_passed.py, so a change cannot make its own verdict pass: decision 0023. # `commits` checks out the base too and runs the base's tools/commits_are_the_maintainers.py, which reads every commit the pull request carries and refuses unless each names the Maintainer ("Identity: every commit a pull request carries is the Maintainer's, checked outside the shell", issue 204). It reads the event's payload too, and refuses unless an account on the configuration's list opened the pull request from this repo: decision 0102. On a merge group, whose payload names no opener, it reads the pull request the queue's ref names, with this job's read-only token. @@ -14,6 +14,16 @@ jobs: timeout-minutes: 28 permissions: contents: read + strategy: + fail-fast: false + matrix: + shard: + - 0 + - 1 + - 2 + env: + CHECKS_SHARD: ${{ strategy.job-index }} + CHECKS_SHARDS: ${{ strategy.job-total }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/docs/standards/workflows.md b/docs/standards/workflows.md index 3ea082bb..5c71c955 100644 --- a/docs/standards/workflows.md +++ b/docs/standards/workflows.md @@ -10,7 +10,7 @@ Any other spelling is refused. An org standard is cited from combsmith/.github a ```yaml # The required check, `factory / verify`: it passes only when `make verify` and every shard of the mutation check pass on the commit, with the history tools/check_decisions.py compares it with. -# `checks` runs `make verify`, its simulators on 300 seeds. On 2026-10-21 the runs that finished took 11 to 14.5 minutes, and six runs in three pull requests were cut at its 15-minute timeout with the tests still running, so its timeout was raised to 25 minutes; by 2026-10-29 a run took up to 23, so the job has 28 and its step 26, two kept for the checkout. Raise both before a run nears the step's. +# `checks` runs `make verify` in three legs, its simulators on 300 seeds: each leg runs every check and every third test, told its place and the count by the matrix itself, never by a number written in, so that across the legs each test runs once. As one job, on 2026-10-21 the runs that finished took 11 to 14.5 minutes, and six runs in three pull requests were cut at its 15-minute timeout with the tests still running, so its timeout was raised to 25 minutes; by 2026-10-29 a run took up to 23, so the job was given 28 and its step 26, two kept for the checkout; and on 2026-11-09 the ten runs before took 19 to 23.4 and a merge group was cut at 26.2, so its tests were dealt into legs. Add a leg before one nears the step's limit. # About 5,200 mutants, in twelve shards of about 434, each kill confirmed by two runs of one test. With six shards of 168, of about 1,000 mutants then, the legs ran 14 to 25 minutes against a 25-minute timeout, and legs were cut at 25:00 in three pull requests, once 22 ms after the last kill had printed. With eight shards of 651 the legs mostly ran 6 to 14 minutes, and on 2026-11-06 one was cut at its step's 27 minutes on a runner that ran it 2.8 times slower than a re-run of the same tree ("CI: twelve mutant shards, so a slow runner does not cut a leg at its limit", issue 583); the first run of twelve took 4.4 to 12.3 minutes a leg, 7.3 the median. Every mutant names the tests that kill it, which run first; the whole suite, its slow modules last, runs only when they pass ("Mutation run: every mutant names its tests, the slow suites run last, and a builder may break a few", issue 217). Add a shard before a leg nears its timeout, or before a leg of the median time would take more than 22 minutes on a runner three times slower. A runner as slow may still cut the longest leg: run it again. # `verify` checks out the base, never the head, and runs the base's tools/needs_passed.py, so a change cannot make its own verdict pass: decision 0023. # `commits` checks out the base too and runs the base's tools/commits_are_the_maintainers.py, which reads every commit the pull request carries and refuses unless each names the Maintainer ("Identity: every commit a pull request carries is the Maintainer's, checked outside the shell", issue 204). It reads the event's payload too, and refuses unless an account on the configuration's list opened the pull request from this repo: decision 0102. On a merge group, whose payload names no opener, it reads the pull request the queue's ref names, with this job's read-only token. @@ -25,6 +25,16 @@ jobs: timeout-minutes: 28 permissions: contents: read + strategy: + fail-fast: false + matrix: + shard: + - 0 + - 1 + - 2 + env: + CHECKS_SHARD: ${{ strategy.job-index }} + CHECKS_SHARDS: ${{ strategy.job-total }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -109,8 +119,8 @@ name and the job's name: `factory / verify`. Renaming either silently stops it b `verify` needs every other job and has `if: always()`, the one job-level `if:` a Lens's job may have: GitHub passes a required check that is skipped, so `tools/needs_passed.py` fails it unless each passed. Its one checkout is of the base, never the head: the verdict is the base's `tools/needs_passed.py`, so a change -cannot make its own pass (decision 0023). It runs on no event without a base. The checks and tests in `checks` and -`mutants` are the head's. `commits` is the base's too: it fetches the head and reads who authored and committed each +cannot make its own pass (decision 0023). It runs on no event without a base. The checks and tests in `checks` and `mutants` are the head's, +run in legs their matrix alone tells their place and count, so each test and mutant runs once and every check in each leg. `commits` is the base's too: it fetches the head and reads who authored and committed each commit the pull request carries, which must be the Maintainer, read from `config/hooks.json` (decision 0041). The eval job is a reserved name, `factory / eval`, in eval.yml: the one job that holds the held-out home's credential @@ -119,7 +129,7 @@ Its shape, each credential named by one step alone and its upload the one `UPLOA `tools/check_workflows.py`, by issue 309, "Workflows: eval.yml runs the review Intent's held-out cases and writes eval-report/1". Org standard: ci-standards.md, "Permissions Policy", "Action Pinning Policy" and "CI Job Naming Convention". Its name, triggers and stages depart from "1. CI Pipeline (ci.yml)": `docs/decisions/0014-where-this-repo-departs-from-the-orgs-standards.md`. -Held by: `tests/test_tools.py::test_a_rule_is_not_dodged_by_writing_the_workflow_another_way`, `tests/test_tools.py::test_an_example_in_the_standard_that_has_drifted_is_caught`, `tests/test_tools.py::test_a_lens_job_that_would_never_report_under_its_name_is_caught`, `tests/test_tools.py::test_the_required_check_fails_unless_every_job_it_needs_passed`, `tests/test_tools.py::test_a_lens_job_that_judges_its_needs_runs_from_the_base`, `tests/test_tools.py::test_the_judging_step_and_its_checkout_are_held_to_their_exact_shape`, `tests/test_tools.py::test_the_mutant_shards_are_told_their_index_and_count_by_the_matrix`, `tests/test_evals.py::test_the_eval_jobs_name_is_reserved`, `tests/test_evals.py::test_the_eval_workflow_names_each_credential_in_one_step_alone` +Held by: `tests/test_tools.py::test_a_rule_is_not_dodged_by_writing_the_workflow_another_way`, `tests/test_tools.py::test_an_example_in_the_standard_that_has_drifted_is_caught`, `tests/test_tools.py::test_a_lens_job_that_would_never_report_under_its_name_is_caught`, `tests/test_tools.py::test_the_required_check_fails_unless_every_job_it_needs_passed`, `tests/test_tools.py::test_a_lens_job_that_judges_its_needs_runs_from_the_base`, `tests/test_tools.py::test_the_judging_step_and_its_checkout_are_held_to_their_exact_shape`, `tests/test_tools.py::test_the_mutant_shards_are_told_their_index_and_count_by_the_matrix`, `tests/test_tools.py::test_the_legs_of_make_verify_are_told_their_place_and_count_by_the_matrix`, `tests/test_tools.py::test_the_legs_of_make_verify_run_every_test_once`, `tests/test_evals.py::test_the_eval_jobs_name_is_reserved`, `tests/test_evals.py::test_the_eval_workflow_names_each_credential_in_one_step_alone` ## Pinned to what was looked up diff --git a/examples/workflows/verify.yml b/examples/workflows/verify.yml index 47aaec5d..733542ec 100644 --- a/examples/workflows/verify.yml +++ b/examples/workflows/verify.yml @@ -1,5 +1,5 @@ # The required check, `factory / verify`: it passes only when `make verify` and every shard of the mutation check pass on the commit, with the history tools/check_decisions.py compares it with. -# `checks` runs `make verify`, its simulators on 300 seeds. On 2026-10-21 the runs that finished took 11 to 14.5 minutes, and six runs in three pull requests were cut at its 15-minute timeout with the tests still running, so its timeout was raised to 25 minutes; by 2026-10-29 a run took up to 23, so the job has 28 and its step 26, two kept for the checkout. Raise both before a run nears the step's. +# `checks` runs `make verify` in three legs, its simulators on 300 seeds: each leg runs every check and every third test, told its place and the count by the matrix itself, never by a number written in, so that across the legs each test runs once. As one job, on 2026-10-21 the runs that finished took 11 to 14.5 minutes, and six runs in three pull requests were cut at its 15-minute timeout with the tests still running, so its timeout was raised to 25 minutes; by 2026-10-29 a run took up to 23, so the job was given 28 and its step 26, two kept for the checkout; and on 2026-11-09 the ten runs before took 19 to 23.4 and a merge group was cut at 26.2, so its tests were dealt into legs. Add a leg before one nears the step's limit. # About 5,200 mutants, in twelve shards of about 434, each kill confirmed by two runs of one test. With six shards of 168, of about 1,000 mutants then, the legs ran 14 to 25 minutes against a 25-minute timeout, and legs were cut at 25:00 in three pull requests, once 22 ms after the last kill had printed. With eight shards of 651 the legs mostly ran 6 to 14 minutes, and on 2026-11-06 one was cut at its step's 27 minutes on a runner that ran it 2.8 times slower than a re-run of the same tree ("CI: twelve mutant shards, so a slow runner does not cut a leg at its limit", issue 583); the first run of twelve took 4.4 to 12.3 minutes a leg, 7.3 the median. Every mutant names the tests that kill it, which run first; the whole suite, its slow modules last, runs only when they pass ("Mutation run: every mutant names its tests, the slow suites run last, and a builder may break a few", issue 217). Add a shard before a leg nears its timeout, or before a leg of the median time would take more than 22 minutes on a runner three times slower. A runner as slow may still cut the longest leg: run it again. # `verify` checks out the base, never the head, and runs the base's tools/needs_passed.py, so a change cannot make its own verdict pass: decision 0023. # `commits` checks out the base too and runs the base's tools/commits_are_the_maintainers.py, which reads every commit the pull request carries and refuses unless each names the Maintainer ("Identity: every commit a pull request carries is the Maintainer's, checked outside the shell", issue 204). It reads the event's payload too, and refuses unless an account on the configuration's list opened the pull request from this repo: decision 0102. On a merge group, whose payload names no opener, it reads the pull request the queue's ref names, with this job's read-only token. @@ -14,6 +14,16 @@ jobs: timeout-minutes: 28 permissions: contents: read + strategy: + fail-fast: false + matrix: + shard: + - 0 + - 1 + - 2 + env: + CHECKS_SHARD: ${{ strategy.job-index }} + CHECKS_SHARDS: ${{ strategy.job-total }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/tests/test_tools.py b/tests/test_tools.py index a17d256b..245bb1eb 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -2789,6 +2789,35 @@ def test_the_mutant_shards_are_told_their_index_and_count_by_the_matrix(self): self.assertTrue(caught(self.lens_problems(GOOD.replace(job, job.replace(run, " - run: make verify\n"))), gone)) self.assertEqual(self.lens_problems(GOOD), []) + def test_the_legs_of_make_verify_are_told_their_place_and_count_by_the_matrix(self): + # On 2026-11-09 `make verify` as one job was cut at its step's 26 minutes in a merge group, so it runs in legs. A + # leg told CHECKS_SHARD as a number would run another's share, leaving its own tests in no leg, and CHECKS_SHARDS + # as a number could differ from the matrix's size; each is the matrix's own expression, as the mutant shards'. + job = GOOD[GOOD.index(" checks:\n"):GOOD.index(" mutants:\n")] + index, total = " CHECKS_SHARD: ${{ strategy.job-index }}\n", " CHECKS_SHARDS: ${{ strategy.job-total }}\n" + run = " - run: make verify\n" + self.assertEqual((job.count(index), job.count(total), job.count(run)), (1, 1, 1)) + told, shape = "not told CHECKS_SHARD", "whose steps are not a checkout of the head" + cases = [(index, " CHECKS_SHARD: 0\n", told), (total, " CHECKS_SHARDS: 3\n", told), + (total, "", told), (index, index + " OTHER: 1\n", told), + (" shard:\n", " leg:\n", "not the one key shard"), + (" - 2\n", " - 2\n exclude:\n - shard: 2\n", "not the one key shard"), + (" env:\n CHECKS_SHARD", " container: ghcr.io/x/y:1\n env:\n CHECKS_SHARD", "keys other than"), + (" contents: read\n", " contents: write\n", "granted other than"), + (run, run + " working-directory: evil\n", shape), (run, run + " env:\n CHECKS_SHARD: 0\n", shape), + (run, run + " - run: make fast\n", shape), + (run, " - run: make fast\n - run: make verify\n", shape), + (" fetch-depth: 0\n - run: make verify\n", " - run: make verify\n", shape)] + for old, new, said in cases: + text = GOOD.replace(job, job.replace(old, new)) + self.assertNotEqual(text, GOOD, said) + self.assertTrue(caught(self.lens_problems(text), said), (old, new, said)) + # No job runs `make verify` at all: the checks and the tests are gone, not reshaped. + gone = "has no job that runs `make verify`" + self.assertTrue(caught(self.lens_problems(GOOD.replace(job, "").replace(" - checks\n", "")), gone)) + self.assertTrue(caught(self.lens_problems(GOOD.replace(job, job.replace(run, " - run: make fast\n"))), gone)) + self.assertEqual(self.lens_problems(GOOD), []) + def test_a_lens_job_that_judges_its_needs_runs_from_the_base(self): # verify's verdict is the base's tools/needs_passed.py: its one checkout is of the base, on a pull request and in # the merge queue, and of nothing else, so a change cannot make its own verdict pass (decision 0023). @@ -10119,6 +10148,55 @@ def test_a_check_that_finds_a_problem_fails_verify(self): self.assertFalse(verify.failed([("never", lambda: [])])) self.assertIn("a problem", said.getvalue()) + def test_the_legs_of_make_verify_run_every_test_once(self): + # CI runs `make verify` in legs (docs/standards/workflows.md): each runs every check and each count-th test from + # its place, so that across the legs every test of the whole suite runs, the seeded simulation's included, and + # none twice. A leg it cannot read stops the run before any check or test. + whole = [test.id() for test in verify._tests(verify.suite(False))] + self.assertTrue({"tests.test_simulator", "tests.test_issue_simulator"} <= {i.rsplit(".", 2)[0] for i in whole}) + for count in (1, 2, 3, 5): + legs = [[test.id() for test in verify._tests(verify.suite(False, (index, count)))] for index in range(count)] + dealt = [i for leg in legs for i in leg] + self.assertEqual((sorted(dealt), len(dealt)), (sorted(whole), len(whole)), count) + self.assertLessEqual(max(map(len, legs)) - min(map(len, legs)), 1, count) + self.assertIsNone(verify.leg({})) + self.assertEqual(verify.leg({"CHECKS_SHARD": "2", "CHECKS_SHARDS": "3"}), (2, 3)) + for shard, count, said in (("1", None, "set both"), (None, "3", "set both"), ("", "3", "set both"), + ("one", "3", "set both"), ("-1", "3", "set both"), ("1.0", "3", "set both"), + ("3", "3", "below the count"), ("0", "0", "below the count")): + given = {k: v for k, v in (("CHECKS_SHARD", shard), ("CHECKS_SHARDS", count)) if v is not None} + with self.assertRaises(ValueError, msg=given) as raised: + verify.leg(given) + self.assertIn(said, str(raised.exception), given) + ran = [] + real = (verify.failed, verify.unittest.TextTestRunner) + + class Runner: + def __init__(self, **_): + pass + + def run(self, suite): + ran.append(sorted(test.id() for test in verify._tests(suite))) + return unittest.TestResult() + + verify.failed, verify.unittest.TextTestRunner = (lambda checks: ran.append("checks") or False), Runner + try: + with contextlib.redirect_stdout(io.StringIO()) as said: + self.assertEqual(verify.main([], {"CHECKS_SHARD": "1", "CHECKS_SHARDS": "3"}), 0) + self.assertEqual(ran, ["checks", sorted(whole[1::3])]) + self.assertIn("leg 1 of 3", said.getvalue()) + del ran[:] + self.assertEqual(verify.main([], {"CHECKS_SHARD": "3", "CHECKS_SHARDS": "3"}), 1) + self.assertEqual(ran, [], "a leg outside the count runs nothing") + finally: + verify.failed, verify.unittest.TextTestRunner = real + # From the front door, as CI's matrix tells it: the leg is read from the environment. + env = dict(os.environ, CHECKS_SHARD="3", CHECKS_SHARDS="3") + done = subprocess.run([sys.executable, "-m", "tools.verify"], cwd=str(ROOT), env=env, capture_output=True, + text=True, timeout=60) + self.assertEqual((done.returncode, done.stdout.strip()), (1, "CHECKS_SHARD is 3 and CHECKS_SHARDS is 3; the leg " + "is counted from 0, below the count")) + def test_make_fast_is_verify_without_the_seeded_simulation(self): """The required check runs every test file at any depth; only `make fast` leaves the slow ones out.""" everything = {".".join(p.relative_to(ROOT).with_suffix("").parts) for p in walk.files(ROOT, "tests") diff --git a/tools/check_workflows.py b/tools/check_workflows.py index 8dac04c3..30d2dcec 100644 --- a/tools/check_workflows.py +++ b/tools/check_workflows.py @@ -14,6 +14,8 @@ with no `strategy.matrix` and no `uses:`. Its one job-level `if:` is exactly `always()`, beside a `needs:` that names every other job in its file, and then it judges them with `tools/needs_passed.py`, the base's: its one checkout is of the base, never the head. Only `jobs..name` is read: a workflow's or a step's `name:` names no check run. +In the required check's workflow, the mutants and `make verify` each run in a matrix whose legs are told their place +and the count by the matrix itself (_shards, _legs), so that no mutant and no test is left in no leg. The secret scan, the exempt `ci.yml`, reads its rules from the base (scan_problems): `.gitleaks.toml` and `.gitleaksignore` from a checkout of the base under `base/`, with the head's `.gitleaksignore` deleted first, since @@ -685,6 +687,8 @@ SHARDED = "python3 -m tools.mutants" INDEX = "${{ strategy.job-index }}" TOTAL = "${{ strategy.job-total }}" +# The legs of `make verify`, dealt its tests as the mutant shards are dealt mutants, and told the same way. +LEGGED = "make verify" JUDGING_KEYS = {"name", "needs", "if", "runs-on", "timeout-minutes", "permissions", "env", "steps"} SHARD_KEYS = {"runs-on", "timeout-minutes", "permissions", "strategy", "env", "steps"} HEAD = "${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}" @@ -1351,6 +1355,44 @@ def _shards(entries: list, jobs: list) -> list: return whys +def _legs(entries: list, jobs: list) -> list: + """Why the workflow would not run every test once: a job that runs `LEGGED` exists, and its env: is exactly + CHECKS_SHARD: INDEX and CHECKS_SHARDS: TOTAL, the matrix's own index and size, so a number written in cannot leave a + share of the tests in no leg; its matrix is one key, shard; and its shape is closed, as the mutant shards' is: the + keys SHARD_KEYS and no other, contents: read, a checkout of the head with the whole history and `LEGGED` on a line + of its own, each step holding no other key.""" + whys, any_leg = [], False + for job in jobs: + own = [e for e in entries if e["path"][:2] == ["jobs", job]] + steps = [e for e in own if e["path"] == ["jobs", job, "steps"] and e["key"] == "run"] + if all(LEGGED not in "\n".join([e["value"]] + (e["body"] or [])) for e in steps): + continue + any_leg = True + said = "has the job %s, which runs `%s`, " % (job, LEGGED) + keys = {e["key"] for e in own if e["path"] == ["jobs", job]} + told = {e["key"]: e["value"] for e in own if e["path"] == ["jobs", job, "env"]} + matrix = [e["key"] for e in own if e["path"] == ["jobs", job, "strategy", "matrix"]] + granted = {e["key"]: e["value"] for e in own if e["path"] == ["jobs", job, "permissions"]} + checkouts = [_with(entries, e) for e in own if e["path"] == ["jobs", job, "steps"] and _checkout(e)] + if keys != SHARD_KEYS: + whys.append(said + "with keys other than %s, or lacking one" % ", ".join(sorted(SHARD_KEYS))) + if told != {"CHECKS_SHARD": INDEX, "CHECKS_SHARDS": TOTAL}: + whys.append(said + "not told CHECKS_SHARD: %s and CHECKS_SHARDS: %s and nothing else, so a leg could run " + "another's tests, or every leg one share" % (INDEX, TOTAL)) + if matrix != ["shard"]: + whys.append(said + "whose strategy.matrix is not the one key shard") + if granted != {"contents": "read"}: + whys.append(said + "granted other than contents: read") + if not ([set(step) for step in _steps(entries, job)] == [{"uses", "with"}, {"run", "timeout-minutes"}] + and [(e["value"], e["body"] or []) for e in steps] == [(LEGGED, [])] + and checkouts == [{"persist-credentials": "false", "fetch-depth": "0"}]): + whys.append(said + "whose steps are not a checkout of the head with the whole history and `%s` on a line " + "of its own with its own timeout-minutes, each holding no other key" % LEGGED) + if not any_leg: + whys.append("has no job that runs `%s`, so no check and no test runs before a merge" % LEGGED) + return whys + + def _commits(text: str, entries: list, jobs: list) -> list: """Why the workflow's job `commits` would not read every commit the pull request carries from the base (decision 0041). Its shape is closed: the keys runs-on, timeout-minutes, permissions (contents: read, pull-requests: read, and @@ -1775,7 +1817,7 @@ def lens_job_problems(text: str, check: str) -> list: problems += ["has the job %s beside other jobs, and %s" % (job, why) for why in _judging(text, entries, job, jobs)] if any("needs" in inside for _, inside in named): # the required check's workflow, whose job judges the rest - problems += _commits(text, entries, jobs) + _shards(entries, jobs) + problems += _commits(text, entries, jobs) + _shards(entries, jobs) + _legs(entries, jobs) for key in ("env", "defaults"): if key in top: problems.append("has a top-level %s:, which every job inherits, where each is told what it is and " diff --git a/tools/mutants.py b/tools/mutants.py index eada9849..15fb1315 100644 --- a/tools/mutants.py +++ b/tools/mutants.py @@ -115,6 +115,8 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: COMMITS_JOB_TESTS = ("tests.test_tools.Workflows.test_the_commits_job_reads_the_pull_requests_commits_from_the_base",) JUDGING_SHAPE_TESTS = ("tests.test_tools.Workflows.test_the_judging_step_and_its_checkout_are_held_to_their_exact_shape",) SHARDS_TESTS = ("tests.test_tools.Workflows.test_the_mutant_shards_are_told_their_index_and_count_by_the_matrix",) +LEGS_TESTS = ("tests.test_tools.Workflows.test_the_legs_of_make_verify_are_told_their_place_and_count_by_the_matrix",) +LEG_TESTS = ("tests.test_tools.Verify.test_the_legs_of_make_verify_run_every_test_once",) LIMITED = ("tests.test_tools.Workflows.test_each_run_step_stops_under_a_limit_of_its_own_below_its_jobs",) NEEDED = ("tests.test_tools.Workflows.test_each_step_limit_holds_what_its_step_needs",) P13 = "tools/trial_p13_cli.sh" @@ -6336,8 +6338,8 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ("the commits job may run another tool", "tools/check_workflows.py", ' if [(e["value"], e["body"]) for e in runs] != [(COMMITS, [])] and', " if False and", COMMITS_JOB_TESTS), ("a workflow without the commits job passes the lens rules", "tools/check_workflows.py", - " problems += _commits(text, entries, jobs) + _shards(entries, jobs)\n", - " problems += _shards(entries, jobs)\n", COMMITS_JOB_TESTS), + " problems += _commits(text, entries, jobs) + _shards(entries, jobs) + _legs(entries, jobs)\n", + " problems += _shards(entries, jobs) + _legs(entries, jobs)\n", COMMITS_JOB_TESTS), # Issue 180: the judging step and the mutant shards held to their exact shape. ("the judging job may have other keys", "tools/check_workflows.py", " if set(inside) - JUDGING_KEYS:\n", " if False:\n", JUDGING_SHAPE_TESTS), @@ -6367,8 +6369,36 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ' if not any("mutants" in "\\n".join([e["value"]] + (e["body"] or [])) for e in runs):\n', ' if not any(e["value"] == SHARDED for e in runs):\n', SHARDS_TESTS), ("the shards are not held at all", "tools/check_workflows.py", - " problems += _commits(text, entries, jobs) + _shards(entries, jobs)\n", - " problems += _commits(text, entries, jobs)\n", SHARDS_TESTS), + " problems += _commits(text, entries, jobs) + _shards(entries, jobs) + _legs(entries, jobs)\n", + " problems += _commits(text, entries, jobs) + _legs(entries, jobs)\n", SHARDS_TESTS), + # The legs of `make verify` (2026-11-09: one job was cut at its step's 26 minutes in a merge group). + ("the legs are not held at all", "tools/check_workflows.py", + " problems += _commits(text, entries, jobs) + _shards(entries, jobs) + _legs(entries, jobs)\n", + " problems += _commits(text, entries, jobs) + _shards(entries, jobs)\n", LEGS_TESTS), + ("a leg may be told its place by a number", "tools/check_workflows.py", + ' if told != {"CHECKS_SHARD": INDEX, "CHECKS_SHARDS": TOTAL}:\n', " if False:\n", LEGS_TESTS), + ("the legs' matrix may be any key", "tools/check_workflows.py", + ' if matrix != ["shard"]:\n', " if False:\n", LEGS_TESTS), + ("the legs' job may have other keys", "tools/check_workflows.py", + " if keys != SHARD_KEYS:\n", " if False:\n", LEGS_TESTS), + ("the legs' job may be granted more", "tools/check_workflows.py", + ' if granted != {"contents": "read"}:\n', " if False:\n", LEGS_TESTS), + ("the legs' steps may be anything", "tools/check_workflows.py", + " and checkouts == [{\"persist-credentials\": \"false\", \"fetch-depth\": \"0\"}]):\n", + " and checkouts == [{\"persist-credentials\": \"false\", \"fetch-depth\": \"0\"}]) and False:\n", LEGS_TESTS), + ("no job need run make verify", "tools/check_workflows.py", + " if not any_leg:\n", " if False:\n", LEGS_TESTS), + ("a leg holds every test", "tools/verify.py", + " return unittest.TestSuite(_tests(whole)[index::count])\n", " return whole\n", LEG_TESTS), + ("a leg skips one test in each count", "tools/verify.py", + " return unittest.TestSuite(_tests(whole)[index::count])\n", + " return unittest.TestSuite(_tests(whole)[index::count + 1])\n", LEG_TESTS), + ("a leg outside the count runs", "tools/verify.py", + " if not 0 <= int(index) < int(count):\n", " if False:\n", LEG_TESTS), + ("one of the pair alone is no leg", "tools/verify.py", + " if index is None and count is None:\n", " if index is None or count is None:\n", LEG_TESTS), + ("a leg CI names is not read", "tools/verify.py", + ' sys.exit(main(sys.argv[1:], os.environ))', ' sys.exit(main(sys.argv[1:]))', LEG_TESTS), ("no job need run the mutants", "tools/check_workflows.py", " if not found:\n whys.append(\"has no job that runs the mutants", " if False:\n whys.append(\"has no job that runs the mutants", SHARDS_TESTS), diff --git a/tools/verify.py b/tools/verify.py index 4cbf8775..fab515b7 100644 --- a/tools/verify.py +++ b/tools/verify.py @@ -3,10 +3,16 @@ `make fast` runs the same checks and every test but the seeded simulation. `make checks` runs the checks alone, half a minute, and is what a session's hooks run after an edit and before a stop (`tools/hooks/after_edit.py`, `tools/hooks/before_stop.py`); the tests are CI's, and each builder's own `make verify`. + +CI runs `make verify` in legs, each told CHECKS_SHARD and CHECKS_SHARDS by its matrix: each leg runs every check and its +share of the tests, so that across the legs every test runs once and no leg nears the step's limit. Neither set: every +test, as at a desk. """ from __future__ import annotations +import os import pathlib +import re import sys import unittest @@ -59,15 +65,39 @@ def _modules(tests) -> set: return found -def suite(fast: bool) -> unittest.TestSuite: - """Every test module under tests/, at any depth; when fast, all but those in SLOW.""" +def _tests(tests) -> list: + return [test for item in tests for test in _tests(item)] if isinstance(tests, unittest.TestSuite) else [tests] + + +def suite(fast: bool, share: tuple = None) -> unittest.TestSuite: + """Every test module under tests/, at any depth; when fast, all but those in SLOW. Given `share`, (a leg, the count + of legs), the tests of the whole one by one, each `count`-th from the leg's place, so that the legs hold each once.""" whole = unittest.defaultTestLoader.discover(str(ROOT / "tests"), top_level_dir=str(ROOT)) + if share is not None: + index, count = share + return unittest.TestSuite(_tests(whole)[index::count]) if not fast: return whole slow = {"tests." + name for name in SLOW} return unittest.TestSuite(item for item in whole if not _modules(item) & slow) +def leg(environ) -> tuple: + """(this leg, the count of legs) from CHECKS_SHARD and CHECKS_SHARDS, which CI's matrix sets, counted from 0; None + when neither is set. Raises ValueError for one without the other, a value that is not a whole number, or a leg + outside the count, which could leave tests in no leg.""" + index, count = environ.get("CHECKS_SHARD"), environ.get("CHECKS_SHARDS") + if index is None and count is None: + return None + if not all(isinstance(said, str) and re.fullmatch(r"[0-9]+", said) for said in (index, count)): + raise ValueError("CHECKS_SHARD is %r and CHECKS_SHARDS is %r; set both, to whole numbers, or neither" + % (index, count)) + if not 0 <= int(index) < int(count): + raise ValueError("CHECKS_SHARD is %s and CHECKS_SHARDS is %s; the leg is counted from 0, below the count" + % (index, count)) + return int(index), int(count) + + def notes() -> list: """What `verify` says of the records before the tests: what they were compared with, and how many Boundary rows name no mutant yet (issue 590).""" @@ -75,16 +105,23 @@ def notes() -> list: "%-22s %s" % ("boundary mutants", check_decisions.unnamed())] -def main(argv: list) -> int: +def main(argv: list, environ=None) -> int: + try: + share = None if "--checks" in argv or "--fast" in argv else leg(environ or {}) + except ValueError as err: + print(err) + return 1 checks_failed = failed(CHECKS) if "--checks" in argv: return 1 if checks_failed else 0 for line in notes(): print(line) unverified(check_trials.unverified()) - result = unittest.TextTestRunner(verbosity=1).run(suite("--fast" in argv)) + if share is not None: + print("%-22s leg %d of %d counted from 0" % ("tests", share[0], share[1])) + result = unittest.TextTestRunner(verbosity=1).run(suite("--fast" in argv, share)) return 1 if checks_failed or not result.wasSuccessful() else 0 if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) + sys.exit(main(sys.argv[1:], os.environ)) From 5524e9db4cf7f7cf780a362de3174a954ddadd34 Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:16:08 -0600 Subject: [PATCH 3/6] Mutants: one its named tests miss is an ERROR, named_test_does_not_kill, never a run of the whole suite The mutant run holds each mutant to the tests it names, alone: when they pass on it, it is an ERROR that names them, where before the whole suite ran after them, 10 to 21 minutes of a 27-minute leg, and hid the wrong name. An edit tools/untested.py tries, through attempt without alone, still runs the whole suite after its tests. The verify workflow's comment, its example and the standard's copy say so; the guard has its test and four mutants. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/verify.yml | 2 +- docs/standards/workflows.md | 2 +- examples/workflows/verify.yml | 2 +- tests/test_tools.py | 36 ++++++++++++++++++++++++++++++++++- tools/AGENTS.md | 6 +++--- tools/mutants.py | 30 ++++++++++++++++++++++++----- 6 files changed, 66 insertions(+), 12 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 733542ec..7f785141 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -1,6 +1,6 @@ # The required check, `factory / verify`: it passes only when `make verify` and every shard of the mutation check pass on the commit, with the history tools/check_decisions.py compares it with. # `checks` runs `make verify` in three legs, its simulators on 300 seeds: each leg runs every check and every third test, told its place and the count by the matrix itself, never by a number written in, so that across the legs each test runs once. As one job, on 2026-10-21 the runs that finished took 11 to 14.5 minutes, and six runs in three pull requests were cut at its 15-minute timeout with the tests still running, so its timeout was raised to 25 minutes; by 2026-10-29 a run took up to 23, so the job was given 28 and its step 26, two kept for the checkout; and on 2026-11-09 the ten runs before took 19 to 23.4 and a merge group was cut at 26.2, so its tests were dealt into legs. Add a leg before one nears the step's limit. -# About 5,200 mutants, in twelve shards of about 434, each kill confirmed by two runs of one test. With six shards of 168, of about 1,000 mutants then, the legs ran 14 to 25 minutes against a 25-minute timeout, and legs were cut at 25:00 in three pull requests, once 22 ms after the last kill had printed. With eight shards of 651 the legs mostly ran 6 to 14 minutes, and on 2026-11-06 one was cut at its step's 27 minutes on a runner that ran it 2.8 times slower than a re-run of the same tree ("CI: twelve mutant shards, so a slow runner does not cut a leg at its limit", issue 583); the first run of twelve took 4.4 to 12.3 minutes a leg, 7.3 the median. Every mutant names the tests that kill it, which run first; the whole suite, its slow modules last, runs only when they pass ("Mutation run: every mutant names its tests, the slow suites run last, and a builder may break a few", issue 217). Add a shard before a leg nears its timeout, or before a leg of the median time would take more than 22 minutes on a runner three times slower. A runner as slow may still cut the longest leg: run it again. +# About 5,200 mutants, in twelve shards of about 434, each kill confirmed by two runs of one test. With six shards of 168, of about 1,000 mutants then, the legs ran 14 to 25 minutes against a 25-minute timeout, and legs were cut at 25:00 in three pull requests, once 22 ms after the last kill had printed. With eight shards of 651 the legs mostly ran 6 to 14 minutes, and on 2026-11-06 one was cut at its step's 27 minutes on a runner that ran it 2.8 times slower than a re-run of the same tree ("CI: twelve mutant shards, so a slow runner does not cut a leg at its limit", issue 583); the first run of twelve took 4.4 to 12.3 minutes a leg, 7.3 the median. Every mutant names the tests that kill it, which run alone ("Mutation run: every mutant names its tests, the slow suites run last, and a builder may break a few", issue 217); a mutant they miss is an ERROR, named_test_does_not_kill, since the whole suite run after them took 10 to 21 minutes of a leg (2026-11-09). Add a shard before a leg nears its timeout, or before a leg of the median time would take more than 22 minutes on a runner three times slower. A runner as slow may still cut the longest leg: run it again. # `verify` checks out the base, never the head, and runs the base's tools/needs_passed.py, so a change cannot make its own verdict pass: decision 0023. # `commits` checks out the base too and runs the base's tools/commits_are_the_maintainers.py, which reads every commit the pull request carries and refuses unless each names the Maintainer ("Identity: every commit a pull request carries is the Maintainer's, checked outside the shell", issue 204). It reads the event's payload too, and refuses unless an account on the configuration's list opened the pull request from this repo: decision 0102. On a merge group, whose payload names no opener, it reads the pull request the queue's ref names, with this job's read-only token. name: factory diff --git a/docs/standards/workflows.md b/docs/standards/workflows.md index 5c71c955..5dd7df6f 100644 --- a/docs/standards/workflows.md +++ b/docs/standards/workflows.md @@ -11,7 +11,7 @@ Any other spelling is refused. An org standard is cited from combsmith/.github a ```yaml # The required check, `factory / verify`: it passes only when `make verify` and every shard of the mutation check pass on the commit, with the history tools/check_decisions.py compares it with. # `checks` runs `make verify` in three legs, its simulators on 300 seeds: each leg runs every check and every third test, told its place and the count by the matrix itself, never by a number written in, so that across the legs each test runs once. As one job, on 2026-10-21 the runs that finished took 11 to 14.5 minutes, and six runs in three pull requests were cut at its 15-minute timeout with the tests still running, so its timeout was raised to 25 minutes; by 2026-10-29 a run took up to 23, so the job was given 28 and its step 26, two kept for the checkout; and on 2026-11-09 the ten runs before took 19 to 23.4 and a merge group was cut at 26.2, so its tests were dealt into legs. Add a leg before one nears the step's limit. -# About 5,200 mutants, in twelve shards of about 434, each kill confirmed by two runs of one test. With six shards of 168, of about 1,000 mutants then, the legs ran 14 to 25 minutes against a 25-minute timeout, and legs were cut at 25:00 in three pull requests, once 22 ms after the last kill had printed. With eight shards of 651 the legs mostly ran 6 to 14 minutes, and on 2026-11-06 one was cut at its step's 27 minutes on a runner that ran it 2.8 times slower than a re-run of the same tree ("CI: twelve mutant shards, so a slow runner does not cut a leg at its limit", issue 583); the first run of twelve took 4.4 to 12.3 minutes a leg, 7.3 the median. Every mutant names the tests that kill it, which run first; the whole suite, its slow modules last, runs only when they pass ("Mutation run: every mutant names its tests, the slow suites run last, and a builder may break a few", issue 217). Add a shard before a leg nears its timeout, or before a leg of the median time would take more than 22 minutes on a runner three times slower. A runner as slow may still cut the longest leg: run it again. +# About 5,200 mutants, in twelve shards of about 434, each kill confirmed by two runs of one test. With six shards of 168, of about 1,000 mutants then, the legs ran 14 to 25 minutes against a 25-minute timeout, and legs were cut at 25:00 in three pull requests, once 22 ms after the last kill had printed. With eight shards of 651 the legs mostly ran 6 to 14 minutes, and on 2026-11-06 one was cut at its step's 27 minutes on a runner that ran it 2.8 times slower than a re-run of the same tree ("CI: twelve mutant shards, so a slow runner does not cut a leg at its limit", issue 583); the first run of twelve took 4.4 to 12.3 minutes a leg, 7.3 the median. Every mutant names the tests that kill it, which run alone ("Mutation run: every mutant names its tests, the slow suites run last, and a builder may break a few", issue 217); a mutant they miss is an ERROR, named_test_does_not_kill, since the whole suite run after them took 10 to 21 minutes of a leg (2026-11-09). Add a shard before a leg nears its timeout, or before a leg of the median time would take more than 22 minutes on a runner three times slower. A runner as slow may still cut the longest leg: run it again. # `verify` checks out the base, never the head, and runs the base's tools/needs_passed.py, so a change cannot make its own verdict pass: decision 0023. # `commits` checks out the base too and runs the base's tools/commits_are_the_maintainers.py, which reads every commit the pull request carries and refuses unless each names the Maintainer ("Identity: every commit a pull request carries is the Maintainer's, checked outside the shell", issue 204). It reads the event's payload too, and refuses unless an account on the configuration's list opened the pull request from this repo: decision 0102. On a merge group, whose payload names no opener, it reads the pull request the queue's ref names, with this job's read-only token. name: factory diff --git a/examples/workflows/verify.yml b/examples/workflows/verify.yml index 733542ec..7f785141 100644 --- a/examples/workflows/verify.yml +++ b/examples/workflows/verify.yml @@ -1,6 +1,6 @@ # The required check, `factory / verify`: it passes only when `make verify` and every shard of the mutation check pass on the commit, with the history tools/check_decisions.py compares it with. # `checks` runs `make verify` in three legs, its simulators on 300 seeds: each leg runs every check and every third test, told its place and the count by the matrix itself, never by a number written in, so that across the legs each test runs once. As one job, on 2026-10-21 the runs that finished took 11 to 14.5 minutes, and six runs in three pull requests were cut at its 15-minute timeout with the tests still running, so its timeout was raised to 25 minutes; by 2026-10-29 a run took up to 23, so the job was given 28 and its step 26, two kept for the checkout; and on 2026-11-09 the ten runs before took 19 to 23.4 and a merge group was cut at 26.2, so its tests were dealt into legs. Add a leg before one nears the step's limit. -# About 5,200 mutants, in twelve shards of about 434, each kill confirmed by two runs of one test. With six shards of 168, of about 1,000 mutants then, the legs ran 14 to 25 minutes against a 25-minute timeout, and legs were cut at 25:00 in three pull requests, once 22 ms after the last kill had printed. With eight shards of 651 the legs mostly ran 6 to 14 minutes, and on 2026-11-06 one was cut at its step's 27 minutes on a runner that ran it 2.8 times slower than a re-run of the same tree ("CI: twelve mutant shards, so a slow runner does not cut a leg at its limit", issue 583); the first run of twelve took 4.4 to 12.3 minutes a leg, 7.3 the median. Every mutant names the tests that kill it, which run first; the whole suite, its slow modules last, runs only when they pass ("Mutation run: every mutant names its tests, the slow suites run last, and a builder may break a few", issue 217). Add a shard before a leg nears its timeout, or before a leg of the median time would take more than 22 minutes on a runner three times slower. A runner as slow may still cut the longest leg: run it again. +# About 5,200 mutants, in twelve shards of about 434, each kill confirmed by two runs of one test. With six shards of 168, of about 1,000 mutants then, the legs ran 14 to 25 minutes against a 25-minute timeout, and legs were cut at 25:00 in three pull requests, once 22 ms after the last kill had printed. With eight shards of 651 the legs mostly ran 6 to 14 minutes, and on 2026-11-06 one was cut at its step's 27 minutes on a runner that ran it 2.8 times slower than a re-run of the same tree ("CI: twelve mutant shards, so a slow runner does not cut a leg at its limit", issue 583); the first run of twelve took 4.4 to 12.3 minutes a leg, 7.3 the median. Every mutant names the tests that kill it, which run alone ("Mutation run: every mutant names its tests, the slow suites run last, and a builder may break a few", issue 217); a mutant they miss is an ERROR, named_test_does_not_kill, since the whole suite run after them took 10 to 21 minutes of a leg (2026-11-09). Add a shard before a leg nears its timeout, or before a leg of the median time would take more than 22 minutes on a runner three times slower. A runner as slow may still cut the longest leg: run it again. # `verify` checks out the base, never the head, and runs the base's tools/needs_passed.py, so a change cannot make its own verdict pass: decision 0023. # `commits` checks out the base too and runs the base's tools/commits_are_the_maintainers.py, which reads every commit the pull request carries and refuses unless each names the Maintainer ("Identity: every commit a pull request carries is the Maintainer's, checked outside the shell", issue 204). It reads the event's payload too, and refuses unless an account on the configuration's list opened the pull request from this repo: decision 0102. On a merge group, whose payload names no opener, it reads the pull request the queue's ref names, with this job's read-only token. name: factory diff --git a/tests/test_tools.py b/tests/test_tools.py index 245bb1eb..30a9e493 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -10388,6 +10388,35 @@ def run(work, **given): self.assertEqual(mutants.attempt(mutant, root, run)[0], "SURVIVED") self.assertEqual(calls, [{"only": ("tests.test_x.T.test_y",)}, {"first": ("tests.test_x.T.test_y",)}]) + def test_a_mutant_its_named_tests_miss_is_an_error_and_never_runs_the_whole_suite(self): + # On 2026-11-09 two mutants named tests that passed on them, so each leg that held one ran the whole suite after + # them: 10 to 21 minutes of a 27-minute step, and a wrong name nobody saw. The mutant run holds each mutant to + # the tests it names, alone: a pass of them is an ERROR that names them, and nothing else runs. + calls = [] + + def run(work, **given): + calls.append(given) + return (0, "OK\n") if (work / "k.py").read_text() == "rule\n" or given.get("only") == ("t.missed",) \ + else (1, FAILS_A_TEST) + root = tree({"k.py": "rule\n"}) + missed = ("missed", "k.py", "rule\n", "x\n", ("t.missed",)) + outcome, why = mutants.guarded(missed, root, run) + self.assertEqual((outcome, calls), ("ERROR", [{"only": ("t.missed",)}])) + self.assertTrue(why.startswith("named_test_does_not_kill: t.missed passed on it") and why.endswith(AGAIN), why) + # One they kill is killed, and confirmed as before. + del calls[:] + self.assertEqual(mutants.guarded(("hit", "k.py", "rule\n", "x\n", ("t.hit",)), root, run), ("killed", "")) + self.assertEqual(calls[0], {"only": ("t.hit",)}) + # From the front door it fails the run. + with mock.patch.object(mutants, "unnamed", return_value=[]), contextlib.redirect_stdout(io.StringIO()) as said: + self.assertEqual(mutants.main([missed], root, run, workers=1), 1) + self.assertIn("ERROR missed (named_test_does_not_kill", said.getvalue()) + # tools/untested.py's edits, through attempt with no `alone`, still run the whole suite after their tests, which + # here fails on it. + del calls[:] + self.assertEqual(mutants.attempt(missed, root, run), ("killed", "")) + self.assertEqual(calls[0], {"first": ("t.missed",)}) + def test_the_summary_counts_only_the_mutants_killed(self): stale = [("gone", "k.py", "not there\n", "x\n")] code, lines, _ = self.check({"fails": (0, 1, FAILS_A_TEST), "passes": (0, 0, "OK\n")}, @@ -10713,7 +10742,12 @@ def run(work, first=(), only=None): mutants.baseline(every[1:2], root / "gone", run, 2, (every, 0, 2)) self.assertEqual(sorted(calls), [("tests.test_x.T", "tests.test_x.T.test_g"), ("tests.test_x.T.test_b",)]) # From the front door, a shard runs its share, and a filtered run its own names on FILTER_WORKERS. The names - # are this throwaway tree's, which this checkout's check of them could not find. + # are this throwaway tree's, which this checkout's check of them could not find. The baseline runs in the tree; + # each mutant, which runs its named tests alone too, in a copy of it. + + def run(work, first=(), only=None): + calls.append(only if work == root else None) + return PASSES[1:] for environ, paths, parts in (({"MUTANTS_SHARD": "1", "MUTANTS_SHARDS": "4"}, [], [("tests.test_x.T.test_b",), ("tests.test_x.T.test_f",)]), ({}, ["k.py"], [tests[0::2], tests[1::2]])): diff --git a/tools/AGENTS.md b/tools/AGENTS.md index 2a77e5f3..f7973ac3 100644 --- a/tools/AGENTS.md +++ b/tools/AGENTS.md @@ -5,8 +5,7 @@ unless each commit names the Maintainer and a listed account opened it. `walk.py it, and reads the index as a file, so no git is needed. CI checks out the commit afresh, so there all else is committed; an export, with no `.git`, cannot tell, and skips it all. `make_corpus.py` records every decision the Kernel makes, under a directory git ignores, to hold another implementation to this one; the required check does not depend on it. `hooks/` holds the session hooks `.claude/settings.json` runs; `hooks/shell.py` reads a shell command as words for any of them. `eval_result.py` scores one Result against one held-out case, where the case is, and is no check; `context_eval.sh` runs the context's test at the Maintainer's terminal and `context_eval.py` records a run, and neither is a check. `untested.py` reports Python no test notices (decision 0166). -`rulesets_drift.sh` reads the repo's live rulesets through whoever is logged in to gh and redacts each answer outside the repo, -and `rulesets_drift.py` compares what is left with `config/rulesets.json`; it is run by hand and is no check. `machine_user_drift.sh` does the same for `config/machine-user.json`, and `environments_drift.sh` holds each environment to `main` alone. +`rulesets_drift.sh` reads the live rulesets through gh, redacted outside the repo, and `rulesets_drift.py` holds them to `config/rulesets.json`, by hand, no check; `machine_user_drift.sh` so for `config/machine-user.json`, `environments_drift.sh` for `main`. | Rule | Held by | |---|---| @@ -32,7 +31,8 @@ and `rulesets_drift.py` compares what is left with `config/rulesets.json`; it is | A stop lists every protected or reported path changed against origin's main; a write to a remote branch asks | `tests/test_hooks.py::test_an_edit_made_through_eval_is_listed`, `tests/test_hooks.py::test_a_write_to_the_base_the_stop_compares_with_asks` | | A test kills a mutant only failing on it, again alone, and passing alone in an unbroken copy; else `FLAKY`, or `ERROR` on a hang: both fail a run | `tests/test_tools.py::test_a_failure_that_is_not_the_mutants_twice_is_flaky_and_fails_the_check` | | A `mutants.py` entry names text found once in its file (else `STALE`; update it) and tests that kill it, each existing, none in `WHOLE_TOO_LONG` nor its module | `mutants.py`, `tests/test_tools.py::test_every_mutant_names_a_test_that_exists` | -| After the named tests, all of `tests/` runs in dealt parts; `mutants PATH...` breaks at most 40, two at once | `tests/test_tools.py::test_the_whole_suite_runs_in_dealt_parts_each_held_to_the_limit`, `tests/test_tools.py::test_a_filtered_run_is_capped` | +| A mutant runs the tests it names alone; one they miss is an ERROR, `named_test_does_not_kill` | `tests/test_tools.py::test_a_mutant_its_named_tests_miss_is_an_error_and_never_runs_the_whole_suite` | +| After an `untested.py` edit's tests, all of `tests/` runs in dealt parts; `mutants PATH...` breaks 40, 2 at once | `tests/test_tools.py::test_the_whole_suite_runs_in_dealt_parts_each_held_to_the_limit`, `tests/test_tools.py::test_a_filtered_run_is_capped` | A check collects every problem and does not stop at the first. Its message says what is wrong and what to do, in one sentence, because an author will read it with nothing else to go on. diff --git a/tools/mutants.py b/tools/mutants.py index 15fb1315..4dc92d74 100644 --- a/tools/mutants.py +++ b/tools/mutants.py @@ -110,6 +110,7 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: NAMED_MUTANTS = ("tests.test_tools.Decisions.test_a_new_or_changed_row_names_the_mutant_its_test_kills",) MAIN_ROWS = ("tests.test_tools.Decisions.test_a_row_main_holds_names_no_mutant_until_its_check_changes",) RUN_ALONE = ("tests.test_tools.Mutants.test_a_mutant_a_boundary_names_is_run_against_its_own_tests_alone",) +NOT_KILLED_TESTS = ("tests.test_tools.Mutants.test_a_mutant_its_named_tests_miss_is_an_error_and_never_runs_the_whole_suite",) OPENER_TESTS = ("tests.test_tools.Workflows.test_a_pull_request_is_opened_from_this_repo_by_an_account_the_configuration_allows",) OPENER_JOB = ("tests.test_tools.Workflows.test_the_job_refuses_for_the_commits_or_the_opener_and_says_what_it_holds",) COMMITS_JOB_TESTS = ("tests.test_tools.Workflows.test_the_commits_job_reads_the_pull_requests_commits_from_the_base",) @@ -4188,6 +4189,15 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ' if not _exempt(cell, root)]\n return "%d of %d', ']\n return "%d of %d', MAIN_ROWS), ('verify says nothing of the mutants', 'tools/verify.py', ',\n "%-22s %s" % ("boundary mutants", check_decisions.unnamed())]\n', ']\n', MAIN_ROWS), + ("a mutant its named tests miss runs the whole suite", 'tools/mutants.py', + " return attempt(mutant, root, run, trees, alone=True)\n", " return attempt(mutant, root, run, trees)\n", + NOT_KILLED_TESTS), + ("the tests a mutant names run before the whole suite though it runs alone", 'tools/mutants.py', + " if alone and first:\n named = {\"only\": first[0]}\n", "", NOT_KILLED_TESTS), + ("a mutant its named tests miss survives", 'tools/mutants.py', + ' if outcome == "SURVIVED" and alone and first:\n', " if False:\n", NOT_KILLED_TESTS), + ("an edit untested.py tries is held to its named tests alone", 'tools/mutants.py', + " if alone and first:\n", " if first:\n", NOT_KILLED_TESTS), ('a mutant a row names is killed by any test', 'tools/mutants.py', '({"only": first[0]} if name in _held_by_rows(root) ' 'else {"first": first[0]})', '{"first": first[0]}', RUN_ALONE), ("the impostor workflow posts from the Lens's own file", "sim/world.py", @@ -15571,7 +15581,7 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: # method by method, shared among its workers, each part one run held to TIMEOUT, never the whole suite, which outgrew # TIMEOUT on a CI runner (issue 396). A shard runs a disjoint share of what every shard's mutants name, so that its # baseline shrinks as shards are added (issue 398). -# The whole suite, which a mutant runs once the tests it names pass, runs in parts of at most PART_MOST tests dealt in +# The whole suite, which an edit tools/untested.py tries runs once the tests it names pass, runs in parts of at most PART_MOST tests dealt in # turn, one after another, each one run held to TIMEOUT, so that no run grows with the suite (issue 398). On a loaded # desk, where the suite took 940 s, its 1,311 tests made 14 parts of 9 to 124 s. # A run filtered by path, `python3 -m tools.mutants PATH...`, is the one local run allowed (the Maintainer, 2026-10-19, @@ -15594,6 +15604,8 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: NAMED_SCAN = 200 # the lines after a failure's head its error is looked for in # Said of a failure not shown to be the mutant's: one more run of the suite on the mutant decides. UNCONFIRMED = "unconfirmed" +# Why a mutant the tests it names pass on is an ERROR: the whole suite does not run to find the test that kills it. +NOT_KILLED = "named_test_does_not_kill: %s passed on it; name the test that fails on it" + AGAIN # How a FLAKY's reason ends. FLAKY fails the run as an ERROR does: nobody has shown that a test notices the mutant. MEND = "; mark or mend that test, then run python3 -m tools.mutants again" @@ -15845,16 +15857,18 @@ def lent(self): STAMPS = iter(range(1, 2 ** 31)) # next() on it is one step, whichever worker asks -def attempt(mutant: tuple, root: pathlib.Path, run, trees: Trees = None) -> tuple: +def attempt(mutant: tuple, root: pathlib.Path, run, trees: Trees = None, alone: bool = False) -> tuple: """Break one rule in a worker's copy of `root` and run the tests there: (outcome, why). A failure kills the mutant only once `confirm` shows it is the mutant's. One that is not runs the suite once more on the mutant: a pass is SURVIVED, and a second failure that is not the mutant's is FLAKY. However the runs end, - the file's own bytes are written back and read again before the copy is lent to another mutant. + the file's own bytes are written back and read again before the copy is lent to another mutant. `alone`, as the + mutant run gives it, holds a mutant to the tests it names: a pass of them is an ERROR, NOT_KILLED, and the whole + suite never runs after them; tools/untested.py's edits run it after their tests, as a mutant naming none does. """ if trees is None: with tempfile.TemporaryDirectory() as where: - return attempt(mutant, root, run, Trees(root, where)) + return attempt(mutant, root, run, Trees(root, where), alone) name, rel, old, new, *first = mutant with trees.lent() as work: target = work / rel @@ -15889,9 +15903,15 @@ def unbroken() -> None: # A mutant a decision's Boundary row names is held to the tests it lists, alone: one any other test kills # would let the row's test look like the one that notices the break (issue 590). named = ({"only": first[0]} if name in _held_by_rows(root) else {"first": first[0]}) if first else {} + # A mutant the tests it names miss ran the whole suite after them, 10 to 21 minutes of a 27-minute leg, and + # hid the wrong name (2026-11-09): run alone, it is told as an ERROR instead. + if alone and first: + named = {"only": first[0]} for _ in range(2): status, output = run(work, **named) outcome, why = verdict(status, output) + if outcome == "SURVIVED" and alone and first: + return "ERROR", NOT_KILLED % ", ".join(first[0]) if outcome != "killed": return outcome, why outcome, why = confirm(output, work, run, broken, unbroken) @@ -15917,7 +15937,7 @@ def _held_by_rows(root: pathlib.Path) -> set: def guarded(mutant: tuple, root: pathlib.Path, run, trees: Trees = None) -> tuple: """`attempt`, with anything it raises made an ERROR for this mutant, so that the others still report.""" try: - return attempt(mutant, root, run, trees) + return attempt(mutant, root, run, trees, alone=True) except Exception as err: # a failed copy, read or run; never a kill return "ERROR", "the run failed with %s: %s%s" % (type(err).__name__, err, AGAIN) From 74753ff6a964eb0b556bdbac2a869cde1922f6b3 Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:32:13 -0600 Subject: [PATCH 4/6] Review: the impostor mutant is killed at every seed count, and a flaky named test stays SURVIVED The Claude review of #666 found that the impostor mutant's new killer failed on it only at the mutant run's 40 seeds, not at the 300 make verify runs. The test the mutant named before now pins what the mutant breaks: the Lens's own App never posts from the Lens's own file, checked after every step, so it fails on the mutant at 5, 40 and 300 seeds; the mutant names it again. And a named test that failed once and then passed was told as named_test_does_not_kill: it is SURVIVED again, as before the guard, with a mutant for the difference. Co-Authored-By: Claude Opus 5.5 --- tests/test_simulator.py | 6 ++++++ tests/test_tools.py | 8 ++++++++ tools/mutants.py | 12 ++++++++---- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/tests/test_simulator.py b/tests/test_simulator.py index c627f31d..de3b1fb6 100644 --- a/tests/test_simulator.py +++ b/tests/test_simulator.py @@ -914,6 +914,12 @@ def recording(f, r, c, cfg, now): w = World(seed) for _ in range(250): w.step() + # The Lens's own App posts from another file, never the Lens's own, which the Kernel could not + # tell from the Lens's run. + own = [c for c in w.impostors for lens in world.CHECKS + if c["name"] == world.CHECKS[lens] and c["app_id"] == world.APPS[lens] + and c["workflow"] == world.WORKFLOWS[lens]] + self.assertEqual(own, [], seed) causes |= {line.split("(")[-1].rstrip(")") for line in w.trace if "apply: dequeued" in line} impostors += sum("another App posted" in line for line in w.trace) workflows += sum("another workflow posted" in line for line in w.trace) diff --git a/tests/test_tools.py b/tests/test_tools.py index 30a9e493..85b0959d 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -10407,6 +10407,14 @@ def run(work, **given): del calls[:] self.assertEqual(mutants.guarded(("hit", "k.py", "rule\n", "x\n", ("t.hit",)), root, run), ("killed", "")) self.assertEqual(calls[0], {"only": ("t.hit",)}) + # A named test that fails once and then passes, alone or again, is flaky, not a wrong name: as before, the + # second pass of the named tests is SURVIVED, which fails the run too. + del calls[:] + flaky = iter([(1, FAILS_A_TEST)]) + self.assertEqual(mutants.guarded(missed, root, lambda work, **given: (calls.append(given), next(flaky, (0, "OK\n")))[1]), + ("SURVIVED", "")) + self.assertEqual(calls, [{"only": ("t.missed",)}, {"only": "tests.test_x.T.test_x"}, {"only": "tests.test_x.T.test_x"}, + {"only": ("t.missed",)}]) # From the front door it fails the run. with mock.patch.object(mutants, "unnamed", return_value=[]), contextlib.redirect_stdout(io.StringIO()) as said: self.assertEqual(mutants.main([missed], root, run, workers=1), 1) diff --git a/tools/mutants.py b/tools/mutants.py index 4dc92d74..cff2110c 100644 --- a/tools/mutants.py +++ b/tools/mutants.py @@ -4195,13 +4195,16 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ("the tests a mutant names run before the whole suite though it runs alone", 'tools/mutants.py', " if alone and first:\n named = {\"only\": first[0]}\n", "", NOT_KILLED_TESTS), ("a mutant its named tests miss survives", 'tools/mutants.py', - ' if outcome == "SURVIVED" and alone and first:\n', " if False:\n", NOT_KILLED_TESTS), + ' if outcome == "SURVIVED" and alone and first and not again:\n', " if False:\n", NOT_KILLED_TESTS), + ("a flaky named test is told as a wrong name", 'tools/mutants.py', + ' if outcome == "SURVIVED" and alone and first and not again:\n', + ' if outcome == "SURVIVED" and alone and first:\n', NOT_KILLED_TESTS), ("an edit untested.py tries is held to its named tests alone", 'tools/mutants.py', " if alone and first:\n", " if first:\n", NOT_KILLED_TESTS), ('a mutant a row names is killed by any test', 'tools/mutants.py', '({"only": first[0]} if name in _held_by_rows(root) ' 'else {"first": first[0]})', '{"first": first[0]}', RUN_ALONE), ("the impostor workflow posts from the Lens's own file", "sim/world.py", - '"workflow": IMPOSTOR_WORKFLOW,', '"workflow": WORKFLOWS[lens],', ("tests.test_simulator.Simulation.test_the_seeds_serve_items_with_an_acceptance_contract_and_without",)), + '"workflow": IMPOSTOR_WORKFLOW,', '"workflow": WORKFLOWS[lens],', ("tests.test_simulator.Simulation.test_the_seeds_take_a_pull_request_out_of_the_queue_for_each_cause",)), ("a neutral check is read as passing", "config/defaults.json", '"neutral": "unreadable"', '"neutral": "passing"', ("tests.test_decide.Classes",)), ("a neutral check is read as failing", "config/defaults.json", @@ -15907,10 +15910,11 @@ def unbroken() -> None: # hid the wrong name (2026-11-09): run alone, it is told as an ERROR instead. if alone and first: named = {"only": first[0]} - for _ in range(2): + for again in range(2): status, output = run(work, **named) outcome, why = verdict(status, output) - if outcome == "SURVIVED" and alone and first: + # A pass after a failure not shown to be the mutant's is a flaky test, not a wrong name. + if outcome == "SURVIVED" and alone and first and not again: return "ERROR", NOT_KILLED % ", ".join(first[0]) if outcome != "killed": return outcome, why From b4b4c15cbe6d11538af37ee759a9d48cc6b98b6f Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:33:33 -0600 Subject: [PATCH 5/6] Tests: a key named in a workflow's env is put in checks' env, which the legs gave one test_a_key_and_its_environment_are_named_by_their_one_file wrote an env: into the first job with contents: read, which was checks; checks now has an env: of its own, so the key is put in it, not in a second. Co-Authored-By: Claude Opus 5.5 --- tests/test_tools.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_tools.py b/tests/test_tools.py index 85b0959d..d97dfef6 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -4164,15 +4164,13 @@ def test_a_key_and_its_environment_are_named_by_their_one_file(self): # decisio "factory-run": "run-jobs.yml", "APPROVER_TOKEN": "approve-job.yml", "factory-approve": "approve-job.yml", "CONTROLLER_APP_KEY": "pass.yml", "factory-poller": "pass.yml"} for key, owner in named.items(): - text = GOOD.replace(" permissions:\n contents: read\n", - " permissions:\n contents: read\n env:\n X: %s\n" % key, 1) + text = GOOD.replace(" env:\n CHECKS_SHARD:", " env:\n X: %s\n CHECKS_SHARD:" % key, 1) self.assertTrue(caught(check_workflows.shim_problems("other.yml", text), "it names a key or the environment that holds it, which only %s names" % owner), key) self.assertFalse(caught(check_workflows.shim_problems(owner, text), "which only %s names" % owner), key) # GitHub's names of these ignore case, and so does the rule. for spelt in ("Pr_Review_App_Key", "FACTORY-APPLY", "Factory-Poller"): - text = GOOD.replace(" permissions:\n contents: read\n", - " permissions:\n contents: read\n env:\n X: %s\n" % spelt, 1) + text = GOOD.replace(" env:\n CHECKS_SHARD:", " env:\n X: %s\n CHECKS_SHARD:" % spelt, 1) self.assertTrue(caught(check_workflows.shim_problems("other.yml", text), "it names a key"), spelt) # A comment that names one names nothing. self.assertFalse(check_workflows.shim_problems("other.yml", "# READER_APP_KEY\n" + GOOD)) From 35673e99afdc701ff9834ba641428cf5f1c18024 Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:05:47 -0600 Subject: [PATCH 6/6] Mutants: the guard's first catch names its killer, and the loop's mutant follows its rename On this pull request's run the guard told "exactly on the bar does not clear it" as named_test_does_not_kill: the gate test it named passes on it, and test_each_kind_is_held_to_its_own_bar_where_the_cases_are, which holds the bar exactly, fails on it. And "an unconfirmed failure is FLAKY with no second run" mutates the loop the review's fix renamed. Co-Authored-By: Claude Opus 5.5 --- tools/mutants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/mutants.py b/tools/mutants.py index cff2110c..aeeb0f53 100644 --- a/tools/mutants.py +++ b/tools/mutants.py @@ -2610,7 +2610,7 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ' return ("killed", "") if outcome == UNCONFIRMED else (outcome, why)\n', UNRELATED_TESTS + UNBROKEN_TESTS), ("an unconfirmed failure is FLAKY with no second run", "tools/mutants.py", - " for _ in range" "(2):\n", " for _ in range(1):\n", UNRELATED_TESTS), + " for again in range" "(2):\n", " for again in range(1):\n", UNRELATED_TESTS), ("FLAKY is counted as killed", "tools/mutants.py", 'killed += outcome' ' == "killed"', 'killed += outcome in ("killed", "FLAKY")', FLAKY_TESTS), ("the failure line of Python 3.10 and earlier is read as the class", "tools/mutants.py", @@ -8436,7 +8436,7 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ("the bar of the cases that expect a Finding is lower", 'harness/eval/__init__.py', 'BAR_OF_A_KIND = {"met": (9, 10), "finding": (3, 10)}', 'BAR_OF_A_KIND = {"met": (9, 10), "finding": (29, 100)}', ("tests.test_evals.EvalJob.test_each_kind_is_held_to_its_own_bar_where_the_cases_are",)), ("the bar of the clean cases is higher", 'harness/eval/__init__.py', 'BAR_OF_A_KIND = {"met": (9, 10), "finding": (3, 10)}', 'BAR_OF_A_KIND = {"met": (91, 100), "finding": (3, 10)}', ("tests.test_evals.EvalJob.test_each_kind_is_held_to_its_own_bar_where_the_cases_are",)), ("the bar of the cases that expect a Finding is higher", 'harness/eval/__init__.py', 'BAR_OF_A_KIND = {"met": (9, 10), "finding": (3, 10)}', 'BAR_OF_A_KIND = {"met": (9, 10), "finding": (31, 100)}', ("tests.test_evals.EvalJob.test_each_kind_is_held_to_its_own_bar_where_the_cases_are",)), - ("exactly on the bar does not clear it", 'harness/eval/__init__.py', 'return passed * bar[1] >= of * bar[0]', 'return passed * bar[1] > of * bar[0]', ("tests.test_evals.EvalJob.test_the_gate_holds_each_kinds_bar_and_a_fault_still_closes_it",)), + ("exactly on the bar does not clear it", 'harness/eval/__init__.py', 'return passed * bar[1] >= of * bar[0]', 'return passed * bar[1] > of * bar[0]', ("tests.test_evals.EvalJob.test_each_kind_is_held_to_its_own_bar_where_the_cases_are",)), ("the set's share is not held", 'harness/eval/__init__.py', 'if passed < least_passed(found["count"], KIND_LEAST if document["split_ok"] is True else 0) \\\n or (kinds', 'if (kinds', ("tests.test_evals.EvalJob.test_the_gate_holds_each_kinds_bar_and_a_fault_still_closes_it",)),