diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 47aaec5d..7f785141 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -1,6 +1,6 @@ # The required check, `factory / verify`: it passes only when `make verify` and every shard of the mutation check pass on the commit, with the history tools/check_decisions.py compares it with. -# `checks` runs `make verify`, its simulators on 300 seeds. On 2026-10-21 the runs that finished took 11 to 14.5 minutes, and six runs in three pull requests were cut at its 15-minute timeout with the tests still running, so its timeout was raised to 25 minutes; by 2026-10-29 a run took up to 23, so the job has 28 and its step 26, two kept for the checkout. Raise both before a run nears the step's. -# About 5,200 mutants, in twelve shards of about 434, each kill confirmed by two runs of one test. With six shards of 168, of about 1,000 mutants then, the legs ran 14 to 25 minutes against a 25-minute timeout, and legs were cut at 25:00 in three pull requests, once 22 ms after the last kill had printed. With eight shards of 651 the legs mostly ran 6 to 14 minutes, and on 2026-11-06 one was cut at its step's 27 minutes on a runner that ran it 2.8 times slower than a re-run of the same tree ("CI: twelve mutant shards, so a slow runner does not cut a leg at its limit", issue 583); the first run of twelve took 4.4 to 12.3 minutes a leg, 7.3 the median. Every mutant names the tests that kill it, which run first; the whole suite, its slow modules last, runs only when they pass ("Mutation run: every mutant names its tests, the slow suites run last, and a builder may break a few", issue 217). Add a shard before a leg nears its timeout, or before a leg of the median time would take more than 22 minutes on a runner three times slower. A runner as slow may still cut the longest leg: run it again. +# `checks` runs `make verify` in three legs, its simulators on 300 seeds: each leg runs every check and every third test, told its place and the count by the matrix itself, never by a number written in, so that across the legs each test runs once. As one job, on 2026-10-21 the runs that finished took 11 to 14.5 minutes, and six runs in three pull requests were cut at its 15-minute timeout with the tests still running, so its timeout was raised to 25 minutes; by 2026-10-29 a run took up to 23, so the job was given 28 and its step 26, two kept for the checkout; and on 2026-11-09 the ten runs before took 19 to 23.4 and a merge group was cut at 26.2, so its tests were dealt into legs. Add a leg before one nears the step's limit. +# About 5,200 mutants, in twelve shards of about 434, each kill confirmed by two runs of one test. With six shards of 168, of about 1,000 mutants then, the legs ran 14 to 25 minutes against a 25-minute timeout, and legs were cut at 25:00 in three pull requests, once 22 ms after the last kill had printed. With eight shards of 651 the legs mostly ran 6 to 14 minutes, and on 2026-11-06 one was cut at its step's 27 minutes on a runner that ran it 2.8 times slower than a re-run of the same tree ("CI: twelve mutant shards, so a slow runner does not cut a leg at its limit", issue 583); the first run of twelve took 4.4 to 12.3 minutes a leg, 7.3 the median. Every mutant names the tests that kill it, which run alone ("Mutation run: every mutant names its tests, the slow suites run last, and a builder may break a few", issue 217); a mutant they miss is an ERROR, named_test_does_not_kill, since the whole suite run after them took 10 to 21 minutes of a leg (2026-11-09). Add a shard before a leg nears its timeout, or before a leg of the median time would take more than 22 minutes on a runner three times slower. A runner as slow may still cut the longest leg: run it again. # `verify` checks out the base, never the head, and runs the base's tools/needs_passed.py, so a change cannot make its own verdict pass: decision 0023. # `commits` checks out the base too and runs the base's tools/commits_are_the_maintainers.py, which reads every commit the pull request carries and refuses unless each names the Maintainer ("Identity: every commit a pull request carries is the Maintainer's, checked outside the shell", issue 204). It reads the event's payload too, and refuses unless an account on the configuration's list opened the pull request from this repo: decision 0102. On a merge group, whose payload names no opener, it reads the pull request the queue's ref names, with this job's read-only token. name: factory @@ -14,6 +14,16 @@ jobs: timeout-minutes: 28 permissions: contents: read + strategy: + fail-fast: false + matrix: + shard: + - 0 + - 1 + - 2 + env: + CHECKS_SHARD: ${{ strategy.job-index }} + CHECKS_SHARDS: ${{ strategy.job-total }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/docs/standards/workflows.md b/docs/standards/workflows.md index 3ea082bb..5dd7df6f 100644 --- a/docs/standards/workflows.md +++ b/docs/standards/workflows.md @@ -10,8 +10,8 @@ Any other spelling is refused. An org standard is cited from combsmith/.github a ```yaml # The required check, `factory / verify`: it passes only when `make verify` and every shard of the mutation check pass on the commit, with the history tools/check_decisions.py compares it with. -# `checks` runs `make verify`, its simulators on 300 seeds. On 2026-10-21 the runs that finished took 11 to 14.5 minutes, and six runs in three pull requests were cut at its 15-minute timeout with the tests still running, so its timeout was raised to 25 minutes; by 2026-10-29 a run took up to 23, so the job has 28 and its step 26, two kept for the checkout. Raise both before a run nears the step's. -# About 5,200 mutants, in twelve shards of about 434, each kill confirmed by two runs of one test. With six shards of 168, of about 1,000 mutants then, the legs ran 14 to 25 minutes against a 25-minute timeout, and legs were cut at 25:00 in three pull requests, once 22 ms after the last kill had printed. With eight shards of 651 the legs mostly ran 6 to 14 minutes, and on 2026-11-06 one was cut at its step's 27 minutes on a runner that ran it 2.8 times slower than a re-run of the same tree ("CI: twelve mutant shards, so a slow runner does not cut a leg at its limit", issue 583); the first run of twelve took 4.4 to 12.3 minutes a leg, 7.3 the median. Every mutant names the tests that kill it, which run first; the whole suite, its slow modules last, runs only when they pass ("Mutation run: every mutant names its tests, the slow suites run last, and a builder may break a few", issue 217). Add a shard before a leg nears its timeout, or before a leg of the median time would take more than 22 minutes on a runner three times slower. A runner as slow may still cut the longest leg: run it again. +# `checks` runs `make verify` in three legs, its simulators on 300 seeds: each leg runs every check and every third test, told its place and the count by the matrix itself, never by a number written in, so that across the legs each test runs once. As one job, on 2026-10-21 the runs that finished took 11 to 14.5 minutes, and six runs in three pull requests were cut at its 15-minute timeout with the tests still running, so its timeout was raised to 25 minutes; by 2026-10-29 a run took up to 23, so the job was given 28 and its step 26, two kept for the checkout; and on 2026-11-09 the ten runs before took 19 to 23.4 and a merge group was cut at 26.2, so its tests were dealt into legs. Add a leg before one nears the step's limit. +# About 5,200 mutants, in twelve shards of about 434, each kill confirmed by two runs of one test. With six shards of 168, of about 1,000 mutants then, the legs ran 14 to 25 minutes against a 25-minute timeout, and legs were cut at 25:00 in three pull requests, once 22 ms after the last kill had printed. With eight shards of 651 the legs mostly ran 6 to 14 minutes, and on 2026-11-06 one was cut at its step's 27 minutes on a runner that ran it 2.8 times slower than a re-run of the same tree ("CI: twelve mutant shards, so a slow runner does not cut a leg at its limit", issue 583); the first run of twelve took 4.4 to 12.3 minutes a leg, 7.3 the median. Every mutant names the tests that kill it, which run alone ("Mutation run: every mutant names its tests, the slow suites run last, and a builder may break a few", issue 217); a mutant they miss is an ERROR, named_test_does_not_kill, since the whole suite run after them took 10 to 21 minutes of a leg (2026-11-09). Add a shard before a leg nears its timeout, or before a leg of the median time would take more than 22 minutes on a runner three times slower. A runner as slow may still cut the longest leg: run it again. # `verify` checks out the base, never the head, and runs the base's tools/needs_passed.py, so a change cannot make its own verdict pass: decision 0023. # `commits` checks out the base too and runs the base's tools/commits_are_the_maintainers.py, which reads every commit the pull request carries and refuses unless each names the Maintainer ("Identity: every commit a pull request carries is the Maintainer's, checked outside the shell", issue 204). It reads the event's payload too, and refuses unless an account on the configuration's list opened the pull request from this repo: decision 0102. On a merge group, whose payload names no opener, it reads the pull request the queue's ref names, with this job's read-only token. name: factory @@ -25,6 +25,16 @@ jobs: timeout-minutes: 28 permissions: contents: read + strategy: + fail-fast: false + matrix: + shard: + - 0 + - 1 + - 2 + env: + CHECKS_SHARD: ${{ strategy.job-index }} + CHECKS_SHARDS: ${{ strategy.job-total }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -109,8 +119,8 @@ name and the job's name: `factory / verify`. Renaming either silently stops it b `verify` needs every other job and has `if: always()`, the one job-level `if:` a Lens's job may have: GitHub passes a required check that is skipped, so `tools/needs_passed.py` fails it unless each passed. Its one checkout is of the base, never the head: the verdict is the base's `tools/needs_passed.py`, so a change -cannot make its own pass (decision 0023). It runs on no event without a base. The checks and tests in `checks` and -`mutants` are the head's. `commits` is the base's too: it fetches the head and reads who authored and committed each +cannot make its own pass (decision 0023). It runs on no event without a base. The checks and tests in `checks` and `mutants` are the head's, +run in legs their matrix alone tells their place and count, so each test and mutant runs once and every check in each leg. `commits` is the base's too: it fetches the head and reads who authored and committed each commit the pull request carries, which must be the Maintainer, read from `config/hooks.json` (decision 0041). The eval job is a reserved name, `factory / eval`, in eval.yml: the one job that holds the held-out home's credential @@ -119,7 +129,7 @@ Its shape, each credential named by one step alone and its upload the one `UPLOA `tools/check_workflows.py`, by issue 309, "Workflows: eval.yml runs the review Intent's held-out cases and writes eval-report/1". Org standard: ci-standards.md, "Permissions Policy", "Action Pinning Policy" and "CI Job Naming Convention". Its name, triggers and stages depart from "1. CI Pipeline (ci.yml)": `docs/decisions/0014-where-this-repo-departs-from-the-orgs-standards.md`. -Held by: `tests/test_tools.py::test_a_rule_is_not_dodged_by_writing_the_workflow_another_way`, `tests/test_tools.py::test_an_example_in_the_standard_that_has_drifted_is_caught`, `tests/test_tools.py::test_a_lens_job_that_would_never_report_under_its_name_is_caught`, `tests/test_tools.py::test_the_required_check_fails_unless_every_job_it_needs_passed`, `tests/test_tools.py::test_a_lens_job_that_judges_its_needs_runs_from_the_base`, `tests/test_tools.py::test_the_judging_step_and_its_checkout_are_held_to_their_exact_shape`, `tests/test_tools.py::test_the_mutant_shards_are_told_their_index_and_count_by_the_matrix`, `tests/test_evals.py::test_the_eval_jobs_name_is_reserved`, `tests/test_evals.py::test_the_eval_workflow_names_each_credential_in_one_step_alone` +Held by: `tests/test_tools.py::test_a_rule_is_not_dodged_by_writing_the_workflow_another_way`, `tests/test_tools.py::test_an_example_in_the_standard_that_has_drifted_is_caught`, `tests/test_tools.py::test_a_lens_job_that_would_never_report_under_its_name_is_caught`, `tests/test_tools.py::test_the_required_check_fails_unless_every_job_it_needs_passed`, `tests/test_tools.py::test_a_lens_job_that_judges_its_needs_runs_from_the_base`, `tests/test_tools.py::test_the_judging_step_and_its_checkout_are_held_to_their_exact_shape`, `tests/test_tools.py::test_the_mutant_shards_are_told_their_index_and_count_by_the_matrix`, `tests/test_tools.py::test_the_legs_of_make_verify_are_told_their_place_and_count_by_the_matrix`, `tests/test_tools.py::test_the_legs_of_make_verify_run_every_test_once`, `tests/test_evals.py::test_the_eval_jobs_name_is_reserved`, `tests/test_evals.py::test_the_eval_workflow_names_each_credential_in_one_step_alone` ## Pinned to what was looked up diff --git a/examples/workflows/verify.yml b/examples/workflows/verify.yml index 47aaec5d..7f785141 100644 --- a/examples/workflows/verify.yml +++ b/examples/workflows/verify.yml @@ -1,6 +1,6 @@ # The required check, `factory / verify`: it passes only when `make verify` and every shard of the mutation check pass on the commit, with the history tools/check_decisions.py compares it with. -# `checks` runs `make verify`, its simulators on 300 seeds. On 2026-10-21 the runs that finished took 11 to 14.5 minutes, and six runs in three pull requests were cut at its 15-minute timeout with the tests still running, so its timeout was raised to 25 minutes; by 2026-10-29 a run took up to 23, so the job has 28 and its step 26, two kept for the checkout. Raise both before a run nears the step's. -# About 5,200 mutants, in twelve shards of about 434, each kill confirmed by two runs of one test. With six shards of 168, of about 1,000 mutants then, the legs ran 14 to 25 minutes against a 25-minute timeout, and legs were cut at 25:00 in three pull requests, once 22 ms after the last kill had printed. With eight shards of 651 the legs mostly ran 6 to 14 minutes, and on 2026-11-06 one was cut at its step's 27 minutes on a runner that ran it 2.8 times slower than a re-run of the same tree ("CI: twelve mutant shards, so a slow runner does not cut a leg at its limit", issue 583); the first run of twelve took 4.4 to 12.3 minutes a leg, 7.3 the median. Every mutant names the tests that kill it, which run first; the whole suite, its slow modules last, runs only when they pass ("Mutation run: every mutant names its tests, the slow suites run last, and a builder may break a few", issue 217). Add a shard before a leg nears its timeout, or before a leg of the median time would take more than 22 minutes on a runner three times slower. A runner as slow may still cut the longest leg: run it again. +# `checks` runs `make verify` in three legs, its simulators on 300 seeds: each leg runs every check and every third test, told its place and the count by the matrix itself, never by a number written in, so that across the legs each test runs once. As one job, on 2026-10-21 the runs that finished took 11 to 14.5 minutes, and six runs in three pull requests were cut at its 15-minute timeout with the tests still running, so its timeout was raised to 25 minutes; by 2026-10-29 a run took up to 23, so the job was given 28 and its step 26, two kept for the checkout; and on 2026-11-09 the ten runs before took 19 to 23.4 and a merge group was cut at 26.2, so its tests were dealt into legs. Add a leg before one nears the step's limit. +# About 5,200 mutants, in twelve shards of about 434, each kill confirmed by two runs of one test. With six shards of 168, of about 1,000 mutants then, the legs ran 14 to 25 minutes against a 25-minute timeout, and legs were cut at 25:00 in three pull requests, once 22 ms after the last kill had printed. With eight shards of 651 the legs mostly ran 6 to 14 minutes, and on 2026-11-06 one was cut at its step's 27 minutes on a runner that ran it 2.8 times slower than a re-run of the same tree ("CI: twelve mutant shards, so a slow runner does not cut a leg at its limit", issue 583); the first run of twelve took 4.4 to 12.3 minutes a leg, 7.3 the median. Every mutant names the tests that kill it, which run alone ("Mutation run: every mutant names its tests, the slow suites run last, and a builder may break a few", issue 217); a mutant they miss is an ERROR, named_test_does_not_kill, since the whole suite run after them took 10 to 21 minutes of a leg (2026-11-09). Add a shard before a leg nears its timeout, or before a leg of the median time would take more than 22 minutes on a runner three times slower. A runner as slow may still cut the longest leg: run it again. # `verify` checks out the base, never the head, and runs the base's tools/needs_passed.py, so a change cannot make its own verdict pass: decision 0023. # `commits` checks out the base too and runs the base's tools/commits_are_the_maintainers.py, which reads every commit the pull request carries and refuses unless each names the Maintainer ("Identity: every commit a pull request carries is the Maintainer's, checked outside the shell", issue 204). It reads the event's payload too, and refuses unless an account on the configuration's list opened the pull request from this repo: decision 0102. On a merge group, whose payload names no opener, it reads the pull request the queue's ref names, with this job's read-only token. name: factory @@ -14,6 +14,16 @@ jobs: timeout-minutes: 28 permissions: contents: read + strategy: + fail-fast: false + matrix: + shard: + - 0 + - 1 + - 2 + env: + CHECKS_SHARD: ${{ strategy.job-index }} + CHECKS_SHARDS: ${{ strategy.job-total }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/tests/test_simulator.py b/tests/test_simulator.py index c627f31d..de3b1fb6 100644 --- a/tests/test_simulator.py +++ b/tests/test_simulator.py @@ -914,6 +914,12 @@ def recording(f, r, c, cfg, now): w = World(seed) for _ in range(250): w.step() + # The Lens's own App posts from another file, never the Lens's own, which the Kernel could not + # tell from the Lens's run. + own = [c for c in w.impostors for lens in world.CHECKS + if c["name"] == world.CHECKS[lens] and c["app_id"] == world.APPS[lens] + and c["workflow"] == world.WORKFLOWS[lens]] + self.assertEqual(own, [], seed) causes |= {line.split("(")[-1].rstrip(")") for line in w.trace if "apply: dequeued" in line} impostors += sum("another App posted" in line for line in w.trace) workflows += sum("another workflow posted" in line for line in w.trace) diff --git a/tests/test_tools.py b/tests/test_tools.py index a17d256b..d97dfef6 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -2789,6 +2789,35 @@ def test_the_mutant_shards_are_told_their_index_and_count_by_the_matrix(self): self.assertTrue(caught(self.lens_problems(GOOD.replace(job, job.replace(run, " - run: make verify\n"))), gone)) self.assertEqual(self.lens_problems(GOOD), []) + def test_the_legs_of_make_verify_are_told_their_place_and_count_by_the_matrix(self): + # On 2026-11-09 `make verify` as one job was cut at its step's 26 minutes in a merge group, so it runs in legs. A + # leg told CHECKS_SHARD as a number would run another's share, leaving its own tests in no leg, and CHECKS_SHARDS + # as a number could differ from the matrix's size; each is the matrix's own expression, as the mutant shards'. + job = GOOD[GOOD.index(" checks:\n"):GOOD.index(" mutants:\n")] + index, total = " CHECKS_SHARD: ${{ strategy.job-index }}\n", " CHECKS_SHARDS: ${{ strategy.job-total }}\n" + run = " - run: make verify\n" + self.assertEqual((job.count(index), job.count(total), job.count(run)), (1, 1, 1)) + told, shape = "not told CHECKS_SHARD", "whose steps are not a checkout of the head" + cases = [(index, " CHECKS_SHARD: 0\n", told), (total, " CHECKS_SHARDS: 3\n", told), + (total, "", told), (index, index + " OTHER: 1\n", told), + (" shard:\n", " leg:\n", "not the one key shard"), + (" - 2\n", " - 2\n exclude:\n - shard: 2\n", "not the one key shard"), + (" env:\n CHECKS_SHARD", " container: ghcr.io/x/y:1\n env:\n CHECKS_SHARD", "keys other than"), + (" contents: read\n", " contents: write\n", "granted other than"), + (run, run + " working-directory: evil\n", shape), (run, run + " env:\n CHECKS_SHARD: 0\n", shape), + (run, run + " - run: make fast\n", shape), + (run, " - run: make fast\n - run: make verify\n", shape), + (" fetch-depth: 0\n - run: make verify\n", " - run: make verify\n", shape)] + for old, new, said in cases: + text = GOOD.replace(job, job.replace(old, new)) + self.assertNotEqual(text, GOOD, said) + self.assertTrue(caught(self.lens_problems(text), said), (old, new, said)) + # No job runs `make verify` at all: the checks and the tests are gone, not reshaped. + gone = "has no job that runs `make verify`" + self.assertTrue(caught(self.lens_problems(GOOD.replace(job, "").replace(" - checks\n", "")), gone)) + self.assertTrue(caught(self.lens_problems(GOOD.replace(job, job.replace(run, " - run: make fast\n"))), gone)) + self.assertEqual(self.lens_problems(GOOD), []) + def test_a_lens_job_that_judges_its_needs_runs_from_the_base(self): # verify's verdict is the base's tools/needs_passed.py: its one checkout is of the base, on a pull request and in # the merge queue, and of nothing else, so a change cannot make its own verdict pass (decision 0023). @@ -4135,15 +4164,13 @@ def test_a_key_and_its_environment_are_named_by_their_one_file(self): # decisio "factory-run": "run-jobs.yml", "APPROVER_TOKEN": "approve-job.yml", "factory-approve": "approve-job.yml", "CONTROLLER_APP_KEY": "pass.yml", "factory-poller": "pass.yml"} for key, owner in named.items(): - text = GOOD.replace(" permissions:\n contents: read\n", - " permissions:\n contents: read\n env:\n X: %s\n" % key, 1) + text = GOOD.replace(" env:\n CHECKS_SHARD:", " env:\n X: %s\n CHECKS_SHARD:" % key, 1) self.assertTrue(caught(check_workflows.shim_problems("other.yml", text), "it names a key or the environment that holds it, which only %s names" % owner), key) self.assertFalse(caught(check_workflows.shim_problems(owner, text), "which only %s names" % owner), key) # GitHub's names of these ignore case, and so does the rule. for spelt in ("Pr_Review_App_Key", "FACTORY-APPLY", "Factory-Poller"): - text = GOOD.replace(" permissions:\n contents: read\n", - " permissions:\n contents: read\n env:\n X: %s\n" % spelt, 1) + text = GOOD.replace(" env:\n CHECKS_SHARD:", " env:\n X: %s\n CHECKS_SHARD:" % spelt, 1) self.assertTrue(caught(check_workflows.shim_problems("other.yml", text), "it names a key"), spelt) # A comment that names one names nothing. self.assertFalse(check_workflows.shim_problems("other.yml", "# READER_APP_KEY\n" + GOOD)) @@ -10119,6 +10146,55 @@ def test_a_check_that_finds_a_problem_fails_verify(self): self.assertFalse(verify.failed([("never", lambda: [])])) self.assertIn("a problem", said.getvalue()) + def test_the_legs_of_make_verify_run_every_test_once(self): + # CI runs `make verify` in legs (docs/standards/workflows.md): each runs every check and each count-th test from + # its place, so that across the legs every test of the whole suite runs, the seeded simulation's included, and + # none twice. A leg it cannot read stops the run before any check or test. + whole = [test.id() for test in verify._tests(verify.suite(False))] + self.assertTrue({"tests.test_simulator", "tests.test_issue_simulator"} <= {i.rsplit(".", 2)[0] for i in whole}) + for count in (1, 2, 3, 5): + legs = [[test.id() for test in verify._tests(verify.suite(False, (index, count)))] for index in range(count)] + dealt = [i for leg in legs for i in leg] + self.assertEqual((sorted(dealt), len(dealt)), (sorted(whole), len(whole)), count) + self.assertLessEqual(max(map(len, legs)) - min(map(len, legs)), 1, count) + self.assertIsNone(verify.leg({})) + self.assertEqual(verify.leg({"CHECKS_SHARD": "2", "CHECKS_SHARDS": "3"}), (2, 3)) + for shard, count, said in (("1", None, "set both"), (None, "3", "set both"), ("", "3", "set both"), + ("one", "3", "set both"), ("-1", "3", "set both"), ("1.0", "3", "set both"), + ("3", "3", "below the count"), ("0", "0", "below the count")): + given = {k: v for k, v in (("CHECKS_SHARD", shard), ("CHECKS_SHARDS", count)) if v is not None} + with self.assertRaises(ValueError, msg=given) as raised: + verify.leg(given) + self.assertIn(said, str(raised.exception), given) + ran = [] + real = (verify.failed, verify.unittest.TextTestRunner) + + class Runner: + def __init__(self, **_): + pass + + def run(self, suite): + ran.append(sorted(test.id() for test in verify._tests(suite))) + return unittest.TestResult() + + verify.failed, verify.unittest.TextTestRunner = (lambda checks: ran.append("checks") or False), Runner + try: + with contextlib.redirect_stdout(io.StringIO()) as said: + self.assertEqual(verify.main([], {"CHECKS_SHARD": "1", "CHECKS_SHARDS": "3"}), 0) + self.assertEqual(ran, ["checks", sorted(whole[1::3])]) + self.assertIn("leg 1 of 3", said.getvalue()) + del ran[:] + self.assertEqual(verify.main([], {"CHECKS_SHARD": "3", "CHECKS_SHARDS": "3"}), 1) + self.assertEqual(ran, [], "a leg outside the count runs nothing") + finally: + verify.failed, verify.unittest.TextTestRunner = real + # From the front door, as CI's matrix tells it: the leg is read from the environment. + env = dict(os.environ, CHECKS_SHARD="3", CHECKS_SHARDS="3") + done = subprocess.run([sys.executable, "-m", "tools.verify"], cwd=str(ROOT), env=env, capture_output=True, + text=True, timeout=60) + self.assertEqual((done.returncode, done.stdout.strip()), (1, "CHECKS_SHARD is 3 and CHECKS_SHARDS is 3; the leg " + "is counted from 0, below the count")) + def test_make_fast_is_verify_without_the_seeded_simulation(self): """The required check runs every test file at any depth; only `make fast` leaves the slow ones out.""" everything = {".".join(p.relative_to(ROOT).with_suffix("").parts) for p in walk.files(ROOT, "tests") @@ -10310,6 +10386,43 @@ def run(work, **given): self.assertEqual(mutants.attempt(mutant, root, run)[0], "SURVIVED") self.assertEqual(calls, [{"only": ("tests.test_x.T.test_y",)}, {"first": ("tests.test_x.T.test_y",)}]) + def test_a_mutant_its_named_tests_miss_is_an_error_and_never_runs_the_whole_suite(self): + # On 2026-11-09 two mutants named tests that passed on them, so each leg that held one ran the whole suite after + # them: 10 to 21 minutes of a 27-minute step, and a wrong name nobody saw. The mutant run holds each mutant to + # the tests it names, alone: a pass of them is an ERROR that names them, and nothing else runs. + calls = [] + + def run(work, **given): + calls.append(given) + return (0, "OK\n") if (work / "k.py").read_text() == "rule\n" or given.get("only") == ("t.missed",) \ + else (1, FAILS_A_TEST) + root = tree({"k.py": "rule\n"}) + missed = ("missed", "k.py", "rule\n", "x\n", ("t.missed",)) + outcome, why = mutants.guarded(missed, root, run) + self.assertEqual((outcome, calls), ("ERROR", [{"only": ("t.missed",)}])) + self.assertTrue(why.startswith("named_test_does_not_kill: t.missed passed on it") and why.endswith(AGAIN), why) + # One they kill is killed, and confirmed as before. + del calls[:] + self.assertEqual(mutants.guarded(("hit", "k.py", "rule\n", "x\n", ("t.hit",)), root, run), ("killed", "")) + self.assertEqual(calls[0], {"only": ("t.hit",)}) + # A named test that fails once and then passes, alone or again, is flaky, not a wrong name: as before, the + # second pass of the named tests is SURVIVED, which fails the run too. + del calls[:] + flaky = iter([(1, FAILS_A_TEST)]) + self.assertEqual(mutants.guarded(missed, root, lambda work, **given: (calls.append(given), next(flaky, (0, "OK\n")))[1]), + ("SURVIVED", "")) + self.assertEqual(calls, [{"only": ("t.missed",)}, {"only": "tests.test_x.T.test_x"}, {"only": "tests.test_x.T.test_x"}, + {"only": ("t.missed",)}]) + # From the front door it fails the run. + with mock.patch.object(mutants, "unnamed", return_value=[]), contextlib.redirect_stdout(io.StringIO()) as said: + self.assertEqual(mutants.main([missed], root, run, workers=1), 1) + self.assertIn("ERROR missed (named_test_does_not_kill", said.getvalue()) + # tools/untested.py's edits, through attempt with no `alone`, still run the whole suite after their tests, which + # here fails on it. + del calls[:] + self.assertEqual(mutants.attempt(missed, root, run), ("killed", "")) + self.assertEqual(calls[0], {"first": ("t.missed",)}) + def test_the_summary_counts_only_the_mutants_killed(self): stale = [("gone", "k.py", "not there\n", "x\n")] code, lines, _ = self.check({"fails": (0, 1, FAILS_A_TEST), "passes": (0, 0, "OK\n")}, @@ -10635,7 +10748,12 @@ def run(work, first=(), only=None): mutants.baseline(every[1:2], root / "gone", run, 2, (every, 0, 2)) self.assertEqual(sorted(calls), [("tests.test_x.T", "tests.test_x.T.test_g"), ("tests.test_x.T.test_b",)]) # From the front door, a shard runs its share, and a filtered run its own names on FILTER_WORKERS. The names - # are this throwaway tree's, which this checkout's check of them could not find. + # are this throwaway tree's, which this checkout's check of them could not find. The baseline runs in the tree; + # each mutant, which runs its named tests alone too, in a copy of it. + + def run(work, first=(), only=None): + calls.append(only if work == root else None) + return PASSES[1:] for environ, paths, parts in (({"MUTANTS_SHARD": "1", "MUTANTS_SHARDS": "4"}, [], [("tests.test_x.T.test_b",), ("tests.test_x.T.test_f",)]), ({}, ["k.py"], [tests[0::2], tests[1::2]])): diff --git a/tools/AGENTS.md b/tools/AGENTS.md index 2a77e5f3..f7973ac3 100644 --- a/tools/AGENTS.md +++ b/tools/AGENTS.md @@ -5,8 +5,7 @@ unless each commit names the Maintainer and a listed account opened it. `walk.py it, and reads the index as a file, so no git is needed. CI checks out the commit afresh, so there all else is committed; an export, with no `.git`, cannot tell, and skips it all. `make_corpus.py` records every decision the Kernel makes, under a directory git ignores, to hold another implementation to this one; the required check does not depend on it. `hooks/` holds the session hooks `.claude/settings.json` runs; `hooks/shell.py` reads a shell command as words for any of them. `eval_result.py` scores one Result against one held-out case, where the case is, and is no check; `context_eval.sh` runs the context's test at the Maintainer's terminal and `context_eval.py` records a run, and neither is a check. `untested.py` reports Python no test notices (decision 0166). -`rulesets_drift.sh` reads the repo's live rulesets through whoever is logged in to gh and redacts each answer outside the repo, -and `rulesets_drift.py` compares what is left with `config/rulesets.json`; it is run by hand and is no check. `machine_user_drift.sh` does the same for `config/machine-user.json`, and `environments_drift.sh` holds each environment to `main` alone. +`rulesets_drift.sh` reads the live rulesets through gh, redacted outside the repo, and `rulesets_drift.py` holds them to `config/rulesets.json`, by hand, no check; `machine_user_drift.sh` so for `config/machine-user.json`, `environments_drift.sh` for `main`. | Rule | Held by | |---|---| @@ -32,7 +31,8 @@ and `rulesets_drift.py` compares what is left with `config/rulesets.json`; it is | A stop lists every protected or reported path changed against origin's main; a write to a remote branch asks | `tests/test_hooks.py::test_an_edit_made_through_eval_is_listed`, `tests/test_hooks.py::test_a_write_to_the_base_the_stop_compares_with_asks` | | A test kills a mutant only failing on it, again alone, and passing alone in an unbroken copy; else `FLAKY`, or `ERROR` on a hang: both fail a run | `tests/test_tools.py::test_a_failure_that_is_not_the_mutants_twice_is_flaky_and_fails_the_check` | | A `mutants.py` entry names text found once in its file (else `STALE`; update it) and tests that kill it, each existing, none in `WHOLE_TOO_LONG` nor its module | `mutants.py`, `tests/test_tools.py::test_every_mutant_names_a_test_that_exists` | -| After the named tests, all of `tests/` runs in dealt parts; `mutants PATH...` breaks at most 40, two at once | `tests/test_tools.py::test_the_whole_suite_runs_in_dealt_parts_each_held_to_the_limit`, `tests/test_tools.py::test_a_filtered_run_is_capped` | +| A mutant runs the tests it names alone; one they miss is an ERROR, `named_test_does_not_kill` | `tests/test_tools.py::test_a_mutant_its_named_tests_miss_is_an_error_and_never_runs_the_whole_suite` | +| After an `untested.py` edit's tests, all of `tests/` runs in dealt parts; `mutants PATH...` breaks 40, 2 at once | `tests/test_tools.py::test_the_whole_suite_runs_in_dealt_parts_each_held_to_the_limit`, `tests/test_tools.py::test_a_filtered_run_is_capped` | A check collects every problem and does not stop at the first. Its message says what is wrong and what to do, in one sentence, because an author will read it with nothing else to go on. diff --git a/tools/check_workflows.py b/tools/check_workflows.py index 8dac04c3..30d2dcec 100644 --- a/tools/check_workflows.py +++ b/tools/check_workflows.py @@ -14,6 +14,8 @@ with no `strategy.matrix` and no `uses:`. Its one job-level `if:` is exactly `always()`, beside a `needs:` that names every other job in its file, and then it judges them with `tools/needs_passed.py`, the base's: its one checkout is of the base, never the head. Only `jobs..name` is read: a workflow's or a step's `name:` names no check run. +In the required check's workflow, the mutants and `make verify` each run in a matrix whose legs are told their place +and the count by the matrix itself (_shards, _legs), so that no mutant and no test is left in no leg. The secret scan, the exempt `ci.yml`, reads its rules from the base (scan_problems): `.gitleaks.toml` and `.gitleaksignore` from a checkout of the base under `base/`, with the head's `.gitleaksignore` deleted first, since @@ -685,6 +687,8 @@ SHARDED = "python3 -m tools.mutants" INDEX = "${{ strategy.job-index }}" TOTAL = "${{ strategy.job-total }}" +# The legs of `make verify`, dealt its tests as the mutant shards are dealt mutants, and told the same way. +LEGGED = "make verify" JUDGING_KEYS = {"name", "needs", "if", "runs-on", "timeout-minutes", "permissions", "env", "steps"} SHARD_KEYS = {"runs-on", "timeout-minutes", "permissions", "strategy", "env", "steps"} HEAD = "${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}" @@ -1351,6 +1355,44 @@ def _shards(entries: list, jobs: list) -> list: return whys +def _legs(entries: list, jobs: list) -> list: + """Why the workflow would not run every test once: a job that runs `LEGGED` exists, and its env: is exactly + CHECKS_SHARD: INDEX and CHECKS_SHARDS: TOTAL, the matrix's own index and size, so a number written in cannot leave a + share of the tests in no leg; its matrix is one key, shard; and its shape is closed, as the mutant shards' is: the + keys SHARD_KEYS and no other, contents: read, a checkout of the head with the whole history and `LEGGED` on a line + of its own, each step holding no other key.""" + whys, any_leg = [], False + for job in jobs: + own = [e for e in entries if e["path"][:2] == ["jobs", job]] + steps = [e for e in own if e["path"] == ["jobs", job, "steps"] and e["key"] == "run"] + if all(LEGGED not in "\n".join([e["value"]] + (e["body"] or [])) for e in steps): + continue + any_leg = True + said = "has the job %s, which runs `%s`, " % (job, LEGGED) + keys = {e["key"] for e in own if e["path"] == ["jobs", job]} + told = {e["key"]: e["value"] for e in own if e["path"] == ["jobs", job, "env"]} + matrix = [e["key"] for e in own if e["path"] == ["jobs", job, "strategy", "matrix"]] + granted = {e["key"]: e["value"] for e in own if e["path"] == ["jobs", job, "permissions"]} + checkouts = [_with(entries, e) for e in own if e["path"] == ["jobs", job, "steps"] and _checkout(e)] + if keys != SHARD_KEYS: + whys.append(said + "with keys other than %s, or lacking one" % ", ".join(sorted(SHARD_KEYS))) + if told != {"CHECKS_SHARD": INDEX, "CHECKS_SHARDS": TOTAL}: + whys.append(said + "not told CHECKS_SHARD: %s and CHECKS_SHARDS: %s and nothing else, so a leg could run " + "another's tests, or every leg one share" % (INDEX, TOTAL)) + if matrix != ["shard"]: + whys.append(said + "whose strategy.matrix is not the one key shard") + if granted != {"contents": "read"}: + whys.append(said + "granted other than contents: read") + if not ([set(step) for step in _steps(entries, job)] == [{"uses", "with"}, {"run", "timeout-minutes"}] + and [(e["value"], e["body"] or []) for e in steps] == [(LEGGED, [])] + and checkouts == [{"persist-credentials": "false", "fetch-depth": "0"}]): + whys.append(said + "whose steps are not a checkout of the head with the whole history and `%s` on a line " + "of its own with its own timeout-minutes, each holding no other key" % LEGGED) + if not any_leg: + whys.append("has no job that runs `%s`, so no check and no test runs before a merge" % LEGGED) + return whys + + def _commits(text: str, entries: list, jobs: list) -> list: """Why the workflow's job `commits` would not read every commit the pull request carries from the base (decision 0041). Its shape is closed: the keys runs-on, timeout-minutes, permissions (contents: read, pull-requests: read, and @@ -1775,7 +1817,7 @@ def lens_job_problems(text: str, check: str) -> list: problems += ["has the job %s beside other jobs, and %s" % (job, why) for why in _judging(text, entries, job, jobs)] if any("needs" in inside for _, inside in named): # the required check's workflow, whose job judges the rest - problems += _commits(text, entries, jobs) + _shards(entries, jobs) + problems += _commits(text, entries, jobs) + _shards(entries, jobs) + _legs(entries, jobs) for key in ("env", "defaults"): if key in top: problems.append("has a top-level %s:, which every job inherits, where each is told what it is and " diff --git a/tools/mutants.py b/tools/mutants.py index cb134af6..aeeb0f53 100644 --- a/tools/mutants.py +++ b/tools/mutants.py @@ -110,11 +110,14 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: NAMED_MUTANTS = ("tests.test_tools.Decisions.test_a_new_or_changed_row_names_the_mutant_its_test_kills",) MAIN_ROWS = ("tests.test_tools.Decisions.test_a_row_main_holds_names_no_mutant_until_its_check_changes",) RUN_ALONE = ("tests.test_tools.Mutants.test_a_mutant_a_boundary_names_is_run_against_its_own_tests_alone",) +NOT_KILLED_TESTS = ("tests.test_tools.Mutants.test_a_mutant_its_named_tests_miss_is_an_error_and_never_runs_the_whole_suite",) OPENER_TESTS = ("tests.test_tools.Workflows.test_a_pull_request_is_opened_from_this_repo_by_an_account_the_configuration_allows",) OPENER_JOB = ("tests.test_tools.Workflows.test_the_job_refuses_for_the_commits_or_the_opener_and_says_what_it_holds",) COMMITS_JOB_TESTS = ("tests.test_tools.Workflows.test_the_commits_job_reads_the_pull_requests_commits_from_the_base",) JUDGING_SHAPE_TESTS = ("tests.test_tools.Workflows.test_the_judging_step_and_its_checkout_are_held_to_their_exact_shape",) SHARDS_TESTS = ("tests.test_tools.Workflows.test_the_mutant_shards_are_told_their_index_and_count_by_the_matrix",) +LEGS_TESTS = ("tests.test_tools.Workflows.test_the_legs_of_make_verify_are_told_their_place_and_count_by_the_matrix",) +LEG_TESTS = ("tests.test_tools.Verify.test_the_legs_of_make_verify_run_every_test_once",) LIMITED = ("tests.test_tools.Workflows.test_each_run_step_stops_under_a_limit_of_its_own_below_its_jobs",) NEEDED = ("tests.test_tools.Workflows.test_each_step_limit_holds_what_its_step_needs",) P13 = "tools/trial_p13_cli.sh" @@ -2607,7 +2610,7 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ' return ("killed", "") if outcome == UNCONFIRMED else (outcome, why)\n', UNRELATED_TESTS + UNBROKEN_TESTS), ("an unconfirmed failure is FLAKY with no second run", "tools/mutants.py", - " for _ in range" "(2):\n", " for _ in range(1):\n", UNRELATED_TESTS), + " for again in range" "(2):\n", " for again in range(1):\n", UNRELATED_TESTS), ("FLAKY is counted as killed", "tools/mutants.py", 'killed += outcome' ' == "killed"', 'killed += outcome in ("killed", "FLAKY")', FLAKY_TESTS), ("the failure line of Python 3.10 and earlier is read as the class", "tools/mutants.py", @@ -4186,6 +4189,18 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ' if not _exempt(cell, root)]\n return "%d of %d', ']\n return "%d of %d', MAIN_ROWS), ('verify says nothing of the mutants', 'tools/verify.py', ',\n "%-22s %s" % ("boundary mutants", check_decisions.unnamed())]\n', ']\n', MAIN_ROWS), + ("a mutant its named tests miss runs the whole suite", 'tools/mutants.py', + " return attempt(mutant, root, run, trees, alone=True)\n", " return attempt(mutant, root, run, trees)\n", + NOT_KILLED_TESTS), + ("the tests a mutant names run before the whole suite though it runs alone", 'tools/mutants.py', + " if alone and first:\n named = {\"only\": first[0]}\n", "", NOT_KILLED_TESTS), + ("a mutant its named tests miss survives", 'tools/mutants.py', + ' if outcome == "SURVIVED" and alone and first and not again:\n', " if False:\n", NOT_KILLED_TESTS), + ("a flaky named test is told as a wrong name", 'tools/mutants.py', + ' if outcome == "SURVIVED" and alone and first and not again:\n', + ' if outcome == "SURVIVED" and alone and first:\n', NOT_KILLED_TESTS), + ("an edit untested.py tries is held to its named tests alone", 'tools/mutants.py', + " if alone and first:\n", " if first:\n", NOT_KILLED_TESTS), ('a mutant a row names is killed by any test', 'tools/mutants.py', '({"only": first[0]} if name in _held_by_rows(root) ' 'else {"first": first[0]})', '{"first": first[0]}', RUN_ALONE), ("the impostor workflow posts from the Lens's own file", "sim/world.py", @@ -6336,8 +6351,8 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ("the commits job may run another tool", "tools/check_workflows.py", ' if [(e["value"], e["body"]) for e in runs] != [(COMMITS, [])] and', " if False and", COMMITS_JOB_TESTS), ("a workflow without the commits job passes the lens rules", "tools/check_workflows.py", - " problems += _commits(text, entries, jobs) + _shards(entries, jobs)\n", - " problems += _shards(entries, jobs)\n", COMMITS_JOB_TESTS), + " problems += _commits(text, entries, jobs) + _shards(entries, jobs) + _legs(entries, jobs)\n", + " problems += _shards(entries, jobs) + _legs(entries, jobs)\n", COMMITS_JOB_TESTS), # Issue 180: the judging step and the mutant shards held to their exact shape. ("the judging job may have other keys", "tools/check_workflows.py", " if set(inside) - JUDGING_KEYS:\n", " if False:\n", JUDGING_SHAPE_TESTS), @@ -6367,8 +6382,36 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ' if not any("mutants" in "\\n".join([e["value"]] + (e["body"] or [])) for e in runs):\n', ' if not any(e["value"] == SHARDED for e in runs):\n', SHARDS_TESTS), ("the shards are not held at all", "tools/check_workflows.py", - " problems += _commits(text, entries, jobs) + _shards(entries, jobs)\n", - " problems += _commits(text, entries, jobs)\n", SHARDS_TESTS), + " problems += _commits(text, entries, jobs) + _shards(entries, jobs) + _legs(entries, jobs)\n", + " problems += _commits(text, entries, jobs) + _legs(entries, jobs)\n", SHARDS_TESTS), + # The legs of `make verify` (2026-11-09: one job was cut at its step's 26 minutes in a merge group). + ("the legs are not held at all", "tools/check_workflows.py", + " problems += _commits(text, entries, jobs) + _shards(entries, jobs) + _legs(entries, jobs)\n", + " problems += _commits(text, entries, jobs) + _shards(entries, jobs)\n", LEGS_TESTS), + ("a leg may be told its place by a number", "tools/check_workflows.py", + ' if told != {"CHECKS_SHARD": INDEX, "CHECKS_SHARDS": TOTAL}:\n', " if False:\n", LEGS_TESTS), + ("the legs' matrix may be any key", "tools/check_workflows.py", + ' if matrix != ["shard"]:\n', " if False:\n", LEGS_TESTS), + ("the legs' job may have other keys", "tools/check_workflows.py", + " if keys != SHARD_KEYS:\n", " if False:\n", LEGS_TESTS), + ("the legs' job may be granted more", "tools/check_workflows.py", + ' if granted != {"contents": "read"}:\n', " if False:\n", LEGS_TESTS), + ("the legs' steps may be anything", "tools/check_workflows.py", + " and checkouts == [{\"persist-credentials\": \"false\", \"fetch-depth\": \"0\"}]):\n", + " and checkouts == [{\"persist-credentials\": \"false\", \"fetch-depth\": \"0\"}]) and False:\n", LEGS_TESTS), + ("no job need run make verify", "tools/check_workflows.py", + " if not any_leg:\n", " if False:\n", LEGS_TESTS), + ("a leg holds every test", "tools/verify.py", + " return unittest.TestSuite(_tests(whole)[index::count])\n", " return whole\n", LEG_TESTS), + ("a leg skips one test in each count", "tools/verify.py", + " return unittest.TestSuite(_tests(whole)[index::count])\n", + " return unittest.TestSuite(_tests(whole)[index::count + 1])\n", LEG_TESTS), + ("a leg outside the count runs", "tools/verify.py", + " if not 0 <= int(index) < int(count):\n", " if False:\n", LEG_TESTS), + ("one of the pair alone is no leg", "tools/verify.py", + " if index is None and count is None:\n", " if index is None or count is None:\n", LEG_TESTS), + ("a leg CI names is not read", "tools/verify.py", + ' sys.exit(main(sys.argv[1:], os.environ))', ' sys.exit(main(sys.argv[1:]))', LEG_TESTS), ("no job need run the mutants", "tools/check_workflows.py", " if not found:\n whys.append(\"has no job that runs the mutants", " if False:\n whys.append(\"has no job that runs the mutants", SHARDS_TESTS), @@ -8393,7 +8436,7 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ("the bar of the cases that expect a Finding is lower", 'harness/eval/__init__.py', 'BAR_OF_A_KIND = {"met": (9, 10), "finding": (3, 10)}', 'BAR_OF_A_KIND = {"met": (9, 10), "finding": (29, 100)}', ("tests.test_evals.EvalJob.test_each_kind_is_held_to_its_own_bar_where_the_cases_are",)), ("the bar of the clean cases is higher", 'harness/eval/__init__.py', 'BAR_OF_A_KIND = {"met": (9, 10), "finding": (3, 10)}', 'BAR_OF_A_KIND = {"met": (91, 100), "finding": (3, 10)}', ("tests.test_evals.EvalJob.test_each_kind_is_held_to_its_own_bar_where_the_cases_are",)), ("the bar of the cases that expect a Finding is higher", 'harness/eval/__init__.py', 'BAR_OF_A_KIND = {"met": (9, 10), "finding": (3, 10)}', 'BAR_OF_A_KIND = {"met": (9, 10), "finding": (31, 100)}', ("tests.test_evals.EvalJob.test_each_kind_is_held_to_its_own_bar_where_the_cases_are",)), - ("exactly on the bar does not clear it", 'harness/eval/__init__.py', 'return passed * bar[1] >= of * bar[0]', 'return passed * bar[1] > of * bar[0]', ("tests.test_evals.EvalJob.test_the_gate_holds_each_kinds_bar_and_a_fault_still_closes_it",)), + ("exactly on the bar does not clear it", 'harness/eval/__init__.py', 'return passed * bar[1] >= of * bar[0]', 'return passed * bar[1] > of * bar[0]', ("tests.test_evals.EvalJob.test_each_kind_is_held_to_its_own_bar_where_the_cases_are",)), ("the set's share is not held", 'harness/eval/__init__.py', 'if passed < least_passed(found["count"], KIND_LEAST if document["split_ok"] is True else 0) \\\n or (kinds', 'if (kinds', ("tests.test_evals.EvalJob.test_the_gate_holds_each_kinds_bar_and_a_fault_still_closes_it",)), @@ -13248,7 +13291,7 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ('P-58 may write OUT inside the repo', "tools/trial_p58_edits.sh", 'case "$out/" in "$root"/*)', 'case "$out/" in /nowhere/*)', ('tests.test_tools.EditsReadTrial.test_each_refusal_comes_before_anything_is_minted',)), ("P-58's OUT is read without its links", "tools/trial_p58_edits.sh", '2>/dev/null && pwd -P); then', '2>/dev/null && pwd); then', ('tests.test_tools.EditsReadTrial.test_an_output_folder_reached_through_a_link_into_the_repo_is_refused',)), ("P-58's raw answers may be kept inside the repo", "tools/trial_p58_edits.sh", 'case "$scratch/" in "$root"/*)', 'case "$scratch/" in /nowhere/*)', ('tests.test_tools.EditsReadTrial.test_a_scratch_directory_inside_the_repo_is_refused_before_anything_is_minted',)), - ("P-58 keeps the first head's status", "tools/trial_p58_edits.sh", 'head && $0 != ""', 'NR == 1 && $0 != ""', ('tests.test_tools.EditsReadTrial.test_the_status_kept_is_the_last_heads',)), + ("P-58 keeps the first head's status", "tools/trial_p58_edits.sh", 'head && $0 != ""', 'NR == 1 && $0 != ""', ('tests.test_tools.ReachScript.test_a_head_ends_where_the_next_line_is_no_head',)), ("P-56 mints for another App's installation", "tools/trial_p56_comments.sh", '["apps"]["controller"]', '["apps"]["reader"]', ("tests.test_tools.CommentsReadTrial.test_each_read_is_made_with_each_token_kept_with_its_status_and_no_token_is_kept",)), ("P-56 keeps a read without its redaction", "tools/trial_p56_comments.sh", ' (cd "$root" && python3 -m tools.redact_trial < "$scratch/raw-$n" > "$out/clean-$n") 2> "$out/redacted-$n"\n', @@ -15541,7 +15584,7 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: # method by method, shared among its workers, each part one run held to TIMEOUT, never the whole suite, which outgrew # TIMEOUT on a CI runner (issue 396). A shard runs a disjoint share of what every shard's mutants name, so that its # baseline shrinks as shards are added (issue 398). -# The whole suite, which a mutant runs once the tests it names pass, runs in parts of at most PART_MOST tests dealt in +# The whole suite, which an edit tools/untested.py tries runs once the tests it names pass, runs in parts of at most PART_MOST tests dealt in # turn, one after another, each one run held to TIMEOUT, so that no run grows with the suite (issue 398). On a loaded # desk, where the suite took 940 s, its 1,311 tests made 14 parts of 9 to 124 s. # A run filtered by path, `python3 -m tools.mutants PATH...`, is the one local run allowed (the Maintainer, 2026-10-19, @@ -15564,6 +15607,8 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: NAMED_SCAN = 200 # the lines after a failure's head its error is looked for in # Said of a failure not shown to be the mutant's: one more run of the suite on the mutant decides. UNCONFIRMED = "unconfirmed" +# Why a mutant the tests it names pass on is an ERROR: the whole suite does not run to find the test that kills it. +NOT_KILLED = "named_test_does_not_kill: %s passed on it; name the test that fails on it" + AGAIN # How a FLAKY's reason ends. FLAKY fails the run as an ERROR does: nobody has shown that a test notices the mutant. MEND = "; mark or mend that test, then run python3 -m tools.mutants again" @@ -15815,16 +15860,18 @@ def lent(self): STAMPS = iter(range(1, 2 ** 31)) # next() on it is one step, whichever worker asks -def attempt(mutant: tuple, root: pathlib.Path, run, trees: Trees = None) -> tuple: +def attempt(mutant: tuple, root: pathlib.Path, run, trees: Trees = None, alone: bool = False) -> tuple: """Break one rule in a worker's copy of `root` and run the tests there: (outcome, why). A failure kills the mutant only once `confirm` shows it is the mutant's. One that is not runs the suite once more on the mutant: a pass is SURVIVED, and a second failure that is not the mutant's is FLAKY. However the runs end, - the file's own bytes are written back and read again before the copy is lent to another mutant. + the file's own bytes are written back and read again before the copy is lent to another mutant. `alone`, as the + mutant run gives it, holds a mutant to the tests it names: a pass of them is an ERROR, NOT_KILLED, and the whole + suite never runs after them; tools/untested.py's edits run it after their tests, as a mutant naming none does. """ if trees is None: with tempfile.TemporaryDirectory() as where: - return attempt(mutant, root, run, Trees(root, where)) + return attempt(mutant, root, run, Trees(root, where), alone) name, rel, old, new, *first = mutant with trees.lent() as work: target = work / rel @@ -15859,9 +15906,16 @@ def unbroken() -> None: # A mutant a decision's Boundary row names is held to the tests it lists, alone: one any other test kills # would let the row's test look like the one that notices the break (issue 590). named = ({"only": first[0]} if name in _held_by_rows(root) else {"first": first[0]}) if first else {} - for _ in range(2): + # A mutant the tests it names miss ran the whole suite after them, 10 to 21 minutes of a 27-minute leg, and + # hid the wrong name (2026-11-09): run alone, it is told as an ERROR instead. + if alone and first: + named = {"only": first[0]} + for again in range(2): status, output = run(work, **named) outcome, why = verdict(status, output) + # A pass after a failure not shown to be the mutant's is a flaky test, not a wrong name. + if outcome == "SURVIVED" and alone and first and not again: + return "ERROR", NOT_KILLED % ", ".join(first[0]) if outcome != "killed": return outcome, why outcome, why = confirm(output, work, run, broken, unbroken) @@ -15887,7 +15941,7 @@ def _held_by_rows(root: pathlib.Path) -> set: def guarded(mutant: tuple, root: pathlib.Path, run, trees: Trees = None) -> tuple: """`attempt`, with anything it raises made an ERROR for this mutant, so that the others still report.""" try: - return attempt(mutant, root, run, trees) + return attempt(mutant, root, run, trees, alone=True) except Exception as err: # a failed copy, read or run; never a kill return "ERROR", "the run failed with %s: %s%s" % (type(err).__name__, err, AGAIN) diff --git a/tools/verify.py b/tools/verify.py index 4cbf8775..fab515b7 100644 --- a/tools/verify.py +++ b/tools/verify.py @@ -3,10 +3,16 @@ `make fast` runs the same checks and every test but the seeded simulation. `make checks` runs the checks alone, half a minute, and is what a session's hooks run after an edit and before a stop (`tools/hooks/after_edit.py`, `tools/hooks/before_stop.py`); the tests are CI's, and each builder's own `make verify`. + +CI runs `make verify` in legs, each told CHECKS_SHARD and CHECKS_SHARDS by its matrix: each leg runs every check and its +share of the tests, so that across the legs every test runs once and no leg nears the step's limit. Neither set: every +test, as at a desk. """ from __future__ import annotations +import os import pathlib +import re import sys import unittest @@ -59,15 +65,39 @@ def _modules(tests) -> set: return found -def suite(fast: bool) -> unittest.TestSuite: - """Every test module under tests/, at any depth; when fast, all but those in SLOW.""" +def _tests(tests) -> list: + return [test for item in tests for test in _tests(item)] if isinstance(tests, unittest.TestSuite) else [tests] + + +def suite(fast: bool, share: tuple = None) -> unittest.TestSuite: + """Every test module under tests/, at any depth; when fast, all but those in SLOW. Given `share`, (a leg, the count + of legs), the tests of the whole one by one, each `count`-th from the leg's place, so that the legs hold each once.""" whole = unittest.defaultTestLoader.discover(str(ROOT / "tests"), top_level_dir=str(ROOT)) + if share is not None: + index, count = share + return unittest.TestSuite(_tests(whole)[index::count]) if not fast: return whole slow = {"tests." + name for name in SLOW} return unittest.TestSuite(item for item in whole if not _modules(item) & slow) +def leg(environ) -> tuple: + """(this leg, the count of legs) from CHECKS_SHARD and CHECKS_SHARDS, which CI's matrix sets, counted from 0; None + when neither is set. Raises ValueError for one without the other, a value that is not a whole number, or a leg + outside the count, which could leave tests in no leg.""" + index, count = environ.get("CHECKS_SHARD"), environ.get("CHECKS_SHARDS") + if index is None and count is None: + return None + if not all(isinstance(said, str) and re.fullmatch(r"[0-9]+", said) for said in (index, count)): + raise ValueError("CHECKS_SHARD is %r and CHECKS_SHARDS is %r; set both, to whole numbers, or neither" + % (index, count)) + if not 0 <= int(index) < int(count): + raise ValueError("CHECKS_SHARD is %s and CHECKS_SHARDS is %s; the leg is counted from 0, below the count" + % (index, count)) + return int(index), int(count) + + def notes() -> list: """What `verify` says of the records before the tests: what they were compared with, and how many Boundary rows name no mutant yet (issue 590).""" @@ -75,16 +105,23 @@ def notes() -> list: "%-22s %s" % ("boundary mutants", check_decisions.unnamed())] -def main(argv: list) -> int: +def main(argv: list, environ=None) -> int: + try: + share = None if "--checks" in argv or "--fast" in argv else leg(environ or {}) + except ValueError as err: + print(err) + return 1 checks_failed = failed(CHECKS) if "--checks" in argv: return 1 if checks_failed else 0 for line in notes(): print(line) unverified(check_trials.unverified()) - result = unittest.TextTestRunner(verbosity=1).run(suite("--fast" in argv)) + if share is not None: + print("%-22s leg %d of %d counted from 0" % ("tests", share[0], share[1])) + result = unittest.TextTestRunner(verbosity=1).run(suite("--fast" in argv, share)) return 1 if checks_failed or not result.wasSuccessful() else 0 if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) + sys.exit(main(sys.argv[1:], os.environ))