diff --git a/.coveragerc b/.coveragerc index 506e627..ef079d6 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,13 +1,17 @@ # Coverage over the gate scripts, measured by the unit-test suite in -# scripts/tests. The positive controls in scripts/tests/controls.py exercise MOST +# scripts/tests. The positive controls in scripts/tests/controls.py exercise most # gates end to end as a subprocess, which this figure does not see — so the -# number below is the floor under the UNIT tier specifically. +# number in scripts/tests/run.py is the floor under the UNIT tier specifically, +# and it is not a floor under the correctness of a gate the controls cover. # -# Most, not every. controls.py exempts the gates that reach a chart registry or an -# API, and prints the split on every run. Those gates therefore have neither unit -# coverage nor a behavioural control, and they are the largest ones — so the -# figure below is not a floor under their correctness at all, and reading it as -# one would be reading a number measured over a different population. +# Most, not every: controls.py exempts the gates whose input arrives over the +# network, and prints the split on every run. For those a unit test is the only +# proof available, so run.py asserts that each one carries unit coverage, reading +# the exemption list out of controls.py rather than repeating it. +# +# The two tiers answer different questions and neither substitutes for the other. +# A control proves a gate rejects a supplied violation; a unit test proves the +# gate computes the right verdict on a case the real tree does not contain. [run] source = ./scripts omit = diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8f3736..f894100 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -254,6 +254,14 @@ jobs: # the tree rather than a list, so the class stays closed as gates are added. - name: No gate reports success over an absent corpus run: ./scripts/tests/empty-corpus.py + # A control proves a GATE rejects a planted defect. This proves the TESTS + # reject a reverted behaviour, and requires the assertion that catches each + # one to be the assertion that names it — a non-zero exit says the suite + # noticed something, not that the test describing the behaviour is what + # noticed. It runs here rather than by hand because it is offline and its + # cost is one test module per probe — no render, no registry, no cluster. + - name: Reverted gate behaviour is named by the test that holds it + run: ./scripts/tests/reverify-tests.sh # Prose that names a path, a task target or a file:line is a claim about # the tree, and nothing else keeps those claims true as the tree moves. - name: Named things in prose resolve diff --git a/Taskfile.yaml b/Taskfile.yaml index adaa816..c81e1ce 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -118,6 +118,16 @@ tasks: cmds: - ./scripts/tests/empty-corpus.py + validate:reverify-tests: + desc: "Mutation harness — revert a gate behaviour and require the suite to name it" + cmds: + - ./scripts/tests/reverify-tests.sh + + reverify:gates: + desc: "Tier-1 re-verification — plant a defect per gate and read each tool's own exit status (slow; runs `task validate` twice)" + cmds: + - ./scripts/tests/reverify-gates.sh + validate:policy-validity: desc: "Policy-validity gate — every rendered policy overlay is one Kyverno will accept" cmds: @@ -200,6 +210,7 @@ tasks: - validate:gate-tests - validate:gate-controls - validate:empty-corpus + - validate:reverify-tests - validate:named-things - validate:ai-config - validate:workflows diff --git a/scripts/check-image-pins.py b/scripts/check-image-pins.py index d6f97d7..8599548 100755 --- a/scripts/check-image-pins.py +++ b/scripts/check-image-pins.py @@ -142,6 +142,49 @@ def classify(ref: str) -> str: return "mutable" if tag.lower() in MUTABLE_TAGS else "tag" +def bare_name(ref: str) -> str: + """A reference with its tag removed, which is what an exemption names. + + Split on the last colon only when it sits in the final path segment: a + registry with a port (`registry:5000/x/y`) carries a colon that is not a tag + separator, and cutting there would produce a key no exemption can match and + no reader can recognise. + """ + name = ref.rsplit("/", 1)[-1] + return ref.rsplit(":", 1)[0] if ":" in name else ref + + +def verdict(images: dict[str, set[str]], allowed: dict[str, str]) -> list[str]: + """Every image-pin problem, over an inventory and an exemption list. + + Two directions, and the second is the one that rots. A mutable reference with + no exemption is the defect the gate exists for. An exemption the fleet no + longer renders mutably is a description that outlived its reason, and an + exemption list nobody re-checks only ever widens. + """ + failures = [] + mutable_seen: set[str] = set() + + for ref in sorted(images): + if classify(ref) != "mutable": + continue + bare = bare_name(ref) + mutable_seen.add(bare) + if bare in allowed: + continue + failures.append( + f"{ref} (via {', '.join(sorted(images[ref]))}) resolves to a moving target. " + f"Pin it in the addon's values.yaml to the chart's appVersion or a digest.") + + for bare, reason in sorted(allowed.items()): + if bare not in mutable_seen: + failures.append( + f"{bare} is on the mutable-tag exemption list but the fleet no longer " + f"renders it mutably — the exemption outlived its reason. Delete it. " + f"(recorded: {reason[:100]})") + return failures + + def main() -> int: ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--list", action="store_true", help="print the image inventory") @@ -165,27 +208,7 @@ def main() -> int: print(f" {path}: {err}") return 2 - failures = [] - mutable_seen: set[str] = set() - - for ref in sorted(images): - if classify(ref) != "mutable": - continue - name = ref.rsplit("/", 1)[-1] - bare = ref.rsplit(":", 1)[0] if ":" in name else ref - mutable_seen.add(bare) - if bare in ALLOWED_MUTABLE: - continue - failures.append( - f"{ref} (via {', '.join(sorted(images[ref]))}) resolves to a moving target. " - f"Pin it in the addon's values.yaml to the chart's appVersion or a digest.") - - for bare, reason in sorted(ALLOWED_MUTABLE.items()): - if bare not in mutable_seen: - failures.append( - f"{bare} is on the mutable-tag exemption list but the fleet no longer " - f"renders it mutably — the exemption outlived its reason. Delete it. " - f"(recorded: {reason[:100]})") + failures = verdict(images, ALLOWED_MUTABLE) # Reported whatever the verdict: a chart that did not render contributed no # images, and counting the rest as the whole fleet is how a partial scan diff --git a/scripts/check-log-volume-budget.py b/scripts/check-log-volume-budget.py index edc79bf..fafe943 100755 --- a/scripts/check-log-volume-budget.py +++ b/scripts/check-log-volume-budget.py @@ -172,6 +172,53 @@ def alert_threshold() -> float | None: return None +def environment_verdict(cfg: dict, warn: float | None, + rel: str) -> tuple[list[str], bool]: + """Everything wrong with one environment's rendered Loki config. + + Separate from the render because the render reaches a chart repository and + this does not. The three assertions are what decide the gate's outcome, so + they are reachable with a config supplied directly rather than only through a + network round trip. + + Returns the problems found and whether the alert leads a declared cutoff — + the second is what the closing line counts, so an environment that failed the + comparison is not also reported as one the comparison covered. + """ + wal = ((cfg.get("ingester") or {}).get("wal") or {}) + limits = cfg.get("limits_config") or {} + comp = cfg.get("compactor") or {} + problems: list[str] = [] + leads = False + + cutoff = wal.get("disk_full_threshold") + if cutoff is None: + problems.append( + f"{rel}: the render sets no ingester.wal.disk_full_threshold, so the " + f"fraction at which Loki stops accepting every push is an upstream " + f"default. The alert's lead time then depends on a number that can " + f"move under a chart bump with nothing here to compare against.") + elif warn is not None and not warn < float(cutoff): + problems.append( + f"{rel}: the fill alert fires at {warn} but ingestion stops at " + f"{cutoff}. There is no window in which to act — and no remedy is " + f"fast: a retention cut must sync, wait a compaction interval, then " + f"clear retention_delete_delay before a byte is freed, and the volume " + f"cannot be grown through this repo at all.") + else: + leads = True + + if limits.get("retention_period") is None: + problems.append( + f"{rel}: the render sets no limits_config.retention_period. Nothing " + f"then deletes on a schedule, and the volume reaches the cutoff.") + if comp.get("retention_enabled") is not True: + problems.append( + f"{rel}: compactor.retention_enabled is not true, so retention_period " + f"deletes nothing however it is set and the cutoff arrives regardless.") + return problems, leads + + def main() -> int: repo, version = chart_pin() warn = alert_threshold() @@ -183,32 +230,11 @@ def main() -> int: checked = 0 for env in envs: cfg = render(repo, version, env) - wal = ((cfg.get("ingester") or {}).get("wal") or {}) - limits = cfg.get("limits_config") or {} - comp = cfg.get("compactor") or {} rel = f"addons/observability/loki/values-{env}.yaml" - - cutoff = wal.get("disk_full_threshold") - if cutoff is None: - fail(f"{rel}: the render sets no ingester.wal.disk_full_threshold, so the " - f"fraction at which Loki stops accepting every push is an upstream " - f"default. The alert's lead time then depends on a number that can " - f"move under a chart bump with nothing here to compare against.") - elif warn is not None and not warn < float(cutoff): - fail(f"{rel}: the fill alert fires at {warn} but ingestion stops at " - f"{cutoff}. There is no window in which to act — and no remedy is " - f"fast: a retention cut must sync, wait a compaction interval, then " - f"clear retention_delete_delay before a byte is freed, and the volume " - f"cannot be grown through this repo at all.") - else: - checked += 1 - - if limits.get("retention_period") is None: - fail(f"{rel}: the render sets no limits_config.retention_period. Nothing " - f"then deletes on a schedule, and the volume reaches the cutoff.") - if comp.get("retention_enabled") is not True: - fail(f"{rel}: compactor.retention_enabled is not true, so retention_period " - f"deletes nothing however it is set and the cutoff arrives regardless.") + problems, leads = environment_verdict(cfg, warn, rel) + for problem in problems: + fail(problem) + checked += 1 if leads else 0 if failures: for f in failures: diff --git a/scripts/render-addons.py b/scripts/render-addons.py index 54e1231..ea063d7 100755 --- a/scripts/render-addons.py +++ b/scripts/render-addons.py @@ -239,14 +239,25 @@ def discover() -> list[Unit]: ) +def registry_answered(err: str) -> bool: + """Whether helm's complaint says the registry replied and refused the pin. + + Separate from the exit path so the two worlds can be told apart without + running helm. A message naming both a missing chart and a broken connection + is the second: an unreachable registry cannot testify about what it holds. + """ + low = err.lower() + return (any(t in low for t in _NOT_FOUND) + and not any(t in low for t in _UNREACHABLE)) + + def helm_or_exit(cmd: list[str], what: str) -> subprocess.CompletedProcess: """Run a helm command; on failure exit with the RIGHT kind of complaint.""" proc = subprocess.run(cmd, capture_output=True, text=True, timeout=NETWORK_TIMEOUT) if proc.returncode == 0: return proc err = ((proc.stderr or "") + (proc.stdout or "")).strip() - low = err.lower() - if any(t in low for t in _NOT_FOUND) and not any(t in low for t in _UNREACHABLE): + if registry_answered(err): print(f"{what}: the registry answered and the pinned chart is not there.") print(err) print("This is a pin that does not resolve — a finding about this repo.") diff --git a/scripts/tests/controls.py b/scripts/tests/controls.py index e459819..8d5bf41 100755 --- a/scripts/tests/controls.py +++ b/scripts/tests/controls.py @@ -239,12 +239,15 @@ def dotted(node) -> str | None: # The harnesses, which cannot control themselves: a suite asserting its own -# ability to reject would be the thing under test and the thing testing it. Both -# are covered instead by their own self-tests, which run on every ordinary +# ability to reject would be the thing under test and the thing testing it. Each +# is covered instead by its own self-assertions, which run on every ordinary # invocation. Asserted like every other exemption — an entry naming a file that -# no longer exists fails. +# no longer exists fails, a present executable on no list fails, and an entry +# nothing in the repo invokes fails. NOT_GATES = { - "tests/controls.py": "the control harness; self_test() runs on every invocation", + "tests/controls.py": "the control harness; self_test() runs on every " + "invocation, and test_controls.py holds the reader " + "this rule asks", "tests/run.py": "the unit-test runner; asserts its own module list and floors", "tests/reverify-gates.sh": "the Tier-1 re-verification harness; it drives the " "gates rather than checking the tree, and asserts " @@ -252,6 +255,10 @@ def dotted(node) -> str | None: "tests/empty-corpus.py": "the vacuity harness; it runs every gate against an " "emptied corpus rather than checking the tree, and " "asserts its own probe floor and exemptions", + "tests/reverify-tests.sh": "the unit-test re-verification harness; it reverts a " + "gate behaviour and requires the suite to name it, " + "asserts the tree is green before and after, and " + "carries its own probe floor", } @@ -606,31 +613,186 @@ def run_gate(gate: str, cwd: pathlib.Path) -> subprocess.CompletedProcess: return subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, timeout=GATE_TIMEOUT) -def check_vacuity() -> list[str]: - """The suite cannot shrink quietly, and no exemption may match nothing.""" +def caller_files(root: pathlib.Path) -> list[pathlib.Path]: + """The files that decide what runs: the Taskfile and every workflow.""" + return [p for p in (root / "Taskfile.yaml", + *sorted((root / ".github" / "workflows").glob("*.y*ml"))) + if p.is_file()] + + +def _task_commands(task) -> list[str]: + """The command strings one Taskfile task executes. + + `{task: other}` is not one: it names another task, whose own commands are + read when that task is walked. `{cmd: ...}` and `{defer: ...}` are. + """ + if isinstance(task, str): + return [task] + if isinstance(task, list): + entries = task + elif isinstance(task, dict): + entries = task.get("cmds") or [] + else: + return [] + out = [] + for entry in entries: + if isinstance(entry, str): + out.append(entry) + elif isinstance(entry, dict): + for key in ("cmd", "defer"): + if isinstance(entry.get(key), str): + out.append(entry[key]) + return out + + +def caller_commands(root: pathlib.Path) -> tuple[list[str], list[str]]: + """(every command the callers execute, callers that would not parse). + + Taken from the parsed documents at the positions a runner reads a command + from — a task's `cmds:` entries, a workflow step's `run:` script. Every + other string in either file is prose as far as this reader is concerned, + and a comment is not in the parse at all. + + `status:` and `preconditions:` hold commands too and are deliberately + absent. They decide whether a task's work runs; a harness reached only + through one of them asserts nothing about the tree. + + A caller that will not parse is returned as its own fact rather than as a + caller with no commands in it. Those are the same value to `any()` and + different facts to a reader: one says nothing runs the harness, the other + says this file could not be asked. + """ + import yaml + + commands: list[str] = [] + unreadable: list[str] = [] + for path in caller_files(root): + try: + doc = yaml.safe_load(path.read_text()) + except yaml.YAMLError as exc: + detail = str(exc).strip().splitlines() + unreadable.append(f"{path.relative_to(root)}: " + f"{detail[0] if detail else exc.__class__.__name__}") + continue + if not isinstance(doc, dict): + continue + tasks = doc.get("tasks") + if isinstance(tasks, dict): + for task in tasks.values(): + commands += _task_commands(task) + jobs = doc.get("jobs") + if isinstance(jobs, dict): + for job in jobs.values(): + if not isinstance(job, dict): + continue + for step in job.get("steps") or []: + if isinstance(step, dict) and isinstance(step.get("run"), str): + commands.append(step["run"]) + return commands, unreadable + + +def command_words(command: str) -> list[str]: + """The words `command` would execute, shell comments and quoting resolved. + + A second parse, for the reason the first one happened: inside a `run: |` + block the runner is a shell, and a `#` comment there is prose exactly as a + YAML comment is. Quoting is read too, so a path inside a quoted string is a + word of that string rather than a word of the command. + """ + import shlex + + text = blank_comments(command) + try: + return shlex.split(text) + except ValueError: + # An unbalanced quote leaves no argv to read. Splitting on whitespace is + # a weaker view — it cannot tell a word from part of a quoted string — + # and it is named as weaker here rather than handed back as the same + # answer. + return text.split() + + +def invoked_anywhere(rel: str, root: pathlib.Path = ROOT) -> bool: + """Whether a command the task runner or a workflow executes runs `scripts/`. + + Asked of the commands, not of the files. The callers are parsed, the + commands are taken from the positions a runner takes them from, and each + command is split into the words it would execute. A comment naming the + path, a `desc:` describing it and a quoted mention inside a command reach + none of those — which is the point, because this rule exists to reject a + harness nothing runs and text naming a harness is the cheapest way to look + like running one. + + Two limits, both toward reporting a harness as UN-invoked. A path assembled + at run time from a variable is not resolved, so a caller that built one + fails here. And a word this finds is not proved to be the command's + executable — an unquoted `echo scripts/x.sh` counts — so what it separates + is prose from command text, not one word of a command from another. + """ + wanted = {f"scripts/{rel}", f"./scripts/{rel}"} + commands, _unreadable = caller_commands(root) + return any(word in wanted for command in commands for word in command_words(command)) + + +def vacuity_problems(present: set[str], controls, network, not_gates, + invoked=None) -> list[str]: + """The list rules, over lists supplied rather than read off the tree. + + Separated from check_vacuity so each rule can be planted against: every one + is a statement about which name appears on which list, and supplying the + lists is how a violation is introduced without editing the repository the + rest of this file is running inside. + """ + invoked = invoked_anywhere if invoked is None else invoked problems = [] - present = set(gate_files()) - for name, reason in sorted(NOT_GATES.items()): + for name, reason in sorted(not_gates.items()): if name not in present: problems.append( f"{name} is exempted as a harness rather than a gate, but no such " f"executable exists under scripts/ — the exemption outlived its file. " f"(recorded: {reason})") + continue + # A harness excuses itself by asserting its own outcome on every ordinary + # invocation, which is a claim about a thing that gets invoked. One + # nothing runs makes the exemption an excuse for an executable that never + # executes, and its recorded reason is free text nothing else checks. + if not invoked(name): + problems.append( + f"{name} is exempted as a harness that asserts its own outcome, but " + f"neither Taskfile.yaml nor a workflow under .github/workflows/ runs " + f"it. A harness nothing invokes asserts nothing. (recorded: {reason})") for gate in sorted(present): - if gate in NOT_GATES: + if gate in not_gates: continue - if gate not in CONTROLS and gate not in NEEDS_NETWORK: + if gate not in controls and gate not in network: problems.append( f"{gate} ships no positive control and is on no exemption list. A gate " f"nobody has shown to fail is an untested assertion about the tree.") - for gate in sorted(set(CONTROLS) | set(NEEDS_NETWORK)): + for gate in sorted(set(controls) | set(network)): if gate not in present: problems.append( f"{gate} is named by a control or an exemption but no longer exists in " f"scripts/ — the reference outlived the gate.") + return problems + + +def check_vacuity() -> list[str]: + """The suite cannot shrink quietly, and no exemption may match nothing.""" + present = set(gate_files()) + problems = vacuity_problems(present, CONTROLS, NEEDS_NETWORK, NOT_GATES) + + # The harness-invocation rule reads what the callers run. A caller that will + # not parse runs nothing as far as that rule can tell, and every harness in + # it then reads as one nobody invokes — which points the reader at the + # exemption list instead of at the one file to fix. + for unreadable in caller_commands(ROOT)[1]: + problems.append( + f"{unreadable} — this file decides what runs, and it could not be " + f"parsed. The harness-invocation rule examined nothing in it, which " + f"is not the same as it invoking nothing.") for gate, call in sorted(NEEDS_NETWORK_PY.items()): p = SCRIPTS / gate @@ -782,6 +944,41 @@ def self_test() -> int: bad += 0 if ok else 1 print() + # check_vacuity's own verdicts, which decide this file's exit code and which + # nothing else reaches. Each is a rule about a LIST \u2014 a gate on no list, an + # exemption naming a file that is gone, a harness nothing runs \u2014 so each is + # planted by supplying the lists rather than by editing the tree. + # + # This is the shape one level down from the one the harness-invocation rule + # was added to reject: a branch that is correct and asserted by nobody. The + # NOT_GATES reason for this file says its self-test runs on every invocation, + # and until these cases existed that sentence covered everything except the + # branch it was written beside. + print("\u2500\u2500 Vacuity-rule self-test \u2500\u2500") + vacuity_cases: list[tuple[str, set[str], set[str], dict, dict, str | None]] = [ + ("a gate on no list at all", + {"check-x.py"}, set(), {}, {}, "ships no positive control"), + ("a control naming a gate that is gone", + set(), {"check-x.py"}, {}, {}, "the reference outlived the gate"), + ("an exemption naming a harness that is gone", + set(), set(), {}, {"tests/gone.py": "a reason"}, + "the exemption outlived its file"), + ("a harness nothing invokes", + {"tests/gone.py"}, set(), {}, {"tests/gone.py": "a reason"}, + "A harness nothing invokes asserts nothing"), + ("a gate covered by a control", + {"check-x.py"}, {"check-x.py"}, {}, {}, None), + ] + for name, present_, controls_, network_, not_gates_, expect_ in vacuity_cases: + found = vacuity_problems(present_, controls_, network_, not_gates_, + invoked=lambda _name: False) + shown = " ".join(found) + ok = (not found) if expect_ is None else any(expect_ in p for p in found) + print(f" {'ok ' if ok else 'FAIL'} {name}: " + f"{(shown[:70] + chr(8230)) if shown else 'accepted'}") + bad += 0 if ok else 1 + print() + print("\u2500\u2500 Mutation-contract self-test \u2500\u2500") for case, before, after, disk, marker, expect in cases: why = mutation_landed("fixture.yaml", before, after, disk, marker) diff --git a/scripts/tests/reverify-tests.sh b/scripts/tests/reverify-tests.sh new file mode 100755 index 0000000..55f68ab --- /dev/null +++ b/scripts/tests/reverify-tests.sh @@ -0,0 +1,593 @@ +#!/usr/bin/env bash +# Revert one gate behaviour at a time and require the suite to NAME it. +# +# scripts/tests/reverify-gates.sh proves a GATE rejects a planted defect. This +# proves the TESTS reject a reverted behaviour, which is the other half and the +# one a passing suite cannot supply on its own: a test that asserts what it just +# constructed passes forever, and so does a test whose subject was quietly +# rewritten underneath it. +# +# Each probe below names the test ids that must fail. Requiring a non-zero exit +# alone is not enough — it proves the suite noticed something, not that the +# assertion which noticed is the one that describes the behaviour. A suite can +# catch a mutant by accident through an unrelated fixture, and then the mutant +# has demonstrated detection without demonstrating coverage. +set -uo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ROOT" +SP="$(mktemp -d)" + +# Backups live outside $SP, for the reason reverify-gates.sh records: between +# planting and restoring, the backup is the only copy of the original, and +# keeping it in the directory the cleanup removes means an interrupt destroys +# the means of restoration. +BK="$(mktemp -d)" +OUT="$SP/rt.out" + +MUT_FILES=() + +_slot() { printf '%s' "$1" | tr '/' '_'; } + +mut() { + local f="$1" s + s="$(_slot "$f")" + [ -e "$BK/$s" ] || cp "$f" "$BK/$s" + case " ${MUT_FILES[*]:-} " in + *" $f "*) ;; + *) MUT_FILES+=("$f") ;; + esac +} + +res() { + local f="$1" s + s="$(_slot "$f")" + [ -e "$BK/$s" ] && cp "$BK/$s" "$f" + return 0 +} + +restore_all() { + [ "${#MUT_FILES[@]}" -eq 0 ] && return 0 + local f + for f in "${MUT_FILES[@]}"; do res "$f"; done +} + +cleanup() { restore_all; rm -rf "$SP" "$BK"; } +trap cleanup EXIT +trap 'cleanup; exit 130' INT +trap 'cleanup; exit 143' TERM + +pass=0; fail=0 + +# Every module this harness plants against. Named rather than discovered: a +# module that stops being loaded would otherwise drop out of the clean-tree and +# restored-tree checks without either of them failing. +# The modules whose assertions this harness reverts behaviour against, plus the +# one holding the vacuity floors those same gates carry. check-platform-crs.py, +# check-policy-admission.py and validate-dashboards.py each assert a floor under +# their corpus AND the verdict over it; a mutant reverting the floor half is held +# by test_corpus_floors and by nothing in the list below, so without it that +# mutant reads as a miss when a test does in fact catch it. +ALL_MODULES="test_policy_admission test_platform_crs test_dashboards \ + test_render_addons test_image_pins test_log_volume_budget test_controls \ + test_corpus_floors" + +# Bytecode caching keys on (mtime, size). A restore and the next mutation inside +# the same second can hand the run the PREVIOUS mutant's module, and the suite +# then reports the previous mutant's failing test against this mutant's label — +# a green line naming the wrong assertion, which is worse than a red one. +_suite() { + rm -rf "$ROOT/scripts/__pycache__" "$ROOT/scripts/tests/__pycache__" + ( cd "$ROOT/scripts/tests" \ + && PYTHONDONTWRITEBYTECODE=1 python3 -m unittest "$@" ) >"$OUT" 2>&1 +} + +# rejects