Prove the gates a positive control cannot reach

.coveragerc

@@ -1,13 +1,17 @@
 # Coverage over the gate scripts, measured by the unit-test suite in
-# scripts/tests. The positive controls in scripts/tests/controls.py exercise MOST
+# scripts/tests. The positive controls in scripts/tests/controls.py exercise most
 # gates end to end as a subprocess, which this figure does not see — so the
-# number below is the floor under the UNIT tier specifically.
+# number in scripts/tests/run.py is the floor under the UNIT tier specifically,
+# and it is not a floor under the correctness of a gate the controls cover.
 #
-# Most, not every. controls.py exempts the gates that reach a chart registry or an
-# API, and prints the split on every run. Those gates therefore have neither unit
-# coverage nor a behavioural control, and they are the largest ones — so the
-# figure below is not a floor under their correctness at all, and reading it as
-# one would be reading a number measured over a different population.
+# Most, not every: controls.py exempts the gates whose input arrives over the
+# network, and prints the split on every run. For those a unit test is the only
+# proof available, so run.py asserts that each one carries unit coverage, reading
+# the exemption list out of controls.py rather than repeating it.
+#
+# The two tiers answer different questions and neither substitutes for the other.
+# A control proves a gate rejects a supplied violation; a unit test proves the
+# gate computes the right verdict on a case the real tree does not contain.
 [run]
 source = ./scripts
 omit =

.github/workflows/ci.yml

@@ -254,6 +254,14 @@ jobs:
       # the tree rather than a list, so the class stays closed as gates are added.
       - name: No gate reports success over an absent corpus
         run: ./scripts/tests/empty-corpus.py
+      # A control proves a GATE rejects a planted defect. This proves the TESTS
+      # reject a reverted behaviour, and requires the assertion that catches each
+      # one to be the assertion that names it — a non-zero exit says the suite
+      # noticed something, not that the test describing the behaviour is what
+      # noticed. It runs here rather than by hand because it is offline and its
+      # cost is one test module per probe — no render, no registry, no cluster.
+      - name: Reverted gate behaviour is named by the test that holds it
+        run: ./scripts/tests/reverify-tests.sh
       # Prose that names a path, a task target or a file:line is a claim about
       # the tree, and nothing else keeps those claims true as the tree moves.
       - name: Named things in prose resolve

Taskfile.yaml

@@ -118,6 +118,16 @@ tasks:
     cmds:
       - ./scripts/tests/empty-corpus.py
 
+  validate:reverify-tests:
+    desc: "Mutation harness — revert a gate behaviour and require the suite to name it"
+    cmds:
+      - ./scripts/tests/reverify-tests.sh
+
+  reverify:gates:
+    desc: "Tier-1 re-verification — plant a defect per gate and read each tool's own exit status (slow; runs `task validate` twice)"
+    cmds:
+      - ./scripts/tests/reverify-gates.sh
+
   validate:policy-validity:
     desc: "Policy-validity gate — every rendered policy overlay is one Kyverno will accept"
     cmds:
@@ -200,6 +210,7 @@ tasks:
       - validate:gate-tests
       - validate:gate-controls
       - validate:empty-corpus
+      - validate:reverify-tests
       - validate:named-things
       - validate:ai-config
       - validate:workflows

scripts/check-image-pins.py

@@ -142,6 +142,49 @@ def classify(ref: str) -> str:
     return "mutable" if tag.lower() in MUTABLE_TAGS else "tag"
 
 
+def bare_name(ref: str) -> str:
+    """A reference with its tag removed, which is what an exemption names.
+
+    Split on the last colon only when it sits in the final path segment: a
+    registry with a port (`registry:5000/x/y`) carries a colon that is not a tag
+    separator, and cutting there would produce a key no exemption can match and
+    no reader can recognise.
+    """
+    name = ref.rsplit("/", 1)[-1]
+    return ref.rsplit(":", 1)[0] if ":" in name else ref
+
+
+def verdict(images: dict[str, set[str]], allowed: dict[str, str]) -> list[str]:
+    """Every image-pin problem, over an inventory and an exemption list.
+
+    Two directions, and the second is the one that rots. A mutable reference with
+    no exemption is the defect the gate exists for. An exemption the fleet no
+    longer renders mutably is a description that outlived its reason, and an
+    exemption list nobody re-checks only ever widens.
+    """
+    failures = []
+    mutable_seen: set[str] = set()
+
+    for ref in sorted(images):
+        if classify(ref) != "mutable":
+            continue
+        bare = bare_name(ref)
+        mutable_seen.add(bare)
+        if bare in allowed:
+            continue
+        failures.append(
+            f"{ref} (via {', '.join(sorted(images[ref]))}) resolves to a moving target. "
+            f"Pin it in the addon's values.yaml to the chart's appVersion or a digest.")
+
+    for bare, reason in sorted(allowed.items()):
+        if bare not in mutable_seen:
+            failures.append(
+                f"{bare} is on the mutable-tag exemption list but the fleet no longer "
+                f"renders it mutably — the exemption outlived its reason. Delete it. "
+                f"(recorded: {reason[:100]})")
+    return failures
+
+
 def main() -> int:
     ap = argparse.ArgumentParser(description=__doc__)
     ap.add_argument("--list", action="store_true", help="print the image inventory")
@@ -165,27 +208,7 @@ def main() -> int:
             print(f"        {path}: {err}")
         return 2
 
-    failures = []
-    mutable_seen: set[str] = set()
-
-    for ref in sorted(images):
-        if classify(ref) != "mutable":
-            continue
-        name = ref.rsplit("/", 1)[-1]
-        bare = ref.rsplit(":", 1)[0] if ":" in name else ref
-        mutable_seen.add(bare)
-        if bare in ALLOWED_MUTABLE:
-            continue
-        failures.append(
-            f"{ref} (via {', '.join(sorted(images[ref]))}) resolves to a moving target. "
-            f"Pin it in the addon's values.yaml to the chart's appVersion or a digest.")
-
-    for bare, reason in sorted(ALLOWED_MUTABLE.items()):
-        if bare not in mutable_seen:
-            failures.append(
-                f"{bare} is on the mutable-tag exemption list but the fleet no longer "
-                f"renders it mutably — the exemption outlived its reason. Delete it. "
-                f"(recorded: {reason[:100]})")
+    failures = verdict(images, ALLOWED_MUTABLE)
 
     # Reported whatever the verdict: a chart that did not render contributed no
     # images, and counting the rest as the whole fleet is how a partial scan

scripts/check-log-volume-budget.py

@@ -172,6 +172,53 @@ def alert_threshold() -> float | None:
     return None
 
 
+def environment_verdict(cfg: dict, warn: float | None,
+                        rel: str) -> tuple[list[str], bool]:
+    """Everything wrong with one environment's rendered Loki config.
+
+    Separate from the render because the render reaches a chart repository and
+    this does not. The three assertions are what decide the gate's outcome, so
+    they are reachable with a config supplied directly rather than only through a
+    network round trip.
+
+    Returns the problems found and whether the alert leads a declared cutoff —
+    the second is what the closing line counts, so an environment that failed the
+    comparison is not also reported as one the comparison covered.
+    """
+    wal = ((cfg.get("ingester") or {}).get("wal") or {})
+    limits = cfg.get("limits_config") or {}
+    comp = cfg.get("compactor") or {}
+    problems: list[str] = []
+    leads = False
+
+    cutoff = wal.get("disk_full_threshold")
+    if cutoff is None:
+        problems.append(
+            f"{rel}: the render sets no ingester.wal.disk_full_threshold, so the "
+            f"fraction at which Loki stops accepting every push is an upstream "
+            f"default. The alert's lead time then depends on a number that can "
+            f"move under a chart bump with nothing here to compare against.")
+    elif warn is not None and not warn < float(cutoff):
+        problems.append(
+            f"{rel}: the fill alert fires at {warn} but ingestion stops at "
+            f"{cutoff}. There is no window in which to act — and no remedy is "
+            f"fast: a retention cut must sync, wait a compaction interval, then "
+            f"clear retention_delete_delay before a byte is freed, and the volume "
+            f"cannot be grown through this repo at all.")
+    else:
+        leads = True
+
+    if limits.get("retention_period") is None:
+        problems.append(
+            f"{rel}: the render sets no limits_config.retention_period. Nothing "
+            f"then deletes on a schedule, and the volume reaches the cutoff.")
+    if comp.get("retention_enabled") is not True:
+        problems.append(
+            f"{rel}: compactor.retention_enabled is not true, so retention_period "
+            f"deletes nothing however it is set and the cutoff arrives regardless.")
+    return problems, leads
+
+
 def main() -> int:
     repo, version = chart_pin()
     warn = alert_threshold()
@@ -183,32 +230,11 @@ def main() -> int:
     checked = 0
     for env in envs:
         cfg = render(repo, version, env)
-        wal = ((cfg.get("ingester") or {}).get("wal") or {})
-        limits = cfg.get("limits_config") or {}
-        comp = cfg.get("compactor") or {}
         rel = f"addons/observability/loki/values-{env}.yaml"
-
-        cutoff = wal.get("disk_full_threshold")
-        if cutoff is None:
-            fail(f"{rel}: the render sets no ingester.wal.disk_full_threshold, so the "
-                 f"fraction at which Loki stops accepting every push is an upstream "
-                 f"default. The alert's lead time then depends on a number that can "
-                 f"move under a chart bump with nothing here to compare against.")
-        elif warn is not None and not warn < float(cutoff):
-            fail(f"{rel}: the fill alert fires at {warn} but ingestion stops at "
-                 f"{cutoff}. There is no window in which to act — and no remedy is "
-                 f"fast: a retention cut must sync, wait a compaction interval, then "
-                 f"clear retention_delete_delay before a byte is freed, and the volume "
-                 f"cannot be grown through this repo at all.")
-        else:
-            checked += 1
-
-        if limits.get("retention_period") is None:
-            fail(f"{rel}: the render sets no limits_config.retention_period. Nothing "
-                 f"then deletes on a schedule, and the volume reaches the cutoff.")
-        if comp.get("retention_enabled") is not True:
-            fail(f"{rel}: compactor.retention_enabled is not true, so retention_period "
-                 f"deletes nothing however it is set and the cutoff arrives regardless.")
+        problems, leads = environment_verdict(cfg, warn, rel)
+        for problem in problems:
+            fail(problem)
+        checked += 1 if leads else 0
 
     if failures:
         for f in failures:

scripts/render-addons.py

@@ -239,14 +239,25 @@ def discover() -> list[Unit]:
 )
 
 
+def registry_answered(err: str) -> bool:
+    """Whether helm's complaint says the registry replied and refused the pin.
+
+    Separate from the exit path so the two worlds can be told apart without
+    running helm. A message naming both a missing chart and a broken connection
+    is the second: an unreachable registry cannot testify about what it holds.
+    """
+    low = err.lower()
+    return (any(t in low for t in _NOT_FOUND)
+            and not any(t in low for t in _UNREACHABLE))
+
+
 def helm_or_exit(cmd: list[str], what: str) -> subprocess.CompletedProcess:
     """Run a helm command; on failure exit with the RIGHT kind of complaint."""
     proc = subprocess.run(cmd, capture_output=True, text=True, timeout=NETWORK_TIMEOUT)
     if proc.returncode == 0:
         return proc
     err = ((proc.stderr or "") + (proc.stdout or "")).strip()
-    low = err.lower()
-    if any(t in low for t in _NOT_FOUND) and not any(t in low for t in _UNREACHABLE):
+    if registry_answered(err):
         print(f"{what}: the registry answered and the pinned chart is not there.")
         print(err)
         print("This is a pin that does not resolve — a finding about this repo.")