No gate reports success over a corpus that is not there

.github/workflows/ci.yml

@@ -239,6 +239,12 @@ jobs:
       # asserted exemption fails this step, so the suite cannot shrink quietly.
       - name: Positive controls — every gate can reject
         run: ./scripts/tests/controls.py
+      # A control proves a gate rejects a violation it was handed. This proves it
+      # rejects the ABSENCE of the thing it reads — a renamed directory, a
+      # narrowed glob, a filter that stopped matching. Every gate is probed, from
+      # the tree rather than a list, so the class stays closed as gates are added.
+      - name: No gate reports success over an absent corpus
+        run: ./scripts/tests/empty-corpus.py
       # Prose that names a path, a task target or a file:line is a claim about
       # the tree, and nothing else keeps those claims true as the tree moves.
       - name: Named things in prose resolve
@@ -400,7 +406,7 @@ jobs:
           kyverno version
 
       - name: Run policy unit tests
-        run: kyverno test policies/kyverno/tests
+        run: ./scripts/kyverno-test.sh policies/kyverno/tests
 
       # `kyverno test` cannot reach Fulcio/Rekor, so it never verifies a Cosign
       # signature offline — the unit tests above only pin verify-images' match/

CLAUDE.md

@@ -104,6 +104,7 @@ task validate:athena-panel-columns # Every column a CUR panel names is one the e
 task validate:fork-safety          # No hardcoded catalog repoURL in applied ApplicationSets (report-only locally)
 task validate:log-volume-budget    # Loki declares the fraction at which it stops ingesting, and the alert leads it
 task validate:falco-rule-floor     # Every Falco rule set installed on a node is one Falco actually loads
+task validate:empty-corpus         # No gate reports success over a corpus that is not there
 ```
 
 ### Local `task validate` is a subset of CI

Taskfile.yaml

@@ -113,6 +113,11 @@ tasks:
     cmds:
       - ./scripts/tests/controls.py
 
+  validate:empty-corpus:
+    desc: "Vacuity gate — no gate reports success over a corpus that is not there"
+    cmds:
+      - ./scripts/tests/empty-corpus.py
+
   validate:policy-validity:
     desc: "Policy-validity gate — every rendered policy overlay is one Kyverno will accept"
     cmds:
@@ -194,6 +199,7 @@ tasks:
       - validate:policy-validity
       - validate:gate-tests
       - validate:gate-controls
+      - validate:empty-corpus
       - validate:named-things
       - validate:ai-config
       - validate:workflows

scripts/check-falco-rule-floor.py

@@ -136,7 +136,7 @@ def rank(priority: str) -> int:
 
 def chart_pin() -> tuple[str, str, str]:
     """Falco's chart coordinates, DERIVED from the ApplicationSet."""
-    doc = yaml.safe_load(APPSET.read_text())
+    doc = gatelib.read_yaml(APPSET)
     spec = doc.get("spec") or {}
     for gen in spec.get("generators") or []:
         for inner in (gen.get("matrix") or {}).get("generators") or []:

scripts/check-hardcoded-org.py

@@ -82,6 +82,12 @@
 ORG = "nanohype"
 CATALOG = "eks-gitops"  # THIS repo — the vended catalog. NOT the product repos.
 
+# A floor on applied ApplicationSets read. Set well under what the catalog ships,
+# so it catches "the glob matched almost nothing" — a renamed directory, a wrong
+# --root, a working tree that never checked out — and never "one appset was
+# retired".
+MIN_APPSETS = 20
+
 # A repoURL whose value points at the CATALOG repo, over either transport ArgoCD
 # accepts. Anchored on `repoURL:` so image refs, oci:// chart repos, and the
 # Kyverno subjectRegExp are structurally out of scope; pinned to the catalog repo
@@ -119,6 +125,18 @@ def main() -> int:
     # the top level can strand a fork. See the opt-in note in the docstring.
     files = sorted(p for p in appsets.glob("*.y*ml") if p.is_file())
 
+    # A floor on what was EXAMINED. The directory existing is not the same as it
+    # holding the fleet: with the glob answering nothing, this printed
+    # "Scanned 0 applied ApplicationSet(s)" and its success line, and exited 0
+    # under --blocking. Every other outcome of this gate is a sentence about what
+    # it read; that one was a sentence about a set it never had.
+    if len(files) < MIN_APPSETS:
+        print(f"FAIL  found {len(files)} applied ApplicationSet(s) under "
+              f"{appsets.relative_to(args.root)}, below the floor of {MIN_APPSETS}.")
+        print("      A scan over almost nothing reports the same as a scan over a")
+        print("      catalog with no hardcoded repoURL in it.")
+        return 2
+
     violations: list[tuple[pathlib.Path, int, str]] = []
     for path in files:
         for lineno, line in enumerate(

scripts/check-log-volume-budget.py

@@ -97,7 +97,7 @@ def fail(msg: str) -> None:
 
 def chart_pin() -> tuple[str, str]:
     """Chart coordinates DERIVED from the ApplicationSet, never re-declared."""
-    doc = yaml.safe_load(APPSET.read_text())
+    doc = gatelib.read_yaml(APPSET)
     sources = ((doc.get("spec") or {}).get("template") or {}).get("spec", {}).get("sources") or []
     for src in sources:
         if isinstance(src, dict) and "chart" in src:

scripts/check-platform-crs.py

@@ -85,6 +85,28 @@
 NETWORK_TIMEOUT = 300
 OPERATOR_APPSET = ROOT / "applicationsets" / "addons-agent-operator.yaml"
 CHART = "oci://ghcr.io/nanohype/eks-agent-platform/charts/operator"
+# The API-group suffix the operator chart's CRDs share. Used to count candidate
+# CRs independently of the schema set, so an empty schema resolution and an empty
+# corpus are distinguishable from a clean one.
+OPERATOR_API_SUFFIX = ".nanohype.dev"
+
+# A floor on CRs found. Set low on purpose: `candidates` counts documents whose
+# API group the operator owns, but the WALK keeps only kinds the pinned chart
+# ships a schema for, so what a healthy run reports depends on what that chart
+# resolves to where the gate runs — a reviewer measured four where this tree
+# walks eight. A floor above the smallest resolution is red somewhere it should
+# be green, which is what a floor above its corpus always is.
+#
+# A constant rather than a derivation, and the reason is worth stating rather
+# than dressing up: every quantity this gate could derive a floor from comes out of
+# the same walk over the same files, so a corpus that shrinks shrinks the floor
+# with it. A completeness assertion — candidates by API group against candidates
+# by schema kind — was written and is circular for exactly that reason: both
+# filters read the documents the walk found. There is no second enumerator of
+# this repo's custom resources, so the floor is a number, and
+# scripts/tests/test_corpus_floors.py holds it against the tree from both sides.
+MIN_CRS = 2
+
 CRD_VERSION = "v1alpha1"
 
 # Directories with no bearing on what a cluster applies.
@@ -99,6 +121,11 @@ def pinned_chart_version() -> str:
     whose repoURL is the operator chart; its sibling `targetRevision: main` is
     the catalog's own git revision and must not be mistaken for it.
     """
+    if not OPERATOR_APPSET.is_file():
+        print(f"Cannot run: {OPERATOR_APPSET.relative_to(ROOT)} does not exist, so the "
+              f"operator chart version this gate resolves its CRDs from is unknown. "
+              f"An unreadable pin is not the same as a catalog with no CRs in it.")
+        sys.exit(gatelib.CANNOT_RUN)
     text = OPERATOR_APPSET.read_text()
     m = re.search(
         r"repoURL:\s*\S*ghcr\.io/nanohype/eks-agent-platform/charts.*?targetRevision:\s*(\S+)",
@@ -394,6 +421,11 @@ def check(listing: bool, offline: bool) -> int:
 
         walked = 0
         skipped_templates = 0
+        # Every CR carrying an operator API group, and every one the walk
+        # reached. Two filters over one corpus: the walk keeps kinds the chart
+        # defines, this keeps the group the chart owns.
+        candidates: set[str] = set()
+        reached: set[str] = set()
         for f in manifests():
             # Chart source is Go-template text, identified structurally rather
             # than by whatever happens to break the parser. A manifest that will
@@ -415,14 +447,20 @@ def check(listing: bool, offline: bool) -> int:
                 if not isinstance(doc, dict):
                     continue
                 kind = doc.get("kind")
+                api = str(doc.get("apiVersion", ""))
+                rel = f.relative_to(ROOT)
+                name = (doc.get("metadata") or {}).get("name", "<unnamed>")
+                ident = f"{rel}: {kind}/{name}"
+                if (api.endswith("/" + CRD_VERSION)
+                        and api.split("/", 1)[0].endswith(OPERATOR_API_SUFFIX)):
+                    candidates.add(ident)
                 if kind not in schemas:
                     continue
-                if not str(doc.get("apiVersion", "")).endswith("/" + CRD_VERSION):
+                if not api.endswith("/" + CRD_VERSION):
                     continue
-                rel = f.relative_to(ROOT)
-                name = (doc.get("metadata") or {}).get("name", "<unnamed>")
+                reached.add(ident)
                 if listing:
-                    print(f"  {rel}: {kind}/{name}")
+                    print(f"  {ident}")
                 walk(doc.get("spec") or {}, schemas[kind], "spec", kind, f"{rel} ({name})", problems)
                 walked += 1
 
@@ -438,6 +476,27 @@ def check(listing: bool, offline: bool) -> int:
         )
         return 1
 
+    # A completeness assertion rather than a floor. `candidates` is the same
+    # corpus filtered by API GROUP, which is independent of the schema-kind
+    # filter the walk uses — so a schema set that resolved short, a renamed kind
+    # or a moved manifest shows up as candidates the walk did not reach, and an
+    # empty corpus shows up as no candidates at all. A number picked here could
+    # be wrong in either direction; this cannot.
+    if len(candidates) < MIN_CRS:
+        print(f"\nFAIL  {len(candidates)} custom resource(s) carry an operator API "
+              f"group, below the floor of {MIN_CRS}. This gate walked almost nothing, "
+              f"which is not the same as the catalog's CRs being admissible.",
+              file=sys.stderr)
+        return gatelib.CANNOT_RUN
+    missed = sorted(candidates - reached)
+    if missed:
+        print(f"\nFAIL  {len(missed)} custom resource(s) carry an operator API group "
+              f"and were not walked — the operator chart shipped no schema for their "
+              f"kind, so nothing checked them:", file=sys.stderr)
+        for item in missed:
+            print(f"  - {item}", file=sys.stderr)
+        return 1
+
     print(f"\nok: {walked} platform CR(s) admissible against operator chart {version}")
     return 0
 

scripts/check-policy-admission.py

@@ -166,6 +166,11 @@
 # Enforce runs on staging + production; both flip the base policies to Enforce.
 ENFORCE_ENVS = ["staging", "production"]
 
+# A floor on addon×env manifests rendered. Set well under what the catalog
+# produces; see the note at the comparison for why it is a number rather than a
+# quantity derived from the same walk.
+MIN_RENDERED = 40
+
 # Kinds Kyverno's workload policies (and their autogen pod-controller variants)
 # evaluate. Namespace-scoped, so they must carry metadata.namespace for the
 # exclusion match to fire — helm -n stamps it on most charts; this backfills any
@@ -322,7 +327,7 @@ def check_exclusion_parity() -> tuple[bool, set[str], set[str]]:
     print("── Exclusion-list parity ──────────────────────────────────────────")
     lists: dict[str, list[str]] = {}
     for group, fname in EXCLUSION_POLICIES:
-        doc = yaml.safe_load((POLICY_DIR / group / "base" / fname).read_text())
+        doc = gatelib.read_yaml(POLICY_DIR / group / "base" / fname)
         for rule in doc["spec"]["rules"]:
             key = f"{doc['metadata']['name']}/{rule['name']}"
             # A rule's exclusion is the union of every `exclude.any` entry's
@@ -682,6 +687,21 @@ def main() -> int:
             return 1
         print(f"  rendered {count} addon×env manifests into their namespaces, "
               f"plus the canary\n")
+        # A constant, not a derivation. Bounding the render by the discovered
+        # units was written first and is circular: discover() reads the same
+        # ApplicationSets the render does, so a discovery that matched almost
+        # nothing lowers the bound by exactly as much as it lowers the count and
+        # the comparison holds. Nothing else in the tree enumerates the fleet.
+        #
+        # The canary and the runtime pod keep every rule-coverage assertion green
+        # regardless of how much fleet was rendered, so without a floor here a run
+        # over almost no fleet is indistinguishable from a clean one.
+        if count < MIN_RENDERED:
+            print(f"  FAIL  {count} manifest(s) rendered, below the floor of "
+                  f"{MIN_RENDERED}. The policies were evaluated against a fleet "
+                  f"this catalog does not have, and 'no addon flagged' is a "
+                  f"statement about that fleet rather than this one.\n")
+            return gatelib.CANNOT_RUN
 
         coverage_ok = check_namespace_coverage(landed, excluded)
 

scripts/gatelib.py

@@ -59,6 +59,24 @@ def read_yaml_all(path) -> list:
         sys.exit(CANNOT_RUN)
 
 
+def read_yaml(path):
+    """The first document in `path`, or exit 2 naming the file. See read_yaml_all.
+
+    For the gates that read one manifest to derive what they check against — a
+    chart pin, an appset's coordinates. An unguarded `read_text()` there raises
+    FileNotFoundError, which exits 1: the status this repo uses for "the gate
+    rejected the tree". By exit code a reader cannot tell that from a finding,
+    and the traceback names a pathlib internal rather than the manifest that is
+    missing.
+    """
+    docs = read_yaml_all(path)
+    if not docs:
+        print(f"Cannot run: {pathlib.Path(path)} holds no YAML document, so the "
+              f"coordinates this gate reads from it are unknown.")
+        sys.exit(CANNOT_RUN)
+    return docs[0]
+
+
 def read_json(path):
     """`path` parsed as JSON, or exit 2 naming the file. See read_yaml_all."""
     import json

scripts/kubeconform-scan.sh

@@ -49,4 +49,27 @@ args=(-strict -summary -schema-location default -schema-location "$DATREE")
 [ -n "${KUBECONFORM_CACHE:-}" ] && args+=(-cache "$KUBECONFORM_CACHE")
 [ -n "$SKIP" ] && args+=(-skip "$SKIP")
 
-exec kubeconform "${args[@]}" "$@"
+# A floor on what was VALIDATED. kubeconform exits 0 over an empty target and
+# prints "0 resource found in 0 file", which is the same status a clean tree
+# gets — so a renamed directory, a wrong argument or a rendered/ that was never
+# written all report success. `exec` gave the caller kubeconform's status and
+# nothing else; the summary is what says whether anything was read.
+out="$(kubeconform "${args[@]}" "$@" 2>&1)"
+rc=$?
+printf '%s\n' "$out"
+[ "$rc" -ne 0 ] && exit "$rc"
+
+found="$(printf '%s' "$out" | sed -n 's/.*Summary: \([0-9][0-9]*\) resource.*/\1/p' | head -1)"
+if [ -z "$found" ]; then
+  echo "Cannot run: kubeconform printed no summary, so how many resources it read"
+  echo "is unknown. A pass here would report the same thing as a clean tree."
+  exit 2
+fi
+if [ "$found" -eq 0 ]; then
+  echo "FAIL  kubeconform validated 0 resources under: $*"
+  echo "      Nothing was schema-checked, which is not the same as everything being"
+  echo "      valid. Check the path — a renamed directory or an unrendered target"
+  echo "      reports exactly this."
+  exit 2
+fi
+exit 0

scripts/kyverno-test.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# `kyverno test` with a floor on what it ran.
+#
+# The CLI exits 0 over a directory holding no tests, printing "No test yamls
+# available" — the same status a passing suite gets. A renamed directory, a
+# narrowed path or a fixture set that stopped matching all report success, and
+# this is the job that stands behind every Kyverno policy in the catalog.
+set -uo pipefail
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+TESTS="${1:-policies/kyverno/tests}"
+
+# A floor on tests EXECUTED. Set under what the suite holds, so it catches
+# "matched almost nothing" rather than one case being retired.
+MIN_TESTS=15
+
+if ! command -v kyverno >/dev/null 2>&1; then
+  echo "Cannot run: kyverno is not on PATH. No policy was tested — that is"
+  echo "different from every policy passing."
+  exit 2
+fi
+
+cd "$ROOT"
+out="$(kyverno test "$TESTS" 2>&1)"
+rc=$?
+printf '%s\n' "$out"
+[ "$rc" -ne 0 ] && exit "$rc"
+
+ran="$(printf '%s' "$out" | sed -n 's/^Test Summary: \([0-9][0-9]*\) tests passed.*/\1/p' | head -1)"
+if [ -z "$ran" ]; then
+  echo "Cannot run: kyverno printed no test summary, so how many tests ran is"
+  echo "unknown. A pass here would report the same thing as a passing suite."
+  exit 2
+fi
+if [ "$ran" -lt "$MIN_TESTS" ]; then
+  echo "FAIL  $ran Kyverno test(s) ran under $TESTS, below the floor of $MIN_TESTS."
+  echo "      Almost nothing was tested, which is not the same as every policy"
+  echo "      behaving. Check the path — a renamed directory reports exactly this."
+  exit 2
+fi
+exit 0

scripts/tests/controls.py

@@ -149,6 +149,7 @@ def __repr__(self) -> str:
 # from an invocation, which is a real gap and not a design choice.
 NEEDS_NETWORK_SH = {
     "kubeconform-scan.sh": "kubeconform",
+    "kyverno-test.sh": "kyverno",
 }
 
 NEEDS_NETWORK = {**NEEDS_NETWORK_PY, **NEEDS_NETWORK_SH}
@@ -248,6 +249,9 @@ def dotted(node) -> str | None:
     "tests/reverify-gates.sh": "the Tier-1 re-verification harness; it drives the "
                                "gates rather than checking the tree, and asserts "
                                "its own pass/fail totals",
+    "tests/empty-corpus.py": "the vacuity harness; it runs every gate against an "
+                             "emptied corpus rather than checking the tree, and "
+                             "asserts its own probe floor and exemptions",
 }