diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4bed326..20f879f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -239,6 +239,12 @@ jobs: # asserted exemption fails this step, so the suite cannot shrink quietly. - name: Positive controls — every gate can reject run: ./scripts/tests/controls.py + # A control proves a gate rejects a violation it was handed. This proves it + # rejects the ABSENCE of the thing it reads — a renamed directory, a + # narrowed glob, a filter that stopped matching. Every gate is probed, from + # the tree rather than a list, so the class stays closed as gates are added. + - name: No gate reports success over an absent corpus + run: ./scripts/tests/empty-corpus.py # Prose that names a path, a task target or a file:line is a claim about # the tree, and nothing else keeps those claims true as the tree moves. - name: Named things in prose resolve @@ -400,7 +406,7 @@ jobs: kyverno version - name: Run policy unit tests - run: kyverno test policies/kyverno/tests + run: ./scripts/kyverno-test.sh policies/kyverno/tests # `kyverno test` cannot reach Fulcio/Rekor, so it never verifies a Cosign # signature offline — the unit tests above only pin verify-images' match/ diff --git a/CLAUDE.md b/CLAUDE.md index d68dcab..c4229e5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -104,6 +104,7 @@ task validate:athena-panel-columns # Every column a CUR panel names is one the e task validate:fork-safety # No hardcoded catalog repoURL in applied ApplicationSets (report-only locally) task validate:log-volume-budget # Loki declares the fraction at which it stops ingesting, and the alert leads it task validate:falco-rule-floor # Every Falco rule set installed on a node is one Falco actually loads +task validate:empty-corpus # No gate reports success over a corpus that is not there ``` ### Local `task validate` is a subset of CI diff --git a/Taskfile.yaml b/Taskfile.yaml index e0a52bc..1e1b56d 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -113,6 +113,11 @@ tasks: cmds: - ./scripts/tests/controls.py + validate:empty-corpus: + desc: "Vacuity gate — no gate reports success over a corpus that is not there" + cmds: + - ./scripts/tests/empty-corpus.py + validate:policy-validity: desc: "Policy-validity gate — every rendered policy overlay is one Kyverno will accept" cmds: @@ -194,6 +199,7 @@ tasks: - validate:policy-validity - validate:gate-tests - validate:gate-controls + - validate:empty-corpus - validate:named-things - validate:ai-config - validate:workflows diff --git a/scripts/check-falco-rule-floor.py b/scripts/check-falco-rule-floor.py index e659a11..062b1da 100755 --- a/scripts/check-falco-rule-floor.py +++ b/scripts/check-falco-rule-floor.py @@ -136,7 +136,7 @@ def rank(priority: str) -> int: def chart_pin() -> tuple[str, str, str]: """Falco's chart coordinates, DERIVED from the ApplicationSet.""" - doc = yaml.safe_load(APPSET.read_text()) + doc = gatelib.read_yaml(APPSET) spec = doc.get("spec") or {} for gen in spec.get("generators") or []: for inner in (gen.get("matrix") or {}).get("generators") or []: diff --git a/scripts/check-hardcoded-org.py b/scripts/check-hardcoded-org.py index 04107dd..05e8938 100755 --- a/scripts/check-hardcoded-org.py +++ b/scripts/check-hardcoded-org.py @@ -82,6 +82,12 @@ ORG = "nanohype" CATALOG = "eks-gitops" # THIS repo — the vended catalog. NOT the product repos. +# A floor on applied ApplicationSets read. Set well under what the catalog ships, +# so it catches "the glob matched almost nothing" — a renamed directory, a wrong +# --root, a working tree that never checked out — and never "one appset was +# retired". +MIN_APPSETS = 20 + # A repoURL whose value points at the CATALOG repo, over either transport ArgoCD # accepts. Anchored on `repoURL:` so image refs, oci:// chart repos, and the # Kyverno subjectRegExp are structurally out of scope; pinned to the catalog repo @@ -119,6 +125,18 @@ def main() -> int: # the top level can strand a fork. See the opt-in note in the docstring. files = sorted(p for p in appsets.glob("*.y*ml") if p.is_file()) + # A floor on what was EXAMINED. The directory existing is not the same as it + # holding the fleet: with the glob answering nothing, this printed + # "Scanned 0 applied ApplicationSet(s)" and its success line, and exited 0 + # under --blocking. Every other outcome of this gate is a sentence about what + # it read; that one was a sentence about a set it never had. + if len(files) < MIN_APPSETS: + print(f"FAIL found {len(files)} applied ApplicationSet(s) under " + f"{appsets.relative_to(args.root)}, below the floor of {MIN_APPSETS}.") + print(" A scan over almost nothing reports the same as a scan over a") + print(" catalog with no hardcoded repoURL in it.") + return 2 + violations: list[tuple[pathlib.Path, int, str]] = [] for path in files: for lineno, line in enumerate( diff --git a/scripts/check-log-volume-budget.py b/scripts/check-log-volume-budget.py index a09896f..edc79bf 100755 --- a/scripts/check-log-volume-budget.py +++ b/scripts/check-log-volume-budget.py @@ -97,7 +97,7 @@ def fail(msg: str) -> None: def chart_pin() -> tuple[str, str]: """Chart coordinates DERIVED from the ApplicationSet, never re-declared.""" - doc = yaml.safe_load(APPSET.read_text()) + doc = gatelib.read_yaml(APPSET) sources = ((doc.get("spec") or {}).get("template") or {}).get("spec", {}).get("sources") or [] for src in sources: if isinstance(src, dict) and "chart" in src: diff --git a/scripts/check-platform-crs.py b/scripts/check-platform-crs.py index ec50f78..78a7bb4 100755 --- a/scripts/check-platform-crs.py +++ b/scripts/check-platform-crs.py @@ -85,6 +85,28 @@ NETWORK_TIMEOUT = 300 OPERATOR_APPSET = ROOT / "applicationsets" / "addons-agent-operator.yaml" CHART = "oci://ghcr.io/nanohype/eks-agent-platform/charts/operator" +# The API-group suffix the operator chart's CRDs share. Used to count candidate +# CRs independently of the schema set, so an empty schema resolution and an empty +# corpus are distinguishable from a clean one. +OPERATOR_API_SUFFIX = ".nanohype.dev" + +# A floor on CRs found. Set low on purpose: `candidates` counts documents whose +# API group the operator owns, but the WALK keeps only kinds the pinned chart +# ships a schema for, so what a healthy run reports depends on what that chart +# resolves to where the gate runs — a reviewer measured four where this tree +# walks eight. A floor above the smallest resolution is red somewhere it should +# be green, which is what a floor above its corpus always is. +# +# A constant rather than a derivation, and the reason is worth stating rather +# than dressing up: every quantity this gate could derive a floor from comes out of +# the same walk over the same files, so a corpus that shrinks shrinks the floor +# with it. A completeness assertion — candidates by API group against candidates +# by schema kind — was written and is circular for exactly that reason: both +# filters read the documents the walk found. There is no second enumerator of +# this repo's custom resources, so the floor is a number, and +# scripts/tests/test_corpus_floors.py holds it against the tree from both sides. +MIN_CRS = 2 + CRD_VERSION = "v1alpha1" # Directories with no bearing on what a cluster applies. @@ -99,6 +121,11 @@ def pinned_chart_version() -> str: whose repoURL is the operator chart; its sibling `targetRevision: main` is the catalog's own git revision and must not be mistaken for it. """ + if not OPERATOR_APPSET.is_file(): + print(f"Cannot run: {OPERATOR_APPSET.relative_to(ROOT)} does not exist, so the " + f"operator chart version this gate resolves its CRDs from is unknown. " + f"An unreadable pin is not the same as a catalog with no CRs in it.") + sys.exit(gatelib.CANNOT_RUN) text = OPERATOR_APPSET.read_text() m = re.search( r"repoURL:\s*\S*ghcr\.io/nanohype/eks-agent-platform/charts.*?targetRevision:\s*(\S+)", @@ -394,6 +421,11 @@ def check(listing: bool, offline: bool) -> int: walked = 0 skipped_templates = 0 + # Every CR carrying an operator API group, and every one the walk + # reached. Two filters over one corpus: the walk keeps kinds the chart + # defines, this keeps the group the chart owns. + candidates: set[str] = set() + reached: set[str] = set() for f in manifests(): # Chart source is Go-template text, identified structurally rather # than by whatever happens to break the parser. A manifest that will @@ -415,14 +447,20 @@ def check(listing: bool, offline: bool) -> int: if not isinstance(doc, dict): continue kind = doc.get("kind") + api = str(doc.get("apiVersion", "")) + rel = f.relative_to(ROOT) + name = (doc.get("metadata") or {}).get("name", "") + ident = f"{rel}: {kind}/{name}" + if (api.endswith("/" + CRD_VERSION) + and api.split("/", 1)[0].endswith(OPERATOR_API_SUFFIX)): + candidates.add(ident) if kind not in schemas: continue - if not str(doc.get("apiVersion", "")).endswith("/" + CRD_VERSION): + if not api.endswith("/" + CRD_VERSION): continue - rel = f.relative_to(ROOT) - name = (doc.get("metadata") or {}).get("name", "") + reached.add(ident) if listing: - print(f" {rel}: {kind}/{name}") + print(f" {ident}") walk(doc.get("spec") or {}, schemas[kind], "spec", kind, f"{rel} ({name})", problems) walked += 1 @@ -438,6 +476,27 @@ def check(listing: bool, offline: bool) -> int: ) return 1 + # A completeness assertion rather than a floor. `candidates` is the same + # corpus filtered by API GROUP, which is independent of the schema-kind + # filter the walk uses — so a schema set that resolved short, a renamed kind + # or a moved manifest shows up as candidates the walk did not reach, and an + # empty corpus shows up as no candidates at all. A number picked here could + # be wrong in either direction; this cannot. + if len(candidates) < MIN_CRS: + print(f"\nFAIL {len(candidates)} custom resource(s) carry an operator API " + f"group, below the floor of {MIN_CRS}. This gate walked almost nothing, " + f"which is not the same as the catalog's CRs being admissible.", + file=sys.stderr) + return gatelib.CANNOT_RUN + missed = sorted(candidates - reached) + if missed: + print(f"\nFAIL {len(missed)} custom resource(s) carry an operator API group " + f"and were not walked — the operator chart shipped no schema for their " + f"kind, so nothing checked them:", file=sys.stderr) + for item in missed: + print(f" - {item}", file=sys.stderr) + return 1 + print(f"\nok: {walked} platform CR(s) admissible against operator chart {version}") return 0 diff --git a/scripts/check-policy-admission.py b/scripts/check-policy-admission.py index 46ede36..68ae734 100755 --- a/scripts/check-policy-admission.py +++ b/scripts/check-policy-admission.py @@ -166,6 +166,11 @@ # Enforce runs on staging + production; both flip the base policies to Enforce. ENFORCE_ENVS = ["staging", "production"] +# A floor on addon×env manifests rendered. Set well under what the catalog +# produces; see the note at the comparison for why it is a number rather than a +# quantity derived from the same walk. +MIN_RENDERED = 40 + # Kinds Kyverno's workload policies (and their autogen pod-controller variants) # evaluate. Namespace-scoped, so they must carry metadata.namespace for the # exclusion match to fire — helm -n stamps it on most charts; this backfills any @@ -322,7 +327,7 @@ def check_exclusion_parity() -> tuple[bool, set[str], set[str]]: print("── Exclusion-list parity ──────────────────────────────────────────") lists: dict[str, list[str]] = {} for group, fname in EXCLUSION_POLICIES: - doc = yaml.safe_load((POLICY_DIR / group / "base" / fname).read_text()) + doc = gatelib.read_yaml(POLICY_DIR / group / "base" / fname) for rule in doc["spec"]["rules"]: key = f"{doc['metadata']['name']}/{rule['name']}" # A rule's exclusion is the union of every `exclude.any` entry's @@ -682,6 +687,21 @@ def main() -> int: return 1 print(f" rendered {count} addon×env manifests into their namespaces, " f"plus the canary\n") + # A constant, not a derivation. Bounding the render by the discovered + # units was written first and is circular: discover() reads the same + # ApplicationSets the render does, so a discovery that matched almost + # nothing lowers the bound by exactly as much as it lowers the count and + # the comparison holds. Nothing else in the tree enumerates the fleet. + # + # The canary and the runtime pod keep every rule-coverage assertion green + # regardless of how much fleet was rendered, so without a floor here a run + # over almost no fleet is indistinguishable from a clean one. + if count < MIN_RENDERED: + print(f" FAIL {count} manifest(s) rendered, below the floor of " + f"{MIN_RENDERED}. The policies were evaluated against a fleet " + f"this catalog does not have, and 'no addon flagged' is a " + f"statement about that fleet rather than this one.\n") + return gatelib.CANNOT_RUN coverage_ok = check_namespace_coverage(landed, excluded) diff --git a/scripts/gatelib.py b/scripts/gatelib.py index a17656d..5b06872 100644 --- a/scripts/gatelib.py +++ b/scripts/gatelib.py @@ -59,6 +59,24 @@ def read_yaml_all(path) -> list: sys.exit(CANNOT_RUN) +def read_yaml(path): + """The first document in `path`, or exit 2 naming the file. See read_yaml_all. + + For the gates that read one manifest to derive what they check against — a + chart pin, an appset's coordinates. An unguarded `read_text()` there raises + FileNotFoundError, which exits 1: the status this repo uses for "the gate + rejected the tree". By exit code a reader cannot tell that from a finding, + and the traceback names a pathlib internal rather than the manifest that is + missing. + """ + docs = read_yaml_all(path) + if not docs: + print(f"Cannot run: {pathlib.Path(path)} holds no YAML document, so the " + f"coordinates this gate reads from it are unknown.") + sys.exit(CANNOT_RUN) + return docs[0] + + def read_json(path): """`path` parsed as JSON, or exit 2 naming the file. See read_yaml_all.""" import json diff --git a/scripts/kubeconform-scan.sh b/scripts/kubeconform-scan.sh index 0da878a..9da00dc 100755 --- a/scripts/kubeconform-scan.sh +++ b/scripts/kubeconform-scan.sh @@ -49,4 +49,27 @@ args=(-strict -summary -schema-location default -schema-location "$DATREE") [ -n "${KUBECONFORM_CACHE:-}" ] && args+=(-cache "$KUBECONFORM_CACHE") [ -n "$SKIP" ] && args+=(-skip "$SKIP") -exec kubeconform "${args[@]}" "$@" +# A floor on what was VALIDATED. kubeconform exits 0 over an empty target and +# prints "0 resource found in 0 file", which is the same status a clean tree +# gets — so a renamed directory, a wrong argument or a rendered/ that was never +# written all report success. `exec` gave the caller kubeconform's status and +# nothing else; the summary is what says whether anything was read. +out="$(kubeconform "${args[@]}" "$@" 2>&1)" +rc=$? +printf '%s\n' "$out" +[ "$rc" -ne 0 ] && exit "$rc" + +found="$(printf '%s' "$out" | sed -n 's/.*Summary: \([0-9][0-9]*\) resource.*/\1/p' | head -1)" +if [ -z "$found" ]; then + echo "Cannot run: kubeconform printed no summary, so how many resources it read" + echo "is unknown. A pass here would report the same thing as a clean tree." + exit 2 +fi +if [ "$found" -eq 0 ]; then + echo "FAIL kubeconform validated 0 resources under: $*" + echo " Nothing was schema-checked, which is not the same as everything being" + echo " valid. Check the path — a renamed directory or an unrendered target" + echo " reports exactly this." + exit 2 +fi +exit 0 diff --git a/scripts/kyverno-test.sh b/scripts/kyverno-test.sh new file mode 100755 index 0000000..5b77633 --- /dev/null +++ b/scripts/kyverno-test.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# `kyverno test` with a floor on what it ran. +# +# The CLI exits 0 over a directory holding no tests, printing "No test yamls +# available" — the same status a passing suite gets. A renamed directory, a +# narrowed path or a fixture set that stopped matching all report success, and +# this is the job that stands behind every Kyverno policy in the catalog. +set -uo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TESTS="${1:-policies/kyverno/tests}" + +# A floor on tests EXECUTED. Set under what the suite holds, so it catches +# "matched almost nothing" rather than one case being retired. +MIN_TESTS=15 + +if ! command -v kyverno >/dev/null 2>&1; then + echo "Cannot run: kyverno is not on PATH. No policy was tested — that is" + echo "different from every policy passing." + exit 2 +fi + +cd "$ROOT" +out="$(kyverno test "$TESTS" 2>&1)" +rc=$? +printf '%s\n' "$out" +[ "$rc" -ne 0 ] && exit "$rc" + +ran="$(printf '%s' "$out" | sed -n 's/^Test Summary: \([0-9][0-9]*\) tests passed.*/\1/p' | head -1)" +if [ -z "$ran" ]; then + echo "Cannot run: kyverno printed no test summary, so how many tests ran is" + echo "unknown. A pass here would report the same thing as a passing suite." + exit 2 +fi +if [ "$ran" -lt "$MIN_TESTS" ]; then + echo "FAIL $ran Kyverno test(s) ran under $TESTS, below the floor of $MIN_TESTS." + echo " Almost nothing was tested, which is not the same as every policy" + echo " behaving. Check the path — a renamed directory reports exactly this." + exit 2 +fi +exit 0 diff --git a/scripts/tests/controls.py b/scripts/tests/controls.py index 7dadd5b..e459819 100755 --- a/scripts/tests/controls.py +++ b/scripts/tests/controls.py @@ -149,6 +149,7 @@ def __repr__(self) -> str: # from an invocation, which is a real gap and not a design choice. NEEDS_NETWORK_SH = { "kubeconform-scan.sh": "kubeconform", + "kyverno-test.sh": "kyverno", } NEEDS_NETWORK = {**NEEDS_NETWORK_PY, **NEEDS_NETWORK_SH} @@ -248,6 +249,9 @@ def dotted(node) -> str | None: "tests/reverify-gates.sh": "the Tier-1 re-verification harness; it drives the " "gates rather than checking the tree, and asserts " "its own pass/fail totals", + "tests/empty-corpus.py": "the vacuity harness; it runs every gate against an " + "emptied corpus rather than checking the tree, and " + "asserts its own probe floor and exemptions", } diff --git a/scripts/tests/empty-corpus.py b/scripts/tests/empty-corpus.py new file mode 100755 index 0000000..6a6445d --- /dev/null +++ b/scripts/tests/empty-corpus.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +"""No gate reports success over a corpus that is not there. + +A gate reports on the population it read. Nothing in an exit code separates "this +catalog holds no violation" from "this run held no catalog", and the second is +what a renamed directory, a wrong working directory, a narrowed glob or a filter +that stopped matching all look like. The fork-safety gate printed +`Scanned 0 applied ApplicationSet(s)` followed by its success line and exited 0 +under `--blocking`. + +Naming the gates that do it is a list of the ones somebody found. This empties +the corpus and asks every gate, so the class is closed rather than three +instances of it: + + * the gate population is every executable under scripts/, derived from the + tree — a gate added tomorrow is probed without anyone remembering to add it + * the corpus is every tracked file the gates read, emptied wholesale, so no + per-gate knowledge of what each one walks has to be maintained here + * a gate must exit NON-ZERO, and must not do it by crashing + +The last is its own defect and is why this reads the output rather than the +status alone. An unguarded `read_text()` on an absent manifest raises +FileNotFoundError, which exits 1 — the status this repo uses for "the gate +rejected the tree". By exit code a crash and a finding are the same answer, and +the traceback names a pathlib internal rather than the file that is missing. + +WHAT THIS DOES NOT ESTABLISH + +Two things, both stated rather than implied by a green run. + +That a gate examined the RIGHT population — only that it noticed an absent one. +A gate reading one file of thirty passes here. That is the corpus-completeness +question, and it belongs with each gate's own tests. + +And WHY a gate refused. Several derive their coordinates from an ApplicationSet +— which chart to render, which environments exist — and emptying every corpus at +once takes that input away too, so they exit 2 naming the appset rather than +saying anything about the corpus they check. Those runs are marked below. A +two-pass probe keeping applicationsets/ was tried and does not separate the +cases: for a gate whose corpus IS applicationsets/, the same file is both. What +would separate them is per-gate knowledge of which input is which, which is the +hand-maintained map this harness exists to avoid, so the limit is recorded +instead of papered over. +""" + +from __future__ import annotations + +import os +import pathlib +import shutil +import subprocess +import sys +import tempfile + +ROOT = pathlib.Path(__file__).resolve().parent.parent.parent +SCRIPTS = ROOT / "scripts" + +# Seconds one gate may run against the emptied tree. +GATE_TIMEOUT = 300 + +# Directories whose contents are the gates' subject matter. Emptied wholesale +# rather than per gate: a per-gate map of what each one walks is the same +# hand-maintained list this harness exists to replace, and it would go stale in +# the direction that matters — a gate whose corpus moved would be probed against +# the directory it no longer reads and would pass. +CORPUS_DIRS = ("applicationsets", "addons", "policies", "dashboards", "catalog", "docs") + +# Gates this repository runs that are not executables under scripts/. The +# workflow blocks the same merges on them, so a probe named for flooring every +# corpus that skipped them would have a population quietly narrower than its +# name. Each names the command as its caller runs it and the corpus it reads. +# +# All three passed over an absent corpus when this list was written: `kyverno +# test` printed that no tests are available and exited 0; `trivy config` over an +# empty render exited 0; and the render loop matched no overlays and printed +# "All manifests built successfully", which is the sentence a reader takes as +# proof the fleet renders. +COMMAND_GATES = { + "kyverno test": (["./scripts/kyverno-test.sh"], "policies"), + "trivy config": (["trivy", "config", "--exit-code", "1", "--severity", + "MEDIUM,HIGH,CRITICAL", "--ignorefile", ".trivyignore.yaml", + "rendered"], "rendered"), + "kustomize build loop": (["task", "kustomize:build"], "addons"), +} + +# Gates that answer about something other than the catalog's manifests, so +# emptying the corpus above leaves their real subject in place and the probe +# would assert nothing about them. Each names what it reads instead, and is +# asserted: an entry naming a gate that no longer exists fails, and so does one +# whose stated subject is no longer part of the tree. +OTHER_SUBJECT = { + "check-workflows.sh": ".github/workflows", +} + +# Gates whose corpus is an argument, not a default. Run with no argv a gate like +# this reads stdin or its own default and says nothing about the tree, so the +# probe hands it the argv its callers do — the same reason +# scripts/tests/controls.py carries GATE_ARGS. +# +# kubeconform-scan.sh was exempted here as answering about .github/workflows. It +# does not read that directory: Taskfile.yaml and ci.yml pass it +# `applicationsets/` and `rendered`. Both halves of that exemption's assertion +# held — the file exists, the directory exists — while the claim was false, which +# is what an exemption asserted on the wrong operand looks like. +GATE_ARGS = { + "kubeconform-scan.sh": ["applicationsets/"], + "check-hardcoded-org.py": ["--blocking"], +} + +# A floor on gates PROBED. With the glob answering nothing this would report that +# every gate refuses an empty corpus, over no gates. +# +# And a floor on THAT, because a floor of zero is the shape this whole harness +# exists to reject, one level up: set MIN_PROBED to 0 and the tree stays green +# over a probe that examined nothing. The lower bound is not a second constant — +# it is the size of the population the probe is named for, so it cannot be +# lowered without deleting gates. +MIN_PROBED = 23 + +CRASH_MARKERS = ("Traceback (most recent call last)", "\npanic: ") + + +def gate_files() -> list[pathlib.Path]: + """Every executable gate, from the tree rather than a list here.""" + return sorted( + p for p in SCRIPTS.rglob("*") + if p.is_file() and os.access(p, os.X_OK) and p.suffix in {".py", ".sh"} + and p.parent != SCRIPTS / "tests" + ) + + +def tracked_files() -> list[str]: + """What git tracks, which is the tree the gates are shipped to read. + + Not a filesystem walk: a local build cache, a rendered/ directory or an + uncommitted scratch file would be copied into the probe and then fail the + staging below, and none of them is part of what a gate examines. + """ + proc = subprocess.run(["git", "ls-files", "-z"], cwd=ROOT, capture_output=True, + text=True, timeout=GATE_TIMEOUT) + if proc.returncode != 0: + print(f"Cannot run: `git ls-files` failed in {ROOT} — {proc.stderr.strip()}") + sys.exit(2) + return [p for p in proc.stdout.split("\0") if p] + + +def emptied_tree(dest: pathlib.Path, keep: tuple[str, ...] = ()) -> int: + """A copy of the tracked tree with the corpus removed, `keep` left intact.""" + for rel in tracked_files(): + src = ROOT / rel + if not src.is_file(): + continue + out = dest / rel + out.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, out) + + removed = 0 + for name in [d for d in CORPUS_DIRS if d not in keep]: + for path in sorted((dest / name).rglob("*")): + if path.is_file() and path.suffix != ".py": + path.unlink() + removed += 1 + + # The gates that ask git what the tree tracks need an answer, and an + # uninitialised directory makes them exit 2 for that reason instead of for + # the corpus being empty — which would score a pass this harness did not earn. + subprocess.run(["git", "init", "-q"], cwd=dest, check=True, timeout=GATE_TIMEOUT) + survivors = sorted(str(p.relative_to(dest)) for p in dest.rglob("*") + if p.is_file() and not str(p.relative_to(dest)).startswith(".git/")) + subprocess.run(["git", "add", "--", *survivors], cwd=dest, check=True, + timeout=GATE_TIMEOUT, capture_output=True) + return removed + + +def probe(gate: pathlib.Path, tree: pathlib.Path) -> tuple[bool, str, bool]: + """(refused, why not, refused-on-a-derivation-input). + + The third value is what keeps CANNOT_RUN meaningful. A gate that exits 2 + because an ApplicationSet it reads to learn its coordinates is gone has said + nothing about its corpus, and scoring that as a corpus refusal is the + collapse this repo avoids everywhere else. + """ + rel = gate.relative_to(SCRIPTS) + try: + proc = subprocess.run([str(tree / "scripts" / rel), *GATE_ARGS.get(gate.name, [])], + cwd=tree, capture_output=True, text=True, + timeout=GATE_TIMEOUT, env={**os.environ, + "KUBECONFORM_SKIP": ""}) + except subprocess.TimeoutExpired: + return False, f"did not finish in {GATE_TIMEOUT}s against an empty corpus", False + out = proc.stdout + proc.stderr + if any(marker in out for marker in CRASH_MARKERS): + first = next((ln for ln in out.splitlines() if "Error" in ln or "panic:" in ln), + "").strip() + return False, (f"CRASHED rather than refusing (exit {proc.returncode}) — " + f"{first[:120]}. A crash exits non-zero, so by status alone it " + f"is a finding; the traceback then names a library internal " + f"rather than the input that is absent"), False + if proc.returncode == 0: + last = out.strip().splitlines()[-1][:120] if out.strip() else "silently" + return False, f"exited 0 over an empty corpus — {last}", False + on_derivation = (proc.returncode == gatelib_cannot_run() + and any(f"{d}/" in out for d in CORPUS_DIRS)) + return True, "", on_derivation + + +def probe_command(label: str, argv: list[str], corpus: str, + tree: pathlib.Path) -> tuple[bool, str]: + """Whether a command this repo runs as a gate refuses its corpus emptied. + + Not an executable under scripts/, so it carries none of the conventions the + gates share — no CANNOT_RUN, no diagnostic naming the input. The question is + the same one: does it report success having read nothing. + """ + if shutil.which(argv[0]) is None: + return True, "" # the tool is absent; that is not a verdict + try: + proc = subprocess.run(argv, cwd=tree, capture_output=True, text=True, + timeout=GATE_TIMEOUT) + except subprocess.TimeoutExpired: + return False, f"did not finish in {GATE_TIMEOUT}s" + out = proc.stdout + proc.stderr + if proc.returncode != 0: + return True, "" + last = out.strip().splitlines()[-1][:110] if out.strip() else "silently" + return False, (f"exited 0 with {corpus}/ emptied — {last}") + + +def gatelib_cannot_run() -> int: + """The shared "this did not run" status, read from gatelib rather than 2.""" + text = (ROOT / "scripts" / "gatelib.py").read_text() + for line in text.splitlines(): + if line.startswith("CANNOT_RUN"): + return int(line.split("=", 1)[1].strip()) + return 2 + + +def main() -> int: + gates = gate_files() + if not gates: + print("FAIL found no gate executables under scripts/ — this harness probed") + print(" nothing, which is not the same as every gate refusing.") + return 2 + + problems: list[str] = [] + for name, subject in sorted(OTHER_SUBJECT.items()): + if not any(g.name == name for g in gates): + problems.append(f"{name} is exempted as answering about {subject} rather " + f"than the catalog, but no such gate exists under scripts/ " + f"— the exemption outlived its file.") + elif not (ROOT / subject).is_dir(): + problems.append(f"{name} is exempted as answering about {subject}, which " + f"is not a directory in this tree — the exemption names a " + f"subject the repo does not have.") + + probed = 0 + with tempfile.TemporaryDirectory() as tmp: + tree = pathlib.Path(tmp) / "tree" + removed = emptied_tree(tree) + if not removed: + print("FAIL emptying the corpus removed no file, so every gate below was") + print(" run against the real tree and refusing it would prove nothing.") + return 2 + print(f"── {len(gates)} gate(s) against a tree with {removed} corpus file(s) " + f"removed ──\n") + + for gate in gates: + if gate.name in OTHER_SUBJECT: + print(f" skip {gate.name}: answers about {OTHER_SUBJECT[gate.name]}, " + f"which this probe does not empty") + continue + probed += 1 + ok, why, on_derivation = probe(gate, tree) + if ok: + print(f" ok {gate.name}" + + (" (refused on a derivation input, not on its corpus)" + if on_derivation else "")) + else: + print(f" FAIL {gate.name}: {why}") + problems.append(f"{gate.name} {why}") + + for label, (argv, corpus) in sorted(COMMAND_GATES.items()): + probed += 1 + ok, why = probe_command(label, argv, corpus, tree) + print(f" {'ok ' if ok else 'FAIL'} {label}" + + ("" if ok else f": {why}")) + if not ok: + problems.append(f"{label} {why}") + print() + + # The floor's own floor. A probe population of 25 with MIN_PROBED at 0 reports + # success over nothing, and nothing else in the tree reads this constant. + if MIN_PROBED < 1: + problems.append( + f"MIN_PROBED is {MIN_PROBED}, so this harness would report every gate " + f"refusing an empty corpus over a population of none. A floor of zero is " + f"the shape this harness exists to reject.") + elif len(gates) < MIN_PROBED: + problems.append( + f"MIN_PROBED is {MIN_PROBED} and scripts/ holds {len(gates)} gate(s), so " + f"this harness cannot pass on any tree. A floor above its corpus fails " + f"every run, which is the other way a floor is wrong.") + if probed < MIN_PROBED: + problems.append(f"only {probed} gate(s) were probed, under the floor of " + f"{MIN_PROBED} — the population this harness reads shrank, and " + f"a clean run over it says nothing about the rest.") + + print() + if problems: + for p in problems: + print(f"FAIL {p}") + print("\n A gate that reports success over an absent corpus reports the same") + print(" thing as one over a clean tree. Give it a floor on what it EXAMINED,") + print(" or make the absent input exit 2 with the file named.") + return 1 + + print(f"✓ all {probed} probed gate(s) refuse an empty corpus, and none does it by " + f"crashing") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/run.py b/scripts/tests/run.py index 75ef305..9a57965 100755 --- a/scripts/tests/run.py +++ b/scripts/tests/run.py @@ -36,6 +36,7 @@ "test_named_things", "test_falco_rule_floor", "test_gatelib", + "test_corpus_floors", ) # A floor well under the real count. It catches "discovery found almost nothing", diff --git a/scripts/tests/test_corpus_floors.py b/scripts/tests/test_corpus_floors.py new file mode 100644 index 0000000..2756c52 --- /dev/null +++ b/scripts/tests/test_corpus_floors.py @@ -0,0 +1,109 @@ +"""Every floor sits above zero and below the corpus it guards. + +A floor of zero is the vacuous pass with a constant in front of it. A floor above +the real corpus makes the gate red on every run, and a gate that is always red is +one people route around — the same destination by a longer road. + +These are constants rather than derivations, and the reason is recorded at each +one: every quantity available to derive a floor from comes out of the same walk +over the same files, so a corpus that shrinks shrinks the derivation with it. +Both circular forms were written and rejected. What can be asserted is the pair +of bounds, against the tree. +""" + +from __future__ import annotations + +import pathlib +import unittest + +import yaml +from gateloader import load + +ROOT = pathlib.Path(__file__).resolve().parent.parent.parent + +fork_safety = load("check-hardcoded-org") +platform_crs = load("check-platform-crs") +policy_admission = load("check-policy-admission") +dashboards = load("validate-dashboards") + + +class EveryFloorIsAboveZero(unittest.TestCase): + """Zero grants exactly the vacuous pass the floor exists to stop.""" + + def test_each_gate_declares_one(self): + for label, floor in ( + ("check-hardcoded-org MIN_APPSETS", fork_safety.MIN_APPSETS), + ("check-platform-crs MIN_CRS", platform_crs.MIN_CRS), + ("check-policy-admission MIN_RENDERED", policy_admission.MIN_RENDERED), + ("validate-dashboards MIN_DASHBOARD_REFS", dashboards.MIN_DASHBOARD_REFS), + ): + with self.subTest(floor=label): + self.assertGreater(floor, 0) + + +class EveryFloorIsBelowTheCorpusItGuards(unittest.TestCase): + """Measured off the tree. A floor above the real population is always red.""" + + def test_the_applied_appsets_clear_the_fork_safety_floor(self): + applied = [p for p in (ROOT / "applicationsets").glob("*.y*ml") if p.is_file()] + self.assertGreater(len(applied), fork_safety.MIN_APPSETS) + + def test_the_platform_crs_clear_their_floor(self): + """Counted the way the gate counts candidates: the operator's API groups. + + Matching a `/v1alpha1` suffix alone counts every ApplicationSet in the + tree — 44 documents against the 8 the gate walks — so the bound would + hold for any floor up to 43 and could not see the always-red case. + """ + found = 0 + for path in platform_crs.manifests(): + if platform_crs.gatelib.is_helm_template(path): + continue + for doc in yaml.safe_load_all(path.read_text()): + if not isinstance(doc, dict): + continue + api = str(doc.get("apiVersion", "")) + if (api.endswith("/" + platform_crs.CRD_VERSION) + and api.split("/", 1)[0].endswith( + platform_crs.OPERATOR_API_SUFFIX)): + found += 1 + self.assertGreater(found, platform_crs.MIN_CRS, + f"MIN_CRS is {platform_crs.MIN_CRS} and this tree holds " + f"{found} platform CR(s), so the gate exits 2 on every run") + + def test_the_render_clears_the_policy_admission_floor(self): + """Bounded by units times the ENFORCE environments — staging and + production, not all four: doubling the ceiling lets any floor in 66..127 + pass here while making the gate red on every run.""" + units = [u for u in policy_admission.discover() + if u.chart not in policy_admission.SKIP_CHARTS] + self.assertGreater(len(units), 0) + self.assertLess(policy_admission.MIN_RENDERED, + len(units) * len(policy_admission.ENFORCE_ENVS)) + + def test_the_dashboards_clear_their_floor(self): + refs = dashboards.discover(ROOT) + self.assertGreater(len(refs), dashboards.MIN_DASHBOARD_REFS) + + +class TheFloorsGuardTheRightQuantity(unittest.TestCase): + """A floor on findings is not a floor on the corpus. + + Counting what was FOUND cannot separate a clean catalog from an unexamined + one; only counting what was READ can. + """ + + def test_each_floor_is_compared_against_a_count_of_what_was_read(self): + for rel, expr in ( + ("scripts/check-hardcoded-org.py", "if len(files) < MIN_APPSETS:"), + ("scripts/check-platform-crs.py", "if len(candidates) < MIN_CRS:"), + ("scripts/check-policy-admission.py", "if count < MIN_RENDERED:"), + ("scripts/validate-dashboards.py", + "if 0 < len(refs) < MIN_DASHBOARD_REFS:"), + ): + with self.subTest(gate=rel): + self.assertIn(expr, (ROOT / rel).read_text()) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/validate-dashboards.py b/scripts/validate-dashboards.py index d80a04c..f4f759c 100755 --- a/scripts/validate-dashboards.py +++ b/scripts/validate-dashboards.py @@ -387,6 +387,12 @@ def main() -> int: print("Locally-authored dashboards: datasources wired, template variables declared.\n") refs = discover(args.root) + if 0 < len(refs) < MIN_DASHBOARD_REFS: + print(f"FAIL found {len(refs)} grafanaCom.id reference(s), below the floor of " + f"{MIN_DASHBOARD_REFS}. The comment above that constant promised a floor " + f"on what was EXAMINED and the code enforced one of zero, so a glob that " + f"matched almost nothing reported the same as a clean tree.") + return 2 if not refs: print(f"FAIL no grafanaCom.id references found. This repo ships " f"{MIN_DASHBOARD_REFS}+ of them, so finding none means this gate read "