diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f894100..dfb6969 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -542,13 +542,77 @@ jobs: - name: Render every addon across every environment run: ./scripts/render-addons.py - # Tag immutability is a fact about the commit under test, so it blocks. - # Whether a CVE landed in one of these images overnight is not, and that - # half stays scheduled — blocking a merge on the world changing is how a - # gate teaches people to route around it. + # Tag immutability is a fact about the commit under test, so it blocks + # here. What the image CONTAINS is a separate question with a separate + # job — image-vulnerabilities — because it has to pull every image and is + # slow enough to be worth running in parallel with this one. - name: Every rendered image carries an immutable reference run: ./scripts/check-image-pins.py + # ── Image vulnerability gate ───────────────────────────────────────── + # Whoever moves a chart pin owns what the image behind it carries, and this + # repo pins independently of any cluster. trivy-operator reports the same + # findings at runtime, but to an operator who cannot change the pin without a + # pull request here — so the decision belongs at the pin, which is this job. + # + # Blocking on a CRITICAL with a published fix that image-advisories.yaml does + # not acknowledge. A new advisory can therefore turn a pull request red for a + # reason the pull request did not cause; that is deliberate and the advisory + # file is the release valve, because the alternative is a scan whose findings + # nobody has to answer. HIGH is counted and printed, not gated. + # + # The run scans a digest-pinned end-of-life image first and requires its known + # CRITICALs back. A scanner with no database returns a clean result for every + # image, which is indistinguishable from a healthy fleet by exit code — so + # without the canary a green run here would prove nothing at all. + image-vulnerabilities: + name: Rendered images carry no unacknowledged CRITICAL + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Install Helm + uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version-file: .python-version + - name: Install Python dependencies + run: pip install --require-hashes -r requirements.txt + + # Same guard the validate job carries: setup-trivy reads an empty version + # as "install the latest release", so an expansion failure would scan with + # an unpinned trivy and pass. + - name: Assert the pinned trivy version resolved + run: | + if [ -z "${{ env.TRIVY_VERSION }}" ]; then + echo "TRIVY_VERSION expanded empty. setup-trivy reads that as 'latest'," + echo "so this job would scan with an unpinned trivy and pass. Restore" + echo "the TRIVY_VERSION key in this workflow's env block." + exit 1 + fi + + - name: Install trivy + uses: aquasecurity/setup-trivy@81e514348e19b6112ce2a7e3ecbafe19c1e1f567 # v0.3.1 + with: + cache: true + version: ${{ env.TRIVY_VERSION }} # zizmor: ignore[unpinned-tools] + + - name: Cache Helm charts + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ~/.cache/helm + key: helm-charts-${{ hashFiles('applicationsets/*.yaml') }} + restore-keys: | + helm-charts- + + - name: Every fixed CRITICAL is acknowledged + run: ./scripts/check-image-vulnerabilities.py + # ── Policy-admission gate ──────────────────────────────────────────── # The best-practice / pod-security policies run in Enforce mode on staging and # production (the overlays flip validationFailureAction to Enforce), so a @@ -903,6 +967,7 @@ jobs: kyverno, fork-safety, helm-render, + image-vulnerabilities, policy-admission, appsets, appset-render, @@ -934,6 +999,7 @@ jobs: 'Kyverno policy tests (+ verify-images contract)': '${{ needs.kyverno.result }}', 'Fork-safety gate (no hardcoded org, blocking)': '${{ needs.fork-safety.result }}', 'Helm-render (every addon × every env)': '${{ needs.helm-render.result }}', + 'Rendered images carry no unacknowledged CRITICAL': '${{ needs.image-vulnerabilities.result }}', 'Policy-admission (Enforce-tier Kyverno vs the fleet)': '${{ needs.policy-admission.result }}', 'ApplicationSet schema + sync waves': '${{ needs.appsets.result }}', 'Appset render (Karpenter subnet selector)': '${{ needs.appset-render.result }}', @@ -989,6 +1055,7 @@ jobs: kyverno, fork-safety, helm-render, + image-vulnerabilities, policy-admission, appsets, appset-render, diff --git a/CLAUDE.md b/CLAUDE.md index 7a3f4ea..054ca41 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -105,6 +105,7 @@ task validate:fork-safety # No hardcoded catalog repoURL in applied App task validate:log-volume-budget # Loki declares the fraction at which it stops ingesting, and the alert leads it task validate:falco-rule-floor # Every Falco rule set installed on a node is one Falco actually loads task validate:empty-corpus # No gate reports success over a corpus that is not there +task validate:image-vulnerabilities # Every fixed CRITICAL in a rendered image is acknowledged (not in `task validate` — see below) ``` ### Local `task validate` is a subset of CI @@ -112,7 +113,8 @@ task validate:empty-corpus # No gate reports success over a corpus that `task validate` runs the structural gates (lint, kustomize build, helm-render, ApplicationSet schema, sync-wave ordering, appset render, policy-admission, dashboards, fork-safety). CI runs those plus several gates that have **no local -`task` target**, so a clean `task validate` is necessary but not sufficient: +`task` target**, and one that has a target but is deliberately outside the +aggregate, so a clean `task validate` is necessary but not sufficient: - **Zero-placeholder gate** — `scripts/no-placeholders.sh` (CI job `placeholders`) - **Renovate config schema + manager-default drift** — @@ -164,6 +166,14 @@ dashboards, fork-safety). CI runs those plus several gates that have **no local `scripts/check-catalog-revision.py` (CI job `catalog-revision`) - **Alert coverage** — `scripts/check-alert-coverage.py`, which runs inside the `dashboards` job alongside the locally-available dashboard and Athena gates +- **Image vulnerabilities** — `scripts/check-image-vulnerabilities.py` (CI job + `image-vulnerabilities`). The one with a target of its own, + `task validate:image-vulnerabilities`, kept out of the aggregate because it + pulls every image the pinned charts render. Blocking on a CRITICAL + with a published fix that `image-advisories.yaml` does not acknowledge; HIGH is + counted and printed. Every run scans a digest-pinned end-of-life image first + and requires its known CRITICALs back, so a clean result is a result rather + than a scanner with no database - **Render → render-assert → kubeconform → `trivy config`** (CI job `validate`); locally this is `task render` then `task scan`, not part of `task validate` diff --git a/Taskfile.yaml b/Taskfile.yaml index c81e1ce..cde71ce 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -93,6 +93,14 @@ tasks: cmds: - ./scripts/check-image-pins.py + validate:image-vulnerabilities: + desc: "Image-CVE gate — every fixed CRITICAL in a rendered image is acknowledged in image-advisories.yaml" + cmds: + # Pulls and scans every image the pinned charts render, so it is slower + # than the structural gates and is NOT in `task validate`. CI runs it as + # its own blocking job; run it here before moving a chart pin. + - ./scripts/check-image-vulnerabilities.py + validate:workflows: desc: "Workflow supply-chain gate — zizmor over .github/workflows, offline, MEDIUM+ blocks" cmds: diff --git a/image-advisories.yaml b/image-advisories.yaml new file mode 100644 index 0000000..b1b2a09 --- /dev/null +++ b/image-advisories.yaml @@ -0,0 +1,267 @@ +# Fixed CRITICAL vulnerabilities this catalog knowingly ships, and why. +# +# scripts/check-image-vulnerabilities.py scans every image the pinned charts +# render and blocks on any CRITICAL with a published fix. An entry here says a +# human read that finding and decided the pin stands for now. Nothing else grants +# that: an image with no entry fails the gate, and an entry that stops matching +# the scan fails it too. +# +# Every entry is checked in four directions on every run, so this file cannot +# quietly become a permanent waiver: +# +# * a finding whose (id, package) no entry names -> unacknowledged, blocks +# * a finding on an image its entry does not list -> the entry does not cover it +# * an entry no scanned image carries -> the entry outlived its reason +# * an entry listing an image with no such finding -> the same, per image +# +# So the way to clear an entry is to move the chart pin to a version whose image +# carries the fixed package. Deleting the entry without doing that fails the +# first rule; doing it and leaving the entry behind fails the third or fourth. +# +# `images` name the image WITHOUT a tag. The acknowledgement is about the image; +# the tag moves with every chart bump, and a file that had to be edited on every +# bump would be edited without being read. +# +# ── What this catalog can and cannot do about these ── +# +# Every image below except one is built by the upstream project whose chart this +# catalog pins. This repo chooses a chart version; it does not build the image, +# and it cannot rebuild it against a patched dependency. The lever it has is the +# pin, and the pin moves when upstream publishes a chart whose image carries the +# fix — which Renovate opens a pull request for, and which this gate then either +# clears or does not. +# +# The exception is ghcr.io/nanohype/eks-agent-platform/eval-runner, which is +# built in this organisation. Its entry is a request to that repo's build, not to +# a chart pin here. + +advisories: + - id: CVE-2026-56854 + package: golang.org/x/crypto + reason: >- + Present in every Go-built image the fleet renders, because it is in the + module graph of almost every controller upstream ships. Fixed in + golang.org/x/crypto 0.55.0, which each project picks up on its own release + cadence; this catalog carries no build for any of them. Clears image by + image as chart pins move to versions built against 0.55.0 or later. + images: + - docker.io/envoyproxy/ai-gateway-controller + - docker.io/envoyproxy/ai-gateway-extproc + - docker.io/envoyproxy/gateway + - docker.io/falcosecurity/falcoctl + - docker.io/grafana/loki + - docker.io/grafana/loki-canary + - docker.io/grafana/tempo + - docker.io/velero/velero + - ghcr.io/external-secrets/external-secrets + - ghcr.io/grafana/grafana-operator + - ghcr.io/kedacore/keda + - ghcr.io/kedacore/keda-admission-webhooks + - ghcr.io/kedacore/keda-metrics-apiserver + - ghcr.io/opencost/opencost + - mirror.gcr.io/aquasec/trivy-operator + - nats + - nats-streaming + - natsio/prometheus-nats-exporter + - otel/opentelemetry-collector-contrib + - prom/memcached-exporter + - quay.io/argoproj/argo-events + - quay.io/argoproj/argo-rollouts + - quay.io/argoproj/argocli + - quay.io/argoproj/argoexec + - quay.io/argoproj/kubectl-argo-rollouts + - quay.io/argoproj/workflow-controller + - quay.io/cilium/cilium + - quay.io/jetstack/cert-manager-cainjector + - quay.io/jetstack/cert-manager-controller + - quay.io/jetstack/cert-manager-startupapicheck + - quay.io/jetstack/cert-manager-webhook + - reg.kyverno.io/kyverno/background-controller + - reg.kyverno.io/kyverno/cleanup-controller + - reg.kyverno.io/kyverno/kyverno + - reg.kyverno.io/kyverno/kyverno-cli + - reg.kyverno.io/kyverno/kyvernopre + - reg.kyverno.io/kyverno/reports-controller + - registry.k8s.io/descheduler/descheduler + - registry.k8s.io/external-dns/external-dns + - registry.k8s.io/kube-state-metrics/kube-state-metrics + - registry.k8s.io/metrics-server/metrics-server + + - id: CVE-2025-68121 + package: stdlib + reason: >- + The Go standard library compiled into the binary, not a dependency the + image can update separately — it moves when upstream rebuilds on a patched + toolchain. Fixed in Go 1.24.13, 1.25.7 and 1.26.0-rc.3. + images: + - docker.io/falcosecurity/k8s-metacollector + - ghcr.io/aquasecurity/node-collector + - nats + - nats-streaming + - natsio/nats-server-config-reloader + - natsio/prometheus-nats-exporter + - quay.io/argoproj/argo-events + - registry.k8s.io/metrics-server/metrics-server + - us-docker.pkg.dev/fairwinds-ops/oss/goldilocks + + - id: CVE-2024-24790 + package: stdlib + reason: >- + The same shape as CVE-2025-68121 and on the same image, against an older + toolchain: k8s-metacollector 0.1.1 is built on Go 1.21.1, and the fix is in + 1.21.11 and 1.22.4. The falco chart pin is what moves it. + images: + - docker.io/falcosecurity/k8s-metacollector + - nats + - nats-streaming + - natsio/nats-server-config-reloader + - natsio/prometheus-nats-exporter + + - id: CVE-2026-33186 + package: google.golang.org/grpc + reason: >- + A transitive module in three upstream controllers, fixed in grpc-go 1.79.3. + Not separable from the binary; it moves when each project rebuilds. + images: + - docker.io/falcosecurity/k8s-metacollector + - quay.io/argoproj/argo-events + - registry.k8s.io/metrics-server/metrics-server + + - id: CVE-2026-33815 + package: github.com/jackc/pgx/v5 + reason: >- + The Postgres driver argo-events links for its Postgres event source, fixed + in pgx 5.9.0. This catalog does not enable that event source, but the + package is in the shipped binary either way, so the finding is real and the + remedy is the same — the argo-events chart pin. + images: + - quay.io/argoproj/argo-events + + - id: CVE-2026-33816 + package: github.com/jackc/pgx/v5 + reason: >- + The second advisory against the same driver version in the same binary, + with the same fixed version and the same remedy as CVE-2026-33815. + images: + - quay.io/argoproj/argo-events + + - id: CVE-2026-31789 + package: libcrypto3 + reason: >- + OpenSSL in the goldilocks image's Alpine base, fixed in 3.5.6-r0. A base + layer this catalog does not build; it moves with the goldilocks chart pin. + images: + - ghcr.io/aquasecurity/node-collector + - us-docker.pkg.dev/fairwinds-ops/oss/goldilocks + + - id: CVE-2026-31789 + package: libssl3 + reason: >- + The same OpenSSL advisory against the sibling package in the same base + layer, cleared by the same chart bump as the libcrypto3 entry. + images: + - ghcr.io/aquasecurity/node-collector + - us-docker.pkg.dev/fairwinds-ops/oss/goldilocks + + - id: CVE-2026-59873 + package: tar + reason: >- + The one first-party image in this list. eval-runner is built in + nanohype/eks-agent-platform, so the remedy is a rebuild on a base carrying + tar 7.5.19 and a new operator chart version here — not a decision this + repository can take on its own. + images: + - ghcr.io/nanohype/eks-agent-platform/eval-runner + + # ── Images a controller starts ──────────────────────────────────────── + # + # Every entry below was invisible until the population took in the images a + # controller is handed rather than only the containers a pod template declares. + # None of them appears in any pod spec this catalog renders; each is a + # reference an operator reads and then creates a pod from. + + - id: CVE-2026-22039 + package: github.com/kyverno/kyverno + reason: >- + A CRITICAL in the policy engine itself, on the CLI image the kyverno chart + carries in a ConfigMap — the same reference ALLOWED_MUTABLE records as + untagged. Fixed in 1.15.3 and 1.16.3; the catalog pins chart 3.8.2, whose + appVersion is 1.18.2, so the ConfigMap's `:latest` is what carries the + finding rather than the running controllers. Clears on a chart that pins + that reference. + images: + - ghcr.io/kyverno/kyverno + + - id: CVE-2025-30215 + package: github.com/nats-io/nats-server/v2 + reason: >- + The NATS server compiled into the exporter image the argo-events eventbus + controller starts. Fixed in 2.11.1 and 2.10.27; the argo-events chart + leaves this image's tag unset, so it moves only when the chart does. + images: + - nats-streaming + - natsio/prometheus-nats-exporter + + - id: CVE-2022-48174 + package: busybox + reason: >- + The Alpine base of the NATS config-reloader sidecar the eventbus controller + creates. Fixed in 1.36.1-r1; this catalog does not build the image and the + argo-events chart pins the tag. + images: + - natsio/nats-server-config-reloader + + - id: CVE-2022-48174 + package: busybox-binsh + reason: >- + The same advisory against the sibling package in the same base layer, + cleared by the same chart bump. + images: + - natsio/nats-server-config-reloader + + - id: CVE-2022-48174 + package: ssl_client + reason: >- + The third package of the same busybox advisory in the same base layer. + images: + - natsio/nats-server-config-reloader + + - id: CVE-2022-37434 + package: zlib + reason: >- + zlib in the same NATS config-reloader base layer, fixed in 1.2.12-r2. The + image is pinned old by the argo-events chart; nothing here rebuilds it. + images: + - nats + - natsio/nats-server-config-reloader + + - id: CVE-2022-23806 + package: stdlib + reason: >- + The Go standard library compiled into the NATS exporter, fixed in 1.16.14 + and 1.17.7. Moves when the argo-events chart ships an image built on a + patched toolchain. + images: + - nats-streaming + - natsio/prometheus-nats-exporter + + - id: CVE-2023-24538 + package: stdlib + reason: >- + The same shape against both NATS images the eventbus controller starts, + fixed in Go 1.19.8 and 1.20.3. + images: + - nats + - nats-streaming + - natsio/nats-server-config-reloader + - natsio/prometheus-nats-exporter + + - id: CVE-2023-24540 + package: stdlib + reason: >- + The same two images and the same remedy, fixed in Go 1.19.9 and 1.20.4. + images: + - nats + - nats-streaming + - natsio/nats-server-config-reloader + - natsio/prometheus-nats-exporter diff --git a/scripts/check-image-pins.py b/scripts/check-image-pins.py index 8599548..d9a6c66 100755 --- a/scripts/check-image-pins.py +++ b/scripts/check-image-pins.py @@ -35,11 +35,32 @@ mutable one (`latest`, `main`, `master`, `edge`, `stable`, `dev`), resolves to something different tomorrow with nothing in this repo changing. -UNSCANNABLE IS REPORTED, NEVER COUNTED CLEAN - -A chart that fails to render is named in the output and its images are absent -from the inventory. A gate that silently skipped it would report the remaining -charts as the whole fleet. +WHAT CANNOT BE READ IS NOT THEREFORE ABSENT + +Two ways this gate can fail to see an image, each with its own verdict, because +a run that examined less than it reports on is not a clean run. + +A chart that fails to render contributes no images, so exit 2: the images that +DID render carry an immutable reference, and that is a result over part of the +fleet rather than over the fleet. + +A reference the classifier cannot place is a chart that rendered perfectly well +and carries a string this gate owes an answer about. It is a failure, exit 1, +naming the two declarations that resolve it — the controller that starts it, or +the reason pulling it runs nothing. Reported as an aside it reached no verdict +at all, and the run's correctness rested on a sibling gate reading the same list +as a refusal; a gate whose result depends on another gate's reading of its +output has not stated its own. + +THE REFERENCE FORMS THIS READS + +An `image:` key is read whole, in every spelling and at every depth, so a +digest, a tag or a bare name reaches the classifier as written. Under that walk +sits a pattern over every string a render carries, which is how an image handed +to a controller in a flag or a ConfigMap is found — and it admits the digest +form, `@sha256:`, with or without a tag alongside it. That is the +form this gate's own remediation text recommends, so being blind to it would be +recommending the one spelling the completeness floor could not see. """ from __future__ import annotations @@ -51,6 +72,8 @@ import subprocess import sys +import yaml + # Shared precondition helper, loaded by path: these are hyphenated executables # run from varying working directories. _gl = pathlib.Path(__file__).resolve().parent / "gatelib.py" @@ -79,6 +102,15 @@ sys.modules["render_addons"] = render_addons _spec.loader.exec_module(render_addons) +# Helm charts occasionally emit the YAML `=` default-value sentinel (e.g. `- =` +# inside a ConfigMap payload). PyYAML's SafeLoader has no constructor for it and +# raises, which would drop a whole chart's render out of the walk below — so map +# the sentinel to its literal scalar rather than lose the chart. +yaml.SafeLoader.add_constructor( + "tag:yaml.org,2002:value", + lambda loader, node: loader.construct_scalar(node), # type: ignore[arg-type] +) + ROOT = render_addons.REPO_ROOT NETWORK_TIMEOUT = 300 @@ -91,24 +123,56 @@ # Exemptions, asserted against the real render. An entry naming an image the # fleet no longer renders mutably FAILS: an exemption that matches nothing is a # description that rots, and it rots toward permissive. -ALLOWED_MUTABLE: dict[str, str] = {} - - -def inventory(env: str) -> tuple[dict[str, set[str]], list[tuple[str, str]]]: - """(image -> charts that render it, [(path, why-not-scanned)]).""" +ALLOWED_MUTABLE: dict[str, str] = { + "ghcr.io/kyverno/kyverno": + "the kyverno chart carries this in a ConfigMap as the CLI image for " + "`kyverno` invocations, with no values key to pin it. Invisible until " + "controller-supplied references entered the population. Clears when the " + "chart exposes the tag, or when this catalog stops shipping that " + "ConfigMap.", + "nats-streaming": + "the argo-events version table's oldest streaming row carries `latest`, " + "alongside rows that pin. An eventbus asking for that version gets a " + "moving image; nothing in this catalog selects it, and nothing here can " + "pin it. Clears on a chart that pins the row.", + "natsio/prometheus-nats-exporter": + "an argo-events eventbus default the controller passes to the NATS " + "StatefulSet it creates. The chart pins two other tags of this image and " + "leaves the exporter's unset. Clears on a chart version that pins it.", +} + + +def inventory(env: str, seen: set[str] | None = None + ) -> tuple[dict[str, set[str]], list[tuple[str, str]], list[tuple[str, str]]]: + """(image -> charts that render it, charts not rendered, references not placed). + + Two lists rather than one, because they are two facts with two repairs and + two verdicts. A chart that did not render leaves the fleet's image set + unknown — the gate examined less than it reports on. A reference the + classifier cannot place is a chart that rendered perfectly well and carries + something this gate cannot answer for; the repair is to classify it. Held in + one list, the second printed under the first's heading and reached no + verdict at all. + + `seen`, if given, collects every image-shaped reference the render carried — + including the ones excluded from the population — so a declaration can be + checked against what the render contains rather than against what survived. + """ gatelib.require('helm') units = render_addons.discover() aliases = render_addons.add_repos(units) images: dict[str, set[str]] = {} - unscannable: list[tuple[str, str]] = [] + unrendered: list[tuple[str, str]] = [] + unclassified: list[tuple[str, str]] = [] + seen = set() if seen is None else seen for u in units: if u.chart in getattr(render_addons, "SKIP_CHARTS", {}): - unscannable.append((u.path, "chart is on render-addons' SKIP_CHARTS list")) + unrendered.append((u.path, "chart is on render-addons' SKIP_CHARTS list")) continue d = ROOT / u.path if not d.is_dir(): - unscannable.append((u.path, "path does not exist")) + unrendered.append((u.path, "path does not exist")) continue vf = [] if (d / "values.yaml").exists(): @@ -124,11 +188,396 @@ def inventory(env: str) -> tuple[dict[str, set[str]], list[tuple[str, str]]]: cmd += vf proc = subprocess.run(cmd, capture_output=True, text=True, timeout=NETWORK_TIMEOUT) if proc.returncode != 0: - unscannable.append((u.path, (proc.stderr.strip() or proc.stdout.strip())[:200])) + unrendered.append((u.path, (proc.stderr.strip() or proc.stdout.strip())[:200])) + continue + # BOTH streams, because which one carries the pull report is a property + # of the helm build rather than of this repository: some versions write + # it to stdout, where it lands in the manifest stream this parses, and + # some to stderr, where nothing sees it. Read from either, so what the + # gate knows about a run does not depend on which helm ran it. + pulled = chart_artifacts(proc.stdout + "\n" + proc.stderr) + for ref_str in extract_images(proc.stdout, u.chart, unrendered, unclassified, + seen, pulled): + images.setdefault(ref_str, set()).add(u.chart) + return images, unrendered, unclassified + + +# helm's own report of an OCI artifact it fetched, which it prints as two lines +# before the manifests: +# +# Pulled: public.ecr.aws/karpenter/karpenter:1.14.0 +# Digest: sha256: +# +# This is the answer to a question a list cannot answer. An OCI registry serves +# charts and container images through one API and one reference grammar, so +# `public.ecr.aws/karpenter/karpenter:1.14.0` is indistinguishable from an image +# by shape — and it is the chart, while the container beside it is +# `public.ecr.aws/karpenter/controller`. helm pulled the first AS A CHART and +# says so, which is an observation about this run rather than a name somebody +# wrote down. +# +# Being an observation is what makes it usable here. A declaration naming these +# would have to be re-checked against the render by declaration_rot, which +# deletes an entry the render does not support — and the render only carries +# them under the helm builds that print to stdout. One rule would demand the +# entry and the other remove it. Nothing is declared, so nothing rots. +HELM_PULLED = re.compile(r"^Pulled:\s*(?P\S+)\s*$", re.M) + + +def chart_artifacts(output: str) -> set[str]: + """Every OCI artifact helm reports pulling as a chart, from either stream.""" + return {m.group("ref") for m in HELM_PULLED.finditer(output)} + + +# An image reference as a controller is handed one: in a flag, a ConfigMap value, +# a CR field. Requires a path separator, which is what separates an image from +# the host:port strings that fill a rendered config — `loki.monitoring.svc:3100` +# and `127.0.0.1:8080` are addresses, not images, and no grammar that admits them +# can be read by a human. +# Two alternatives, because an official image carries neither a registry nor an +# organisation: `nats:2.10.10` is the whole reference. Requiring a path separator +# made that shape invisible, and the argo-events event-bus controller declares +# exactly it — `natsImage` beside the two `natsio/*` sidecars in the same rows, +# so one StatefulSet had its helpers scanned and its main container silent. +# +# The single-segment alternative needs two discriminators the prefixed one does +# not, because a rendered config is full of strings with that shape: +# +# * the match may not begin part-way through a longer token. Without that the +# alternative starts after a dot and `vault.example.com:8200` yields +# `com:8200`, which is a hostname's last label and a port. +# * the NAME must start with a letter and carry no dot. That excludes a +# timestamp (`00:00Z`), an address (`10.0.0.5:8080`, +# `telemetry.monitoring.svc.cluster.local:4318`), an IPv6 fragment and a +# ratio (`1:1`). +# * the TAG must be `latest` or begin with a digit, optionally after a `v`. +# That excludes the RBAC names — `kyverno:admission-controller`, +# `system:auth-delegator` — which are the shape's other common occupant. +# +# Together they admit every official-image reference this fleet pins and no +# string in the render that is not one. +# +# A THIRD alternative for the digest form, which the two tag alternatives cannot +# spell and which is the one this gate's own remediation text tells an operator +# to write. `@sha256:` carries no tag for them to match, so the whole +# reference yielded nothing but its `sha256:` tail — a single-segment shape +# whose bare name is `sha256`, and an entry by that name swallowed it. A +# reference the walk cannot spell is not therefore absent, and one this gate +# recommends is the worst shape to be blind to. +# +# Tried first, so `:@sha256:` is captured whole rather than as +# its `repo:tag` head, and the digest a container actually runs is what reaches +# the classifier. The 64-hex suffix is discriminator enough on its own, so the +# path separator and the tag restrictions the tag alternatives need are not +# repeated here: nothing else in a render has that shape. +# The registry grammar's own definition of a host, written once because both +# alternatives below need it and two spellings of one grammar drift. +# +# A domain COMPONENT carries no dot: dots are the separators, and a component +# neither starts nor ends with a dash. That is what makes the host unambiguous, +# and unambiguous is what makes it safe. Spelled as a starred class containing +# `.` and `-` followed by a plus-quantified group beginning with `.` over that +# same class, every way of splitting a run of `-.` between the two quantifiers +# is a distinct parse, and the engine tries all of them before failing — 43 +# characters of `0.` and `-.` took a third of a second, 83 would not finish. +# The input here is rendered content from charts this repository does not +# author and the gate runs on every render, so a string that makes the matcher +# climb arrives with a chart bump, and the symptom is a job that never returns +# rather than one that fails. +# +# Here `[a-z0-9]+` and `-+` are disjoint, so a run of dashes can only be a +# separator inside one component and a dot can only end one. Every input has +# exactly one parse or none. +DOMAIN_COMPONENT = r"[a-z0-9]+(?:-+[a-z0-9]+)*" +REGISTRY = rf"(?:{DOMAIN_COMPONENT}(?:\.{DOMAIN_COMPONENT})+(?::\d+)?/)?" + +IMAGE_REF = re.compile( + r"(?`, only the value. Excluded by + # SHAPE rather than by a declaration, because the two rules this repo + # already has cannot both hold over one: an entry named `sha256` would have + # to excuse it wherever it appears, and declaration_rot deletes an entry the + # render does not support. A shape that can never be a reference is not an + # exemption to keep re-reading — it is not a reference. + r"(?!sha(?:256|512):[0-9a-f]{32,}(?![0-9a-z]))" + r"(?:" + rf"({REGISTRY}" + r"[a-z0-9][a-z0-9._-]*(?:/[a-z0-9][a-z0-9._-]*)*" + r"(?::[a-zA-Z0-9][\w.-]*)?@sha256:[0-9a-f]{64})" + rf"|({REGISTRY}" + r"[a-z0-9][a-z0-9._-]*(?:/[a-z0-9][a-z0-9._-]*)+:[a-zA-Z0-9][\w.-]*)" + r"|([a-z][a-z0-9]*(?:[_-][a-z0-9]+)*:(?:latest|v?[0-9][\w.-]*))" + r")\b") + +# Images a CONTROLLER starts, which no pod template in this render declares. The +# controller is handed the reference and creates the pod later, so the deployable +# surface is strictly larger than what `helm template` shows as a container. +# +# Whether a given string is one of these is a fact about a controller's behaviour, +# and this gate reads manifests. So it is declared rather than inferred, and the +# declaration is what the assertion is against: every image-shaped string the +# render carries must be in the structural population, on this list, or on +# NOT_A_CONTAINER. An unclassified one is reported, so the answer to "is this +# deployed?" is never silence. +CONTROLLER_IMAGES = { + "nats": + "the NATS server the argo-events eventbus controller starts, declared as " + "`natsImage` in its own version table beside the two natsio/* sidecars it " + "creates in the same StatefulSet", + "nats-streaming": + "the same table's streaming variant, for eventbus resources that ask for " + "it", + "docker.io/envoyproxy/ratelimit": + "the Envoy Gateway controller reads it from its own EnvoyGateway " + "ConfigMap and creates the rate-limit Deployment", + "docker.io/envoyproxy/ai-gateway-extproc": + "the AI Gateway controller injects it as the ext-proc sidecar on every " + "gateway pod it manages", + "ghcr.io/aquasecurity/node-collector": + "trivy-operator creates one node-collector pod per node scan", + "quay.io/argoproj/argoexec": + "the workflow controller injects it as the init and wait container of " + "every workflow step pod", + "quay.io/jetstack/cert-manager-acmesolver": + "cert-manager creates one solver pod per HTTP-01 challenge", + "natsio/nats-server-config-reloader": + "the argo-events eventbus controller creates the NATS StatefulSet with " + "this sidecar", + "natsio/prometheus-nats-exporter": + "the same controller, same StatefulSet", + "ghcr.io/kyverno/kyverno": + "the chart's own CLI image reference, carried in a ConfigMap for " + "`kyverno` invocations rather than as a container", +} + +# Image-shaped strings that are not container images. An OCI artifact reference +# resolves through a registry and looks identical, and pulling one starts no +# container — so scanning it for a running-container CVE would report on +# something nothing runs. +NOT_A_CONTAINER = { + "falco-rules": + "a falcoctl rulesfile OCI artifact; check-falco-rule-floor.py resolves " + "these against the registry and asserts Falco loads what they install", + "falco-incubating-rules": + "the same, the incubating tier", + "falco-sandbox-rules": + "the same, the sandbox tier", + "localhost": + "a memcached address in a Loki config value; `localhost:11211` is a host " + "and a port", + "hubble-relay": + "a Kubernetes Service address in a Cilium config value — `hubble-relay:80` " + "has the shape of a single-segment image and is a host and a port", + "ghcr.io/falcosecurity/plugins/plugin/container": + "a falcoctl rules/plugin OCI artifact, unpacked into an emptyDir", + "ghcr.io/falcosecurity/plugins/plugin/k8smeta": + "the same, the k8smeta plugin", + "mirror.gcr.io/aquasec/trivy-checks": + "the trivy checks bundle, an OCI artifact trivy-operator downloads", +} + + +# Charts that render no container image because they ship only CustomResource +# definitions. Asserted in both directions by chart_coverage below: one that +# starts shipping a workload fails here, and one that is named and no longer +# rendered fails too. +IMAGELESS_CHARTS = { + "ai-gateway-crds-helm": "the Envoy AI Gateway CRDs, applied ahead of the " + "controller that renders one of their kinds", + "prometheus-operator-crds": "the Prometheus operator CRDs, applied ahead of " + "the charts that render ServiceMonitors", +} + + +def chart_coverage(images: dict[str, set[str]], units, + seen: set[str] | None = None) -> list[str]: + """Every chart the render covered contributed an image, or is declared not to. + + A per-chart floor rather than a total, because a total cannot see the shape + that actually happened: an extractor missed the ordinary `- image:` list-item + spelling, two charts' entire workloads fell out of the inventory, and the + count stayed large enough to look healthy. One image per chart is derived + from the corpus — no constant to pick, and it moves with the catalog. + """ + problems: list[str] = [] + # Keyed by chart NAME, which is what `images` records, and this catalog pins + # opentelemetry-collector three times — otel-agent, otel-gateway and + # otel-gateway-floor. Any one of those can render nothing while the other two + # keep the name contributing, so the floor is per chart name and says so + # rather than reading as per render unit. + rendered = {u.chart for u in units if u.chart not in getattr( + render_addons, "SKIP_CHARTS", {})} + contributing = set().union(*images.values()) if images else set() + + for chart in sorted(rendered - contributing): + if chart in IMAGELESS_CHARTS: + continue + problems.append( + f"{chart} rendered and contributed no image. Every chart shipping a " + f"workload contributes at least one, so either the extraction stopped " + f"seeing a shape this chart uses, or the chart ships only CRDs and " + f"belongs in IMAGELESS_CHARTS with the reason.") + + problems += declaration_rot(seen if seen is not None + else {bare_name(ref) for ref in images}) + + for chart, reason in sorted(IMAGELESS_CHARTS.items()): + if chart not in rendered: + problems.append( + f"{chart} is declared imageless but the fleet no longer renders it — " + f"the entry outlived its chart. (recorded: {reason})") + elif chart in contributing: + problems.append( + f"{chart} is declared imageless and now contributes an image. It " + f"ships a workload; delete the entry so its images are scanned like " + f"every other chart's. (recorded: {reason})") + return problems + + +def bare_name(ref: str) -> str: + """A reference with tag and digest removed, which is what an entry names. + + The tag separator is the last colon in the FINAL path segment: a registry + with a port carries a colon that is not one, and cutting there produces a key + no entry can match and no reader can recognise. + """ + ref = ref.split("@", 1)[0] + name = ref.rsplit("/", 1)[-1] + return ref.rsplit(":", 1)[0] if ":" in name else ref + + +def string_scalars(node) -> list[str]: + """Every string a rendered document carries, at any depth. + + A controller is handed an image the same way it is handed anything else — a + flag in `args`, a value in a ConfigMap, a field on a custom resource — so the + surface an image can arrive through is every string, not a key name. + """ + out: list[str] = [] + if isinstance(node, dict): + for value in node.values(): + out.extend(string_scalars(value)) + elif isinstance(node, list): + for value in node: + out.extend(string_scalars(value)) + elif isinstance(node, str): + out.append(node) + return out + + +def keyed_images(node) -> set[str]: + """Every `image:` key anywhere in the documents, whatever declares it. + + Structure, not text. The pattern this replaced was anchored on `image:` + preceded only by whitespace, so the ordinary list-item form `- image: ` + under `containers:` never matched it and two charts' entire workloads were + absent from a scan reporting fifty-five images. + + Deliberately not restricted to pod templates. A walk over pod owners is a + strict SUBSET of this one — every container's image is an `image:` key — so + it would be mechanism with no effect, and it would drop the custom resources + that name an image an operator then runs. + """ + found: set[str] = set() + if isinstance(node, dict): + for key, value in node.items(): + if key == "image" and isinstance(value, str): + found.add(value) + else: + found |= keyed_images(value) + elif isinstance(node, list): + for value in node: + found |= keyed_images(value) + return found + + +def extract_images(rendered: str, chart: str, + unrendered: list[tuple[str, str]], + unclassified: list[tuple[str, str]], + seen: set[str] | None = None, + pulled: set[str] | None = None) -> set[str]: + """Every image one chart's render deploys, and an assertion that it is every one. + + Structural first, because a pattern is a claim about spelling. IMAGE is + anchored on `image:` preceded only by whitespace, so the ordinary list-item + form `- image: ` never matched it: against this catalog's render the + pattern yielded 55 images where the pod specs hold 57, and the four it missed + were the whole workload of two charts. A scanner that omits images silently + is worse than none, because its green result becomes evidence. + + The text scan is kept as an independent floor under the walks rather than + replaced. A parser and a pattern fail on different inputs, so a structural + walk that stops seeing a shape is caught by the one that reads the bytes — + and an image ONLY the pattern finds is either a string payload a controller + reads (declared in TEXT_ONLY_IMAGES) or a walk that has drifted. + """ + try: + docs = list(yaml.safe_load_all(rendered)) + except yaml.YAMLError as exc: + first = str(exc).strip().splitlines()[0] + unrendered.append((chart, f"rendered YAML this gate could not parse — {first}")) + return set() + + structural = keyed_images(docs) + textual = {m.group(1) for m in IMAGE.finditer(rendered)} + candidates = {c for s in string_scalars(docs) + for groups in IMAGE_REF.findall(s) for c in groups if c} + if seen is not None: + # Every image-shaped reference the render carried, including the ones + # excluded below. A declaration is about what the render CONTAINS, so + # checking it against the surviving population would report every + # NOT_A_CONTAINER entry as stale by construction. + seen |= {bare_name(r) for r in structural | textual | candidates} + + # Compared on the bare name, because the same image reaches a render twice: + # once as a container reference carrying a digest, and once as a bare + # `repo:tag` in an annotation or a config value. Those are one image, and + # reporting the second as unreachable would be reporting the first. + structural_bare = {bare_name(r) for r in structural} + # Compared whole, not on the bare name. A registry can serve a chart and a + # container image from one repository path, and passing over every reference + # sharing a pulled chart's name would take the image with it — which is the + # one direction this gate must never fail in. + artifacts = set(pulled or ()) + controller: set[str] = set() + for ref_str in sorted((textual | candidates) - structural): + bare = bare_name(ref_str) + if bare in structural_bare or bare in NOT_A_CONTAINER: continue - for m in IMAGE.finditer(proc.stdout): - images.setdefault(m.group(1), set()).add(u.chart) - return images, unscannable + if ref_str in artifacts or ref_str.split("@", 1)[0] in artifacts: + # helm reported fetching this exact reference as a chart. Pulling a + # chart starts no container, so there is nothing here to scan. + continue + if bare in CONTROLLER_IMAGES: + controller.add(ref_str) + continue + unclassified.append(( + chart, + f"{ref_str} is image-shaped and reaches no pod template or `image:` key. " + f"Either a structural walk stopped seeing a shape, or a controller is " + f"handed it and starts a pod later — in which case it belongs in " + f"CONTROLLER_IMAGES with the controller that starts it, or in " + f"NOT_A_CONTAINER if pulling it runs nothing")) + return structural | textual | controller + + +def declaration_rot(seen: set[str]) -> list[str]: + """Declarations the render no longer supports. + + Both lists say something about images this catalog renders. An entry for one + it does not is an excuse for nothing, and an exemption list nobody re-reads + only ever widens. + """ + problems = [] + for table, label in ((CONTROLLER_IMAGES, "started by a controller"), + (NOT_A_CONTAINER, "not a container image")): + for bare, reason in sorted(table.items()): + if bare not in seen: + problems.append( + f"{bare} is declared {label} and the fleet renders no reference to " + f"it — the declaration outlived its image. (recorded: {reason})") + return problems def classify(ref: str) -> str: @@ -142,16 +591,15 @@ def classify(ref: str) -> str: return "mutable" if tag.lower() in MUTABLE_TAGS else "tag" -def bare_name(ref: str) -> str: - """A reference with its tag removed, which is what an exemption names. - - Split on the last colon only when it sits in the final path segment: a - registry with a port (`registry:5000/x/y`) carries a colon that is not a tag - separator, and cutting there would produce a key no exemption can match and - no reader can recognise. - """ - name = ref.rsplit("/", 1)[-1] - return ref.rsplit(":", 1)[0] if ":" in name else ref +# What an operator is told to write. Every reference form named here in +# backticks is one the walk above reads, asserted rather than assumed: advice +# pointing at a spelling this gate is blind to is worse than no advice, because +# following it moves an image out of the population and the run stays green. +MUTABLE_REMEDIATION = ( + "Pin it in the addon's values.yaml to the chart's appVersion, or to a digest — " + "`@sha256:` is a form this gate reads whole, whether it carries a " + "tag as well or not." +) def verdict(images: dict[str, set[str]], allowed: dict[str, str]) -> list[str]: @@ -173,8 +621,8 @@ def verdict(images: dict[str, set[str]], allowed: dict[str, str]) -> list[str]: if bare in allowed: continue failures.append( - f"{ref} (via {', '.join(sorted(images[ref]))}) resolves to a moving target. " - f"Pin it in the addon's values.yaml to the chart's appVersion or a digest.") + f"{ref} (via {', '.join(sorted(images[ref]))}) resolves to a moving " + f"target. {MUTABLE_REMEDIATION}") for bare, reason in sorted(allowed.items()): if bare not in mutable_seen: @@ -191,40 +639,63 @@ def main() -> int: ap.add_argument("--env", default="production") args = ap.parse_args() - images, unscannable = inventory(args.env) + seen: set[str] = set() + images, unrendered, unclassified = inventory(args.env, seen) if args.list: for ref in sorted(images): print(f"{classify(ref):8} {ref} ({', '.join(sorted(images[ref]))})") - print(f"\n{len(images)} image(s); {len(unscannable)} chart(s) unscannable") - for path, err in unscannable: - print(f" unscannable {path}: {err}") + print(f"\n{len(images)} image(s); {len(unrendered)} chart(s) not rendered; " + f"{len(unclassified)} reference(s) not placed") + for path, err in unrendered: + print(f" not rendered {path}: {err}") + for path, err in unclassified: + print(f" not placed {path}: {err}") return 0 if not images: print("FAIL rendered no images at all. Every chart failed, or the extractor " "stopped matching — either way this reports the same as a clean fleet.") - for path, err in unscannable: + for path, err in unrendered: print(f" {path}: {err}") return 2 - failures = verdict(images, ALLOWED_MUTABLE) + failures = chart_coverage(images, render_addons.discover(), seen) + failures += verdict(images, ALLOWED_MUTABLE) - # Reported whatever the verdict: a chart that did not render contributed no - # images, and counting the rest as the whole fleet is how a partial scan - # reads as a complete one. - if unscannable: - print(f"{len(unscannable)} chart(s) could not be rendered and were NOT scanned:") - for path, err in unscannable: - print(f" {path}: {err}") - print() + # A reference the classifier cannot place is a verdict this gate owes and has + # not given: the chart rendered, the string is image-shaped, and whether + # anything runs it is unanswered. Printed under a heading about charts that + # did not render, it reached no verdict at all and the run exited 0 — + # correctness resting on a sibling gate reading the same list as a refusal, + # which is that gate holding the line rather than this one stating a result. + failures.extend(f"{chart}: {detail}" for chart, detail in unclassified) if failures: + if unrendered: + print(f"{len(unrendered)} chart(s) could not be rendered and were NOT " + f"scanned, so the population below is a subset of the fleet:") + for path, err in unrendered: + print(f" {path}: {err}") + print() print(f"{len(failures)} image-pin problem(s) across {len(images)} rendered image(s):\n") for f in failures: print(f" {f}") return 1 + # No failure among the images that WERE derived is not a verdict over the + # fleet when a chart contributed none. Exit 2 rather than 0: a chart that did + # not render is a gate that examined less than it reports on, which is not + # the same as finding nothing. + if unrendered: + print(f"Cannot run: {len(unrendered)} chart(s) could not be rendered and were " + f"NOT scanned:") + for path, err in unrendered: + print(f" {path}: {err}") + print(f"The {len(images)} image(s) that did render carry an immutable " + f"reference. That is a result over part of the fleet, not the fleet.") + return gatelib.CANNOT_RUN + print(f"✓ all {len(images)} rendered image(s) across {len(images and set().union(*images.values()) or [])} " f"chart(s) carry an immutable reference " f"({len(ALLOWED_MUTABLE)} exemption(s), each still matching the render)") diff --git a/scripts/check-image-vulnerabilities.py b/scripts/check-image-vulnerabilities.py new file mode 100755 index 0000000..ee76b26 --- /dev/null +++ b/scripts/check-image-vulnerabilities.py @@ -0,0 +1,416 @@ +#!/usr/bin/env python3 +"""Every image the production render carries is scanned, and a CRITICAL is a decision. + + scripts/check-image-vulnerabilities.py # blocking gate + scripts/check-image-vulnerabilities.py --list # print every finding + scripts/check-image-vulnerabilities.py --self-test # the canary alone + +WHOSE QUESTION THIS IS + +An adopter of this catalog inherits every chart pin in it without reading. The +version a pin resolves to decides which image lands on their nodes, so whoever +moves the pin owns what the image carries. Nothing else in the pipeline can: +trivy-operator reports at runtime, on a cluster that already pulled the image, +to an operator who cannot change the pin without a PR here. + +WHAT IS BLOCKING AND WHAT IS NOT + +Blocking: a CRITICAL vulnerability WITH A FIX AVAILABLE, in an image the fleet +renders, that `image-advisories.yaml` does not acknowledge. Those three +qualifiers are the whole bar. A CRITICAL with no fix published is not a decision +anyone here can act on; a HIGH is counted and printed, and the count is not +gated, because holding a merge on it would gate this repo on the rate at which +upstream chart images accumulate advisories rather than on anything the commit +changed. + +That leaves a real objection, and the answer to it is the advisory file rather +than a softer bar. What this gate reads is not a function of the commit alone: it +is the commit against a vulnerability database and against whatever the pinned +tags resolve to today. Both move, and they move in both directions. + +A database that GAINS an advisory turns a tree red that passed yesterday. The +failure names the image, the CVE and the fixed version, and clears by bumping the +chart or by recording why the finding stands. + +A database that LOSES one — a reclassification, a withdrawn advisory — turns a +tree red that passed for the opposite reason: rules 3 and 4 fire, the entry +acknowledging it no longer matches any scanned image, and the clearing action is +deleting an entry. The same happens with no database change at all, because this +fleet is pinned by version tag and a tag is not immutable (see +check-image-pins.py on what counts as pinned): a publisher re-pushing the same +tag on a patched base removes the finding underneath a standing entry. + +Neither direction is silent and neither is permanent, and both are decisions with +an author. That is the whole of what the advisory file buys. + +THE ADVISORY FILE IS ASSERTED IN FOUR DIRECTIONS + +An acknowledgement list nobody re-checks only ever widens, so every entry is +checked against the scan rather than trusted: + + * a finding whose (id, package) no entry names -> the gate's own question + * a finding on an image its entry does not list -> the entry does not cover it + * an entry whose (id, package) no image carries -> the entry outlived its reason + * an entry listing an image with no such finding -> the same, per image + +Entries name images WITHOUT a tag, because the acknowledgement is about the +image and a chart bump moves the tag. A bump that FIXES the finding therefore +fails the third or fourth rule rather than leaving a stale excuse behind. + +THE CANARY, WHICH IS WHAT MAKES A CLEAN RESULT MEAN ANYTHING + +A vulnerability scanner that has lost its database, or been handed a flag it +reads as "report nothing", returns a clean result for every image. That is +indistinguishable from a healthy fleet by exit code and by output alone. + +So every run first scans a digest-pinned image with historical, fixed CRITICALs +and requires them to come back. A canary that comes back clean exits 2: it is a +statement about the scanner, not about the catalogue, and the two must not print +the same thing. + +CHART-SOURCED APPLICATIONS ONLY, SAID PLAINLY + +The population comes from check-image-pins.inventory, which walks the +ApplicationSets that pin a Helm chart. A kustomize-sourced Application renders +workloads through no chart and contributes nothing — check-policy-admission.py +names one by path in KUSTOMIZE_WORKLOADS (dashboards/base, the grafana-operator +namespace), and its images run on every full-tier cluster and are not scanned +here. Taking them in means a second render path; until that lands the boundary is +written down, the same way the environment one below is. + +ONE ENVIRONMENT, SAID PLAINLY + +The population is the production render. render-addons produces four — +development, staging, production and hub — and nothing here asserts they agree, +so a component enabled only in development or on the hub is deployed by this +fleet and is not scanned. That is a narrower claim than "every image the fleet +renders", and it is the claim this gate holds: every image the PRODUCTION render +carries. Widening it means four scans of seventy images per run, which is a cost +decision rather than a technical one, and until it is taken the boundary belongs +in writing rather than in the reader's assumption. + +UNSCANNABLE IS NOT CLEAN + +An image that could not be pulled contributed no findings. Counting the rest as +the whole fleet is how a partial scan reads as a complete one, so any image that +fails to scan exits 2 with the image named. +""" + +from __future__ import annotations + +import argparse +import collections +import importlib.util +import json +import pathlib +import subprocess +import sys +from typing import NoReturn + +# Shared precondition helper, loaded by path: these are hyphenated executables +# run from varying working directories. +_gl = pathlib.Path(__file__).resolve().parent / "gatelib.py" +_gs = importlib.util.spec_from_file_location("gatelib", _gl) +assert _gs and _gs.loader, f"{_gl} is not loadable as a module" +gatelib = importlib.util.module_from_spec(_gs) +sys.modules["gatelib"] = gatelib +_gs.loader.exec_module(gatelib) + +# The image inventory, from the gate that already derives it. Re-deriving the +# population here would let the two disagree about what the fleet is, and the +# one that scanned fewer images would be the one printing a clean result. +_ip_path = pathlib.Path(__file__).resolve().parent / "check-image-pins.py" +_ip_spec = importlib.util.spec_from_file_location("check_image_pins", _ip_path) +assert _ip_spec and _ip_spec.loader, f"{_ip_path} is not loadable as a module" +image_pins = importlib.util.module_from_spec(_ip_spec) +sys.modules["check_image_pins"] = image_pins +_ip_spec.loader.exec_module(image_pins) + +ROOT = pathlib.Path(__file__).resolve().parent.parent +ADVISORIES = ROOT / "image-advisories.yaml" + +# Seconds one trivy invocation may run. A scan with no deadline turns a stalled +# registry into a job that hangs until the CI runner's own ceiling, with no +# diagnostic naming the image that stalled. +SCAN_TIMEOUT = 600 + +# The severity that blocks. HIGH is counted and printed; see the header. +BLOCKING_SEVERITY = "CRITICAL" +REPORTED_SEVERITIES = ["CRITICAL", "HIGH"] + +# The floor on images scanned is DERIVED, not picked: one image per chart the +# render covers, asserted per chart by check-image-pins.chart_coverage. A total +# cannot see the shape that actually happened — an extractor missed the ordinary +# `- image:` list-item spelling, two charts' whole workloads left the inventory, +# and the count stayed large enough to look healthy. The per-chart form catches +# exactly that, and it moves with the catalog rather than with a constant. + +# A digest-pinned image with historical CRITICALs that have published fixes. +# Alpine 3.10 is end-of-life, so these cannot be patched away underneath the +# canary, and the digest means the tag cannot be repointed at a clean build. +# +# mirror.gcr.io rather than docker.io: this pulls on every CI run, and Docker +# Hub's anonymous limits are the problem that mirror exists to solve. The +# catalog already pulls trivy-operator through it. +CANARY = ("mirror.gcr.io/library/alpine@sha256:" + "ca1c944a4f8486a153024d9965aafbe24f5723c1d5c02f4964c045a16d19dc54") + +Finding = collections.namedtuple("Finding", "image bare id package installed fixed severity") + +failures: list[str] = [] + + +def fail(msg: str) -> None: + failures.append(msg) + + +def cannot_run(*lines: str) -> NoReturn: + for line in lines: + print(line) + print("This gate examined nothing, which is not the same as finding nothing.") + sys.exit(gatelib.CANNOT_RUN) + + +def bare(ref: str) -> str: + """An image reference with tag and digest removed. + + What an advisory names. The tag moves with every chart bump and the digest + moves with every rebuild; the image is what a reader recognises and what the + acknowledgement is actually about. + + The tag separator is the last colon in the FINAL path segment. A registry + with a port carries a colon that is not one, and cutting there produces a key + no advisory can match and no reader can recognise. + """ + ref = ref.split("@", 1)[0] + name = ref.rsplit("/", 1)[-1] + return ref.rsplit(":", 1)[0] if ":" in name else ref + + +def scan(ref: str) -> list[Finding] | None: + """Fixed findings at the reported severities, or None if the image did not scan.""" + cmd = ["trivy", "image", "--quiet", "--scanners", "vuln", + "--severity", ",".join(REPORTED_SEVERITIES), + "--ignore-unfixed", "--format", "json", ref] + try: + proc = subprocess.run(cmd, capture_output=True, text=True, timeout=SCAN_TIMEOUT) + except subprocess.TimeoutExpired: + return None + if proc.returncode != 0: + return None + try: + report = json.loads(proc.stdout) + except json.JSONDecodeError: + return None + out = [] + for result in report.get("Results") or []: + for v in result.get("Vulnerabilities") or []: + out.append(Finding(ref, bare(ref), v["VulnerabilityID"], v.get("PkgName", ""), + v.get("InstalledVersion", ""), v.get("FixedVersion", ""), + v.get("Severity", ""))) + return out + + +def read_advisories() -> list[dict]: + """The acknowledged findings, or exit 2 naming what is wrong with the file.""" + if not ADVISORIES.is_file(): + cannot_run(f"Cannot run: {ADVISORIES.name} does not exist, so every finding " + f"below would be reported as unacknowledged whether or not a " + f"decision had been recorded about it.") + doc = gatelib.read_yaml_all(ADVISORIES) + entries = (doc[0] or {}).get("advisories") if doc else None + if entries is None: + cannot_run(f"Cannot run: {ADVISORIES.name} declares no `advisories` key.") + return list(entries) + + +def verdict(findings: list[Finding], advisories: list[dict]) -> list[str]: + """Everything wrong, over a scan and an acknowledgement list. + + Four directions, and only the first is the question the gate is named for. + The other three are what stop the acknowledgement list from becoming a + permanent waiver nobody re-reads. + + Deduplicated in order. One image can carry the same advisory in several + scanned artifacts — a Go binary and the layer it sits in — and printing the + identical sentence twice reads as two findings needing two decisions. + """ + problems: list[str] = [] + blocking = [f for f in findings if f.severity == BLOCKING_SEVERITY] + + by_key: dict[tuple[str, str], dict] = {} + for entry in advisories: + if not isinstance(entry, dict): + problems.append( + f"image-advisories.yaml: an entry under `advisories:` is a " + f"{type(entry).__name__}, not a mapping — nothing can be read from it, " + f"and the findings it was meant to acknowledge are unacknowledged.") + continue + key = (str(entry.get("id", "")), str(entry.get("package", ""))) + # str() around the read, not just the strip: a `reason:` key written and + # left empty parses as None, and `None.strip()` raises — reaching a + # traceback instead of the sentence two lines below, which is written for + # exactly this case. id and package were already read defensively. + if not str(entry.get("reason") or "").strip(): + problems.append( + f"image-advisories.yaml: {key[0]} in {key[1]} is acknowledged with no " + f"reason recorded, so nothing states what would let it be removed.") + by_key[key] = entry + + # 1. A blocking finding nothing acknowledges — the gate's own question. + # 2. A blocking finding on an image its entry does not name. + for f in blocking: + key = (f.id, f.package) + acknowledged = by_key.get(key) + if acknowledged is None: + problems.append( + f"{f.image}: {f.id} in {f.package} {f.installed} is CRITICAL and fixed " + f"in {f.fixed}, and no entry in image-advisories.yaml names it. Move the " + f"chart pin to a version carrying the fix, or record why the finding " + f"stands.") + continue + listed = [str(i) for i in (acknowledged.get("images") or [])] + if f.bare not in listed: + problems.append( + f"{f.image}: {f.id} in {f.package} is acknowledged, but the entry does " + f"not list {f.bare}. An acknowledgement covers the images it names — a " + f"new image acquiring a known CRITICAL is a decision, not an inheritance.") + continue + + # 3. An entry no image carries. 4. An entry listing an image with no such finding. + carried = {(f.id, f.package, f.bare) for f in blocking} + # The mappings only: a non-mapping entry is already reported above, and + # reaching `.get` on it here is the traceback that sentence exists to + # replace. + for entry in [e for e in advisories if isinstance(e, dict)]: + key = (str(entry.get("id", "")), str(entry.get("package", ""))) + listed = [str(i) for i in (entry.get("images") or [])] + if not listed: + problems.append( + f"image-advisories.yaml: {key[0]} in {key[1]} lists no images, so it " + f"acknowledges nothing while reading as a considered decision.") + continue + if not any((key[0], key[1], img) in carried for img in listed): + problems.append( + f"image-advisories.yaml: {key[0]} in {key[1]} is acknowledged but no " + f"scanned image carries it — the entry outlived its reason. Delete it.") + continue + for img in listed: + if (key[0], key[1], img) not in carried: + problems.append( + f"image-advisories.yaml: {key[0]} in {key[1]} lists {img}, which no " + f"longer carries it. Drop that image from the entry.") + return list(dict.fromkeys(problems)) + + +def run_canary() -> int: + """Prove the scanner reports something before believing it reported nothing.""" + found = scan(CANARY) + if found is None: + cannot_run("Cannot run: the canary image could not be scanned.", + f" {CANARY}", + "An unreachable registry is a fact about the network, not about " + "the images this catalog pins.") + critical = [f for f in found if f.severity == BLOCKING_SEVERITY] + if not critical: + cannot_run("Cannot run: the canary returned no fixed CRITICAL findings.", + f" {CANARY}", + "The canary is pinned by digest to an image whose CRITICALs have " + "published fixes and cannot be patched away underneath it, so an " + "empty result here is a fact about the scanner. One with no " + "database, or one reading a flag as 'report nothing', returns a " + "clean result for every image — including every image below.") + print(f"canary OK: {len(critical)} fixed CRITICAL finding(s) from the pinned " + f"end-of-life image, so a clean result below is a result rather than a " + f"silence — " + + ", ".join(sorted({f"{f.id} ({f.package})" for f in critical}))) + return 0 + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--list", action="store_true", + help="print every finding at the reported severities, then exit 0") + ap.add_argument("--self-test", action="store_true", + help="run the canary alone") + ap.add_argument("--env", default="production") + args = ap.parse_args() + + gatelib.require("trivy", "helm") + run_canary() + if args.self_test: + return 0 + + seen: set[str] = set() + images, unrendered, unclassified = image_pins.inventory(args.env, seen) + if unrendered: + cannot_run("Cannot run: charts that did not render contributed no images, so " + "the scan below would cover part of the fleet and report on all of " + "it.", + *(f" {path}: {err}" for path, err in unrendered)) + # A reference the classifier could not place is one this scan cannot decide + # to include or exclude. Excluded silently it is an unscanned image inside a + # population reported as the fleet's — the same shape as a chart that did + # not render, one string down. + if unclassified: + cannot_run("Cannot run: the render carries image-shaped references that reach " + "no pod template and are on no declaration, so whether the scan " + "below covers them is unknown:", + *(f" {chart}: {detail}" for chart, detail in unclassified)) + units = image_pins.render_addons.discover() + coverage = image_pins.chart_coverage(images, units, seen) + if coverage: + cannot_run("Cannot run: the image population is smaller than the charts that " + "rendered it, so a scan over it says less than it appears to:", + *(f" {problem}" for problem in coverage)) + if len(images) < len(units): + cannot_run(f"Cannot run: the render produced {len(images)} image(s) from " + f"{len(units)} chart(s). Every chart shipping a workload " + f"contributes at least one, so the walk got smaller rather than " + f"the catalog.") + + findings: list[Finding] = [] + unscannable: list[str] = [] + for ref in sorted(images): + got = scan(ref) + if got is None: + unscannable.append(ref) + continue + findings.extend(got) + if unscannable: + cannot_run("Cannot run: these images could not be scanned, and the ones that " + "were say nothing about them:", + *(f" {ref} (via {', '.join(sorted(images[ref]))})" + for ref in unscannable)) + + if args.list: + for f in sorted(findings, key=lambda f: (f.severity, f.image, f.id)): + print(f"{f.severity:8} {f.image} {f.id} {f.package} {f.installed} " + f"-> {f.fixed}") + print(f"\n{len(findings)} fixed finding(s) at {'/'.join(REPORTED_SEVERITIES)} " + f"across {len(images)} image(s)") + return 0 + + for problem in verdict(findings, read_advisories()): + fail(problem) + + critical = [f for f in findings if f.severity == BLOCKING_SEVERITY] + high = [f for f in findings if f.severity == "HIGH"] + + if failures: + print(f"{len(failures)} problem(s) across {len(images)} scanned image(s):\n") + for problem in failures: + print(f" {problem}") + return 1 + + print(f"image vulnerabilities OK: {len(images)} image(s) scanned across " + f"{len(set().union(*images.values()))} chart(s). " + f"{len(critical)} fixed CRITICAL finding(s), every one acknowledged in " + f"image-advisories.yaml against the image carrying it, and every " + f"acknowledgement still matched by the scan. {len(high)} fixed HIGH " + f"finding(s) counted and not gated — run --list to read them.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/controls.py b/scripts/tests/controls.py index 8d5bf41..dcb279b 100755 --- a/scripts/tests/controls.py +++ b/scripts/tests/controls.py @@ -142,6 +142,7 @@ def __repr__(self) -> str: "check-image-pins.py": "subprocess.run", "check-log-volume-budget.py": "subprocess.run", "check-falco-rule-floor.py": "subprocess.run", + "check-image-vulnerabilities.py": "subprocess.run", } # Shell gates: no syntax tree available, so this is a text check over the diff --git a/scripts/tests/reverify-gates.sh b/scripts/tests/reverify-gates.sh index 16908e3..a1ea7e2 100755 --- a/scripts/tests/reverify-gates.sh +++ b/scripts/tests/reverify-gates.sh @@ -131,6 +131,29 @@ run 0 "task validate" task validate run 0 "controls floor" ./scripts/tests/controls.py run 0 "check-workflows.sh (zizmor)" ./scripts/check-workflows.sh run 0 "check-image-pins.py" ./scripts/check-image-pins.py + +# The same tree under the other helm stream shape. Some helm builds write the +# OCI pull report to stdout, where it lands in the manifest stream the gate +# parses and every OCI-sourced chart puts its own coordinates there; some write +# it to stderr, where nothing sees it. A gate reading one only is green on the +# machine that renders and red in the job that installs the other build, with +# nothing in the tree different — which is not a verdict about the tree. +# +# Reproduced rather than described: a shim runs the real helm and folds stderr +# into stdout, with a cold OCI cache, because helm reports nothing on a hit. +mkdir -p "$SP/oldhelm" +REAL_HELM="$(command -v helm)" +cat > "$SP/oldhelm/helm" <&1 +SHIM +chmod +x "$SP/oldhelm/helm" +run 0 "image-pins: a helm reporting OCI pulls on stdout" \ + env "PATH=$SP/oldhelm:$PATH" \ + "HELM_CACHE_HOME=$SP/oldhelm/cache" \ + "HELM_CONFIG_HOME=$SP/oldhelm/config" \ + "HELM_DATA_HOME=$SP/oldhelm/data" \ + ./scripts/check-image-pins.py run 0 "check-renovate-coverage.py" ./scripts/check-renovate-coverage.py run 0 "check-ai-config.py" ./scripts/check-ai-config.py run 0 "check-env-coverage.py" ./scripts/check-env-coverage.py @@ -164,6 +187,40 @@ PY run nonzero "image-pins: unpinned readiness-checker" ./scripts/check-image-pins.py res $F +# What the gate cannot read is not therefore absent, and the two ways it can +# fail to read are two verdicts. A reference the classifier cannot place is a +# chart that rendered and a question this gate owes an answer to; a chart that +# did not render leaves the fleet's image set unknown. +# +# The digest form is the one the gate's own remediation recommends. Written with +# no tag it matched neither tag alternative, so the whole reference yielded only +# its `sha256:` tail — a single-segment shape a declaration by that name +# passed over. +F=addons/networking/external-dns/values.yaml; mut $F +python3 - "$F" <<'PY' +import pathlib,sys +p=pathlib.Path(sys.argv[1]); s=p.read_text() +a="podAnnotations:\n" +assert a in s, "podAnnotations anchor not found" +ref="ghcr.io/example/probe-worker@sha256:" + "9f8e" + "a"*60 +m=s.replace(a, a + f' probe/image: "{ref}"\n', 1) +assert m!=s, "mutation did not land" +p.write_text(m) +print(f" planted {ref[:52]}...") +PY +run nonzero "image-pins: a digest-only reference nothing declares" ./scripts/check-image-pins.py +res $F + +# A chart that contributes no images and is DECLARED to contribute none, so the +# per-chart floor passes over it and the unrendered verdict is what is being +# read. Any other chart would be caught by the floor first, which would score +# this probe green for a reason it did not plant. +F=addons/ai-platform/envoy-ai-gateway-crds/values.yaml; mut $F +printf '\nbroken: [unclosed\n' >> $F +echo " planted unparseable values for a chart declared imageless" +run nonzero "image-pins: a chart that did not render cannot report a clean fleet" ./scripts/check-image-pins.py +res $F + F=applicationsets/addons-agent-operator.yaml; mut $F python3 - "$F" <<'PY' import pathlib,sys @@ -389,7 +446,7 @@ echo "RESULT pass=$pass fail=$fail" # The harness owes the same assertion it demands of the gates: with every `run` # line deleted it would report pass=0 fail=0 and exit 0, which is a green run # over nothing checked. -MIN_CHECKS=32 +MIN_CHECKS=35 total=$((pass + fail)) if [ "$total" -lt "$MIN_CHECKS" ]; then echo "FAIL ran $total check(s), under the floor of $MIN_CHECKS — this harness" diff --git a/scripts/tests/run.py b/scripts/tests/run.py index c803116..eeebf8c 100755 --- a/scripts/tests/run.py +++ b/scripts/tests/run.py @@ -52,6 +52,12 @@ "test_render_addons", "test_image_pins", "test_log_volume_budget", + # The image population both image gates read, and the acknowledgement file + # the vulnerability verdict is decided against. One walk answers every + # question either gate asks, so an extractor that stops seeing a shape + # removes those images from all of them at once. + "test_image_extraction", + "test_image_vulnerabilities", # The floors that keep an emptied corpus from reading as a clean one, # asserted apart from the gates that carry them. "test_corpus_floors", @@ -116,7 +122,20 @@ # two differ enough to matter, so the name here says combined and the printed # line says so too. Calling a combined figure "line coverage" would be a # measurement mislabelled as the one the rubric asks about. -COMBINED_FLOOR = 37 +# +# What the number does NOT capture: scripts/tests/controls.py runs most gates end +# to end as a subprocess against a mutated tree, so those gates have behavioural +# coverage this line count cannot see. Neither figure substitutes for the other — +# the controls prove a gate rejects, the unit tests prove it computes the right +# answer on a case the real tree does not contain. +# +# MOST, and the exceptions are the ones that matter to this floor. controls.py +# exempts every gate that reaches a chart registry or an API, and its own run +# prints the split. Those gates have neither kind of coverage, they are the +# largest in the tree, and among them are the gates on the paths testing-rubric +# calls security-critical. So this figure being low is not offset by behavioural +# coverage for precisely the files where that offset was being claimed. +COMBINED_FLOOR = 40 # A ceiling on gate scripts carrying NO unit coverage at all, complementing the # floors below. The floors stop a covered file regressing; nothing stopped a NEW @@ -124,6 +143,16 @@ # # Ratchets downward only. Adding a gate without tests fails here rather than # diluting the combined figure by a percentage point nobody notices. +# +# Two of the files this counts as covered are covered by IMPORT, not by tests. +# check-image-vulnerabilities.py loads check-image-pins.py by path so the two +# cannot disagree about what the fleet is, and that loads render-addons.py in +# turn; a test module importing the first executes the module bodies of all +# three. Coverage cannot tell that from a test, so both read as non-zero while +# carrying no assertions of their own. The ceiling is lowered anyway, because a +# ratchet that declines to move is one nobody can regress against — but it is +# the weaker half of this file's claim, and the per-gate floors and the controls +# in scripts/tests/controls.py are where the real one lives. MAX_UNCOVERED_GATES = 11 PER_GATE_FLOORS = { @@ -138,9 +167,10 @@ "scripts/check-platform-crs.py": 51, "scripts/validate-dashboards.py": 49, "scripts/render-addons.py": 40, - "scripts/check-image-pins.py": 38, "scripts/check-log-volume-budget.py": 58, "scripts/check-falco-rule-floor.py": 28, + "scripts/check-image-vulnerabilities.py": 41, + "scripts/check-image-pins.py": 88, } diff --git a/scripts/tests/test_corpus_floors.py b/scripts/tests/test_corpus_floors.py index 2756c52..d042091 100644 --- a/scripts/tests/test_corpus_floors.py +++ b/scripts/tests/test_corpus_floors.py @@ -1,14 +1,22 @@ """Every floor sits above zero and below the corpus it guards. -A floor of zero is the vacuous pass with a constant in front of it. A floor above -the real corpus makes the gate red on every run, and a gate that is always red is -one people route around — the same destination by a longer road. - -These are constants rather than derivations, and the reason is recorded at each -one: every quantity available to derive a floor from comes out of the same walk -over the same files, so a corpus that shrinks shrinks the derivation with it. -Both circular forms were written and rejected. What can be asserted is the pair -of bounds, against the tree. +A gate reports on the population it read. Nothing in an exit code distinguishes +"this catalog holds no violation" from "this run held no catalog", and the second +is what a renamed directory, a wrong `--root`, a narrowed glob or a filter that +stopped matching all look like. The fork-safety gate printed +`Scanned 0 applied ApplicationSet(s)` followed by its success line and exited 0 +under `--blocking`. + +A floor of zero is that vacuous pass with a constant in front of it. A floor +above the real corpus makes the gate red on every run, and a gate that is always +red is one people route around — the same destination by a longer road. + +The floors are constants rather than derivations, and the reason is recorded at +each one: every quantity available to derive a floor from comes out of the same +walk over the same files, so a corpus that shrinks shrinks the derivation with +it. Both circular forms were written and rejected. What can be asserted is the +pair of bounds, against the tree — and, where a second enumerator does exist, +the derivation instead of a number. """ from __future__ import annotations @@ -21,10 +29,12 @@ ROOT = pathlib.Path(__file__).resolve().parent.parent.parent +render_addons = load("render-addons") fork_safety = load("check-hardcoded-org") platform_crs = load("check-platform-crs") policy_admission = load("check-policy-admission") dashboards = load("validate-dashboards") +image_pins = load("check-image-pins") class EveryFloorIsAboveZero(unittest.TestCase): @@ -81,10 +91,36 @@ def test_the_render_clears_the_policy_admission_floor(self): self.assertLess(policy_admission.MIN_RENDERED, len(units) * len(policy_admission.ENFORCE_ENVS)) + def test_the_policy_admission_floor_exceeds_a_degenerate_render(self): + """The other bound, and the reason the floor is not merely > 0. + + A render producing one manifest per unit and nothing per environment is + the exact shape the floor exists to catch, so the floor has to sit above + it — between the degenerate render and the largest one, not merely + somewhere above zero. + """ + units = [u for u in policy_admission.discover() + if u.chart not in policy_admission.SKIP_CHARTS] + self.assertGreaterEqual(policy_admission.MIN_RENDERED, len(units)) + def test_the_dashboards_clear_their_floor(self): refs = dashboards.discover(ROOT) self.assertGreater(len(refs), dashboards.MIN_DASHBOARD_REFS) + def test_the_imageless_charts_are_charts_the_fleet_renders(self): + """The image floor is per chart, so its exemption is where it can rot. + + A chart declared imageless that the catalog no longer pins is an entry + excusing nothing, and the per-chart floor is exactly as strong as that + list is short. + """ + charts = {u.chart for u in render_addons.discover()} + for chart in image_pins.IMAGELESS_CHARTS: + with self.subTest(chart=chart): + self.assertIn(chart, charts, + f"{chart} is declared to render no image but the fleet " + f"does not pin it — the entry outlived its chart") + class TheFloorsGuardTheRightQuantity(unittest.TestCase): """A floor on findings is not a floor on the corpus. @@ -104,6 +140,17 @@ def test_each_floor_is_compared_against_a_count_of_what_was_read(self): with self.subTest(gate=rel): self.assertIn(expr, (ROOT / rel).read_text()) + def test_the_image_floor_is_derived_per_chart(self): + """Not a constant, because here a second enumerator exists: every chart + the render covers must contribute an image, which is the shape a total + cannot see.""" + source = (ROOT / "scripts" / "check-image-vulnerabilities.py").read_text() + self.assertIn("coverage = image_pins.chart_coverage(images, units, seen)", source) + self.assertIn("if len(images) < len(units):", source) + self.assertNotIn("MIN_IMAGES", source, + "the picked constant is back; the per-chart derivation is " + "what catches an extractor that stopped seeing a shape") + if __name__ == "__main__": unittest.main() diff --git a/scripts/tests/test_image_extraction.py b/scripts/tests/test_image_extraction.py new file mode 100644 index 0000000..49e43a0 --- /dev/null +++ b/scripts/tests/test_image_extraction.py @@ -0,0 +1,926 @@ +"""Unit tests for how the fleet's image population is derived from a render. + +Both gates that ask anything about images read this one inventory, so an +extractor that misses a spelling removes those images from every question at +once — and the count that remains stays large enough to look healthy. That +happened: the pattern was anchored on `image:` preceded only by whitespace, the +ordinary list-item form `- image: ` never matched, and two charts' +entire workloads were absent from a scan reporting fifty-five images. + +So the walk is structural and the pattern is kept under it as an independent +floor. A parser and a regex fail on different inputs; each is planted against +here, and so is the per-chart floor that catches a chart contributing nothing. +""" + +from __future__ import annotations + +import contextlib +import io +import pathlib +import re +import subprocess +import sys +import tempfile +import unittest + +from gateloader import load + +gate = load("check-image-pins") +GATE = pathlib.Path(gate.__file__) + + +class Unit: + """The fields chart_coverage reads off a render unit.""" + + def __init__(self, chart: str): + self.chart = chart + + +def render(*docs: str) -> str: + return "\n---\n".join(docs) + + +DEPLOYMENT = """ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app +spec: + template: + spec: + containers: + - name: app + image: ghcr.io/example/app:1.0.0 +""" + + +class ExtractingFromARender(unittest.TestCase): + def images(self, rendered, chart="c"): + """(population, references the classifier could not place). + + The two out-parameters are separate because they are separate verdicts: + a render that will not parse leaves the fleet unknown, a reference that + will not classify is a question this gate owes an answer to. Every case + below that asserts `problems == []` is also asserting the render parsed, + so the two are checked apart. + """ + unrendered: list[tuple[str, str]] = [] + unclassified: list[tuple[str, str]] = [] + found = gate.extract_images(rendered, chart, unrendered, unclassified) + self.assertEqual(unrendered, [], "the fixture render did not parse") + return found, unclassified + + def both(self, rendered, chart="c"): + """(population, not-rendered, not-placed), for the cases that plant one.""" + unrendered: list[tuple[str, str]] = [] + unclassified: list[tuple[str, str]] = [] + found = gate.extract_images(rendered, chart, unrendered, unclassified) + return found, unrendered, unclassified + + def test_the_list_item_spelling_is_found(self): + """The shape the pattern never matched, and the whole defect. + + `- image:` under a `containers:` list is the ordinary Kubernetes form. + Anchored on whitespace-then-`image:`, a pattern sees nothing here. + """ + found, problems = self.images(DEPLOYMENT) + self.assertEqual(found, {"ghcr.io/example/app:1.0.0"}) + self.assertEqual(problems, []) + + def test_every_container_list_is_walked(self): + rendered = """ +apiVersion: apps/v1 +kind: DaemonSet +spec: + template: + spec: + initContainers: + - name: init + image: init:1 + containers: + - name: main + image: main:2 + ephemeralContainers: + - name: debug + image: debug:3 +""" + found, _ = self.images(rendered) + self.assertEqual(found, {"init:1", "main:2", "debug:3"}) + + def test_a_cronjob_pod_template_is_reached(self): + """Nesting depth is not a case the walk has to know about.""" + rendered = """ +apiVersion: batch/v1 +kind: CronJob +spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: rotate + image: aws-cli:2 +""" + found, _ = self.images(rendered) + self.assertEqual(found, {"aws-cli:2"}) + + def test_an_image_key_outside_a_pod_template_is_found(self): + """A custom resource can name an image its operator then runs.""" + rendered = """ +apiVersion: agents.nanohype.dev/v1alpha1 +kind: AgentFleet +spec: + agents: + - name: a + image: ghcr.io/example/runner:0.1.0 +""" + found, _ = self.images(rendered) + self.assertEqual(found, {"ghcr.io/example/runner:0.1.0"}) + + def test_an_image_only_the_text_scan_finds_is_reported(self): + """The extraction-drift assertion: a structural walk that stopped seeing + a shape looks identical to a controller reading a string payload, so the + second must be declared and the first is reported.""" + rendered = """ +apiVersion: v1 +kind: ConfigMap +data: + envoy-gateway.yaml: | + rateLimitDeployment: + container: + image: docker.io/example/undeclared:1.0 +""" + found, problems = self.images(rendered) + self.assertIn("docker.io/example/undeclared:1.0", found) + self.assertEqual(len(problems), 1) + self.assertIn("stopped seeing a shape", problems[0][1]) + + def test_a_declared_controller_image_is_not_reported(self): + """A controller is handed the reference and creates the pod later, so no + pod template in this render declares it.""" + declared = next(iter(gate.CONTROLLER_IMAGES)) + rendered = f""" +apiVersion: v1 +kind: ConfigMap +data: + cfg: | + image: {declared}:abc123 +""" + found, problems = self.images(rendered) + self.assertIn(f"{declared}:abc123", found) + self.assertEqual(problems, []) + + def test_an_image_in_a_controller_flag_is_a_candidate(self): + """The shape neither the key walk nor the line-anchored pattern can see. + + A controller handed `--worker-image=` creates the pod later; the + reference is an argument, not a key, and it is not at line start. + """ + rendered = """ +apiVersion: apps/v1 +kind: Deployment +spec: + template: + spec: + containers: + - name: c + image: ghcr.io/example/app:1.0.0 + args: ["--worker-image=ghcr.io/example/undeclared-worker:2.0"] +""" + found, problems = self.images(rendered) + self.assertEqual(len(problems), 1) + self.assertIn("ghcr.io/example/undeclared-worker:2.0", problems[0][1]) + self.assertIn("a controller is handed it", problems[0][1]) + + def test_a_single_segment_official_image_is_a_candidate(self): + """`nats:2.10.10` is the whole reference — no registry, no organisation. + + The argo-events event-bus controller declares exactly this beside the two + natsio/* sidecars it starts in the same StatefulSet, so requiring a path + separator scanned the helpers and left the main container silent. + """ + rendered = """ +apiVersion: v1 +kind: ConfigMap +data: + controller-config.yaml: | + nats: + versions: + - natsImage: nats:2.10.10 + - natsImage: undeclared-single-segment:9.9.9 +""" + found, problems = self.images(rendered) + # `nats` is declared in CONTROLLER_IMAGES, so it enters the population. + self.assertIn("nats:2.10.10", found) + # An undeclared one is reported rather than silently absent. + self.assertEqual(len(problems), 1) + self.assertIn("undeclared-single-segment:9.9.9", problems[0][1]) + + def test_an_rbac_name_is_not_an_image(self): + """`kyverno:admission-controller` has the shape and is a role name; the + tag is the discriminator.""" + rendered = ("apiVersion: v1\nkind: ConfigMap\ndata:\n" + " x: |\n role: kyverno:admission-controller\n" + " other: system:auth-delegator\n") + found, problems = self.images(rendered) + self.assertEqual(problems, []) + + def test_a_hostname_does_not_yield_its_last_label(self): + """Without an anchor the alternative starts after a dot and + `vault.example.com:8200` yields `com:8200`.""" + rendered = ("apiVersion: v1\nkind: ConfigMap\ndata:\n" + " x: |\n server: vault.example.com:8200\n") + found, problems = self.images(rendered) + self.assertEqual(problems, []) + + def test_a_host_and_port_is_not_an_image(self): + """A rendered config is full of addresses; a grammar admitting them + cannot be read, and every one would demand a declaration.""" + rendered = """ +apiVersion: v1 +kind: ConfigMap +data: + cfg: | + endpoint: loki.monitoring.svc.cluster.local:3100 + bind: 127.0.0.1:8080 +""" + found, problems = self.images(rendered) + self.assertEqual(problems, []) + + def test_a_render_that_will_not_parse_is_reported_not_dropped(self): + """A chart silently absent from the inventory is the whole failure mode.""" + found, unrendered, unclassified = self.both("kind: Deployment\n bad: [unclosed\n") + self.assertEqual(found, set()) + self.assertEqual(len(unrendered), 1) + self.assertIn("could not parse", unrendered[0][1]) + self.assertEqual(unclassified, [], + "a render that did not parse cannot also have produced a " + "reference the classifier could not place") + + def test_the_helm_value_sentinel_does_not_lose_the_chart(self): + """Charts emit `- =` inside ConfigMap payloads; PyYAML has no constructor + for it, and raising there would drop the whole render.""" + rendered = DEPLOYMENT + "\n---\napiVersion: v1\nkind: ConfigMap\ndata:\n a: |\n - =\n" + found, problems = self.images(rendered) + self.assertEqual(found, {"ghcr.io/example/app:1.0.0"}) + self.assertEqual(problems, []) + + +class EveryChartContributesAnImage(unittest.TestCase): + """A per-chart floor, because a total cannot see two charts falling out.""" + + def coverage(self, images, charts, seen=None): + """Every declared imageless chart is rendered, and every declared image + is seen, unless a case says otherwise — so a fixture does not trip a rot + rule it is not about. + """ + units = [Unit(c) for c in charts] + [ + Unit(c) for c in gate.IMAGELESS_CHARTS if c not in charts] + if seen is None: + seen = set(gate.CONTROLLER_IMAGES) | set(gate.NOT_A_CONTAINER) + return gate.chart_coverage(images, units, seen) + + def test_declaration_rot_reaches_the_verdict(self): + """chart_coverage is where the rot rule is consulted, so gutting the call + must fail here and not only where the rule itself is tested.""" + problems = self.coverage({"x:1": {"a"}}, ["a"], seen=set()) + self.assertTrue(any("outlived its image" in p for p in problems)) + + def test_a_chart_contributing_nothing_is_reported(self): + problems = self.coverage({"x:1": {"a"}}, ["a", "b"]) + self.assertEqual(len(problems), 1) + self.assertIn("b rendered and contributed no image", problems[0]) + + def test_every_chart_contributing_passes(self): + self.assertEqual( + self.coverage({"x:1": {"a"}, "y:2": {"b"}}, ["a", "b"]), []) + + def test_a_declared_imageless_chart_is_not_reported(self): + declared = next(iter(gate.IMAGELESS_CHARTS)) + self.assertEqual(self.coverage({"x:1": {"a"}}, ["a", declared]), []) + + def test_a_declared_chart_the_fleet_stopped_rendering_is_reported(self): + """The exemption is where a per-chart floor rots: an entry excusing a + chart nobody pins excuses nothing and reads as considered.""" + seen = set(gate.CONTROLLER_IMAGES) | set(gate.NOT_A_CONTAINER) + problems = gate.chart_coverage({"x:1": {"a"}}, [Unit("a")], seen) + self.assertEqual(len(problems), len(gate.IMAGELESS_CHARTS)) + self.assertIn("outlived its chart", problems[0]) + + def test_a_declared_chart_that_starts_shipping_a_workload_is_reported(self): + declared = next(iter(gate.IMAGELESS_CHARTS)) + problems = self.coverage({"x:1": {declared}}, [declared]) + self.assertTrue(any("now contributes an image" in p for p in problems)) + + def test_an_empty_inventory_reports_every_chart(self): + """The vacuous case: no images at all is not a fleet with no images.""" + problems = self.coverage({}, ["a", "b"]) + self.assertEqual(len(problems), 2) + + +if __name__ == "__main__": + unittest.main() + + +class DeclarationsMatchTheRender(unittest.TestCase): + """Both declaration tables say something about images this catalog renders. + + An entry for one it does not is an excuse for nothing, and an exemption list + nobody re-reads only ever widens. This is the reverse direction the + controller-image comment claims and did not have. + """ + + def test_a_controller_image_the_render_dropped_is_reported(self): + declared = next(iter(gate.CONTROLLER_IMAGES)) + problems = gate.declaration_rot( + (set(gate.CONTROLLER_IMAGES) | set(gate.NOT_A_CONTAINER)) - {declared}) + self.assertEqual(len(problems), 1) + self.assertIn(declared, problems[0]) + self.assertIn("outlived its image", problems[0]) + + def test_a_non_container_declaration_the_render_dropped_is_reported(self): + declared = next(iter(gate.NOT_A_CONTAINER)) + problems = gate.declaration_rot( + (set(gate.CONTROLLER_IMAGES) | set(gate.NOT_A_CONTAINER)) - {declared}) + self.assertEqual(len(problems), 1) + self.assertIn(declared, problems[0]) + + def test_every_declaration_matched_passes(self): + self.assertEqual( + gate.declaration_rot(set(gate.CONTROLLER_IMAGES) | set(gate.NOT_A_CONTAINER)), + []) + + def test_an_empty_render_reports_every_declaration(self): + problems = gate.declaration_rot(set()) + self.assertEqual(len(problems), + len(gate.CONTROLLER_IMAGES) + len(gate.NOT_A_CONTAINER)) + + +DIGEST = "sha256:" + "9f8e7d6c5b4a3f2e1d0c9b8a7f6e5d4c3b2a1f0e9d8c7b6a5f4e3d2c1b0a9f8e" + + +class ADigestPinnedReferenceIsInThePopulation(unittest.TestCase): + """`@sha256:` is a reference this gate recommends and could not read. + + The pattern under the structural walk had two alternatives and both required + a tag, so a digest-pinned reference in a controller flag or a config value + yielded nothing but its `sha256:` tail — a single-segment shape whose + bare name is `sha256`, and an entry by that name passed over it. The gate's + own remediation tells an operator to pin to a digest, so the one spelling it + recommends was the one its completeness floor could not see. + """ + + def candidates(self, s: str) -> list[str]: + return [c for groups in gate.IMAGE_REF.findall(s) for c in groups if c] + + def test_a_digest_only_reference_is_matched_whole(self): + self.assertEqual(self.candidates(f"ghcr.io/example/thing@{DIGEST}"), + [f"ghcr.io/example/thing@{DIGEST}"]) + + def test_a_tag_and_digest_reference_is_matched_whole(self): + """Not as its `repo:tag` head: what a container runs is the digest, and + the head alone reaches the classifier as a tag.""" + ref = f"ghcr.io/example/thing:1.2.3@{DIGEST}" + self.assertEqual(self.candidates(ref), [ref]) + self.assertEqual(gate.classify(ref), "digest") + + def test_a_single_segment_official_image_carries_a_digest(self): + """`nats@sha256:...` — no registry, no organisation, and the shape the + event-bus controller declares.""" + self.assertEqual(self.candidates(f"nats@{DIGEST}"), [f"nats@{DIGEST}"]) + + def test_the_bare_name_drops_both_tag_and_digest(self): + for ref in (f"ghcr.io/example/thing@{DIGEST}", + f"ghcr.io/example/thing:1.2.3@{DIGEST}", + "ghcr.io/example/thing:1.2.3"): + with self.subTest(ref=ref): + self.assertEqual(gate.bare_name(ref), "ghcr.io/example/thing") + + def test_a_digest_with_no_repository_before_it_is_not_a_reference(self): + """A `digest:` field carries one, and it names no image. + + Excluded by shape, not by a declaration, because the two rules this + repository already has cannot both hold over it. A NOT_A_CONTAINER entry + excuses whatever carries its bare name, so an entry for `sha256` would + excuse every reference whose only matched token is a digest — and + `declaration_rot` deletes an entry the render does not support, which is + every tree that happens not to carry a bare digest that day. One rule + demands the entry and the other deletes it. + + There is nothing lost: a digest that belongs to a repository is matched + with that repository, whole, by the alternative above. + """ + self.assertEqual(self.candidates(DIGEST), []) + self.assertNotIn("sha256", gate.NOT_A_CONTAINER, + "a declaration matching nothing in the render is an " + "exemption that only ever widens") + + def test_a_repository_digest_is_unaffected_by_that_exclusion(self): + """The exclusion is anchored at the start of a token, so it removes the + bare form and nothing else.""" + for ref in (f"ghcr.io/example/app@{DIGEST}", + f"ghcr.io/example/app:1.0.0@{DIGEST}"): + with self.subTest(ref=ref): + self.assertEqual(self.candidates(ref), [ref]) + + def test_a_digest_in_a_rendered_field_contributes_nothing(self): + """End to end: a `digest:` value is not an unplaceable reference, so it + does not put a gate that reads this population into a refusal.""" + rendered = f""" +apiVersion: v1 +kind: ConfigMap +data: + provenance.yaml: | + chart: example + digest: {DIGEST} +""" + unrendered: list[tuple[str, str]] = [] + unclassified: list[tuple[str, str]] = [] + found = gate.extract_images(rendered, "c", unrendered, unclassified) + self.assertEqual((found, unrendered, unclassified), (set(), [], [])) + + def test_a_digest_shaped_string_that_is_not_a_digest_is_not_matched(self): + """64 lowercase hex, because the suffix is the only discriminator the + digest alternative has — it drops the path separator and the tag rules + the others need.""" + for s in (f"ghcr.io/example/thing@sha256:{'a' * 63}", + "ghcr.io/example/thing@sha256:not-a-digest", + f"ghcr.io/example/thing@sha512:{'a' * 64}"): + with self.subTest(s=s): + self.assertNotIn(f"ghcr.io/example/thing@{s.split('@')[1]}", + self.candidates(s)) + + def test_a_digest_pinned_controller_reference_is_reported_when_undeclared(self): + """The finding, end to end. A controller is handed a digest-pinned + reference; no pod template declares it, no list names it, and before the + digest alternative the whole string produced nothing to report.""" + rendered = f""" +apiVersion: v1 +kind: ConfigMap +data: + controller.yaml: | + workerImage: ghcr.io/example/undeclared-worker@{DIGEST} +""" + unrendered: list[tuple[str, str]] = [] + unclassified: list[tuple[str, str]] = [] + gate.extract_images(rendered, "c", unrendered, unclassified) + self.assertEqual(unrendered, []) + self.assertEqual(len(unclassified), 1) + self.assertIn(f"ghcr.io/example/undeclared-worker@{DIGEST}", unclassified[0][1]) + + def test_a_digest_pinned_declared_controller_reference_is_in_the_population(self): + declared = next(iter(gate.CONTROLLER_IMAGES)) + rendered = f""" +apiVersion: v1 +kind: ConfigMap +data: + controller.yaml: | + image: {declared}@{DIGEST} +""" + unrendered: list[tuple[str, str]] = [] + unclassified: list[tuple[str, str]] = [] + found = gate.extract_images(rendered, "c", unrendered, unclassified) + self.assertEqual((unrendered, unclassified), ([], [])) + self.assertIn(f"{declared}@{DIGEST}", found) + self.assertEqual(gate.classify(f"{declared}@{DIGEST}"), "digest") + + def test_a_digest_pinned_container_is_read_by_the_structural_walk(self): + rendered = f""" +apiVersion: apps/v1 +kind: Deployment +spec: + template: + spec: + containers: + - name: app + image: ghcr.io/example/app@{DIGEST} +""" + unrendered: list[tuple[str, str]] = [] + unclassified: list[tuple[str, str]] = [] + found = gate.extract_images(rendered, "c", unrendered, unclassified) + self.assertEqual(found, {f"ghcr.io/example/app@{DIGEST}"}) + self.assertEqual((unrendered, unclassified), ([], [])) + + def test_every_form_the_remediation_recommends_is_one_this_reads(self): + """The sentence and the pattern, checked against each other. + + Read out of the gate's own remediation string rather than retyped here, + so rewording the advice to name a spelling the pattern does not read + fails — which is the defect this closes, one revision later. + """ + recommended = re.findall(r"`([^`]+)`", gate.MUTABLE_REMEDIATION) + self.assertTrue(recommended, + f"the remediation names no reference form: " + f"{gate.MUTABLE_REMEDIATION}") + for form in recommended: + with self.subTest(form=form): + concrete = (form.replace("", "ghcr.io/example/app") + .replace("", DIGEST.split(":", 1)[1])) + self.assertEqual(self.candidates(concrete), [concrete], + f"the remediation recommends {form}, which the " + f"pattern under the structural walk does not read") + + +class WhatCannotBeReadReachesAVerdict(unittest.TestCase): + """Neither silence, and not the same verdict. + + A chart that did not render leaves the fleet's image set unknown; a + reference the classifier cannot place is a chart that rendered and a + question this gate owes an answer to. Held in one list, the second printed + under the first's heading and reached no verdict at all — the run exited 0, + and CI blocked only because a sibling gate read the same list as a refusal. + """ + + IMAGES = {"ghcr.io/example/app:1.0.0": {"c"}} + MUTABLE = {"ghcr.io/example/app:latest": {"c"}} + + def verdict(self, images, unrendered=(), unclassified=()): + """main() over a planted inventory. + + chart_coverage and ALLOWED_MUTABLE are neutralised because both assert + against the real fleet: a fixture of two images would trip every + per-chart floor and every exemption-rot check, and the case would then + pass for a reason it did not plant. Each has its own tests elsewhere in + this file. + """ + saved = (gate.inventory, gate.chart_coverage, dict(gate.ALLOWED_MUTABLE), + sys.argv) + gate.inventory = lambda env, seen=None: (images, list(unrendered), + list(unclassified)) + gate.chart_coverage = lambda *a, **k: [] + gate.ALLOWED_MUTABLE.clear() + sys.argv = ["check-image-pins.py"] + try: + with contextlib.redirect_stdout(io.StringIO()) as out: + rc = gate.main() + finally: + gate.inventory, gate.chart_coverage, argv = saved[0], saved[1], saved[3] + gate.ALLOWED_MUTABLE.update(saved[2]) + sys.argv = argv + return rc, out.getvalue() + + def test_a_clean_fleet_passes(self): + """The control: without it every case below could be failing for a + reason the case did not introduce.""" + rc, out = self.verdict(self.IMAGES) + self.assertEqual(rc, 0, out) + + def test_a_reference_that_cannot_be_placed_fails(self): + rc, out = self.verdict(self.IMAGES, unclassified=[ + ("c", "ghcr.io/example/mystery:1.0 is image-shaped and reaches no pod " + "template or `image:` key.")]) + self.assertEqual(rc, 1) + self.assertIn("ghcr.io/example/mystery:1.0", out) + + def test_a_reference_that_cannot_be_placed_is_not_a_scope_caveat(self): + """It is printed as a problem, not under a heading about charts that did + not render — the chart rendered.""" + _, out = self.verdict(self.IMAGES, unclassified=[("c", "mystery")]) + self.assertNotIn("could not be rendered", out) + self.assertIn("image-pin problem", out) + + def test_a_chart_that_did_not_render_cannot_report_a_clean_fleet(self): + """Every image that DID render is immutable, and that is a result over + part of the fleet rather than over the fleet.""" + rc, out = self.verdict(self.IMAGES, unrendered=[("addons/x", "helm failed")]) + self.assertEqual(rc, gate.gatelib.CANNOT_RUN) + self.assertIn("Cannot run", out) + self.assertIn("addons/x", out) + + def test_a_chart_that_did_not_render_is_named_alongside_a_real_failure(self): + """The failure is the verdict, and the unrendered chart still bounds what + the verdict covers — dropping either one loses something a reader needs.""" + rc, out = self.verdict(self.MUTABLE, unrendered=[("addons/x", "helm failed")]) + self.assertEqual(rc, 1) + self.assertIn("addons/x", out) + self.assertIn("subset of the fleet", out) + self.assertIn("moving target", out) + + def test_an_empty_population_cannot_run(self): + rc, out = self.verdict({}, unrendered=[("addons/x", "helm failed")]) + self.assertEqual(rc, gate.gatelib.CANNOT_RUN) + self.assertIn("rendered no images at all", out) + + +class WhatHelmReportsPulling(unittest.TestCase): + """An OCI registry serves charts and container images through one grammar. + + `public.ecr.aws/karpenter/karpenter:1.14.0` is indistinguishable from an + image by shape, and it is the chart — the container beside it is + `public.ecr.aws/karpenter/controller`. Nothing in the reference says which, + and no walk over the render can, because a chart coordinate reaches no pod + template by construction. + + helm answers it, about this run: it prints what it fetched as a chart. That + is an observation rather than a name somebody wrote down, which matters + because a declaration here could not survive. `declaration_rot` removes an + entry the render does not support, and the render carries these only under + the helm builds that print the report to stdout — so one rule would demand + the entry and the other remove it. + """ + + REPORT = ("Pulled: public.ecr.aws/karpenter/karpenter:1.14.0\n" + "Digest: sha256:" + "4c1e" + "a" * 60 + "\n") + + def test_the_report_names_the_artifact(self): + self.assertEqual(gate.chart_artifacts(self.REPORT), + {"public.ecr.aws/karpenter/karpenter:1.14.0"}) + + def test_a_render_with_no_report_names_nothing(self): + self.assertEqual(gate.chart_artifacts(DEPLOYMENT), set()) + + def test_the_digest_line_is_not_an_artifact(self): + """It names the artifact above it, not another one.""" + self.assertNotIn("sha256:" + "4c1e" + "a" * 60, + gate.chart_artifacts(self.REPORT)) + + def test_a_pulled_chart_is_not_an_unplaceable_reference(self): + """The whole finding, end to end. The report leads the manifests, so on + the helm builds that print it to stdout every OCI-sourced chart puts its + own coordinates into the stream this parses.""" + rendered = self.REPORT + DEPLOYMENT + unrendered: list[tuple[str, str]] = [] + unclassified: list[tuple[str, str]] = [] + found = gate.extract_images(rendered, "karpenter", unrendered, unclassified, + None, gate.chart_artifacts(rendered)) + self.assertEqual((unrendered, unclassified), ([], [])) + self.assertEqual(found, {"ghcr.io/example/app:1.0.0"}) + + def test_without_the_report_the_same_stream_is_unplaceable(self): + """The control on the case above: it passes because the report was read, + not because the reference stopped being image-shaped.""" + rendered = self.REPORT + DEPLOYMENT + unrendered: list[tuple[str, str]] = [] + unclassified: list[tuple[str, str]] = [] + gate.extract_images(rendered, "karpenter", unrendered, unclassified) + self.assertEqual(len(unclassified), 1) + self.assertIn("public.ecr.aws/karpenter/karpenter:1.14.0", unclassified[0][1]) + + def test_a_container_sharing_the_repository_path_is_not_passed_over(self): + """Compared whole, not on the bare name. + + A registry can serve a chart and an image from one path, and passing + over every reference sharing a pulled chart's name would take the image + with it — the one direction this gate must never fail in. + """ + rendered = ("Pulled: ghcr.io/example/thing:1.0.0\n" + "Digest: sha256:" + "b" * 64 + "\n" + "apiVersion: v1\nkind: ConfigMap\ndata:\n" + " cfg: |\n image: ghcr.io/example/thing:9.9.9\n") + unrendered: list[tuple[str, str]] = [] + unclassified: list[tuple[str, str]] = [] + gate.extract_images(rendered, "thing", unrendered, unclassified, + None, gate.chart_artifacts(rendered)) + self.assertEqual(len(unclassified), 1) + self.assertIn("ghcr.io/example/thing:9.9.9", unclassified[0][1]) + + def test_the_same_artifact_carrying_a_digest_is_recognised(self): + """`@sha256:` and `` are one artifact; the report names + the second and the render can carry either.""" + ref = "ghcr.io/example/chart:1.0.0" + rendered = (f"Pulled: {ref}\n" + "apiVersion: v1\nkind: ConfigMap\ndata:\n" + f" cfg: |\n chart: {ref}@sha256:{'c' * 64}\n") + unrendered: list[tuple[str, str]] = [] + unclassified: list[tuple[str, str]] = [] + gate.extract_images(rendered, "chart", unrendered, unclassified, + None, gate.chart_artifacts(rendered)) + self.assertEqual(unclassified, []) + + def test_a_pulled_artifact_still_reaches_the_declaration_record(self): + """`seen` is what the render CONTAINED, so declaration_rot keeps reading + the same corpus whether or not a reference was passed over.""" + rendered = self.REPORT + DEPLOYMENT + seen: set[str] = set() + gate.extract_images(rendered, "karpenter", [], [], seen, + gate.chart_artifacts(rendered)) + self.assertIn("public.ecr.aws/karpenter/karpenter", seen) + + +class TheReportIsReadFromEitherStream(unittest.TestCase): + """Which stream carries it is a property of the helm build, not of this repo. + + Some helm builds write the OCI pull report to stdout, where it lands in the + manifest stream this gate parses; some write it to stderr, where nothing sees + it. Reading one only makes the gate's answer depend on which helm ran it — + green on the machine that renders, red in the job that installs a different + build, with the same tree. + + So `inventory` is exercised here against both stream shapes, with helm itself + stubbed: the tool is the external input, and what varies is the input. + """ + + REPORT = ("Pulled: public.ecr.aws/karpenter/karpenter:1.14.0\n" + "Digest: sha256:" + "4c1e" + "a" * 60 + "\n") + + class Unit: + chart = "karpenter" + path = "addons/operations/karpenter" + repo = "https://example.com" + version = "1.14.0" + namespace = "kube-system" + params: tuple = () + is_oci = True + + def oci_ref(self): + return "oci://public.ecr.aws/karpenter/karpenter" + + def inventory_with(self, stdout, stderr): + """inventory() over one unit whose helm run produced these two streams.""" + root = pathlib.Path(tempfile.mkdtemp()) + (root / self.Unit.path).mkdir(parents=True) + + class Completed: + returncode = 0 + + def __init__(self, out, err): + self.stdout, self.stderr = out, err + + saved = (gate.ROOT, gate.render_addons.discover, gate.render_addons.add_repos, + gate.subprocess.run, gate.gatelib.require) + gate.ROOT = root + gate.render_addons.discover = lambda *a, **k: [self.Unit()] + gate.render_addons.add_repos = lambda *a, **k: {} + gate.subprocess.run = lambda *a, **k: Completed(stdout, stderr) + gate.gatelib.require = lambda *a, **k: None + try: + return gate.inventory("production") + finally: + (gate.ROOT, gate.render_addons.discover, gate.render_addons.add_repos, + gate.subprocess.run, gate.gatelib.require) = saved + + def test_a_report_on_stdout_is_read(self): + """The shape that fails: the report is in the manifest stream, so the + chart's own coordinates are a reference the classifier must place.""" + _, unrendered, unclassified = self.inventory_with(self.REPORT + DEPLOYMENT, "") + self.assertEqual((unrendered, unclassified), ([], [])) + + # A chart carrying its own OCI coordinate in its rendered content. Whether + # the pull report reaches the gate then decides the verdict on the SAME + # render, which is what makes the stream choice observable at all. + SELF_REFERENCING = ("apiVersion: v1\nkind: ConfigMap\ndata:\n" + " cfg: |\n" + " chart: public.ecr.aws/karpenter/karpenter:1.14.0\n") + + def test_a_report_on_stderr_is_read(self): + """The render carries the coordinate and the report does not, so a gate + reading stdout alone has no evidence and reports the chart's own + reference as one it cannot place.""" + _, unrendered, unclassified = self.inventory_with( + DEPLOYMENT + "---\n" + self.SELF_REFERENCING, self.REPORT) + self.assertEqual((unrendered, unclassified), ([], [])) + + def test_both_stream_shapes_produce_the_same_verdict(self): + """The property, stated directly: one tree, two helm builds, one answer.""" + body = DEPLOYMENT + "---\n" + self.SELF_REFERENCING + on_out = self.inventory_with(self.REPORT + body, "") + on_err = self.inventory_with(body, self.REPORT) + self.assertEqual(on_out[0], on_err[0]) + self.assertEqual((on_out[1], on_out[2]), (on_err[1], on_err[2])) + self.assertEqual(on_err[2], []) + + def test_no_report_at_all_leaves_the_render_unchanged(self): + images, unrendered, unclassified = self.inventory_with(DEPLOYMENT, "") + self.assertEqual((unrendered, unclassified), ([], [])) + self.assertEqual(images, {"ghcr.io/example/app:1.0.0": {"karpenter"}}) + + +# Run the pattern in a child, so a pattern that does not return FAILS instead of +# hanging the suite. An in-process timing assertion cannot be reached by a +# matcher that never finishes — the very case it exists to catch. +_MATCH_IN_CHILD = """ +import importlib.util, sys +spec = importlib.util.spec_from_file_location("g", sys.argv[1]) +g = importlib.util.module_from_spec(spec) +spec.loader.exec_module(g) +found = [c for grp in g.IMAGE_REF.findall(sys.argv[2]) for c in grp if c] +print(len(found)) +""" + + +class TheHostGrammarHasOneParse(unittest.TestCase): + """A reference either parses one way or not at all, and it says so in time. + + Written as a starred class containing `.` and `-` followed by a + plus-quantified group whose body is that same class, a run of `-.` can be + split between the two quantifiers every possible way and the engine tries + all of them before failing. That is not a wrong answer, it is an answer that + never arrives: 43 characters took two thirds of a second and every four + further repetitions multiplied it by fifteen. + + The input is rendered content from charts this repository does not author + and this gate runs on every render, so the string arrives with a chart bump + rather than with an attacker, and the symptom is a job that never returns. + + So the registry grammar's own definition is used instead. A domain + COMPONENT carries no dot — dots are the separators — and neither starts nor + ends with a dash, which leaves every input exactly one parse or none. + """ + + # 32 repetitions, 67 characters. Under the ambiguous spelling the + # 43-character case multiplied by fifteen three times over, which is most of + # an hour; the fixed one answers in microseconds. Nothing between those two + # is reachable by a slow machine. + HOSTILE = "0." + "-." * 32 + "!" + BUDGET_SECONDS = 20 + + def completes(self, text): + """The number of references found, or a failure if the match hangs.""" + try: + proc = subprocess.run( + [sys.executable, "-c", _MATCH_IN_CHILD, str(GATE), text], + capture_output=True, text=True, timeout=self.BUDGET_SECONDS) + except subprocess.TimeoutExpired: + self.fail(f"IMAGE_REF did not finish on {len(text)} characters within " + f"{self.BUDGET_SECONDS}s — the host alternatives have more " + f"than one parse again") + self.assertEqual(proc.returncode, 0, proc.stderr) + return int(proc.stdout.strip()) + + def test_a_hostile_host_string_completes_and_matches_nothing(self): + self.assertEqual(self.completes(self.HOSTILE), 0) + + def test_ten_times_the_hostile_input_still_completes(self): + """One length can pass on a fast machine with a matcher that is merely + slow; ten times the input inside the same budget cannot.""" + self.assertEqual(self.completes("0." + "-." * 320 + "!"), 0) + + def test_a_hostile_string_inside_a_render_completes(self): + """Delivered the way a chart delivers one — a value in a ConfigMap.""" + rendered = ("apiVersion: v1\\nkind: ConfigMap\\ndata:\\n" + f" cfg: |\\n upstream: {self.HOSTILE}\\n") + unclassified: list[tuple[str, str]] = [] + gate.extract_images(rendered, "c", [], unclassified) + self.assertEqual(unclassified, []) + + def test_the_references_this_fleet_renders_still_parse(self): + """The grammar narrowed, so what it must still admit is read off the + population rather than a list written here.""" + for ref in sorted(gate.CONTROLLER_IMAGES) + sorted(gate.ALLOWED_MUTABLE): + with self.subTest(ref=ref): + concrete = f"{ref}:1.0.0" + self.assertEqual( + [c for grp in gate.IMAGE_REF.findall(concrete) for c in grp if c], + [concrete]) + + def test_a_domain_component_carries_no_dot_of_its_own(self): + """Dots separate components and never sit inside one. That is the + registry grammar, and it is what leaves a single parse.""" + self.assertRegex("ghcr.io", f"^{gate.DOMAIN_COMPONENT}\\.{gate.DOMAIN_COMPONENT}$") + self.assertNotRegex("gh.cr", f"^{gate.DOMAIN_COMPONENT}$") + + def test_a_component_neither_starts_nor_ends_with_a_dash(self): + for bad in ("-ghcr", "ghcr-", "-", "gh--cr-"): + with self.subTest(component=bad): + self.assertNotRegex(bad, f"^{gate.DOMAIN_COMPONENT}$") + self.assertRegex("gh--cr", f"^{gate.DOMAIN_COMPONENT}$") + + def test_a_dotless_host_is_not_a_registry(self): + """A host needs more than one component, and that is a decision with a + consequence rather than a detail of the grammar. + + `localhost:11211` in a Loki config is a host and a port, and it reaches + this walk as a single-segment token — which is why NOT_A_CONTAINER + carries an entry for it. Admitting a dotless host as a registry would + make `localhost:5000/x/y:1.0` one reference instead, and the entry that + records why the address is not an image would stop matching anything. + """ + self.assertEqual( + [c for grp in gate.IMAGE_REF.findall("localhost:5000/x/y:1.0") + for c in grp if c], + ["localhost:5000"]) + self.assertIn("localhost", gate.NOT_A_CONTAINER) + self.assertEqual( + [c for grp in gate.IMAGE_REF.findall("registry.local:5000/x/y:1.0") + for c in grp if c], + ["registry.local:5000/x/y:1.0"]) + + +class OneCorpus(unittest.TestCase): + """`inventory` and the per-chart floor read the same units, not two views. + + `discover()` is asserted per file elsewhere — every ApplicationSet pinning a + chart contributes a unit, every matrix element pinning one appears among + that appset's units. `inventory` walks exactly those units, and + `chart_coverage` then requires each of them to have contributed an image. + + Passing over a reference helm reported pulling sits between the two, so the + question is whether it can take a chart's only contribution with it. It + cannot take a real one: a chart artifact is not an image the chart deploys, + and it never entered the population in the first place. What it must not do + is leave a chart looking covered when it contributed nothing — so that is + asserted here rather than reasoned about. + """ + + def test_a_chart_contributing_only_its_own_coordinate_is_reported(self): + """The whole risk in one case. The reference is passed over, the chart + contributed no image, and the per-chart floor says so — silence here + would be a chart dropping out of every question both gates ask.""" + rendered = ("Pulled: ghcr.io/example/chart:1.0.0\n" + "apiVersion: v1\nkind: ConfigMap\ndata: {}\n") + found = gate.extract_images(rendered, "chart", [], [], + None, gate.chart_artifacts(rendered)) + self.assertEqual(found, set()) + problems = gate.chart_coverage({}, [Unit("chart")]) + self.assertTrue(any("chart rendered and contributed no image" in p + for p in problems), problems) diff --git a/scripts/tests/test_image_vulnerabilities.py b/scripts/tests/test_image_vulnerabilities.py new file mode 100644 index 0000000..25bd9da --- /dev/null +++ b/scripts/tests/test_image_vulnerabilities.py @@ -0,0 +1,205 @@ +"""Unit tests for the image-vulnerability gate's verdict. + +The gate pulls and scans every image the pinned charts render, so it reaches a +registry and the positive-control sweep exempts it. What that leaves untested is +the part that decides the outcome: whether a finding is acknowledged, and whether +an acknowledgement still describes the scan. + +Both directions matter and they fail for opposite reasons. Too strict and the +gate reports decisions that were already taken; too loose and +`image-advisories.yaml` becomes a permanent waiver, which is the one outcome that +would make a green run worse than no gate at all. + +Everything here is offline. The trivy invocation and the canary are exercised by +running the gate, not from here. +""" + +from __future__ import annotations + +import pathlib +import unittest + +import yaml +from gateloader import load + +gate = load("check-image-vulnerabilities") + +ROOT = pathlib.Path(__file__).resolve().parent.parent.parent + + +def finding(image="quay.io/argoproj/argo-events:v1.9.11", cve="CVE-2026-33815", + package="github.com/jackc/pgx/v5", installed="v5.7.5", fixed="5.9.0", + severity="CRITICAL"): + return gate.Finding(image, gate.bare(image), cve, package, installed, fixed, + severity) + + +def advisory(cve="CVE-2026-33815", package="github.com/jackc/pgx/v5", + reason="upstream rebuild", images=("quay.io/argoproj/argo-events",)): + return {"id": cve, "package": package, "reason": reason, "images": list(images)} + + +class TheImageAnAdvisoryNames(unittest.TestCase): + """An acknowledgement is about the image; the tag moves with every bump.""" + + def test_a_tag_is_dropped(self): + self.assertEqual(gate.bare("quay.io/argoproj/argo-events:v1.9.11"), + "quay.io/argoproj/argo-events") + + def test_a_digest_is_dropped(self): + self.assertEqual(gate.bare("quay.io/cilium/cilium@sha256:" + "0" * 64), + "quay.io/cilium/cilium") + + def test_a_tag_and_a_digest_together_are_dropped(self): + self.assertEqual( + gate.bare("ghcr.io/opencost/opencost:1.121.1@sha256:" + "5" * 64), + "ghcr.io/opencost/opencost") + + def test_an_untagged_reference_is_its_own_name(self): + self.assertEqual(gate.bare("otel/opentelemetry-collector-contrib"), + "otel/opentelemetry-collector-contrib") + + def test_a_registry_port_is_not_read_as_a_tag(self): + """Cutting there produces a key no advisory can match.""" + self.assertEqual(gate.bare("registry:5000/nanohype/agent"), + "registry:5000/nanohype/agent") + self.assertEqual(gate.bare("registry:5000/nanohype/agent:1.2.3"), + "registry:5000/nanohype/agent") + + +class AnUnacknowledgedFinding(unittest.TestCase): + """The question the gate is named for.""" + + def test_a_critical_with_no_entry_blocks(self): + problems = gate.verdict([finding()], []) + self.assertEqual(len(problems), 1) + self.assertIn("no entry in image-advisories.yaml names it", problems[0]) + self.assertIn("CVE-2026-33815", problems[0]) + self.assertIn("fixed in 5.9.0", problems[0]) + + def test_an_acknowledged_critical_passes(self): + self.assertEqual(gate.verdict([finding()], [advisory()]), []) + + def test_a_high_is_not_blocking(self): + """Counted and printed; gating it would gate on upstream's advisory rate.""" + self.assertEqual(gate.verdict([finding(severity="HIGH")], []), []) + + def test_an_entry_matching_only_the_cve_does_not_cover_another_package(self): + """One CVE id can be filed against several packages with different fixes.""" + problems = gate.verdict([finding(package="golang.org/x/crypto")], + [advisory(package="github.com/jackc/pgx/v5")]) + self.assertIn("no entry in image-advisories.yaml names it", problems[0]) + + def test_the_same_finding_reported_twice_is_one_problem(self): + """One image carries an advisory in the binary and in the layer around it.""" + problems = gate.verdict([finding(), finding()], []) + self.assertEqual(len(problems), 1) + + +class AnAcknowledgementCoversTheImagesItNames(unittest.TestCase): + """A new image acquiring a known CRITICAL is a decision, not an inheritance.""" + + def test_a_finding_on_an_unlisted_image_blocks(self): + """The listed image is covered; the one beside it on the same CVE is not.""" + problems = gate.verdict( + [finding(image="quay.io/cilium/cilium:v1.19.6"), + finding(image="quay.io/argoproj/argo-events:v1.9.11")], + [advisory(images=("quay.io/argoproj/argo-events",))]) + self.assertEqual(len(problems), 1) + self.assertIn("the entry does not list quay.io/cilium/cilium", problems[0]) + + def test_a_tag_bump_on_a_listed_image_still_passes(self): + self.assertEqual( + gate.verdict([finding(image="quay.io/argoproj/argo-events:v1.9.12")], + [advisory()]), []) + + +class AnAcknowledgementThatStoppedDescribingTheScan(unittest.TestCase): + """A list nobody re-checks only ever widens, and it widens permissively.""" + + def test_an_entry_no_image_carries_blocks(self): + problems = gate.verdict([], [advisory()]) + self.assertEqual(len(problems), 1) + self.assertIn("outlived its reason", problems[0]) + + def test_an_entry_listing_an_image_without_the_finding_blocks(self): + """The shape a chart bump leaves behind when it fixes one image of many.""" + problems = gate.verdict( + [finding(image="quay.io/argoproj/argo-events:v1.9.11")], + [advisory(images=("quay.io/argoproj/argo-events", "quay.io/cilium/cilium"))]) + self.assertEqual(len(problems), 1) + self.assertIn("lists quay.io/cilium/cilium, which no longer carries it", + problems[0]) + + def test_an_entry_with_no_images_acknowledges_nothing(self): + problems = gate.verdict([finding()], [advisory(images=())]) + joined = " ".join(problems) + self.assertIn("lists no images", joined) + + def test_an_entry_with_no_reason_blocks(self): + problems = gate.verdict([finding()], [advisory(reason=" ")]) + self.assertIn("acknowledged with no reason recorded", problems[0]) + + def test_a_high_only_finding_does_not_keep_a_critical_entry_alive(self): + """The entry is about the blocking severity; a HIGH match is not a match.""" + problems = gate.verdict([finding(severity="HIGH")], [advisory()]) + self.assertEqual(len(problems), 1) + self.assertIn("outlived its reason", problems[0]) + + +class TheShippedAdvisoryFile(unittest.TestCase): + """It is read on every run, so its shape is part of the gate.""" + + @classmethod + def setUpClass(cls): + doc = yaml.safe_load((ROOT / "image-advisories.yaml").read_text()) + cls.entries = doc["advisories"] + + def test_every_entry_carries_the_four_keys_the_verdict_reads(self): + for entry in self.entries: + with self.subTest(entry=entry.get("id")): + for key in ("id", "package", "reason", "images"): + self.assertIn(key, entry) + + def test_every_entry_records_a_reason(self): + for entry in self.entries: + with self.subTest(entry=entry.get("id")): + self.assertTrue(str(entry["reason"]).strip()) + + def test_every_listed_image_is_named_without_a_tag_or_digest(self): + """A tagged entry would stop matching at the next chart bump.""" + for entry in self.entries: + for image in entry["images"]: + with self.subTest(image=image): + self.assertEqual(gate.bare(image), image) + + def test_no_two_entries_share_an_id_and_package(self): + """The verdict keys on that pair, so a duplicate silently shadows one.""" + keys = [(e["id"], e["package"]) for e in self.entries] + self.assertEqual(len(keys), len(set(keys))) + + def test_the_file_acknowledges_something(self): + """An empty list makes every rule below hold over nothing.""" + self.assertTrue(self.entries) + + +class TheFloorsThatStopAVacuousPass(unittest.TestCase): + """A scan over almost nothing reports the same as a scan over a clean fleet.""" + + def test_the_floor_on_images_scanned_is_derived_per_chart(self): + """A constant can be set above the render and make the gate always red, + or below it and let a shrunken walk through. One image per chart the + render covers is neither: it moves with the catalog.""" + self.assertFalse(hasattr(gate, "MIN_IMAGES")) + + def test_the_canary_is_pinned_by_digest(self): + """A tag can be repointed at a rebuilt, clean image, and then the canary + stops proving the scanner reports anything.""" + self.assertIn("@sha256:", gate.CANARY) + + def test_the_blocking_severity_is_among_the_ones_scanned_for(self): + self.assertIn(gate.BLOCKING_SEVERITY, gate.REPORTED_SEVERITIES) + + +if __name__ == "__main__": + unittest.main()