From fbcee1481ab3ee329edbfa6281ed4e6cbb2daaac Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Sat, 7 Nov 2026 12:55:52 +0000 Subject: [PATCH] Replace policy-violating CI helpers with tested native Rust contracts --- .github/workflows/ci-integrity.yml | 7 + .github/workflows/proven-tests.yml | 17 +- .github/workflows/recon.yml | 2 +- .gitignore | 1 + Justfile | 4 +- docs/CI-CD.adoc | 2 +- docs/native-ci-helper-migration.adoc | 39 +++ proven/ledger.json | 2 +- scripts/ci/test-inventory-failure.sh | 33 +++ scripts/proven/inventory.sh | 16 +- scripts/proven/verify-ledger.py | 259 -------------------- scripts/proven/verify-ledger.sh | 5 + scripts/recon/compare.py | 166 ------------- scripts/recon/compare.sh | 5 + tools/ci/Cargo.lock | 288 ++++++++++++++++++++++ tools/ci/Cargo.toml | 15 ++ tools/ci/src/bin/recon-compare.rs | 126 ++++++++++ tools/ci/src/bin/verify-ledger.rs | 41 ++++ tools/ci/src/lib.rs | 346 +++++++++++++++++++++++++++ tools/ci/tests/cli.rs | 136 +++++++++++ 20 files changed, 1061 insertions(+), 449 deletions(-) create mode 100644 docs/native-ci-helper-migration.adoc create mode 100644 scripts/ci/test-inventory-failure.sh delete mode 100644 scripts/proven/verify-ledger.py create mode 100644 scripts/proven/verify-ledger.sh delete mode 100644 scripts/recon/compare.py create mode 100644 scripts/recon/compare.sh create mode 100644 tools/ci/Cargo.lock create mode 100644 tools/ci/Cargo.toml create mode 100644 tools/ci/src/bin/recon-compare.rs create mode 100644 tools/ci/src/bin/verify-ledger.rs create mode 100644 tools/ci/src/lib.rs create mode 100644 tools/ci/tests/cli.rs diff --git a/.github/workflows/ci-integrity.yml b/.github/workflows/ci-integrity.yml index c8c0959..6a2576f 100644 --- a/.github/workflows/ci-integrity.yml +++ b/.github/workflows/ci-integrity.yml @@ -117,6 +117,13 @@ jobs: done exit "$fail" + - name: Native CI helper contracts + run: | + set -euo pipefail + cargo test --locked --manifest-path tools/ci/Cargo.toml + cargo clippy --locked --manifest-path tools/ci/Cargo.toml --all-targets -- -D warnings + bash scripts/ci/test-inventory-failure.sh + - name: Repository path portability run: | set -euo pipefail diff --git a/.github/workflows/proven-tests.yml b/.github/workflows/proven-tests.yml index 2d6cbac..82ab20c 100644 --- a/.github/workflows/proven-tests.yml +++ b/.github/workflows/proven-tests.yml @@ -81,7 +81,7 @@ jobs: - name: Derive the run report (cells must select real tests) run: | bash scripts/ci/annotate.sh "proven:ledger" -- \ - python3 scripts/proven/verify-ledger.py \ + bash scripts/proven/verify-ledger.sh \ --inventory /tmp/inventory.json \ --ledger proven/ledger.json \ --root . \ @@ -101,19 +101,8 @@ jobs: --proven-tests-root . \ --fail-on-defect \ --output /tmp/proven-aggregate.json - python3 - <<'PY' - import json - agg = json.load(open('/tmp/proven-aggregate.json')) - records = agg['proven_tests'] - assert len(records) == 1, records - record = records[0] - assert record['defects'] == [], record['defects'] - summary = record['summary'] - assert summary['total'] > 0, summary - print(f"proven-tests: {summary['passed']}/{summary['total']} passed, " - f"{summary['covered_cells']}/{summary['cat_aspect_cells']} cells, " - f"{summary['actually_proven']} actually-proven") - PY + jq -e '.proven_tests | type == "array" and length == 1 and (.[0].defects == []) and (.[0].summary.total | type == "number" and . > 0)' /tmp/proven-aggregate.json > /dev/null + jq -r '.proven_tests[0].summary | "proven-tests: \(.passed)/\(.total) passed, \(.covered_cells)/\(.cat_aspect_cells) cells, \(.actually_proven) actually-proven"' /tmp/proven-aggregate.json - name: Publish to the job summary if: always() diff --git a/.github/workflows/recon.yml b/.github/workflows/recon.yml index f76c42a..8317bd7 100644 --- a/.github/workflows/recon.yml +++ b/.github/workflows/recon.yml @@ -142,7 +142,7 @@ jobs: if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ inputs.fail_on_new }}" = "false" ]; then args="--no-fail" fi - python3 scripts/recon/compare.py \ + bash scripts/recon/compare.sh \ --baseline recon/baseline.json \ --assail "$SARIF_DIR/assail.json" \ --pons "$SARIF_DIR/pons.json" \ diff --git a/.gitignore b/.gitignore index 7faad10..0f4c573 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # RSR-compliant .gitignore /target +/tools/ci/target/ /reports/ /verisimdb-data/ /runtime/ diff --git a/Justfile b/Justfile index 365c6d9..88f0d6d 100644 --- a/Justfile +++ b/Justfile @@ -305,7 +305,7 @@ proven: set -euo pipefail cargo build --locked --release bash scripts/proven/inventory.sh /tmp/panic-attack-tests.json - python3 scripts/proven/verify-ledger.py \ + bash scripts/proven/verify-ledger.sh \ --inventory /tmp/panic-attack-tests.json \ --ledger proven/ledger.json \ --root . \ @@ -324,7 +324,7 @@ recon: set -euo pipefail mkdir -p recon-out cargo run --release --quiet -- assail . --output recon-out/assail.json --quiet - python3 scripts/recon/compare.py \ + bash scripts/recon/compare.sh \ --baseline recon/baseline.json \ --assail recon-out/assail.json \ --pons /dev/null \ diff --git a/docs/CI-CD.adoc b/docs/CI-CD.adoc index ddae0c9..29a1713 100644 --- a/docs/CI-CD.adoc +++ b/docs/CI-CD.adoc @@ -201,7 +201,7 @@ never declared**, so: * `scripts/proven/inventory.sh` asks the test runner what tests exist, per binary; -* `scripts/proven/verify-ledger.py` fails when a cell selects no executed test, +* `scripts/proven/verify-ledger.sh` fails when a cell selects no executed test, when an executed test is claimed by nothing, or when a stronger tier is claimed than the evidence supports; * the resulting report is fed to `panic-attack aggregate --proven-tests diff --git a/docs/native-ci-helper-migration.adoc b/docs/native-ci-helper-migration.adoc new file mode 100644 index 0000000..ba2534c --- /dev/null +++ b/docs/native-ci-helper-migration.adoc @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 += Native CI helper migration — 7 November 2026 + +Fresh governance execution exposed two production Python helpers that violate +the estate language policy. They are replaced by the Rust programs in +`tools/ci`, called through the existing-purpose shell entrypoints. Workflow, +Justfile and ledger pointers are updated together; no language-policy waiver +or scanner exclusion is added. The workflow's inline aggregate check uses +its already required jq, with the original nonempty/defect-free assertions. + +The standalone dev-tool workspace has its own generated lockfile and is not +an application binary or a new root test binary. It retains the ledger's +coordinate selectors, first matching buckets, remainder ratchet, declared +absence conflict check and proof-file requirements. Test identities now name +both the test binary and test name so equal names cannot collapse across +binaries. Empty inventories, missing coordinates and invalid proof tiers +remain failures; a failed claim does not overwrite an existing report. + +Recon preserves category-debt comparison, bootstrap baselines and explicit +`--no-fail` advisory mode. Missing, invalid or unrecognised scanner reports +are unavailable, never clean: strict mode fails and no replacement baseline +is proposed. SARIF rule identifiers are recognised alongside JSON categories. + +The inventory producer now preserves a failed Cargo enumeration instead of +turning it into an empty successful list. It does not write an empty/partial +inventory. A dedicated CI fault test injects a failing test binary and an +empty suite; this Bash control is a hosted Linux check, not a claimed Windows +execution. + +Local Windows evidence: six library controls and three CLI integration tests +pass. They cover stale/empty claims, binary-name collisions, absent/covered +conflicts, missing proof warrants, remainder bounds, malformed scanner data, +debt regressions and unavailable/advisory behaviour. The actual repository +ledger's taxonomy and selectors are checked. CI runs these controls and +all-targets Clippy with warnings denied before testing the real inventory. + +Creusot with Pearlite contracts: `OPEN`, not run for these CI utilities. +Verus: `OPEN`, not run. Executable contract tests are not formal proofs; +framework provenance tiers are retained and no independent witness is claimed. diff --git a/proven/ledger.json b/proven/ledger.json index ae56509..835bc90 100644 --- a/proven/ledger.json +++ b/proven/ledger.json @@ -42,7 +42,7 @@ }, "doctrine": [ "A cell is covered only when a test that actually ran selected it. Nothing", - "here declares coverage; scripts/proven/verify-ledger.py derives it from a", + "here declares coverage; scripts/proven/verify-ledger.sh derives it from a", "test inventory produced by `cargo test -- --list` in the same commit.", "Tiers are the framework's three: Actually-Proven (a machine-checked proof", "ladder exists), Provisionally-Proven (typed assertions, example/property", diff --git a/scripts/ci/test-inventory-failure.sh b/scripts/ci/test-inventory-failure.sh new file mode 100644 index 0000000..f6b6148 --- /dev/null +++ b/scripts/ci/test-inventory-failure.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +set -euo pipefail +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +mkdir -p "$work/bin" "$work/repo/tests" +printf '' > "$work/repo/tests/broken.rs" +cat > "$work/bin/cargo" <<'MOCK' +#!/usr/bin/env bash +case " $* " in + *' --lib '*) printf 'library_control: test\n' ;; + *) exit 42 ;; +esac +MOCK +chmod +x "$work/bin/cargo" +cd "$work/repo" +set +e +PATH="$work/bin:$PATH" bash "$root/scripts/proven/inventory.sh" "$work/inventory.json" +status=$? +set -e +[ "$status" -eq 42 ] || { echo 'failed enumeration was not preserved' >&2; exit 1; } +[ ! -e "$work/inventory.json" ] || { echo 'partial inventory was emitted' >&2; exit 1; } +cat > "$work/bin/cargo" <<'MOCK' +#!/usr/bin/env bash +exit 0 +MOCK +set +e +PATH="$work/bin:$PATH" bash "$root/scripts/proven/inventory.sh" "$work/empty.json" +status=$? +set -e +[ "$status" -eq 1 ] && [ ! -e "$work/empty.json" ] || { echo 'empty inventory was certified' >&2; exit 1; } +echo 'Inventory failure and empty-suite controls passed.' diff --git a/scripts/proven/inventory.sh b/scripts/proven/inventory.sh index e43295f..3a07d8b 100644 --- a/scripts/proven/inventory.sh +++ b/scripts/proven/inventory.sh @@ -30,7 +30,10 @@ add() { } # Library unit tests (module::path::name). -add "panic_attack" "$(cargo test --locked --lib -- --list 2>/dev/null || true)" +names="$(cargo test --locked --lib -- --list)" || { + code=$?; echo 'inventory: library test enumeration failed' >&2; exit "$code"; +} +add "panic_attack" "$names" # Integration test binaries, one per tests/*.rs file. for f in tests/*.rs; do @@ -40,11 +43,14 @@ for f in tests/*.rs; do # shell-driven fixtures, not Rust test binaries verisimdb_e2e) continue ;; esac - add "$bin" "$(cargo test --locked --test "$bin" -- --list 2>/dev/null || true)" + names="$(cargo test --locked --test "$bin" -- --list)" || { + code=$?; echo "inventory: test enumeration failed for $bin" >&2; exit "$code"; + } + add "$bin" "$names" done +total="$(jq '[.[] | length] | add // 0' "$tmp")" +bins="$(jq 'length' "$tmp")" +[ "$total" -gt 0 ] || { echo "inventory: no tests found — refusing to certify an empty suite" >&2; exit 1; } jq -S . "$tmp" > "$OUT" -total="$(jq '[.[] | length] | add // 0' "$OUT")" -bins="$(jq 'length' "$OUT")" echo "inventory: $total test(s) in $bins binar(ies) -> $OUT" -[ "$total" -gt 0 ] || { echo "inventory: no tests found — refusing to certify an empty suite" >&2; exit 1; } diff --git a/scripts/proven/verify-ledger.py b/scripts/proven/verify-ledger.py deleted file mode 100644 index f5134a8..0000000 --- a/scripts/proven/verify-ledger.py +++ /dev/null @@ -1,259 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MPL-2.0 -# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell -"""Derive a proven-tests run report from tests that ACTUALLY RAN. - -The framework's organising rule (proven-tests-and-benches) is that coverage is -derived from executed tests at typed coordinates, never declared. A JSON file -saying "we cover Security" is a claim; a JSON file that can only be produced -from a test inventory the test runner emitted is evidence. - -So this verifier: - - 1. refuses to emit anything unless every ledger cell selects at least one - test present in the inventory (a cell with no test is a stale claim); - 2. refuses to emit anything unless every test in the inventory reaches a - typed coordinate: either an explicit cell, or a `buckets` rule (module - prefix or binary -> category x aspect), or `declared_unmapped`. A test that - reaches none of those is an invisible gap and fails the run; - 2b. caps how much of the suite may reach the LAST bucket (the declared - remainder) via `max_unmapped`. That is a ratchet: the number can be lowered - in review, and a suite that grows past it fails rather than quietly - claiming more un-triaged ground; - 3. refuses `Actually-Proven` unless the entry names a proof file that exists; - 4. writes a `schema_version: 1` run report, the format panic-attack's - `aggregate --proven-tests` reads, so the two tools agree on one artefact - rather than on a paragraph in a design doc. - -Inventory format (produced by scripts/proven/inventory.sh): - - {"lib": ["a::b::test", ...], "tests/foo.rs": ["test_name", ...], ...} - -Exit codes: 0 report written and self-consistent, 1 claim not supported, -2 usage or input error. -""" -from __future__ import annotations - -import argparse -import json -import re -import sys -from pathlib import Path - -TERSE = re.compile(r"^(?P[^\s:]+(?:::[^\s:]+)*): test$") - - -def load_inventory(path: Path) -> dict[str, list[str]]: - data = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(data, dict): - raise SystemExit("inventory: expected an object of {binary: [test names]}") - clean: dict[str, list[str]] = {} - for binary, names in data.items(): - if not isinstance(names, list): - raise SystemExit(f"inventory: {binary} is not a list") - clean[binary] = sorted({str(n) for n in names}) - if not clean: - raise SystemExit("inventory: no test binaries — refusing to certify an empty run") - return clean - - -def select(entries: list[dict], inventory: dict[str, list[str]]) -> dict[str, dict]: - """Map test name -> the ledger cell that claims it.""" - claimed: dict[str, dict] = {} - for entry in entries: - bins = entry.get("bins") or [] - pattern = entry.get("pattern") - regex = re.compile(pattern) if pattern else None - matched = 0 - for binary, names in inventory.items(): - if binary not in bins: - continue - for name in names: - if regex and not regex.search(name): - continue - matched += 1 - claimed.setdefault(name, entry) - if matched == 0: - raise SystemExit( - f"ledger cell {entry['category']} x {entry['aspect']} selects no " - f"test in the inventory (bins={bins}, pattern={pattern!r}): the " - "claim is stale — fix the selection or declare the cell absent" - ) - return claimed - - -def main() -> int: - ap = argparse.ArgumentParser() - ap.add_argument("--inventory", required=True) - ap.add_argument("--ledger", default="proven/ledger.json") - ap.add_argument("--root", default=".") - ap.add_argument("--out", default="proven-report.json") - ap.add_argument("--allow-absent", action="store_true", - help="report declared-absent cells as informational instead of failing") - args = ap.parse_args() - - inventory = load_inventory(Path(args.inventory)) - ledger = json.loads(Path(args.ledger).read_text(encoding="utf-8")) - entries = ledger.get("entries", []) - absent = ledger.get("declared_absent", []) - unmapped_decl = ledger.get("declared_unmapped", []) - - claimed = select(entries, inventory) - - # Buckets: every executed test that no explicit cell claims is assigned a - # typed coordinate by the first matching rule. This is how "every test is - # accounted for" stays true as the suite grows without hand-writing a cell - # per test — and the last rule's population is capped, so it cannot become a - # place where untriaged work hides. - buckets = ledger.get("buckets", []) - compiled = [ - (re.compile(b["match"]) if b.get("match") else None, set(b.get("bins") or []), b) - for b in buckets - ] - bucketed: dict[str, dict] = {} - remainder: list[str] = [] - for binary, names in inventory.items(): - for name in names: - if name in claimed: - continue - chosen = None - for regex, bins, bucket in compiled: - if bins and binary not in bins: - continue - if regex is not None and not regex.search(name): - continue - chosen = bucket - break - if chosen is None: - continue - bucketed[name] = chosen - if buckets and chosen is buckets[-1]: - remainder.append(name) - - all_tests = {name for names in inventory.values() for name in names} - unaccounted = sorted(all_tests - set(claimed) - set(bucketed)) - allowed_unmapped = [] - for decl in unmapped_decl: - regex = re.compile(decl["match"]) - allowed_unmapped.extend(t for t in unaccounted if regex.search(t)) - unaccounted = [t for t in unaccounted if t not in set(allowed_unmapped)] - - problems: list[str] = [] - if unaccounted: - problems.append( - f"{len(unaccounted)} executed test(s) are claimed by no ledger cell and " - f"declared nowhere, e.g. {', '.join(unaccounted[:5])}" - ) - - report_entries = [] - cells_covered: set[tuple[str, str]] = set() - tiers = {"Actually-Proven": 0, "Provisionally-Proven": 0, "Unproven": 0} - root = Path(args.root) - - for name in sorted(claimed): - entry = claimed[name] - tier = entry["tier"] - if tier not in tiers: - problems.append(f"{name}: unknown tier {tier!r}") - continue - proof_file = entry.get("proof_file") - if tier == "Actually-Proven": - if not proof_file: - problems.append( - f"{name}: claims Actually-Proven with no proof_file " - "(the framework's type makes this claim impossible to make honestly)" - ) - elif not (root / proof_file).is_file(): - problems.append(f"{name}: cited proof_file {proof_file} does not exist") - elif proof_file: - problems.append( - f"{name}: tier {tier} must not carry a proof_file — that would " - "borrow the warrant of a stronger tier" - ) - tiers[tier] += 1 - cells_covered.add((entry["category"], entry["aspect"])) - report_entries.append( - { - "test_id": name, - "category": entry["category"], - "aspect": entry["aspect"], - "provenance": tier, - "proof_ladder": [], - "result": "passed", - } - ) - - # The bucketed remainder, folded in at its declared tier. - for name in sorted(bucketed): - bucket = bucketed[name] - tier = bucket.get("tier", "Provisionally-Proven") - tiers[tier] += 1 - cells_covered.add((bucket["category"], bucket["aspect"])) - report_entries.append( - { - "test_id": name, - "category": bucket["category"], - "aspect": bucket["aspect"], - "provenance": tier, - "proof_ladder": [], - "result": "passed", - } - ) - - max_unmapped = ledger.get("max_unmapped") - if remainder and max_unmapped is not None and len(remainder) > int(max_unmapped): - problems.append( - f"{len(remainder)} test(s) fall through to the declared remainder bucket, " - f"above the ratchet of {max_unmapped}: either map them to a coordinate " - "in `buckets`, or raise the cap in review with the reason" - ) - - categories = ledger.get("taxonomy", {}).get("categories", []) - aspects = ledger.get("taxonomy", {}).get("aspects", []) - for a in absent: - if (a["category"], a["aspect"]) in cells_covered: - problems.append( - f"{a['category']} x {a['aspect']} is declared absent but is also covered" - ) - cat_aspect_cells = len(categories) * len(aspects) - - if problems: - for p in problems: - print(f"::error::{p}", file=sys.stderr) - return 1 - - report = { - "schema_version": 1, - "suite": ledger.get("suite", "unknown"), - "summary": { - "passed": len(report_entries), - "total": len(report_entries), - "actually_proven": tiers["Actually-Proven"], - "provisionally_proven": tiers["Provisionally-Proven"], - "unproven": tiers["Unproven"], - "covered_cells": len(cells_covered), - "cat_aspect_cells": cat_aspect_cells, - "lattice_covered": len(cells_covered), - }, - "entries": report_entries, - } - Path(args.out).write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") - - print( - f"proven: {len(report_entries)} executed test(s) across " - f"{len(cells_covered)} of {cat_aspect_cells} cat×aspect cells " - f"({tiers['Actually-Proven']} actually-proven, " - f"{tiers['Provisionally-Proven']} provisionally-proven, " - f"{tiers['Unproven']} unproven); {len(absent)} cell(s) declared absent" - ) - if remainder: - print( - f"proven: {len(remainder)} test(s) in the declared remainder bucket " - f"(cap {max_unmapped}) — the to-do list, not a coverage claim" - ) - print(f"proven: wrote {args.out}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/proven/verify-ledger.sh b/scripts/proven/verify-ledger.sh new file mode 100644 index 0000000..71cc550 --- /dev/null +++ b/scripts/proven/verify-ledger.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +set -euo pipefail +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +exec cargo run --quiet --locked --manifest-path "$root/tools/ci/Cargo.toml" --bin verify-ledger -- "$@" diff --git a/scripts/recon/compare.py b/scripts/recon/compare.py deleted file mode 100644 index 1cd9c18..0000000 --- a/scripts/recon/compare.py +++ /dev/null @@ -1,166 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MPL-2.0 -# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell -"""Compare two independent scans of this repository against a frozen baseline. - -The baseline is a *debt ledger*, not a target. It exists so that: - - * a category that did not exist yesterday fails the job today (a regression), - * a count that rises above what was accepted fails (new debt), - * a count that FALLS is reported, so paying debt down is visible and the - baseline can be tightened in the same PR that fixed it. - -Findings that are simply standing debt never fail the job. Failing on them -would create pressure to suppress them — the failure mode this estate keeps -paying for (see panic-attack's own user-classification registry doctrine). - -Both scanner outputs are read defensively on purpose. `pons` is v0.1 and its -JSON envelope is still settling, so findings are located structurally rather -than by a hard-coded path: any list named `findings` at any depth, or the -`weak_points` array panic-attack emits. If a shape is not recognised the run -says so and does not silently count zero. -""" -from __future__ import annotations - -import argparse -import json -import sys -from collections import Counter -from pathlib import Path - - -def walk_lists(node, wanted: set[str], path=""): - """Yield (json path, list) for every list whose key is in `wanted`.""" - if isinstance(node, dict): - for key, value in node.items(): - here = f"{path}.{key}" if path else key - if key in wanted and isinstance(value, list): - yield here, value - yield from walk_lists(value, wanted, here) - elif isinstance(node, list): - for i, value in enumerate(node): - yield from walk_lists(value, wanted, f"{path}[{i}]") - - -def label(finding: dict) -> str: - for key in ("category", "rule_id", "rule", "id", "code", "check"): - value = finding.get(key) - if isinstance(value, str): - return value - return "unknown" - - -def counts_from(path: Path, wanted: set[str], name: str) -> tuple[Counter, list[str]]: - notes: list[str] = [] - if not path.exists() or path.stat().st_size == 0: - notes.append(f"{name}: no report at {path} — counted as absent, not as zero") - return Counter(), notes - try: - data = json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError as exc: - notes.append(f"{name}: report is not JSON ({exc})") - return Counter(), notes - - counter: Counter = Counter() - found = False - for where, findings in walk_lists(data, wanted): - found = True - for finding in findings: - if isinstance(finding, dict): - counter[label(finding)] += 1 - notes.append(f"{name}: {len(findings)} finding(s) at {where}") - if not found: - notes.append( - f"{name}: no {'/'.join(sorted(wanted))} array in the report — shape not " - "recognised; counted as absent rather than as clean" - ) - return counter, notes - - -def main() -> int: - ap = argparse.ArgumentParser() - ap.add_argument("--baseline", default="recon/baseline.json") - ap.add_argument("--assail", required=True) - ap.add_argument("--pons", required=True) - ap.add_argument("--summary") - ap.add_argument("--no-fail", action="store_true") - args = ap.parse_args() - - baseline = ( - json.loads(Path(args.baseline).read_text(encoding="utf-8")) - if Path(args.baseline).exists() - else {} - ) - scanners = { - "assail": (Path(args.assail), {"weak_points"}), - "pons": (Path(args.pons), {"findings", "results"}), - } - - lines: list[str] = ["## Recon: two independent scans", ""] - regressions = 0 - observed: dict[str, dict[str, int]] = {} - - for name, (path, wanted) in scanners.items(): - counter, notes = counts_from(path, wanted, name) - observed[name] = dict(sorted(counter.items())) - frozen = baseline.get(name) or {} - lines.append(f"### {name}") - lines.append("") - for note in notes: - lines.append(f"- {note}") - if counter: - lines.append("") - lines.append("| category | observed | baseline | verdict |") - lines.append("|---|---|---|---|") - for category, count in sorted(counter.items()): - allowed = frozen.get(category) - if allowed is None: - verdict = "**NEW (fails)**" if frozen else "new (baseline empty)" - if frozen: - regressions += 1 - elif count > allowed: - verdict = f"**+{count - allowed} (fails)**" - regressions += 1 - elif count < allowed: - verdict = f"−{allowed - count} (debt paid; tighten the baseline)" - else: - verdict = "at baseline" - lines.append(f"| {category} | {count} | {allowed if allowed is not None else '—'} | {verdict} |") - else: - lines.append("- no findings counted") - missing = sorted(set(frozen) - set(counter)) - if missing: - lines.append("") - lines.append(f"- baseline categories no longer reported: {', '.join(missing)}") - lines.append("") - - lines.append("### Freeze or tighten the baseline") - lines.append("") - lines.append("```json") - lines.append(json.dumps(observed, indent=2, sort_keys=True)) - lines.append("```") - - out = "\n".join(lines) - print(out) - if args.summary: - Path(args.summary).write_text( - (Path(args.summary).read_text(encoding="utf-8") if Path(args.summary).exists() else "") - + out - + "\n", - encoding="utf-8", - ) - - if args.no_fail: - print("recon: --no-fail, reporting only", file=sys.stderr) - return 0 - if regressions: - print( - f"::error::{regressions} category regression(s) against {args.baseline}", - file=sys.stderr, - ) - return 1 - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/recon/compare.sh b/scripts/recon/compare.sh new file mode 100644 index 0000000..6452c6c --- /dev/null +++ b/scripts/recon/compare.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +set -euo pipefail +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +exec cargo run --quiet --locked --manifest-path "$root/tools/ci/Cargo.toml" --bin recon-compare -- "$@" diff --git a/tools/ci/Cargo.lock b/tools/ci/Cargo.lock new file mode 100644 index 0000000..1942e68 --- /dev/null +++ b/tools/ci/Cargo.lock @@ -0,0 +1,288 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "clap" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "panic-ci-tools" +version = "0.1.0" +dependencies = [ + "clap", + "regex", + "serde", + "serde_json", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/tools/ci/Cargo.toml b/tools/ci/Cargo.toml new file mode 100644 index 0000000..893efd3 --- /dev/null +++ b/tools/ci/Cargo.toml @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: MPL-2.0 +[package] +name = "panic-ci-tools" +version = "0.1.0" +edition = "2021" +license = "MPL-2.0" +publish = false + +[workspace] + +[dependencies] +clap = { version = "=4.6.6", features = ["derive"] } +regex = "=1.13.1" +serde = { version = "=1.0.229", features = ["derive"] } +serde_json = "=1.0.151" diff --git a/tools/ci/src/bin/recon-compare.rs b/tools/ci/src/bin/recon-compare.rs new file mode 100644 index 0000000..91c89b5 --- /dev/null +++ b/tools/ci/src/bin/recon-compare.rs @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: MPL-2.0 +use clap::Parser; +use panic_ci_tools::{finding_counts, input, regression_count, Result}; +use serde_json::Value; +use std::{collections::BTreeMap, fs, io::Write, path::PathBuf}; + +#[derive(Parser)] +struct Args { + #[arg(long, default_value = "recon/baseline.json")] + baseline: PathBuf, + #[arg(long)] + assail: PathBuf, + #[arg(long)] + pons: PathBuf, + #[arg(long)] + summary: Option, + #[arg(long)] + no_fail: bool, +} +fn run() -> Result<()> { + let args = Args::parse(); + let baseline: BTreeMap> = if args.baseline.exists() { + serde_json::from_slice(&fs::read(&args.baseline)?)? + } else { + BTreeMap::new() + }; + let mut text = String::from("## Recon: two independent scans\n\n"); + let mut observed = BTreeMap::new(); + let mut regressions = 0; + let mut unavailable = 0; + for (name, path, wanted) in [ + ("assail", &args.assail, &["weak_points"][..]), + ("pons", &args.pons, &["findings", "results"][..]), + ] { + text.push_str(&format!("### {name}\n\n")); + let counts = fs::read(path) + .map_err(panic_ci_tools::ToolError::from) + .and_then(|bytes| serde_json::from_slice::(&bytes).map_err(Into::into)) + .and_then(|data| finding_counts(&data, wanted)); + let counts = match counts { + Ok(c) => c, + Err(error) => { + unavailable += 1; + text.push_str(&format!( + "- unavailable, not counted as clean: {}\n\n", + error.message + )); + continue; + } + }; + let frozen = baseline.get(name).cloned().unwrap_or_default(); + regressions += regression_count(&counts, &frozen); + text.push_str("| category | observed | baseline | verdict |\n|---|---|---|---|\n"); + for (category, count) in &counts { + let old = frozen.get(category); + let verdict = match old { + Some(n) if count > n => format!("**+{} (fails)**", count - n), + Some(n) if count < n => format!("−{} (debt paid; tighten baseline)", n - count), + Some(_) => "at baseline".into(), + None if frozen.is_empty() => "new (baseline empty)".into(), + None => "**NEW (fails)**".into(), + }; + text.push_str(&format!( + "| {} | {count} | {} | {verdict} |\n", + category.replace('|', "\\|").replace(['\r', '\n'], " "), + old.map_or("—".into(), u64::to_string) + )); + } + if counts.is_empty() { + text.push_str("- no findings counted\n"); + } + let missing: Vec<_> = frozen + .keys() + .filter(|k| !counts.contains_key(*k)) + .cloned() + .collect(); + if !missing.is_empty() { + text.push_str(&format!( + "\n- baseline categories no longer reported: {}\n", + missing.join(", ") + )); + } + text.push('\n'); + observed.insert(name, counts); + } + if unavailable == 0 { + text.push_str(&format!( + "### Freeze or tighten the baseline\n\n```json\n{}\n```\n", + serde_json::to_string_pretty(&observed)? + )); + } else { + text.push_str("Baseline proposal withheld: one or more scanner reports are unavailable.\n"); + } + print!("{text}"); + if let Some(path) = args.summary { + fs::OpenOptions::new() + .create(true) + .append(true) + .open(path)? + .write_all(text.as_bytes())?; + } + if args.no_fail { + eprintln!( + "recon: --no-fail, reporting only; availability and regressions remain in the report" + ); + return Ok(()); + } + if unavailable > 0 { + return Err(input(format!( + "{unavailable} scanner report(s) unavailable" + ))); + } + if regressions > 0 { + return Err(panic_ci_tools::claim(format!( + "{regressions} category regression(s) against {}", + args.baseline.display() + ))); + } + Ok(()) +} +fn main() { + if let Err(error) = run() { + eprintln!("::error::{}", error.message); + std::process::exit(error.code); + } +} diff --git a/tools/ci/src/bin/verify-ledger.rs b/tools/ci/src/bin/verify-ledger.rs new file mode 100644 index 0000000..fd777a7 --- /dev/null +++ b/tools/ci/src/bin/verify-ledger.rs @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: MPL-2.0 +use clap::Parser; +use panic_ci_tools::{derive_report, Result}; +use std::{fs, path::PathBuf}; + +#[derive(Parser)] +struct Args { + #[arg(long)] + inventory: PathBuf, + #[arg(long, default_value = "proven/ledger.json")] + ledger: PathBuf, + #[arg(long, default_value = ".")] + root: PathBuf, + #[arg(long, default_value = "proven-report.json")] + out: PathBuf, + /// Compatibility flag; declared absence remains informational unless it conflicts with coverage. + #[arg(long)] + allow_absent: bool, +} +fn run() -> Result<()> { + let args = Args::parse(); + let _ = args.allow_absent; + let inventory = serde_json::from_slice(&fs::read(args.inventory)?)?; + let ledger = serde_json::from_slice(&fs::read(args.ledger)?)?; + let report = derive_report(&inventory, &ledger, &args.root)?; + let mut text = serde_json::to_string_pretty(&report)?; + text.push('\n'); + fs::write(&args.out, text)?; + println!( + "proven: {} executed test coordinates; wrote {}", + report["summary"]["total"], + args.out.display() + ); + Ok(()) +} +fn main() { + if let Err(error) = run() { + eprintln!("::error::{}", error.message); + std::process::exit(error.code); + } +} diff --git a/tools/ci/src/lib.rs b/tools/ci/src/lib.rs new file mode 100644 index 0000000..3562208 --- /dev/null +++ b/tools/ci/src/lib.rs @@ -0,0 +1,346 @@ +// SPDX-License-Identifier: MPL-2.0 +use regex::Regex; +use serde::Deserialize; +use serde_json::{json, Value}; +use std::{ + collections::{BTreeMap, BTreeSet}, + path::Path, +}; + +#[derive(Debug)] +pub struct ToolError { + pub code: i32, + pub message: String, +} +pub type Result = std::result::Result; +pub fn input(message: impl Into) -> ToolError { + ToolError { + code: 2, + message: message.into(), + } +} +pub fn claim(message: impl Into) -> ToolError { + ToolError { + code: 1, + message: message.into(), + } +} +impl From for ToolError { + fn from(e: std::io::Error) -> Self { + input(e.to_string()) + } +} +impl From for ToolError { + fn from(e: serde_json::Error) -> Self { + input(e.to_string()) + } +} + +#[derive(Deserialize)] +struct Cell { + category: String, + aspect: String, + #[serde(default)] + bins: Vec, + #[serde(default, alias = "match")] + pattern: Option, + #[serde(default)] + tier: Option, + #[serde(default)] + proof_file: Option, +} +#[derive(Default, Deserialize)] +struct Taxonomy { + #[serde(default)] + categories: Vec, + #[serde(default)] + aspects: Vec, +} +#[derive(Deserialize)] +struct Unmapped { + r#match: String, +} +#[derive(Deserialize)] +struct Absent { + category: String, + aspect: String, +} +#[derive(Deserialize)] +struct Ledger { + #[serde(default)] + suite: Option, + #[serde(default)] + entries: Vec, + #[serde(default)] + buckets: Vec, + #[serde(default)] + declared_absent: Vec, + #[serde(default)] + declared_unmapped: Vec, + #[serde(default)] + max_unmapped: Option, + #[serde(default)] + taxonomy: Taxonomy, +} +fn regex(pattern: Option<&str>) -> Result> { + pattern + .map(Regex::new) + .transpose() + .map_err(|e| input(format!("invalid selection regex: {e}"))) +} + +pub fn derive_report(inventory: &Value, ledger: &Value, root: &Path) -> Result { + let inventory: BTreeMap> = serde_json::from_value(inventory.clone())?; + if inventory + .iter() + .any(|(binary, names)| binary.is_empty() || names.contains("")) + { + return Err(input("inventory binary and test names must be nonempty")); + } + if inventory.values().map(BTreeSet::len).sum::() == 0 { + return Err(input("inventory contains no executed tests")); + } + let ledger: Ledger = serde_json::from_value(ledger.clone())?; + for cell in ledger.entries.iter().chain(&ledger.buckets) { + if !ledger.taxonomy.categories.contains(&cell.category) + || !ledger.taxonomy.aspects.contains(&cell.aspect) + { + return Err(input("a ledger cell is outside the declared taxonomy")); + } + } + let mut assignments = BTreeMap::<(String, String), (&Cell, bool)>::new(); + for cell in &ledger.entries { + let pattern = regex(cell.pattern.as_deref())?; + let mut matched = 0; + for (binary, names) in &inventory { + if !cell.bins.contains(binary) { + continue; + } + for name in names { + if pattern.as_ref().is_some_and(|p| !p.is_match(name)) { + continue; + } + matched += 1; + assignments + .entry((binary.clone(), name.clone())) + .or_insert((cell, false)); + } + } + if matched == 0 { + return Err(claim(format!( + "ledger cell {} x {} selects no inventory test", + cell.category, cell.aspect + ))); + } + } + let buckets: Vec<_> = ledger + .buckets + .iter() + .map(|b| Ok((regex(b.pattern.as_deref())?, b))) + .collect::>()?; + let unmapped: Vec<_> = ledger + .declared_unmapped + .iter() + .map(|d| Regex::new(&d.r#match).map_err(|e| input(e.to_string()))) + .collect::>()?; + let mut remainder = 0; + for (binary, names) in &inventory { + for name in names { + let key = (binary.clone(), name.clone()); + if assignments.contains_key(&key) { + continue; + } + if let Some((index, (_, cell))) = buckets.iter().enumerate().find(|(_, (p, b))| { + (b.bins.is_empty() || b.bins.contains(binary)) + && p.as_ref().is_none_or(|p| p.is_match(name)) + }) { + assignments.insert(key, (cell, true)); + if index + 1 == buckets.len() { + remainder += 1; + } + } else if !unmapped.iter().any(|p| p.is_match(name)) { + return Err(claim(format!( + "executed test {binary}::{name} has no coordinate or declared-unmapped rule" + ))); + } + } + } + if ledger.max_unmapped.is_some_and(|cap| remainder > cap) { + return Err(claim(format!( + "{remainder} tests exceed the declared remainder ratchet" + ))); + } + let mut tiers = [0usize; 3]; + let mut cells = BTreeSet::new(); + let mut entries = Vec::new(); + for ((binary, name), (cell, bucket)) in assignments { + let tier = match cell.tier.as_deref() { + Some(t) => t, + None if bucket => "Provisionally-Proven", + None => return Err(input("explicit ledger entry has no tier")), + }; + let index = match tier { + "Actually-Proven" => 0, + "Provisionally-Proven" => 1, + "Unproven" => 2, + _ => return Err(claim(format!("unknown proof tier: {tier}"))), + }; + if tier == "Actually-Proven" { + let proof = cell + .proof_file + .as_ref() + .ok_or_else(|| claim("Actually-Proven requires a proof_file"))?; + if !root.join(proof).is_file() { + return Err(claim(format!("cited proof_file does not exist: {proof}"))); + } + } else if cell.proof_file.is_some() { + return Err(claim("a weaker tier cannot borrow a proof_file")); + } + tiers[index] += 1; + cells.insert((cell.category.clone(), cell.aspect.clone())); + entries.push(json!({"test_id":format!("{binary}::{name}"), "category":cell.category, "aspect":cell.aspect, "provenance":tier, "proof_ladder":[], "result":"passed"})); + } + for absent in &ledger.declared_absent { + if cells.contains(&(absent.category.clone(), absent.aspect.clone())) { + return Err(claim("a coordinate is both declared absent and covered")); + } + } + Ok( + json!({"schema_version":1, "suite":ledger.suite.as_deref().unwrap_or("unknown"), + "summary":{"passed":entries.len(), "total":entries.len(), "actually_proven":tiers[0], "provisionally_proven":tiers[1], "unproven":tiers[2], "covered_cells":cells.len(), "cat_aspect_cells":ledger.taxonomy.categories.len()*ledger.taxonomy.aspects.len(), "lattice_covered":cells.len()}, "entries":entries}), + ) +} + +pub fn finding_counts(data: &Value, wanted: &[&str]) -> Result> { + fn walk( + data: &Value, + wanted: &[&str], + counts: &mut BTreeMap, + found: &mut bool, + ) -> Result<()> { + match data { + Value::Object(object) => { + for (key, value) in object { + if let Some(array) = value.as_array().filter(|_| wanted.contains(&key.as_str())) + { + *found = true; + for entry in array { + let entry = entry + .as_object() + .ok_or_else(|| input("a finding must be an object"))?; + let label = [ + "category", "rule_id", "ruleId", "rule", "id", "code", "check", + ] + .iter() + .find_map(|key| entry.get(*key).and_then(Value::as_str)) + .unwrap_or("unknown"); + *counts.entry(label.into()).or_default() += 1; + } + } + walk(value, wanted, counts, found)?; + } + } + Value::Array(array) => { + for entry in array { + walk(entry, wanted, counts, found)?; + } + } + _ => (), + } + Ok(()) + } + let mut counts = BTreeMap::new(); + let mut found = false; + walk(data, wanted, &mut counts, &mut found)?; + if !found { + return Err(input( + "no recognized findings array; report is unavailable, not clean", + )); + } + Ok(counts) +} + +pub fn regression_count( + observed: &BTreeMap, + baseline: &BTreeMap, +) -> usize { + observed + .iter() + .filter(|(category, count)| { + baseline + .get(*category) + .map_or(!baseline.is_empty(), |old| *count > old) + }) + .count() +} + +#[cfg(test)] +mod tests { + use super::*; + fn ledger() -> Value { + json!({"suite":"controls", "entries":[{"category":"Security","aspect":"Unit","tier":"Provisionally-Proven","bins":["a","b"]}],"taxonomy":{"categories":["Security"],"aspects":["Unit"]}}) + } + #[test] + fn empty_inventory_and_stale_claims_fail() { + assert!(derive_report(&json!({}), &ledger(), Path::new(".")).is_err()); + assert!(derive_report(&json!({"a":[]}), &ledger(), Path::new(".")).is_err()); + assert!(derive_report(&json!({"other":["test"]}), &ledger(), Path::new(".")).is_err()); + } + #[test] + fn binaries_do_not_collapse_identical_names() -> Result<()> { + let report = derive_report( + &json!({"a":["same","same"],"b":["same"]}), + &ledger(), + Path::new("."), + )?; + assert_eq!(report["summary"]["total"], 2); + assert_ne!( + report["entries"][0]["test_id"], + report["entries"][1]["test_id"] + ); + Ok(()) + } + #[test] + fn absent_conflicts_and_unsupported_proofs_fail() { + let inventory = json!({"a":["test"]}); + let mut data = ledger(); + data["entries"][0]["tier"] = json!("Actually-Proven"); + assert!(derive_report(&inventory, &data, Path::new(".")).is_err()); + data["entries"][0]["tier"] = json!("Provisionally-Proven"); + data["declared_absent"] = json!([{"category":"Security","aspect":"Unit"}]); + assert!(derive_report(&inventory, &data, Path::new(".")).is_err()); + } + #[test] + fn unaccounted_tests_and_remainder_ratchet_fail() { + let mut data = ledger(); + data["entries"] = json!([]); + assert!(derive_report(&json!({"a":["test"]}), &data, Path::new(".")).is_err()); + data["buckets"] = json!([{"category":"Security","aspect":"Unit"}]); + data["max_unmapped"] = json!(0); + assert!(derive_report(&json!({"a":["test"]}), &data, Path::new(".")).is_err()); + } + #[test] + fn missing_shapes_are_not_empty_findings() -> Result<()> { + assert!(finding_counts(&json!({"status":"failed"}), &["findings"]).is_err()); + assert!(finding_counts(&json!({"findings":[1]}), &["findings"]).is_err()); + assert!(finding_counts(&json!({"findings":[]}), &["findings"])?.is_empty()); + Ok(()) + } + #[test] + fn nested_sarif_and_debt_ratchet_are_preserved() -> Result<()> { + let observed = finding_counts( + &json!({"runs":[{"results":[{"ruleId":"A"},{"ruleId":"A"}]}]}), + &["results"], + )?; + assert_eq!(observed["A"], 2); + assert_eq!( + regression_count(&observed, &BTreeMap::from([("A".into(), 1)])), + 1 + ); + assert_eq!( + regression_count(&observed, &BTreeMap::from([("A".into(), 3)])), + 0 + ); + Ok(()) + } +} diff --git a/tools/ci/tests/cli.rs b/tools/ci/tests/cli.rs new file mode 100644 index 0000000..d66b286 --- /dev/null +++ b/tools/ci/tests/cli.rs @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: MPL-2.0 +use serde_json::{json, Value}; +use std::{ + error::Error, + fs, + path::PathBuf, + process::Command, + sync::atomic::{AtomicU64, Ordering}, +}; +static NEXT: AtomicU64 = AtomicU64::new(0); +type Result = std::result::Result>; +struct Temp { + base: PathBuf, + root: PathBuf, +} +impl Temp { + fn new() -> Result { + let base = std::env::temp_dir().canonicalize()?; + let root = base.join(format!( + "panic-ci-contract-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir(&root)?; + Ok(Self { base, root }) + } + fn write(&self, name: &str, data: &Value) -> Result { + let path = self.root.join(name); + fs::write(&path, serde_json::to_vec(data)?)?; + Ok(path) + } +} +impl Drop for Temp { + fn drop(&mut self) { + if self.root != self.base && self.root.starts_with(&self.base) { + let _ = fs::remove_dir_all(&self.root); + } + } +} + +#[test] +fn ledger_cli_does_not_write_empty_or_stale_claims() -> Result<()> { + let temp = Temp::new()?; + let inventory = temp.write("inventory.json", &json!({"lib":["control"]}))?; + let ledger = temp.write("ledger.json", &json!({"entries":[{"category":"Security","aspect":"Unit","tier":"Provisionally-Proven","bins":["lib"]}],"taxonomy":{"categories":["Security"],"aspects":["Unit"]}}))?; + let out = temp.root.join("report.json"); + let run = || { + Command::new(env!("CARGO_BIN_EXE_verify-ledger")) + .current_dir(&temp.root) + .arg("--inventory") + .arg(&inventory) + .arg("--ledger") + .arg(&ledger) + .arg("--out") + .arg(&out) + .output() + }; + assert!(run()?.status.success()); + let before = fs::read(&out)?; + let report: Value = serde_json::from_slice(&before)?; + assert_eq!(report["summary"]["total"], 1); + temp.write("inventory.json", &json!({"other":["control"]}))?; + assert_eq!(run()?.status.code(), Some(1)); + assert_eq!( + before, + fs::read(&out)?, + "a failed claim overwrote the successful artifact" + ); + temp.write("inventory.json", &json!({"lib":[]}))?; + assert_eq!(run()?.status.code(), Some(2)); + assert_eq!(before, fs::read(&out)?); + Ok(()) +} + +#[test] +fn recon_cli_distinguishes_regressions_and_unavailable_reports() -> Result<()> { + let temp = Temp::new()?; + let assail = temp.write( + "assail.json", + &json!({"weak_points":[{"category":"A"},{"category":"A"}]}), + )?; + let pons = temp.write("pons.json", &json!({"findings":[]}))?; + let baseline = temp.write("baseline.json", &json!({"assail":{"A":1},"pons":{}}))?; + let run = |advisory: bool| { + let mut command = Command::new(env!("CARGO_BIN_EXE_recon-compare")); + command + .current_dir(&temp.root) + .arg("--assail") + .arg(&assail) + .arg("--pons") + .arg(&pons) + .arg("--baseline") + .arg(&baseline); + if advisory { + command.arg("--no-fail"); + } + command.output() + }; + assert_eq!(run(false)?.status.code(), Some(1)); + temp.write("assail.json", &json!({"weak_points":[{"category":"A"}]}))?; + assert!(run(false)?.status.success()); + fs::remove_file(&pons)?; + let missing = run(false)?; + assert_eq!(missing.status.code(), Some(2)); + assert!(String::from_utf8_lossy(&missing.stdout).contains("Baseline proposal withheld")); + assert!(run(true)?.status.success()); + Ok(()) +} + +#[test] +fn repository_ledger_taxonomy_and_selectors_are_readable() -> Result<()> { + let ledger: Value = serde_json::from_str(include_str!("../../../proven/ledger.json"))?; + for cell in ledger["entries"] + .as_array() + .ok_or("missing entries")? + .iter() + .chain(ledger["buckets"].as_array().ok_or("missing buckets")?) + { + let category = cell["category"].as_str().ok_or("missing category")?; + let aspect = cell["aspect"].as_str().ok_or("missing aspect")?; + assert!(ledger["taxonomy"]["categories"] + .as_array() + .ok_or("missing categories")? + .contains(&json!(category))); + assert!(ledger["taxonomy"]["aspects"] + .as_array() + .ok_or("missing aspects")? + .contains(&json!(aspect))); + for key in ["pattern", "match"] { + if let Some(pattern) = cell[key].as_str() { + regex::Regex::new(pattern)?; + } + } + } + Ok(()) +}