diff --git a/.github/workflows/anchor-corpus.yml b/.github/workflows/anchor-corpus.yml new file mode 100644 index 00000000..750832c6 --- /dev/null +++ b/.github/workflows/anchor-corpus.yml @@ -0,0 +1,59 @@ +name: Anchor corpus + +# Runs this repository's TR-ANC-002 over an external registry anchor corpus +# pinned by commit and SHA-256 in conformance/interop/trace-anchor/pin.json. +# The corpus is read as data; nothing from it is installed or executed. A pin +# bump is a reviewed change to pin.json, never a floating reference. + +on: + push: + branches: [main] + paths: + - "conformance/src/trace_tests/modules/tr_anc.py" + - "conformance/src/trace_tests/inclusion.py" + - "conformance/interop/trace-anchor/**" + - "spec/registry-anchor-v1.md" + - ".github/workflows/anchor-corpus.yml" + pull_request: + branches: [main] + paths: + - "conformance/src/trace_tests/modules/tr_anc.py" + - "conformance/src/trace_tests/inclusion.py" + - "conformance/interop/trace-anchor/**" + - "spec/registry-anchor-v1.md" + - ".github/workflows/anchor-corpus.yml" + schedule: + - cron: "17 6 * * 1" + workflow_dispatch: + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + +permissions: + contents: read + +jobs: + grade: + name: TR-ANC-002 against the pinned anchor corpus + runs-on: ubuntu-latest + defaults: + run: + working-directory: conformance + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v5.3.0 + with: + python-version: "3.12" + - name: Install the suite + run: | + pip install --require-hashes -r requirements/test.txt + pip install --no-deps -e . + - name: Grade + run: python interop/trace-anchor/grade.py --report build/anchor-corpus.json + - name: Upload the report + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: anchor-corpus-report + path: conformance/build/anchor-corpus.json + if-no-files-found: ignore diff --git a/conformance/docs/modules/tr-anc.md b/conformance/docs/modules/tr-anc.md index d0de06ad..3ed01949 100644 --- a/conformance/docs/modules/tr-anc.md +++ b/conformance/docs/modules/tr-anc.md @@ -15,3 +15,7 @@ Checks that the record was entered in a public, append-only log, so it cannot be |---------|-------------|---------------|---------------| | TR-ANC-001 | `transparency` is an `https://` URI with a host. Not resolved | `https://transparency.example/entries/abc123` | missing field, empty string, non-string, `http://`, bare path, `ipfs://` | | TR-ANC-002 | The record's inclusion proof replays to the committed `merkle_root`, per RFC 9162 over an RFC 6962 tree. Offline, no network | a receipt whose `audit_path` reproduces `merkle_root` from the record's leaf | no receipt supplied, missing `leaf_index`/`audit_path`/`leaf_count`/`merkle_root`, non-hex audit node, out-of-range `leaf_index`, proof for a different record, record modified after anchoring | + +## External anchor corpus + +The `Anchor corpus` workflow also runs TR-ANC-002 over an external registry anchor corpus, pinned by commit and SHA-256 in [`interop/trace-anchor/pin.json`](https://github.com/agentrust-io/trace-spec/blob/main/conformance/interop/trace-anchor/pin.json). Each case records its expected decision and the `spec/registry-anchor-v1.md` section it rests on. The job fails on a disagreement that `pin.json` does not list, and reports a listed one that now agrees so the list can be shortened. Moving the pin is a reviewed change to that file. diff --git a/conformance/interop/trace-anchor/grade.py b/conformance/interop/trace-anchor/grade.py new file mode 100644 index 00000000..b80d0863 --- /dev/null +++ b/conformance/interop/trace-anchor/grade.py @@ -0,0 +1,112 @@ +"""Grade TR-ANC-002 against a pinned external registry anchor corpus. + +The corpus is data. This script downloads its MANIFEST.json and each case file +from the commit named in pin.json, refuses any file whose SHA-256 differs from +its pin, and answers every case with this repository's own +``trace_tests.modules.tr_anc.check``. Nothing from the corpus repository is +installed or executed. + +A TR-ANC-002 pass is read as ``anchored`` and anything else as ``refused``; +each case's expected decision comes from the manifest. The job fails when the +corpus does not hold exactly the pinned number of cases, when a file does not +match its digest, or when a case disagrees and is not listed in +``known_disagreements``. A listed case that now agrees is reported so that the +list can be shortened; it does not fail the job. + + python interop/trace-anchor/grade.py [--report build/anchor-corpus.json] +""" + +from __future__ import annotations + +import argparse +import hashlib +import inspect +import json +import sys +import urllib.request +from pathlib import Path +from typing import Any + +from trace_tests.modules import tr_anc + +PIN = Path(__file__).with_name("pin.json") +RAW = "https://raw.githubusercontent.com" + + +def fetch(repository: str, commit: str, path: str, sha256: str) -> bytes: + owner_repo = repository.removeprefix("https://github.com/") + url = f"{RAW}/{owner_repo}/{commit}/{path}" + with urllib.request.urlopen(url, timeout=30) as response: # noqa: S310 (fixed https host) + data: bytes = response.read() + digest = hashlib.sha256(data).hexdigest() + if digest != sha256: + raise SystemExit(f"digest mismatch for {path}: got {digest}, pinned {sha256}") + return data + + +def decide(case: dict[str, Any]) -> tuple[str, str]: + claim, receipt, entry = case.get("claim"), case.get("receipt"), case.get("entry") + if "entry" in inspect.signature(tr_anc.check).parameters: + findings = tr_anc.check(claim, receipt, entry) # type: ignore[call-arg] + else: + findings = tr_anc.check(claim, receipt) + for finding in findings: + if finding.code == "TR-ANC-002": + passed = str(getattr(finding.status, "value", finding.status)).lower() == "pass" + return ("anchored" if passed else "refused"), finding.message + return "refused", findings[0].message if findings else "no TR-ANC finding" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--report", type=Path, help="write per-case results as JSON") + args = parser.parse_args() + + pin = json.loads(PIN.read_text(encoding="utf-8")) + repository, commit = pin["repository"], pin["commit"] + manifest = json.loads(fetch(repository, commit, pin["manifest"], pin["manifest_sha256"])) + base = pin["manifest"].rsplit("/", 1)[0] + vectors = manifest["vectors"] + if len(vectors) != pin["cases"]: + print(f"corpus holds {len(vectors)} cases, pinned {pin['cases']}") + return 1 + + known: dict[str, str] = pin["known_disagreements"] + results, unexpected, resolved = [], [], [] + for vector in vectors: + path = f"{base}/{vector['path']}/case.json" + data = fetch(repository, commit, path, vector["files"]["case.json"]) + got, message = decide(json.loads(data)) + want = vector["expected"]["decision"] + agrees = got == want + results.append( + { + "id": vector["id"], + "rule": vector["rule"], + "expected": want, + "got": got, + "message": message, + } + ) + if not agrees and vector["id"] not in known: + unexpected.append(f"{vector['id']}: expected {want}, TR-ANC-002 says {got} ({message})") + if agrees and vector["id"] in known: + resolved.append(vector["id"]) + + agreed = sum(r["expected"] == r["got"] for r in results) + print(f"{repository}@{commit[:12]} {pin['corpus']}") + print(f"TR-ANC-002 agrees on {agreed} of {len(results)}") + print(f"known disagreements listed: {len(known)}, unexpected: {len(unexpected)}") + for case_id in resolved: + print(f"::notice::{case_id} now agrees; remove it from known_disagreements in pin.json") + for line in unexpected: + print(f"::error::{line}") + if args.report: + args.report.parent.mkdir(parents=True, exist_ok=True) + report = json.dumps({"pin": pin, "results": results}, indent=2) + "\n" + args.report.write_text(report, encoding="utf-8") + return 1 if unexpected else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/conformance/interop/trace-anchor/pin.json b/conformance/interop/trace-anchor/pin.json new file mode 100644 index 00000000..5532b523 --- /dev/null +++ b/conformance/interop/trace-anchor/pin.json @@ -0,0 +1,18 @@ +{ + "corpus": "vectors-trace-anchor", + "repository": "https://github.com/probityai/agent-evidence-vectors", + "commit": "0c265d65bd0003eea2ac7641c856abfd80e3143a", + "manifest": "vectors-trace-anchor/MANIFEST.json", + "manifest_sha256": "3f22497829372efa6c5df887c59904c56050a502b160beeffe7d999fc93733f0", + "cases": 27, + "known_disagreements": { + "anchored-proof-only-receipt": "TR-ANC-002 requires merkle_root and leaf_count on the receipt; registry-anchor-v1 section 5.1 takes both from the entry", + "entry-absent": "TR-ANC-002 takes no registry entry and replays against the receipt's own root (#533)", + "entry-missing-ts": "TR-ANC-002 takes no registry entry, so an entry's defects are not seen (#533)", + "entry-producer-empty": "TR-ANC-002 takes no registry entry, so an entry's defects are not seen (#533)", + "receipt-count-lies": "TR-ANC-002 replays with the receipt's own leaf_count (#533)", + "receipt-subtree-root": "TR-ANC-002 replays against the receipt's own root, here an interior node (#533)", + "self-anchored-no-entry": "a one-leaf receipt naming the record's own leaf hash as root passes with no entry (#533)", + "self-anchored-with-real-entry": "the self-anchored record passes beside a genuine entry it is not in (#533)" + } +}