From 677c8f5ad8f5a111ff4e5bbbd38e3969d57c5feb Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Mon, 26 Oct 2026 07:50:47 -0400 Subject: [PATCH] fix(ci): verify served plugin checksums (D-1483) --- .github/docs/d-1483-investigation.md | 63 ++++++++++ .../docs/registry-download-verification.md | 28 +++++ .github/workflows/registry-sync.yml | 16 ++- .github/workflows/tarball-checksum-gate.yml | 7 +- .../ci/tests/test_registry_sync_triggers.py | 11 ++ .../tests/test_verify_registry_downloads.py | 110 +++++++++++++++++ scripts/ci/verify_registry_downloads.py | 112 ++++++++++++++++++ 7 files changed, 343 insertions(+), 4 deletions(-) create mode 100644 .github/docs/d-1483-investigation.md create mode 100644 .github/docs/registry-download-verification.md create mode 100644 scripts/ci/tests/test_verify_registry_downloads.py create mode 100644 scripts/ci/verify_registry_downloads.py diff --git a/.github/docs/d-1483-investigation.md b/.github/docs/d-1483-investigation.md new file mode 100644 index 000000000..dcb204e67 --- /dev/null +++ b/.github/docs/d-1483-investigation.md @@ -0,0 +1,63 @@ +# D-1483: Golden Path checksum failure + +Public read-only investigation on 2026-10-26, against fresh plugins main +`99abe4448118a2f8b30ca49441baa3c28e6aa38a`. + +## Evidence + +`gh run view 40261854806 -R nself-org/cli --log-failed` reports that step 9, +`nself plugin install ai`, expected +`d2c8c7345babbb367c42fbef9efa1a91e424b401f36452bdcccb005b686599e6` +but downloaded +`03071f9d6c13cca200ca43c2c22dc5ad6c9f722c94d388f89e2a55feb9264598`. + +The public `https://plugins.nself.org/registry.json` still advertises the +expected hash for `ai`, including with `?nocache=1`. Its row has version +`1.1.3`, releaseTag `v1.2.10`, tier `max`, requires_license `true`, and +download_url `https://ping.nself.org/plugins/ai/download`. + +The public `https://plugins.nself.org/catalog.json` advertises exactly the +hash CI downloaded for licensed `ai` version `1.1.3`. The committed catalog +agrees. Its release_signature is null. This is a proven discrepancy between +the registry and catalog metadata, not proof that either checksum can safely +replace the other without checking the signed release of record. + +`ai` is absent from this repository's free registry and releases.json. The +public free release `v1.2.10` has no `ai-*` asset. Both the worker tarball route +and the ping download refuse unauthenticated access (HTTP 401). No credentials +were used to bypass that boundary. + +All 130 live free registry entries were downloaded through their advertised +download_url and hashed with SHA-256. All matched the live registry, checked-out +registry, and free catalog entries (catalog comparisons must include tier, +because cron and notify each have a separate licensed entry). All downloads +resolved to GitHub release assets. No free plugin mismatched or failed to fetch. + +## Cause and owner action + +Licensed `ai` registry/catalog metadata has diverged. Stale licensed registry +data is plausible; public evidence cannot distinguish stale upstream data from +a deployed cache, or establish whether a licensed tarball was rebuilt. The +nocache request did not change the advertised hash. Do not rebuild or replace +release assets to make a checksum pass. + +The owner must reconcile the private bundles `ai` release metadata with the +exact bytes its authenticated download serves, preserving or regenerating the +release signature over the chosen published artifact. This investigation cannot +provide a verified private-repository repair command. After that correction is +reviewed and merged, refresh the worker using the existing authorized workflow: + +```sh +gh workflow run registry-sync.yml -R nself-org/plugins --ref main +``` + +Then re-read the public registry and catalog and retry Golden Path: + +```sh +gh run rerun 40261854806 -R nself-org/cli --failed +``` + +These owner commands were not executed. No release asset, worker, KV, secret, +tag, version, or production configuration was changed. The public free-plugin +verification gate prevents a separate class of regression; it cannot repair +this licensed download failure. diff --git a/.github/docs/registry-download-verification.md b/.github/docs/registry-download-verification.md new file mode 100644 index 000000000..88cfe71dd --- /dev/null +++ b/.github/docs/registry-download-verification.md @@ -0,0 +1,28 @@ +# Verify public plugin downloads + +The registry checksum describes the bytes published on the release, including an +asset uploaded while the release was a draft. Never regenerate it from a local +rebuild. Different gzip implementations can produce different archive bytes. + +Run `python3 scripts/ci/verify_registry_downloads.py` to download and hash every +installable free plugin in the public registry. It follows each `download_url`, +including the worker's R2 redirect, and checks both checksum aliases when present. +Paid plugins and explicitly noninstallable entries are excluded. Missing metadata, +download failures, mismatches, and an empty free-plugin set fail the check. + +`--registry registry.json` checks candidate repository metadata against those same +served download URLs. Registry KV sync runs this after refreshing the worker, +because versionless worker URLs can serve the old release until the sync completes. +A dependent read-only job checks live registry metadata daily and on projection +changes. It runs even when sync fails, while waiting for sync to finish to avoid a +race against the cache refresh. + +The release workflow already downloads published assets to generate release data +with `scripts/verify-published-checksums.sh`. If only repository checksums are wrong, +the owner can run `bash scripts/verify-published-checksums.sh --write ` followed +by `bash scripts/plugin-counts.sh` and review the diff. A changed signed checksum +requires the normal signing flow. If GitHub bytes match but R2 bytes differ, report +the object URL and both hashes to the release owner. Repairing R2, replacing release +assets, or refreshing production caches requires a separate authorized operation. + +Offline tests: `python3 -m unittest discover -s scripts/ci/tests -p 'test_verify_registry_downloads.py'`. diff --git a/.github/workflows/registry-sync.yml b/.github/workflows/registry-sync.yml index 2cb7ec9be..dd660f33c 100644 --- a/.github/workflows/registry-sync.yml +++ b/.github/workflows/registry-sync.yml @@ -26,11 +26,23 @@ concurrency: cancel-in-progress: false jobs: + verify-live-downloads: + name: Verify public free plugin downloads + needs: sync + if: always() + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v7.0.1 + - name: Check live registry against served bytes (read-only) + run: python3 scripts/ci/verify_registry_downloads.py + sync: name: POST /api/sync runs-on: ubuntu-latest - timeout-minutes: 5 + timeout-minutes: 20 steps: + - uses: actions/checkout@v7.0.1 - name: Refresh the worker caches env: CLOUDFLARE_WORKER_SYNC_TOKEN: ${{ secrets.CLOUDFLARE_WORKER_SYNC_TOKEN }} @@ -46,3 +58,5 @@ jobs: -d "{\"ref\": \"${GITHUB_REF}\", \"sha\": \"${GITHUB_SHA}\"}") echo "$response" | jq '{success, message, freeCount, proCount, catalogSynced, timestamp}' echo "$response" | jq -e '.success == true and .catalogSynced == true' >/dev/null + - name: Verify refreshed downloads against candidate registry + run: python3 scripts/ci/verify_registry_downloads.py --registry registry.json diff --git a/.github/workflows/tarball-checksum-gate.yml b/.github/workflows/tarball-checksum-gate.yml index f9072db27..9c2a3ab83 100644 --- a/.github/workflows/tarball-checksum-gate.yml +++ b/.github/workflows/tarball-checksum-gate.yml @@ -200,9 +200,10 @@ jobs: if [ "$FAIL" -ne 0 ]; then echo "" echo "::error::one or more registry.json checksums do not match the corresponding published release asset" - echo "Rebuild locally with scripts/build-tarballs.sh , upload it to the" >&2 - echo "release, then re-run scripts/verify-published-checksums.sh --write" >&2 - echo "to bring registry.json back in sync with what is actually published." >&2 + echo "Do not rebuild or replace the release asset. Derive checksums from published bytes:" >&2 + echo "bash scripts/verify-published-checksums.sh --write " >&2 + echo "bash scripts/plugin-counts.sh" >&2 + echo "Review the release-data diff; changed signed bytes require the normal release signing flow." >&2 exit 1 fi diff --git a/scripts/ci/tests/test_registry_sync_triggers.py b/scripts/ci/tests/test_registry_sync_triggers.py index 05745f7f7..1f6ec451b 100644 --- a/scripts/ci/tests/test_registry_sync_triggers.py +++ b/scripts/ci/tests/test_registry_sync_triggers.py @@ -47,6 +47,17 @@ def test_no_new_secret(self): secrets = set(re.findall(r"secrets\.([A-Z0-9_]+)", self.text)) self.assertEqual(secrets, {"CLOUDFLARE_WORKER_SYNC_TOKEN"}) + def test_candidate_downloads_are_checked_after_refresh(self): + post = self.text.index('-X POST "https://plugins.nself.org/api/sync"') + verify = self.text.index('python3 scripts/ci/verify_registry_downloads.py --registry registry.json') + self.assertLess(post, verify, "versionless URLs must refresh before candidate verification") + + def test_live_verification_waits_for_sync_even_on_failure(self): + live = self.text.split(" verify-live-downloads:", 1)[1].split("\n sync:", 1)[0] + self.assertIn(" needs: sync\n", live) + self.assertIn(" if: always()\n", live) + self.assertIn("run: python3 scripts/ci/verify_registry_downloads.py\n", live) + if __name__ == "__main__": unittest.main() diff --git a/scripts/ci/tests/test_verify_registry_downloads.py b/scripts/ci/tests/test_verify_registry_downloads.py new file mode 100644 index 000000000..d9c24614c --- /dev/null +++ b/scripts/ci/tests/test_verify_registry_downloads.py @@ -0,0 +1,110 @@ +"""Offline fixtures for public download verification (D-1483).""" + +import hashlib +import importlib.util +import io +from pathlib import Path +import unittest + +SCRIPT = Path(__file__).resolve().parents[1] / "verify_registry_downloads.py" +SPEC = importlib.util.spec_from_file_location("verify_registry_downloads", SCRIPT) +VERIFIER = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(VERIFIER) +PUBLISHED = b"draft release bytes, never a local rebuild" +SHA = hashlib.sha256(PUBLISHED).hexdigest() + + +class Response(io.BytesIO): + def geturl(self): + return "https://cdn.example/plugin.tar.gz?token=temporary#secret" + + +class RegistryDownloads(unittest.TestCase): + def row(self, name="demo", **overrides): + return dict(name=name, tier="free", checksum=SHA, + download_url=f"https://registry.example/{name}/tarball", **overrides) + + def run_verify(self, plugins, data=PUBLISHED): + self.urls = [] + self.output = [] + + def opener(url): + self.urls.append(url) + return Response(data) + + return VERIFIER.verify({"plugins": plugins}, opener, self.output.append) + + def test_keyed_and_array_formats(self): + for plugins in ([self.row()], {"demo": self.row()}): + self.assertEqual(self.run_verify(plugins), 0) + self.assertEqual(self.urls, [self.row()["download_url"]]) + + def test_worker_url_wins_over_release_fallback(self): + row = self.row(tarball="https://github.example/draft.tar.gz") + self.assertEqual(self.run_verify([row]), 0) + self.assertEqual(self.urls, [row["download_url"]]) + + def test_tarball_fallback(self): + row = self.row() + del row["download_url"] + row["tarball"] = "https://github.example/draft.tar.gz" + self.assertEqual(self.run_verify([row]), 0) + self.assertEqual(self.urls, [row["tarball"]]) + + def test_uppercase_tarball_url_alias(self): + row = self.row() + del row["download_url"] + row["tarballURL"] = "https://github.example/draft.tar.gz" + self.assertEqual(self.run_verify([row]), 0) + self.assertEqual(self.urls, [row["tarballURL"]]) + + def test_final_url_has_no_temporary_query_or_fragment(self): + for data in (PUBLISHED, b"wrong bytes"): + self.run_verify([self.row()], data) + self.assertIn("https://cdn.example/plugin.tar.gz", self.output[0]) + self.assertNotIn("token=", self.output[0]) + self.assertNotIn("#secret", self.output[0]) + + def test_rebuilt_or_corrupted_bytes_fail(self): + self.assertEqual(self.run_verify([self.row()], b"rebuilt bytes"), 1) + self.assertIn("SHA256 mismatch", self.output[0]) + + def test_all_checksum_aliases_are_checked(self): + row = self.row(checksums={"sha256": "0" * 64}) + self.assertEqual(self.run_verify([row]), 1) + + def test_missing_invalid_checksum_and_url_fail(self): + for field, value in (("checksum", None), ("checksum", "bad"), + ("download_url", None), ("download_url", "http://example/x")): + row = self.row() + row[field] = value + self.assertEqual(self.run_verify([row]), 1) + self.assertEqual(self.urls, []) + + def test_paid_and_noninstallable_entries_never_downloaded(self): + rows = [self.row(), self.row("disabled", installable=False), + self.row("licensed", requires_license=True)] + paid = self.row("paid") + paid["tier"] = "max" + self.assertEqual(self.run_verify(rows + [paid]), 0) + self.assertEqual(len(self.urls), 1) + + def test_failure_does_not_skip_other_plugins(self): + bad = self.row("bad") + bad["checksum"] = "bad" + self.assertEqual(self.run_verify([bad, self.row()]), 1) + self.assertTrue(any(line.startswith("OK demo") for line in self.output)) + + def test_empty_registry_fails(self): + self.assertEqual(self.run_verify([]), 1) + + def test_download_failure_fails(self): + def unavailable(url): + raise OSError("HTTP 503") + output = [] + self.assertEqual(VERIFIER.verify({"plugins": [self.row()]}, unavailable, output.append), 1) + self.assertIn("HTTP 503", output[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/verify_registry_downloads.py b/scripts/ci/verify_registry_downloads.py new file mode 100644 index 000000000..c2e1af4ec --- /dev/null +++ b/scripts/ci/verify_registry_downloads.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Verify free plugin checksums against the bytes their public download URLs serve. + +Inputs: a local registry path or public HTTPS registry URL. +Outputs: per-plugin results and a nonzero exit on invalid metadata/download/hash. +Constraints: read-only, no credentials, no rebuilt archives or paid downloads. +""" + +import argparse +import hashlib +import json +import re +import sys +from pathlib import Path +from urllib.parse import urlparse, urlunparse +from urllib.request import Request, urlopen + + +def open_public(url): + """Open a public HTTPS URL without authentication, following CDN redirects.""" + if urlparse(url).scheme != "https": + raise ValueError("download URL must use HTTPS") + request = Request(url, headers={"User-Agent": "nself-registry-check/1.0"}) + return urlopen(request, timeout=60) + + +def read_registry(source): + """Load local candidate metadata or the public registry served to clients.""" + if source.startswith("https://"): + with open_public(source) as response: + return json.load(response) + return json.loads(Path(source).read_text()) + + +def plugin_rows(registry): + """Normalize the repository's keyed and the worker's array wire formats.""" + plugins = registry.get("plugins") + if isinstance(plugins, dict): + return [dict(row, name=name) for name, row in plugins.items()] + if isinstance(plugins, list): + return plugins + raise ValueError("registry plugins must be an object or array") + + +def verify_plugin(row, opener): + """Hash served bytes and check every recorded source checksum alias.""" + name = row.get("name") + if not isinstance(name, str) or not name: + raise ValueError("plugin has no name") + checksums = [row.get("checksum"), row.get("checksums", {}).get("sha256")] + checksums = [value for value in checksums if value is not None] + if not checksums or any(not isinstance(value, str) or + not re.fullmatch(r"[0-9a-fA-F]{64}", value) + for value in checksums): + raise ValueError("missing or malformed SHA256") + url = (row.get("download_url") or row.get("tarballUrl") or + row.get("tarballURL") or row.get("tarball")) + if not isinstance(url, str) or urlparse(url).scheme != "https": + raise ValueError("missing public HTTPS download URL") + digest = hashlib.sha256() + with opener(url) as response: + while chunk := response.read(1024 * 1024): + digest.update(chunk) + served_url = urlunparse(urlparse(response.geturl())._replace(query="", fragment="")) + actual = digest.hexdigest() + if any(actual != expected.lower() for expected in checksums): + raise ValueError(f"SHA256 mismatch: recorded={','.join(checksums)} " + f"served={actual} url={served_url}") + return actual, served_url + + +def verify(registry, opener=open_public, output=print): + """Check all installable free entries, continuing after each failure.""" + checked = errors = 0 + seen = set() + for row in plugin_rows(registry): + if row.get("tier", "free") != "free" or row.get("requires_license") is True: + continue + if row.get("installable") is False: + continue + name = row.get("name", "") + try: + if name in seen: + raise ValueError("duplicate plugin name") + seen.add(name) + actual, url = verify_plugin(row, opener) + checked += 1 + output(f"OK {name}: sha256={actual} url={url}") + except Exception as error: + errors += 1 + output(f"ERROR {name}: {error}") + if not checked and not errors: + errors += 1 + output("ERROR: no installable free plugins were checked") + output(f"Verified {checked} free plugin download(s); errors={errors}") + return 1 if errors else 0 + + +def main(): + """CLI entry point; unexpected registry/download errors fail closed.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--registry", default="https://plugins.nself.org/registry.json") + args = parser.parse_args() + try: + return verify(read_registry(args.registry)) + except Exception as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main())