From 48de76a5bcf466f46e4e973a21ca911ebec1b23c Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Sat, 7 Nov 2026 02:32:26 +0000 Subject: [PATCH 1/5] Refresh governance policy and migrate supported action locks Adopt published standards ea2f4d8 and supported gh-actions-lock scope. Only two setup paths; reader proposal excluded. --- .github/workflows/actions.lock | 79 ++------------------------------ .github/workflows/governance.yml | 2 +- 2 files changed, 6 insertions(+), 75 deletions(-) diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock index c5cbddb0..d8e7dbaa 100644 --- a/.github/workflows/actions.lock +++ b/.github/workflows/actions.lock @@ -29,12 +29,10 @@ workflows: '.github/workflows/fleet-report-back.yml': - 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' - 'actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333' - '.github/workflows/governance.yml': - - 'metadatastician/standards@75bd5994470661a099df562d6d5ad2fe744dbd66' + '.github/workflows/governance.yml': [] '.github/workflows/hypatia-dispatch-intake.yml': - 'actions/checkout@v7.0.1' - '.github/workflows/hypatia-scan.yml': - - 'metadatastician/standards@75bd5994470661a099df562d6d5ad2fe744dbd66' + '.github/workflows/hypatia-scan.yml': [] '.github/workflows/inbox-steward.yml': - 'actions/checkout@v7.0.1' '.github/workflows/instant-sync.yml': @@ -45,8 +43,7 @@ workflows: - 'actions/checkout@v7.0.1' - 'actions/upload-artifact@v7.0.2' '.github/workflows/lock-sync-gate.yml': [] - '.github/workflows/mirror.yml': - - 'metadatastician/standards@75bd5994470661a099df562d6d5ad2fe744dbd66' + '.github/workflows/mirror.yml': [] '.github/workflows/pages.yml': - 'actions/checkout@v7.0.1' - 'actions/deploy-pages@v5.0.1' @@ -62,10 +59,8 @@ workflows: - 'actions/checkout@v7.0.1' '.github/workflows/rust.yml': - 'actions/checkout@v7.0.1' - '.github/workflows/scorecard.yml': - - 'metadatastician/standards@75bd5994470661a099df562d6d5ad2fe744dbd66' - '.github/workflows/secret-scanner.yml': - - 'hyperpolymath/cicd-suite@4404c3c0474e17753fe7689f3485c02890aac113' + '.github/workflows/scorecard.yml': [] + '.github/workflows/secret-scanner.yml': [] '.github/workflows/supervised-fleet-scan.yml': - 'actions/checkout@v7.0.1' '.github/workflows/vendored-bot-drift.yml': @@ -133,36 +128,11 @@ dependencies: repo_id: 496012378 uses: - 'actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f' - 'denoland/setup-deno@22d081ff2d3a40755e97629de92e3bcbfa7cf2ed': - ref: 'v2.0.5' - commit: 'sha1-22d081ff2d3a40755e97629de92e3bcbfa7cf2ed' - owner_id: 42048915 - repo_id: 356423100 - 'dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f': - ref: 'v1' - commit: 'sha1-e2a55d2ffb04f378e9626c28d38b36d230d1e12f' - owner_id: 1940490 - repo_id: 260749683 'dtolnay/rust-toolchain@v1': ref: 'v1' commit: 'sha1-02cb101ec7c40f2c49e1d9714d64511d8e1b74de' owner_id: 1940490 repo_id: 260749683 - 'editorconfig-checker/action-editorconfig-checker@51f63319f592f97930c73d9c46184d20bd206393': - ref: 'v3.0.0' - commit: 'sha1-51f63319f592f97930c73d9c46184d20bd206393' - owner_id: 26415196 - repo_id: 297874902 - 'erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124': - ref: 'v1.24.1' - commit: 'sha1-54075bcc5e249e4758d363f27d099f55d843f124' - owner_id: 47606891 - repo_id: 331103973 - 'github/codeql-action@b96794f015dfd88f77b49b1c93e0fa7110f94c63': - ref: 'v4.38.0' - commit: 'sha1-b96794f015dfd88f77b49b1c93e0fa7110f94c63' - owner_id: 9919 - repo_id: 259445878 'github/codeql-action@v4.38.2': ref: 'v4.38.2' commit: 'sha1-2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2' @@ -173,13 +143,6 @@ dependencies: commit: 'sha1-0f8e8c99d88aeb3fbfd523f1ef2c6f762d10d64d' owner_id: 75048950 repo_id: 623796603 - 'hyperpolymath/cicd-suite@4404c3c0474e17753fe7689f3485c02890aac113': - ref: '4404c3c0474e17753fe7689f3485c02890aac113' - commit: 'sha1-4404c3c0474e17753fe7689f3485c02890aac113' - owner_id: 6759885 - repo_id: 1326697643 - uses: - - 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' 'hyperpolymath/deed-ecosystem@main': ref: 'main' commit: 'sha1-ed83d6927e8fb21431e403dbf6d7a4af96772746' @@ -195,33 +158,6 @@ dependencies: commit: 'sha1-c1c9fa07992a02c1fd3d67a0dc1b08cccb852aef' owner_id: 6759885 repo_id: 1352485172 - 'metadatastician/standards@75bd5994470661a099df562d6d5ad2fe744dbd66': - ref: '75bd5994470661a099df562d6d5ad2fe744dbd66' - commit: 'sha1-75bd5994470661a099df562d6d5ad2fe744dbd66' - owner_id: 297579217 - repo_id: 1116521501 - uses: - - 'actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9' - - 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' - - 'actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9' - - 'denoland/setup-deno@22d081ff2d3a40755e97629de92e3bcbfa7cf2ed' - - 'dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f' - - 'editorconfig-checker/action-editorconfig-checker@51f63319f592f97930c73d9c46184d20bd206393' - - 'erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124' - - 'github/codeql-action@b96794f015dfd88f77b49b1c93e0fa7110f94c63' - - 'ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc' - - 'oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6' - - 'webfactory/ssh-agent@e83874834305fe9a4a2997156cb26c5de65a8555' - 'ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc': - ref: 'v2.4.4' - commit: 'sha1-2d1146689b8cda280b9bc96326124645441f03bc' - owner_id: 67707773 - repo_id: 421101922 - 'oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6': - ref: 'v2.2.0' - commit: 'sha1-0c5077e51419868618aeaa5fe8019c62421857d6' - owner_id: 108928776 - repo_id: 512644635 'peter-evans/repository-dispatch@v4.0.1': ref: 'v4.0.1' commit: 'sha1-28959ce8df70de7be546dd1250a005dd32156697' @@ -232,8 +168,3 @@ dependencies: commit: 'sha1-d1527c8f67ef9af357290475d4af4b0fab8f12e0' owner_id: 580492 repo_id: 298565987 - 'webfactory/ssh-agent@e83874834305fe9a4a2997156cb26c5de65a8555': - ref: 'v0.10.0' - commit: 'sha1-e83874834305fe9a4a2997156cb26c5de65a8555' - owner_id: 135788 - repo_id: 208510314 diff --git a/.github/workflows/governance.yml b/.github/workflows/governance.yml index 3d1fc85a..512d2491 100644 --- a/.github/workflows/governance.yml +++ b/.github/workflows/governance.yml @@ -35,7 +35,7 @@ permissions: jobs: governance: - uses: metadatastician/standards/.github/workflows/governance-reusable.yml@75bd5994470661a099df562d6d5ad2fe744dbd66 + uses: metadatastician/standards/.github/workflows/governance-reusable.yml@ea2f4d8a886585ed442687ae63d4ad7493379ba4 permissions: actions: read contents: read From c1bf3260f06e7fba95e0086df39528e9a677267a Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Sat, 7 Nov 2026 10:03:38 +0000 Subject: [PATCH 2/5] Preserve pinned reusable closure with authenticated source checks --- .github/workflows/e2e.yml | 30 +- docs/migrations/pinned-source-closure.adoc | 170 ++++ .../secret-scanner-cicd-suite/preflight.sh | 15 +- scripts/audit-lock-closure.sh | 109 ++- scripts/audit-pinned-reusable-sources.sh | 344 ++++++++ scripts/audit-workflow-lock-map.py | 14 +- scripts/lib/lock-closure-receipts.tsv | 9 + scripts/lib/pinned-reusable-sources.json | 827 ++++++++++++++++++ scripts/tests/lock-closure-test.sh | 6 + scripts/tests/pinned-reusable-sources-test.sh | 270 ++++++ .../tests/pinned-source-entrypoints-test.sh | 81 ++ tests/workflow-lock-map-test.py | 45 +- 12 files changed, 1896 insertions(+), 24 deletions(-) create mode 100644 docs/migrations/pinned-source-closure.adoc create mode 100644 scripts/audit-pinned-reusable-sources.sh create mode 100644 scripts/lib/pinned-reusable-sources.json create mode 100644 scripts/tests/pinned-reusable-sources-test.sh create mode 100644 scripts/tests/pinned-source-entrypoints-test.sh diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 6762a099..2e25fe3a 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -14,9 +14,19 @@ permissions: jobs: e2e: name: E2E tests + needs: pinned-source-closure + if: ${{ always() }} runs-on: ubuntu-latest timeout-minutes: 30 steps: + - name: Require verified pinned-source closure + env: + CLOSURE_RESULT: ${{ needs.pinned-source-closure.result }} + run: | + if [ "$CLOSURE_RESULT" != success ]; then + echo "Pinned-source closure did not pass: $CLOSURE_RESULT" >&2 + exit 1 + fi - uses: actions/checkout@v7.0.1 - name: Install PyYAML (for workflow YAML validation) run: pip install pyyaml --quiet @@ -41,10 +51,26 @@ jobs: run: bash tests/fleet-report-back-test.sh - name: Fleet enrolment fixer (report-back rollout) run: bash tests/fix-fleet-enrolment-test.sh - - name: Lock closure contracts (dropped edges, malformed refs) + pinned-source-closure: + name: Pinned source closure + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + env: + PINNED_SOURCE_CACHE: ${{ runner.temp }}/fleet-pinned-sources + steps: + - uses: actions/checkout@v7.0.1 + - name: Test pinned-source failure modes + run: bash scripts/tests/pinned-reusable-sources-test.sh + - name: Authenticate pinned reusable sources + run: bash scripts/audit-pinned-reusable-sources.sh --fetch-sources + - name: Preserve lock closure contracts run: bash scripts/tests/lock-closure-test.sh - - name: Audit this repository's lock closure + - name: Audit vendor lock and pinned-source closure run: ./scripts/audit-lock-closure.sh + - name: Test source closure through migration preflight + run: bash scripts/tests/pinned-source-entrypoints-test.sh - name: Prepared secret-scanner migration stays gated run: | # 0 = every precondition holds, 2 = evidence still missing (expected diff --git a/docs/migrations/pinned-source-closure.adoc b/docs/migrations/pinned-source-closure.adoc new file mode 100644 index 00000000..6e257af4 --- /dev/null +++ b/docs/migrations/pinned-source-closure.adoc @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 +// SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell += Pinned reusable source closure: implementation and evidence +:revdate: 2026-11-07 + +The local implementation verifies reviewed direct reusable-workflow edges +against exact Git source objects and joins their reviewed dependency records +to the ordinary vendor lock for closure auditing. Local acceptance checks pass; +fresh hosted checks are pending. This records work based on Fleet draft +`48de76a5bcf466f46e4e973a21ca911ebec1b23c`, with main/base +`34908580b98cd354caa3bfc5a000bc7935957fce`. It does not certify the entire fleet +or declare the secret-scanner migration ready. + +== Vendor lock and source-attested records + +Official `gh-actions-lock` v0.4.1 excludes cross-repository `jobs..uses` +from locking. The supported v0.0.3 vendor lock retains all 20 ordinary records +unchanged. The source manifest separately preserves the ten original records +removed by that migration: two reusable parent records and eight action records, +including their original ref, commit, owner/repository IDs and nested edges. +The verifier derives one additional EA governance parent, yielding 11 +source-attested records in the integrated audit. The vendor lock itself is not +rewritten to pretend that the official tool locks these external reusables. + +`scripts/lib/pinned-reusable-sources.json` binds each active caller/job to an +exact repository, full commit, workflow path, Git blob and direct `uses` set. +Two old workflows remain explicitly labelled `retained-review`. The manifest's +reviewed inventory SHA-256 is +`63ffe6145c51bf5c44ac5f7d2ae9f304feb4acf7e9e3b52c6891e752ba126046`. +No dependency SHA or caller pin is changed by this closure integration. + +== Active and retained source identities + +The five active remote callers are: + +* `.github/workflows/governance.yml`, job `governance`: standards governance at + `ea2f4d8a886585ed442687ae63d4ad7493379ba4`. +* `.github/workflows/hypatia-scan.yml`, job `hypatia`: standards Hypatia at 75bd. +* `.github/workflows/mirror.yml`, job `mirror`: standards mirror at 75bd. +* `.github/workflows/scorecard.yml`, job `scorecard`: standards Scorecard at 75bd. +* `.github/workflows/secret-scanner.yml`, job `scan`: cicd-suite scanner at + `4404c3c0474e17753fe7689f3485c02890aac113`. + +Here 75bd means `75bd5994470661a099df562d6d5ad2fe744dbd66`. The retained-review +roots are standards governance and secret-scanner at that old commit; neither +is represented as an active call there. The same-repository report-back +workflow remains outside the seven-source remote-reusable manifest and is +covered by the ordinary root-workflow audit. + +The table preserves the exact collected Git identities and content SHA-256 +receipts. Independent collection rehashed 21 Git blobs: these seven remote +workflow sources, one local report-back workflow and thirteen action metadata +objects. Those collection hashes describe source bytes, not execution. + +[cols="1,3,3,3,4",options="header"] +|=== +|Role |Repository / source |Commit |Git blob |Content SHA-256 +|active |metadatastician/standards / .github/workflows/governance-reusable.yml |`ea2f4d8a886585ed442687ae63d4ad7493379ba4` |`8ac530b97d18f7d91972de91c448b7b44c055dfb` |`3b7b753935d0d40f8b854e1b337d7aced884eef60e139292533fdf9f4517577d` +|active |metadatastician/standards / .github/workflows/hypatia-scan-reusable.yml |`75bd5994470661a099df562d6d5ad2fe744dbd66` |`0f1c5ba55861c84f6665a103deb6ae184bb89996` |`eb72f4afd7bf17d2b31b808131ad697bb808a6ae0c8aa07773b5e1dbb4b7df7c` +|active |metadatastician/standards / .github/workflows/mirror-reusable.yml |`75bd5994470661a099df562d6d5ad2fe744dbd66` |`ec52acc3a975fbd528d9a38741bd868fb55de6e5` |`bc7443e325e255dd19baf48722583f07f7be1cd0cdce0f1ce229cfbeab3d17a1` +|active |metadatastician/standards / .github/workflows/scorecard-reusable.yml |`75bd5994470661a099df562d6d5ad2fe744dbd66` |`4e4fd33a6259b34cdc8c5f08d0d4280a216fb492` |`db6ae19dcb09329533d1537e63c329d469dc602badcb4cc514af5da7b9b3a9b6` +|active |hyperpolymath/cicd-suite / .github/workflows/secret-scanner-reusable.yml |`4404c3c0474e17753fe7689f3485c02890aac113` |`81b2ba1dfb074f6f8913baa81eb6768b7497cc2a` |`4745e9318346b1a3eae2e0bf65c6732f73d2ead78ef29de9e20659a71f8057d8` +|retained-review |metadatastician/standards / .github/workflows/governance-reusable.yml |`75bd5994470661a099df562d6d5ad2fe744dbd66` |`b041be08dc971f789ef3bc4d7849ac336ab30206` |`2e8bab47c6fcdfa0b94cc866214c337950a20c13a8af4da179f06485bf494f0c` +|retained-review |metadatastician/standards / .github/workflows/secret-scanner-reusable.yml |`75bd5994470661a099df562d6d5ad2fe744dbd66` |`2eee815b0e4c4c70a777b9c1d8c899b1fb7026c9` |`0c132f45da9832caf7cdb5dcfa164a727208e397f97e51f6a335a17464f2a0fd` +|=== + +== Object verification and evidence boundary + +`scripts/audit-pinned-reusable-sources.sh` fetches only when explicitly invoked +with `--fetch-sources`. It reads published sources over HTTPS into a separate +bare Git cache. Fetched code is never checked out or executed. Git configuration, +replacement refs, source-rewriting settings and source-path file modes are +checked within the verifier's supported profile. + +The verifier hashes the raw commit object, reads its tree identity, verifies +each tree component, and requires the manifest's regular-file blob at the exact +path. It hashes that blob's raw Git object content before parsing direct edges. +It independently discovers active caller/job/path/commit bindings in the local +workflows and compares them with the manifest. Receipt unions, dependency +identities, overlaps and graph targets must also agree before any supplemental +record is exported to the integrated audit. + +Absent cached objects produce nonzero `NOT-RUN`; corrupt or mismatched objects +produce findings. Duplicate JSON keys, malformed dependency identities and +ambiguous vendor fields fail closed. Unsupported YAML aliases, merges, flow +forms, dynamic/local references or nested reusable calls cannot establish a +source proof. Evidence unavailable to the parser is reported without a fabricated +pass. An unsuccessful evidence pass exports no source attestation. + +The reviewed bound is the direct `uses` sets of the named reusable workflows +plus the existing composite-action receipts and dependency graph. It does not +establish arbitrary recursive provenance of invoked scripts, runtime downloads, +container images or program dependencies. Collected metadata confirms the +existing upload-pages composite's upload-artifact edge; the Rust toolchain +composite has no nested `uses`. + +== Legacy receipt preservation + +All 13 original receipt rows remain byte-for-byte; six EA governance rows are +appended, yielding 19. The old standards 75bd group retains its 11-reference +union through three active and two retained-review source paths. Current active +callers at 75bd alone cover eight refs. Deno, editorconfig-checker and Bun occur +in the EA governance source instead, whose six direct refs are represented by +the preserved ordinary records and supplemental records. The EA group has its +own revision-bound receipt; the old group is not used as its evidence. + +== Local validation and CI entrypoints + +The final local acceptance results are: + +[cols="3,2",options="header"] +|=== +|Check |Result +|Pinned-source controls |55 pass +|Preserved lock-closure controls |47 pass +|Preflight entrypoint controls |9 pass +|Workflow-map controls |22 pass +|Explicit HTTPS fetch followed by offline helper |5 verified roots; 11 records +|Integrated closure audit |fatal 0; review 0; not_run 0; accepted 3; source_attested_records 11 +|Vendor verify-local, full verify and lock sync |Pass +|=== + +The source job runs helper fixtures, explicit source fetching, the preserved +closure controls, the integrated offline audit, preflight entrypoint controls +and the prepared-migration preflight. Its cache is available to the proposed-tree +preflight, which snapshots both the helper and manifest. A missing vendor parent +is accepted only through successful integrated source verification; manifest +presence alone is not proof. Existing secret, trigger, context and canary +requirements remain enforced. Missing canary evidence still makes standalone +preflight exit 2, so migration readiness is not asserted. + +The E2E job declares `needs: pinned-source-closure`, uses `always()`, and begins +with a guard requiring that dependency's result to equal `success`. Failure, +not-run, cancellation or skipping cannot quietly satisfy that guard. This is +E2E dependency coupling. The live branch rules did not require the E2E context +at the last observation; no branch-ruleset requirement or ruleset change is +claimed. + +Reproduce from the repository root, with a separate source cache: + +[source,bash] +---- +export PINNED_SOURCE_CACHE=/absolute/path/to/source-cache +bash scripts/audit-pinned-reusable-sources.sh --fetch-sources --json +bash scripts/audit-pinned-reusable-sources.sh --json +bash scripts/tests/pinned-reusable-sources-test.sh +bash scripts/tests/lock-closure-test.sh +bash scripts/tests/pinned-source-entrypoints-test.sh +python3 tests/workflow-lock-map-test.py +bash scripts/audit-lock-closure.sh --json +---- + +The helper defaults to the tracked manifest. It also accepts explicit +`--manifest`, `--repo-root`, `--lock`, `--receipts` and `--cache` paths. The first +helper invocation fetches; the second uses cached objects offline. These checks +inspect source data and fixtures, not fetched executable code. + +== Remaining observations + +Local Debian `en_GB.UTF-8` report-back runs have three ordering failures on both +the base and candidate; this closure change does not claim to repair them or to +make the full repository E2E suite pass. Nine governance findings remained at the +last observation. Fresh hosted checks remain pending, so local acceptance is +not a claim that every hosted repository check is green. + +No normative policy, standards draft 1419, fleet draft 651, ruleset, mirror or +credential is changed by this closure integration. Source caches and raw local +logs are local-only evidence. The tracked manifest and receipt rows carry the +reviewable identities; local absolute paths are not presented as downloadable +GitHub artifacts. diff --git a/docs/migrations/secret-scanner-cicd-suite/preflight.sh b/docs/migrations/secret-scanner-cicd-suite/preflight.sh index 42c60ec5..c8a05042 100755 --- a/docs/migrations/secret-scanner-cicd-suite/preflight.sh +++ b/docs/migrations/secret-scanner-cicd-suite/preflight.sh @@ -76,6 +76,12 @@ cp -R "$ROOT/.github" "$TMP/tree/" 2>/dev/null || finding TREE "$ROOT/.github mi cp "$ROOT/scripts/audit-lock-closure.sh" "$TMP/tree/scripts/" 2>/dev/null || finding TREE "audit-lock-closure.sh missing" cp "$ROOT/scripts/lib/lock-closure-receipts.tsv" "$TMP/tree/scripts/lib/" 2>/dev/null || true cp "$ROOT/scripts/lib/lock-audit-baseline.tsv" "$TMP/tree/scripts/lib/" 2>/dev/null || true +# Preserve the independent source proof in the proposed tree when the vendor +# lock omits remote reusables. The closure audit below must still verify it. +if [ -f "$ROOT/scripts/lib/pinned-reusable-sources.json" ]; then + cp "$ROOT/scripts/lib/pinned-reusable-sources.json" "$TMP/tree/scripts/lib/" || finding TREE "source manifest could not be copied" + cp "$ROOT/scripts/audit-pinned-reusable-sources.sh" "$TMP/tree/scripts/" || finding TREE "source verifier could not be copied" +fi # A prepared migration can subsequently be applied by an approved PR. # In that case old patch hunks no longer apply; check the final artifacts. @@ -85,10 +91,13 @@ if [ -f "$TMP/tree/.github/workflows/secret-scanner.yml" ] && grep -Fq "uses: $CALLEE" "$TMP/tree/.github/workflows/secret-scanner.yml"; then already_applied=1 source_key="'hyperpolymath/cicd-suite@$SUITE_SHA'" - # Both caller's workflow map and dependency record must exist. + # Legacy representation needs both vendor records. Supported vendor locks + # may instead carry a source manifest; its presence is not proof. The + # integrated audit below must authenticate every active binding and edge, + # and missing evidence remains nonzero NOT RUN, never a migration pass. count=$(grep -Fc "$source_key" "$TMP/tree/.github/workflows/actions.lock" 2>/dev/null || true) - if [ "$count" -lt 2 ]; then - finding LOCK-MISSING "applied scanner lacks both pinned suite caller and dependency records" + if [ "$count" -lt 2 ] && [ ! -f "$TMP/tree/scripts/lib/pinned-reusable-sources.json" ]; then + finding LOCK-MISSING "applied scanner lacks both pinned suite vendor records and a source-verification route" apply_ok=0 fi receipt=$(printf 'hyperpolymath/cicd-suite\t%s\tactions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' "$SUITE_SHA") diff --git a/scripts/audit-lock-closure.sh b/scripts/audit-lock-closure.sh index 21a9c8c4..cb6ae2b0 100755 --- a/scripts/audit-lock-closure.sh +++ b/scripts/audit-lock-closure.sh @@ -71,6 +71,10 @@ ONLINE=0 QUIET=0 EMIT_KEY="" EMIT_SOURCES=() +DUMP_LOCK_RECORDS=0 +SOURCE_MANIFEST="" +SOURCE_CACHE="${PINNED_SOURCE_CACHE:-}" +FETCH_SOURCES=0 while [ $# -gt 0 ]; do case "$1" in @@ -84,6 +88,10 @@ while [ $# -gt 0 ]; do --repo-root) REPO_ROOT="${2:?--repo-root needs a directory}"; shift ;; --emit-receipt) EMIT_KEY="${2:?--emit-receipt needs owner/repo@sha}"; shift ;; --source) EMIT_SOURCES+=("${2:?--source needs a file or directory}"); shift ;; + --dump-lock-records) DUMP_LOCK_RECORDS=1 ;; + --source-manifest) SOURCE_MANIFEST="${2:?--source-manifest needs a file}"; shift ;; + --source-cache) SOURCE_CACHE="${2:?--source-cache needs a directory}"; shift ;; + --fetch-sources) FETCH_SOURCES=1 ;; -h|--help) sed -n '2,55p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; *) echo "audit-lock-closure: unknown argument: $1" >&2; exit 64 ;; esac @@ -386,6 +394,21 @@ cat > "$TMP/lock.awk" <<'AWK' } AWK "$AWK" -f "$TMP/lock.awk" "$LOCK" > "$TMP/lock.tsv" +# The record parser deliberately ignores unrelated top-level lines; the +# trusted dump must still reject missing/duplicate sections and non-mapping +# section shapes rather than treating malformed YAML as an empty lock. +"$AWK" ' + { sub(/\r$/, "") } + /^version:/ { + versions++; value=$0; sub(/^version:[[:space:]]*/, "", value) + sub(/[[:space:]]*#.*/, "", value); sub(/[[:space:]]*$/, "", value) + quote=substr(value,1,1); if((quote=="\"" || quote=="\047") && substr(value,length(value),1)==quote) value=substr(value,2,length(value)-2) + if(value!="v0.0.2" && value!="v0.0.3") print "PARSE\tunsupported lock version" + } + /^workflows:/ { workflows++; if($0 !~ /^workflows:[[:space:]]*(\{\})?[[:space:]]*$/) print "PARSE\tworkflows must be a mapping" } + /^dependencies:/ { dependencies++; if($0 !~ /^dependencies:[[:space:]]*(\{\})?[[:space:]]*$/) print "PARSE\tdependencies must be a mapping" } + END { if(versions!=1 || workflows!=1 || dependencies!=1) print "PARSE\tlock requires one version, workflows and dependencies section" } +' "$LOCK" >> "$TMP/lock.tsv" # GitHub resolves owner and repository case-insensitively; the ref is case # sensitive. Fold exactly the owner/repo half everywhere a set is keyed, so a @@ -404,6 +427,8 @@ declare -A LOCK_ENTRIES=() # path -> " folded-ref ..." declare -A WF_ORDER=() declare -A DEP_REF=() declare -A DEP_COMMIT=() +declare -A DEP_OWNER_ID=() +declare -A DEP_REPO_ID=() declare -A DEP_USES=() declare -A DEP_HASFIELD=() declare -A DEP_SEEN=() @@ -457,12 +482,17 @@ while IFS=$'\t' read -r tag a b c; do ;; DEPFIELD) key="$a"; field="$b"; val="$c" + if [ -n "${DEP_HASFIELD["$key|$field"]:-}" ]; then + add_finding FATAL LOCK-DUP-FIELD "$key" "dependency field $field appears more than once" "Keep one unambiguous value per field" + fi + DEP_HASFIELD["$key|$field"]=1 case "$field" in ref) DEP_REF["$key"]="$val" ;; commit) DEP_COMMIT["$key"]="$val" ;; uses) DEP_USES["$key"]="${DEP_USES["$key"]:-} $val" ;; owner_id|repo_id) [[ "$val" =~ ^[0-9]+$ ]] || add_finding FATAL REF-FIELD "$key" "$field is '$val', not an integer" "Record the numeric GitHub id or omit the field" + if [ "$field" = owner_id ]; then DEP_OWNER_ID["$key"]="$val"; else DEP_REPO_ID["$key"]="$val"; fi ;; *) add_finding REVIEW REF-FIELD "$key" "unknown dependency field '$field'" "Remove it or extend the audit" ;; esac @@ -509,6 +539,73 @@ for key in "${DEP_KEYS[@]}"; do done # --------------------------------------------------------- parse the workflows -- +# Internal data-only interface used by the independent source verifier. It +# exits before source verification, avoiding recursion, and never emits a +# successful record dump for malformed lock/receipt input. +if [ "$DUMP_LOCK_RECORDS" -eq 1 ]; then + if [ "${#F_SEV[@]}" -gt 0 ] || [ "${#N_RULE[@]}" -gt 0 ]; then + printf '%s\n' 'audit-lock-closure: cannot dump invalid or incomplete input' >&2 + exit 1 + fi + : > "$TMP/records.jsonl" + for key in "${DEP_KEYS[@]}"; do + edges="$(printf '%s\n' "${DEP_USES[$key]:-}" | jq -Rc 'split(" ") | map(select(length > 0))')" + jq -nc --arg key "$key" --arg ref "${DEP_REF[$key]}" \ + --arg commit "sha1-${DEP_COMMIT[$key]}" --argjson uses "$edges" \ + --argjson owner_id "${DEP_OWNER_ID[$key]:-null}" --argjson repo_id "${DEP_REPO_ID[$key]:-null}" \ + '{($key):({ref:$ref,commit:$commit,uses:$uses} + (if $owner_id==null then {} else {owner_id:$owner_id} end) + (if $repo_id==null then {} else {repo_id:$repo_id} end))}' >> "$TMP/records.jsonl" + done + jq -sc 'add // {}' "$TMP/records.jsonl" + exit 0 +fi + +# Vendor locks deliberately omit remote job calls. A separate verifier may +# supply additional records only after exact commit/tree/blob, caller/path, +# reviewed-edge and receipt checks all succeed. These are source-attested +# records, not a claim that gh-actions-lock generated or verified them. +if [ -z "$SOURCE_MANIFEST" ] && { [ -f "$REPO_ROOT/scripts/lib/pinned-reusable-sources.json" ] || [ -f "$REPO_ROOT/scripts/audit-pinned-reusable-sources.sh" ]; }; then + SOURCE_MANIFEST="$REPO_ROOT/scripts/lib/pinned-reusable-sources.json" +fi +SOURCE_ATTESTED_COUNT=0 +if [ -n "$SOURCE_MANIFEST" ]; then + helper="$(dirname "${BASH_SOURCE[0]}")/audit-pinned-reusable-sources.sh" + args=(--manifest "$SOURCE_MANIFEST" --repo-root "$REPO_ROOT" --lock "$LOCK" --receipts "$RECEIPTS" --json) + [ -z "$SOURCE_CACHE" ] || args+=(--cache "$SOURCE_CACHE") + [ "$FETCH_SOURCES" -eq 0 ] || args+=(--fetch-sources) + source_rc=0 + bash "$helper" "${args[@]}" > "$TMP/source-report.json" || source_rc=$? + if ! jq -e '(.findings | type == "array") and (.not_run | type == "array") and (.dependencies | type == "object")' "$TMP/source-report.json" >/dev/null 2>&1; then + add_notrun SOURCE-REPORT "$SOURCE_MANIFEST" "pinned-source verifier did not return a valid report" "Restore the verifier and its dependencies; no supplemental record is trusted" + else + while IFS= read -r item; do + add_finding "$(jq -r .severity <<< "$item")" "$(jq -r .rule <<< "$item")" "$(jq -r .subject <<< "$item")" "$(jq -r .detail <<< "$item")" "$(jq -r .fix <<< "$item")" + done < <(jq -c '.findings[]' "$TMP/source-report.json") + while IFS= read -r item; do + add_notrun "$(jq -r .rule <<< "$item")" "$(jq -r .subject <<< "$item")" "$(jq -r .detail <<< "$item")" "$(jq -r .fix <<< "$item")" + done < <(jq -c '.not_run[]' "$TMP/source-report.json") + if [ "$source_rc" -eq 0 ] && jq -e '(.findings | length == 0) and (.not_run | length == 0)' "$TMP/source-report.json" >/dev/null; then + while IFS= read -r item; do + key="$(jq -r .key <<< "$item")" + f="$(fold_ref "$key")" + # Overlaps have already been compared by the verifier. Keep every + # existing vendor record untouched, including its spelling. + [ -z "${DEP_IDX[$f]:-}" ] || continue + DEP_KEYS+=("$key"); DEP_IDX["$f"]="$key"; DISP["$f"]="$key" + DEP_REF["$key"]="$(jq -r .value.ref <<< "$item")" + commit="$(jq -r .value.commit <<< "$item")" + DEP_COMMIT["$key"]="${commit#sha1-}" + owner_id="$(jq -r '.value.owner_id//empty' <<< "$item")"; repo_id="$(jq -r '.value.repo_id//empty' <<< "$item")" + [ -z "$owner_id" ] || DEP_OWNER_ID["$key"]="$owner_id" + [ -z "$repo_id" ] || DEP_REPO_ID["$key"]="$repo_id" + DEP_USES["$key"]="$(jq -r '(.value.uses // []) | join(" ")' <<< "$item")" + SOURCE_ATTESTED_COUNT=$((SOURCE_ATTESTED_COUNT + 1)) + done < <(jq -c '.dependencies | to_entries[]' "$TMP/source-report.json") + elif [ "$(jq '(.findings | length) + (.not_run | length)' "$TMP/source-report.json")" -eq 0 ]; then + add_notrun SOURCE-REPORT "$SOURCE_MANIFEST" "source verifier exited $source_rc without a diagnostic" "Inspect the verifier; no supplemental record is trusted" + fi + fi +fi + declare -A WF_FILES=() declare -A STEP_SET=() # path|folded-ref declare -A STEP_LIST=() # path -> " folded-ref ..." @@ -701,7 +798,10 @@ done # One reviewed ref must not resolve to two immutable commits: the snapshot is # then ambiguous, and a regeneration may pick either. declare -A TAG_DIGEST=() -for key in "${DEP_KEYS[@]}"; do +# Supplemental records are appended after vendor parsing. Preserve a stable +# ordering so the same reviewed ambiguity keeps the same subject regardless +# of which artifact carries its records (and baseline matches do not drift). +while IFS= read -r key; do repo="$(printf '%s' "${key%@*}" | tr '[:upper:]' '[:lower:]')" ref="${DEP_REF["$key"]:-${key##*@}}" f="$repo@$ref" @@ -712,7 +812,7 @@ for key in "${DEP_KEYS[@]}"; do add_finding REVIEW REF-AMBIGUOUS "$key" "the same reviewed ref resolves to two immutable commits (${TAG_DIGEST["$f"]} and $digest)" "Keep one reviewed snapshot per ref; delete or re-resolve the stale record" fi TAG_DIGEST["$f"]="$digest" -done +done < <(printf '%s\n' "${DEP_KEYS[@]}" | LC_ALL=C sort) # A mutable branch at step level is not an immutable pin: the lock freezes a # snapshot, but resolution follows the branch. Flag only conventional branch @@ -813,10 +913,11 @@ n_accepted=${#ACC_RULE[@]} if [ "$JSON" -eq 1 ]; then jq -n --argjson fatal "$n_fatal" --argjson review "$n_review" \ --argjson not_run "$n_notrun" --argjson accepted "$n_accepted" \ + --argjson source_attested "$SOURCE_ATTESTED_COUNT" \ --arg lock "$LOCK" --arg wf "$WF_DIR" \ --slurpfile findings "$TMP/findings.jsonl" --slurpfile notrun "$TMP/notrun.jsonl" \ '{lock:$lock,workflows:$wf, - summary:{fatal:$fatal,review:$review,accepted:$accepted,not_run:$not_run}, + summary:{fatal:$fatal,review:$review,accepted:$accepted,not_run:$not_run,source_attested_records:$source_attested}, findings:$findings, not_run:$notrun}' else echo "audit-lock-closure: $LOCK" @@ -855,7 +956,7 @@ else echo "INCOMPLETE: no findings, but $n_notrun check(s) did not run; this is not a pass." fi echo - echo "summary: fatal=$n_fatal review=$n_review accepted=$n_accepted not-run=$n_notrun" + echo "summary: fatal=$n_fatal review=$n_review accepted=$n_accepted not-run=$n_notrun source-attested-records=$SOURCE_ATTESTED_COUNT" fi if [ $((n_fatal + n_review)) -gt 0 ] && [ "$n_notrun" -gt 0 ]; then exit 3; fi diff --git a/scripts/audit-pinned-reusable-sources.sh b/scripts/audit-pinned-reusable-sources.sh new file mode 100644 index 00000000..cd4b944b --- /dev/null +++ b/scripts/audit-pinned-reusable-sources.sh @@ -0,0 +1,344 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# Verify reviewed direct reusable-workflow edges against immutable Git objects. +# This reads source as data; it never checks out or executes fetched content. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +MANIFEST=""; LOCK=""; RECEIPTS=""; CACHE="${PINNED_SOURCE_CACHE:-${XDG_CACHE_HOME:-${HOME:-/tmp}/.cache}/fleet-pinned-sources}" +FETCH=0; JSON=0 +while [ "$#" -gt 0 ]; do + case "$1" in + --manifest|--source-manifest) MANIFEST="${2:?manifest file required}"; shift ;; + --repo-root) ROOT="${2:?repository root required}"; shift ;; + --lock) LOCK="${2:?lock file required}"; shift ;; + --receipts) RECEIPTS="${2:?receipt file required}"; shift ;; + --cache|--source-cache) CACHE="${2:?cache directory required}"; shift ;; + --fetch-sources) FETCH=1 ;; + --json) JSON=1 ;; + -h|--help) printf '%s\n' 'Usage: audit-pinned-reusable-sources.sh [--manifest FILE] [--repo-root DIR] [--lock FILE] [--receipts FILE] [--cache DIR] [--fetch-sources] [--json]'; exit 0 ;; + *) printf 'Unknown argument: %s\n' "$1" >&2; exit 64 ;; + esac + shift +done +MANIFEST="${MANIFEST:-$ROOT/scripts/lib/pinned-reusable-sources.json}" +LOCK="${LOCK:-$ROOT/.github/workflows/actions.lock}" +RECEIPTS="${RECEIPTS:-$ROOT/scripts/lib/lock-closure-receipts.tsv}" +for tool in git jq awk sha256sum; do + command -v "$tool" >/dev/null || { printf 'Missing verifier prerequisite: %s\n' "$tool" >&2; exit 2; } +done +TMP="$(mktemp -d "${TMPDIR:-/tmp}/pinned-sources.XXXXXX")" +trap 'rm -rf "$TMP"' EXIT +: > "$TMP/findings"; : > "$TMP/notrun"; : > "$TMP/roots"; : > "$TMP/groups" +printf '{}\n' > "$TMP/supplement.json" +finding() { + jq -nc --arg rule "$1" --arg subject "$2" --arg detail "$3" \ + '{severity:"FATAL",rule:$rule,subject:$subject,detail:$detail,fix:"Review the source-bound manifest and immutable objects; do not waive closure evidence"}' >> "$TMP/findings" +} +notrun() { + jq -nc --arg rule "$1" --arg subject "$2" --arg detail "$3" \ + '{rule:$rule,subject:$subject,detail:$detail,fix:"Provide authenticated source objects and supported parseable input; fetched source is never executed"}' >> "$TMP/notrun" +} +finish() { + local f n rc=0 + f="$(jq -s length "$TMP/findings")"; n="$(jq -s length "$TMP/notrun")" + [ "$f" -eq 0 ] || rc=1 + [ "$n" -eq 0 ] || rc=$((rc + 2)) + if [ "$rc" -ne 0 ]; then + : > "$TMP/roots"; : > "$TMP/groups"; printf '{}\n' > "$TMP/supplement.json" + fi + jq -n --slurpfile findings "$TMP/findings" --slurpfile not_run "$TMP/notrun" \ + --slurpfile roots "$TMP/roots" --slurpfile groups "$TMP/groups" --slurpfile deps "$TMP/supplement.json" \ + '{findings:$findings,not_run:$not_run,verified_roots:$roots,verified_receipt_sources:($groups|unique),dependencies:$deps[0]}' > "$TMP/report.json" + if [ "$JSON" -eq 1 ]; then cat "$TMP/report.json"; else + jq -r '.findings[] | "FATAL\t\(.rule)\t\(.subject)\t\(.detail)"' "$TMP/report.json" + jq -r '.not_run[] | "NOT-RUN\t\(.rule)\t\(.subject)\t\(.detail)"' "$TMP/report.json" + printf 'pinned reusable sources: findings=%s not-run=%s verified-roots=%s (direct source edges; no fetched execution)\n' "$f" "$n" "$(jq '.verified_roots|length' "$TMP/report.json")" + fi + return "$rc" +} +if [ ! -f "$MANIFEST" ]; then notrun SOURCE-MISSING "$MANIFEST" 'No source manifest'; finish; exit $?; fi +# Streaming paths retain duplicate object keys which ordinary jq object +# parsing would discard. Container-close events identify their parent path. +if ! jq --stream -es 'map(if length==2 then .[0] else .[0][0:-1] end) as $paths | ($paths|length)==($paths|unique|length)' "$MANIFEST" >/dev/null 2>&1; then + finding SOURCE-MANIFEST "$MANIFEST" 'Duplicate JSON keys, multiple documents or malformed JSON'; finish; exit $? +fi +if ! jq -e ' + def sha: type=="string" and test("^[0-9a-f]{40}$"); + def repo: type=="string" and test("^[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*$"); + def path: type=="string" and test("^\\.github/workflows/[A-Za-z0-9_.-]+\\.ya?ml$"); + .version==1 and (.sources|type=="array" and length>0) and (.dependencies|type=="object") and + all(.sources[]; + (.repository|repo) and (.commit|sha) and (.blob|sha) and (.path|path) and + (.role=="active" or .role=="retained-review") and (.callers|type=="array") and + (if .role=="active" then (.callers|length>0) else (.callers|length==0) end) and + all(.callers[]; (.workflow|path) and (.job|type=="string" and test("^[A-Za-z_][A-Za-z0-9_-]*$"))) and + (.uses|type=="array") and all(.uses[]; (.kind=="step" or .kind=="job") and (.ref|type=="string" and length>0))) and + ([.sources[]|[(.repository|ascii_downcase),.commit,.path]]|length)==([.sources[]|[(.repository|ascii_downcase),.commit,.path]]|unique|length) and + all(.dependencies|to_entries[]; + (.key|test("^[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*@[^[:space:]@]+$")) and + (.value|type=="object" and (.ref|type=="string" and test("^[^[:space:]@]+$")) and + (.commit|type=="string" and test("^sha1-[0-9a-f]{40}$")) and + (if has("uses") then (.uses|type=="array" and all(.[];type=="string")) else true end) and + (if has("owner_id") then (.owner_id|type=="number" and .>=0 and floor==.) else true end) and + (if has("repo_id") then (.repo_id|type=="number" and .>=0 and floor==.) else true end) and + (if has("original_lock_record") then (.original_lock_record|type=="string") else true end))) +' "$MANIFEST" >/dev/null 2>&1; then + finding SOURCE-MANIFEST "$MANIFEST" 'Malformed, duplicate or unsupported source manifest'; finish; exit $? +fi + +# Preserve the strict vendor parser rather than introducing another YAML +# dependency. Its data-only mode exits before calling this verifier. +if ! bash "$SCRIPT_DIR/audit-lock-closure.sh" --dump-lock-records --lock "$LOCK" \ + --workflows "$ROOT/.github/workflows" --repo-root "$ROOT" --receipts "$RECEIPTS" --baseline /dev/null > "$TMP/vendor.json" 2> "$TMP/vendor.err"; then + finding SOURCE-DEPENDENCY "$LOCK" 'Vendor lock or receipt input cannot be parsed safely'; finish; exit $? +fi +jq '.dependencies | with_entries(.value |= with_entries(select(.key=="ref" or .key=="commit" or .key=="uses" or .key=="owner_id" or .key=="repo_id")))' "$MANIFEST" > "$TMP/supplement.json" +# Repository names are case-insensitive; ref spelling is not. Reject a +# collision before normalization, then compare records in one key space. +normalize_records() { + jq -e ' + def key: capture("^(?[^@]+)@(?[^@]+)$") | ((.repository|ascii_downcase)+"@"+.ref); + (keys|map(key)) as $keys | ($keys|length)==($keys|unique|length) + ' "$1" >/dev/null 2>&1 || return 1 + jq ' + def key: capture("^(?[^@]+)@(?[^@]+)$") | ((.repository|ascii_downcase)+"@"+.ref); + with_entries(.key |= key | if (.value|has("uses")) then .value.uses |= map(key) else . end) + ' "$1" > "$2" +} +for graph in vendor supplement; do + if ! normalize_records "$TMP/$graph.json" "$TMP/$graph.normalized.json"; then + finding SOURCE-DEPENDENCY "$graph" 'Malformed or case-colliding dependency keys'; finish; exit $? + fi + mv "$TMP/$graph.normalized.json" "$TMP/$graph.json" +done +jq -s '.[0] * .[1]' "$TMP/vendor.json" "$TMP/supplement.json" > "$TMP/graph.json" + +normalize_ref() { + local raw="$1" path ref + [[ "$raw" != *[[:space:]]* && "$raw" != *'${{'* ]] || return 1 + [[ "$raw" =~ ^([A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*)(/[^@]+)?@([^@]+)$ ]] || return 1 + path="${BASH_REMATCH[1]}"; ref="${BASH_REMATCH[3]}" + printf '%s@%s\n' "${path,,}" "$ref" +} + +# Conservative block-YAML reader matching the existing auditor's profile. +# A flow/scalar/anchor spelling we cannot read is evidence missing, not green. +extract_uses() { + awk ' + function uncomment(s, i,c,q,out) { + q="";out="" + for(i=1;i<=length(s);i++) {c=substr(s,i,1);if(q=="") {if(c=="\""||c=="\047")q=c;else if(c=="#"&&(i==1||substr(s,i-1,1)~/[[:space:]]/))break} else if(c==q)q="";out=out c}return out + } + { + sub(/\r$/, "");line=$0 + if(line~/^[[:space:]]*$/||line~/^[[:space:]]*#/)next + if(line~/\t/){print "UNPARSED\ttab in YAML";next} + match(line,/^[[:space:]]*/);ind=RLENGTH + if(block){if(ind>blockind)next;block=0} + if(line~/^[[:space:]]*<<[[:space:]]*:/||line~/(^|[[:space:]])[&*][A-Za-z0-9_-]+/){print "UNPARSED\tunsupported YAML alias or merge";next} + if(line~/^["\047]?jobs["\047]?[[:space:]]*:[[:space:]]*$/){jobs=1;ji=ind;jind=-1;job="";next} + if(jobs&&ind<=ji){jobs=0;job=""} + if(jobs&&ind>ji){ + if(jind<0)jind=ind + if(ind==jind){s=uncomment(line);sub(/^[[:space:]]+/,"",s);sub(/:[[:space:]]*$/,"",s);gsub(/^["\047]|["\047]$/,"",s);if(s!~/^[A-Za-z_][A-Za-z0-9_-]*$/){print "UNPARSED\tjob key not a block scalar key";job=""}else job=s} + } + if(steps&&ind<=si)steps=0 + if(line~/^[[:space:]]*(-[[:space:]]+)?["\047]?steps["\047]?[[:space:]]*:[[:space:]]*$/){steps=1;si=ind;next} + if(match(line,/^[[:space:]]*(-[[:space:]]+)?["\047]?uses["\047]?[[:space:]]*:/)){ + head=substr(line,1,RLENGTH);kind=(head~/-/||(steps&&ind>si))?"step":"job";raw=uncomment(substr(line,RLENGTH+1));sub(/^[[:space:]]+/,"",raw);sub(/[[:space:]]+$/,"",raw) + if(raw==""||raw~/^[|>*&\[{]/||raw~/[[:space:]]/){print "UNPARSED\tuses value not a single plain scalar";next} + if(raw~/^["\047]/){q=substr(raw,1,1);if(substr(raw,length(raw),1)!=q){print "UNPARSED\tunterminated uses quote";next};raw=substr(raw,2,length(raw)-2)} + print "USE\t"kind"\t"raw"\t"job;next + } + if(line~/(^|[[:space:],{])["\047]?uses["\047]?[[:space:]]*:/)print "UNPARSED\tuses in unsupported YAML shape" + s=uncomment(line);if(s~/(^|[[:space:]-])["\047]?[A-Za-z0-9_.-]+["\047]?[[:space:]]*:[[:space:]]*[|>][-+]?[0-9]?[[:space:]]*$/){block=1;blockind=ind} + } + ' "$1" +} + +# Discover the exact live caller/job/path roots independently of the catalog. +shopt -s nullglob +for workflow in "$ROOT"/.github/workflows/*.yml "$ROOT"/.github/workflows/*.yaml; do + caller=".github/workflows/$(basename "$workflow")" + while IFS=$'\t' read -r tag kind raw job; do + if [ "$tag" = UNPARSED ]; then notrun SOURCE-PARSE "$caller" "$kind"; continue; fi + [ "$kind" = job ] || continue + case "$raw" in ./*) continue ;; esac + if [[ ! "$raw" =~ ^([A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*)/(\.github/workflows/[A-Za-z0-9_.-]+\.ya?ml)@([0-9a-f]{40})$ ]] || [ -z "$job" ]; then + finding SOURCE-PIN "$caller/$job" "Remote reusable must have an exact workflow path and full lowercase 40-hex source pin: $raw"; continue + fi + jq -nc --arg workflow "$caller" --arg job "$job" --arg repository "${BASH_REMATCH[1],,}" --arg path "${BASH_REMATCH[2]}" --arg commit "${BASH_REMATCH[3]}" \ + '{workflow:$workflow,job:$job,repository:$repository,path:$path,commit:$commit}' >> "$TMP/roots" + done < <(extract_uses "$workflow") +done +shopt -u nullglob +jq -s 'sort_by(.workflow,.job)' "$TMP/roots" > "$TMP/actual-roots.json" +jq '[.sources[]|select(.role=="active")|. as $s|.callers[]|{workflow,job,repository:($s.repository|ascii_downcase),path:$s.path,commit:$s.commit}]|sort_by(.workflow,.job)' "$MANIFEST" > "$TMP/expected-roots.json" +if ! cmp -s "$TMP/actual-roots.json" "$TMP/expected-roots.json"; then + finding SOURCE-ACTIVE-MISSING "$MANIFEST" 'Active caller/job/path/commit bindings differ from the actual workflow roots (historical paths cannot satisfy active calls)' +fi + +# Isolate Git configuration and transports; raw object hashes are checked in +# addition to tree membership. Replacement refs, grafts and symlinks fail shut. +git_data() { + env -u GIT_DIR -u GIT_WORK_TREE -u GIT_INDEX_FILE -u GIT_OBJECT_DIRECTORY -u GIT_ALTERNATE_OBJECT_DIRECTORIES \ + -u GIT_CONFIG -u GIT_CONFIG_PARAMETERS -u GIT_CONFIG_COUNT GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1 GIT_NO_REPLACE_OBJECTS=1 GIT_NO_LAZY_FETCH=1 \ + git --no-replace-objects -c core.hooksPath=/dev/null -c core.fsmonitor=false -c credential.helper= -c http.extraHeader= \ + -c protocol.allow=never -c protocol.https.allow=always -c http.followRedirects=false -c fetch.fsckObjects=true \ + --git-dir "$BARE" "$@" +} +verify_object() { + local type="$1" oid="$2" dest="$3" actual + git_data cat-file "$type" "$oid" > "$dest" 2>/dev/null || return 1 + actual="$(git_data hash-object -t "$type" --stdin < "$dest")" || return 1 + [ "$actual" = "$oid" ] +} +object_failure() { + local type="$1" oid="$2" subject="$3" + if git_data cat-file -e "$oid" 2>/dev/null || [ -e "$BARE/objects/${oid:0:2}/${oid:2}" ]; then + finding SOURCE-ORIGIN "$subject" "Cached $type object is present but corrupt, has the wrong type, or fails its raw content hash" + else + notrun SOURCE-MISSING "$subject" "Offline cache lacks required $type object $oid" + fi +} +source_bytes() { + local repository="$1" commit="$2" path="$3" blob="$4" dest="$5" tree entry mode type oid name part i + BARE="$CACHE/$repository.git" + for entry in "$CACHE" "$CACHE/${repository%/*}" "$BARE"; do + if [ -L "$entry" ]; then finding SOURCE-ORIGIN "$repository@$commit" 'Symlink in source-cache path'; return 1; fi + done + if [ ! -d "$BARE" ] && [ "$FETCH" -eq 1 ]; then + mkdir -p "$(dirname "$BARE")" + if ! env GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1 git -c core.hooksPath=/dev/null init --bare --object-format=sha1 "$BARE" >/dev/null 2>&1; then + notrun SOURCE-MISSING "$repository@$commit" 'Cannot initialize isolated bare source cache'; return 1 + fi + fi + if [ ! -d "$BARE" ]; then notrun SOURCE-MISSING "$repository@$commit" 'No authenticated bare cache; bootstrap with --fetch-sources'; return 1; fi + if [ "$(git_data rev-parse --is-bare-repository 2>/dev/null)" != true ]; then finding SOURCE-ORIGIN "$repository@$commit" 'Source cache is not a bare Git repository'; return 1; fi + if [ -e "$BARE/info/grafts" ] || [ -L "$BARE/info/grafts" ] || [ -n "$(git_data for-each-ref --format='%(refname)' refs/replace/ 2>/dev/null)" ] || + git_data config --local --get-regexp '^(include\.|includeif\.|url\.|extensions\.partialclone|remote\..*\.promisor)' >/dev/null 2>&1; then + finding SOURCE-ORIGIN "$repository@$commit" 'Cache has grafts, replacement refs or source-rewriting configuration'; return 1 + fi + if ! verify_object commit "$commit" "$TMP/commit"; then + if git_data cat-file -e "$commit" 2>/dev/null || [ -e "$BARE/objects/${commit:0:2}/${commit:2}" ]; then + finding SOURCE-ORIGIN "$repository@$commit" 'Cached commit exists but its raw content/type/hash is corrupt or mismatched'; return 1 + fi + if [ "$FETCH" -eq 1 ]; then + if ! git_data fetch --quiet --no-tags --depth=1 --no-recurse-submodules "https://github.com/$repository.git" "$commit" > "$TMP/fetch.out" 2> "$TMP/fetch.err"; then + notrun SOURCE-MISSING "$repository@$commit" 'Exact public HTTPS source fetch failed'; return 1 + fi + else notrun SOURCE-MISSING "$repository@$commit" 'Requested commit absent or corrupt in offline cache'; return 1; fi + fi + if ! verify_object commit "$commit" "$TMP/commit"; then finding SOURCE-ORIGIN "$repository@$commit" 'Commit content hash does not match requested origin'; return 1; fi + tree="$(sed -n '1s/^tree //p' "$TMP/commit")" + [[ "$tree" =~ ^[0-9a-f]{40}$ ]] || { finding SOURCE-ORIGIN "$repository@$commit" 'Invalid commit tree'; return 1; } + IFS=/ read -r -a parts <<< "$path" + for ((i=0;i<${#parts[@]};i++)); do + part="${parts[$i]}" + if ! verify_object tree "$tree" "$TMP/tree"; then object_failure tree "$tree" "$repository@$commit/$path"; return 1; fi + entry="$(git_data ls-tree "$tree" -- "$part")" + IFS=$' \t' read -r mode type oid name <<< "$entry" + if [ "$name" != "$part" ]; then finding SOURCE-BLOB "$repository@$commit/$path" 'Requested source path absent from exact commit tree'; return 1; fi + if [ "$i" -lt "$((${#parts[@]}-1))" ]; then + if [ "$mode" != 040000 ] || [ "$type" != tree ]; then finding SOURCE-BLOB "$repository@$commit/$path" 'Non-tree or symlink in source path'; return 1; fi + tree="$oid" + else + if [[ "$mode" != 100644 && "$mode" != 100755 ]] || [ "$type" != blob ] || [ "$oid" != "$blob" ]; then + finding SOURCE-BLOB "$repository@$commit/$path" 'Blob identity or regular-file mode differs from manifest'; return 1 + fi + if ! verify_object blob "$blob" "$dest"; then object_failure blob "$blob" "$repository@$commit/$path"; return 1; fi + fi + done +} + +: > "$TMP/observed.jsonl" +index=0 +while IFS= read -r source; do + repository="$(jq -r '.repository|ascii_downcase' <<< "$source")"; commit="$(jq -r .commit <<< "$source")" + path="$(jq -r .path <<< "$source")"; blob="$(jq -r .blob <<< "$source")"; subject="$repository@$commit/$path" + index=$((index+1)); bytes="$TMP/source-$index.yml" + source_bytes "$repository" "$commit" "$path" "$blob" "$bytes" || continue + parse_incomplete=0 + : > "$TMP/uses" + while IFS=$'\t' read -r tag kind raw job; do + if [ "$tag" = UNPARSED ]; then notrun SOURCE-PARSE "$subject" "$kind"; parse_incomplete=1; continue; fi + if [ "$kind" = job ]; then notrun SOURCE-LOCAL "$subject" 'Nested reusable calls are outside this reviewed direct-action source profile'; parse_incomplete=1; continue; fi + if ! ref="$(normalize_ref "$raw")"; then notrun SOURCE-LOCAL "$subject" "Unsupported local, Docker, dynamic or malformed uses: $raw"; parse_incomplete=1; continue; fi + if [[ ! "${ref##*@}" =~ ^[0-9a-f]{40}$ ]]; then finding SOURCE-PIN "$subject" "Authenticated direct remote uses must have a literal full 40-hex source pin: $raw"; continue; fi + jq -nc --arg kind "$kind" --arg ref "$raw" '{kind:$kind,ref:$ref}' >> "$TMP/uses" + done < <(extract_uses "$bytes") + [ "$parse_incomplete" -eq 0 ] || continue + jq -s 'unique_by(.kind,.ref)|sort_by(.kind,.ref)' "$TMP/uses" > "$TMP/actual-uses" + jq '.uses|map({kind,ref})|unique_by(.kind,.ref)|sort_by(.kind,.ref)' <<< "$source" > "$TMP/expected-uses" + if ! cmp -s "$TMP/actual-uses" "$TMP/expected-uses"; then + finding SOURCE-EDGE-DROPPED "$subject" 'Reviewed direct edge set differs from the exact immutable source'; continue + fi + jq -nc --arg repository "$repository" --arg commit "$commit" --slurpfile uses "$TMP/actual-uses" \ + '{repository:$repository,commit:$commit,uses:$uses[0]}' >> "$TMP/observed.jsonl" +done < <(jq -c '.sources[]' "$MANIFEST") + +# Do not turn absent authenticated data into a misleading edge mismatch. +# No source-derived records are exported unless the whole evidence pass is +# complete, including every retained-review path. +if [ -s "$TMP/findings" ] || [ -s "$TMP/notrun" ]; then finish; exit $?; fi + +# A retained source may extend an existing reviewed union, but cannot create +# an unrelated historical dependency population with no current caller. +while IFS= read -r group; do + repository="$(jq -r .repository <<< "$group")"; commit="$(jq -r .commit <<< "$group")"; rid="$repository@$commit" + if ! jq -e --arg r "$repository" --arg c "$commit" 'any(.sources[]; (.repository|ascii_downcase)==$r and .commit==$c and .role=="active")' "$MANIFEST" >/dev/null; then + finding SOURCE-ACTIVE-MISSING "$rid" 'Retained-review evidence has no active source group'; continue + fi + : > "$TMP/group-edges" + while IFS= read -r raw; do normalize_ref "$raw" >> "$TMP/group-edges" || finding SOURCE-DEPENDENCY "$rid" "Invalid edge: $raw"; done < <(jq -r '.uses[].ref' <<< "$group") + LC_ALL=C sort -u "$TMP/group-edges" > "$TMP/group-normalized" + awk -F '\t' -v repo="$repository" -v sha="$commit" '$0!~/^#/ && tolower($1)==repo && $2==sha {print $3}' "$RECEIPTS" | LC_ALL=C sort -u > "$TMP/receipt-edges" + if ! cmp -s "$TMP/group-normalized" "$TMP/receipt-edges"; then finding SOURCE-RECEIPT "$rid" 'Authenticated reviewed source union differs from the active receipt rows'; fi + edges="$(jq -Rsc 'split("\n")|map(select(length>0))' "$TMP/group-normalized")" + if jq -e --arg k "$rid" 'has($k)' "$TMP/graph.json" >/dev/null; then + prior="$(jq -c --arg k "$rid" '.[$k].uses//[]|sort|unique' "$TMP/graph.json")" + if [ "$prior" != "$(jq -c 'sort|unique' <<< "$edges")" ]; then finding SOURCE-DEPENDENCY "$rid" 'Source parent record differs from its authenticated direct-edge union'; fi + else + jq --arg k "$rid" --arg sha "$commit" --argjson edges "$edges" '.+{($k):{ref:$sha,commit:("sha1-"+$sha),uses:$edges}}' "$TMP/supplement.json" > "$TMP/next.json" + mv "$TMP/next.json" "$TMP/supplement.json" + jq -s '.[0]*.[1]' "$TMP/vendor.json" "$TMP/supplement.json" > "$TMP/graph.json" + fi + jq -nc --arg rid "$rid" '$rid' >> "$TMP/groups" +done < <(jq -sc 'sort_by(.repository,.commit)|group_by(.repository,.commit)|map({repository:.[0].repository,commit:.[0].commit,uses:[.[].uses[]]})[]' "$TMP/observed.jsonl") + +# Validate identities, exact overlaps and all dependency targets without +# claiming recursive source authentication beyond the reviewed direct sets. +while IFS= read -r entry; do + key="$(jq -r .key <<< "$entry")"; normalized="$(normalize_ref "$key")" || normalized="" + if [ "$normalized" != "$key" ] || ! jq -e '.value|(.ref|type=="string" and test("^[^[:space:]@]+$")) and (.commit|type=="string" and test("^sha1-[0-9a-f]{40}$")) and ((.uses//[])|type=="array")' <<< "$entry" >/dev/null; then + finding SOURCE-DEPENDENCY "$key" 'Malformed supplemental dependency identity'; continue + fi + ref="$(jq -r .value.ref <<< "$entry")"; digest="$(jq -r .value.commit <<< "$entry")"; named="${key##*@}" + if [[ "$ref" =~ ^[0-9a-f]{40}$ ]] && [ "$digest" != "sha1-$ref" ]; then finding SOURCE-DEPENDENCY "$key" 'Full-SHA ref and resolved commit disagree'; fi + if [[ "$ref" == sha1-* ]]; then finding SOURCE-DEPENDENCY "$key" 'Dependency ref must not use the commit digest prefix'; fi + if [[ "$named" =~ ^[0-9a-f]{40}$ ]]; then + [ "$digest" = "sha1-$named" ] || finding SOURCE-DEPENDENCY "$key" 'SHA key and resolved commit disagree' + elif [ "$named" != "$ref" ]; then finding SOURCE-DEPENDENCY "$key" 'Named ref and dependency ref disagree'; fi + if jq -e --arg k "$key" 'has($k)' "$TMP/vendor.json" >/dev/null; then + left="$(jq -cS --arg k "$key" '.[$k]|{ref,commit,uses:(.uses//[]|sort)} + (if has("owner_id") then {owner_id} else {} end) + (if has("repo_id") then {repo_id} else {} end)' "$TMP/vendor.json")" + right="$(jq -cS '.value|{ref,commit,uses:(.uses//[]|sort)} + (if has("owner_id") then {owner_id} else {} end) + (if has("repo_id") then {repo_id} else {} end)' <<< "$entry")" + [ "$left" = "$right" ] || finding SOURCE-DEPENDENCY "$key" 'Shared vendor and supplemental records disagree' + fi +done < <(jq -c 'to_entries[]' "$TMP/supplement.json") +declare -A VISITING=() DONE=() +visit() { + local key="$1" depth="$2" child + [ -z "${DONE[$key]:-}" ] || return 0 + if [ "$depth" -gt 100 ] || [ -n "${VISITING[$key]:-}" ]; then finding SOURCE-DEPENDENCY "$key" 'Cyclic or excessively deep supplemental graph'; return 0; fi + if ! jq -e --arg k "$key" 'has($k)' "$TMP/graph.json" >/dev/null; then finding SOURCE-DEPENDENCY "$key" 'Required direct/transitive dependency record is missing'; return 0; fi + VISITING["$key"]=1 + while IFS= read -r child; do + if ! normalized="$(normalize_ref "$child")" || [ "$child" != "$normalized" ]; then finding SOURCE-DEPENDENCY "$key" "Invalid nested edge: $child"; continue; fi + visit "$child" "$((depth+1))" + done < <(jq -r --arg k "$key" '.[$k].uses//[]|.[]' "$TMP/graph.json") + unset 'VISITING[$key]'; DONE["$key"]=1 +} +while IFS= read -r key; do visit "$key" 0; done < <(jq -r 'keys[]' "$TMP/supplement.json") +finish diff --git a/scripts/audit-workflow-lock-map.py b/scripts/audit-workflow-lock-map.py index 46794ce5..f2fa46e3 100755 --- a/scripts/audit-workflow-lock-map.py +++ b/scripts/audit-workflow-lock-map.py @@ -97,7 +97,7 @@ def inspect(root, unmanaged=()): doc = load(file) if not isinstance(doc, dict) or not isinstance(doc.get('jobs'), dict): raise ValueError(f'{name}: expected jobs mapping') - refs, local_calls, unsupported = set(), [], [] + refs, remote_calls, local_calls, unsupported = set(), set(), [], [] for job in doc['jobs'].values(): if not isinstance(job, dict): raise ValueError(f'{name}: invalid job') @@ -115,9 +115,17 @@ def inspect(root, unmanaged=()): for raw, is_job in entries: normalized = ref_key(raw) if normalized: - refs.add(normalized) if is_job: + # Official gh-actions-lock v0.4.1 excludes external + # jobs..uses. Check their source pins separately; + # do not infer reusable dependency closure from the lock. + remote_calls.add(normalized) + if not re.fullmatch(r'[0-9a-fA-F]{40}', normalized.rsplit('@', 1)[1]): + findings.append(f'REMOTE-REUSABLE-NOT-SHA {name} {normalized}') notes.append(f'REMOTE-REUSABLE-SOURCE-REVIEW {name} {normalized}') + notes.append(f'UNSUPPORTED-LOCK-SCOPE {name} {normalized} (external reusable dependencies not locked)') + else: + refs.add(normalized) elif raw.startswith('./') and is_job: local = (root / raw).resolve() if not local.is_relative_to(wfdir) or local.suffix not in ('.yml', '.yaml') or not local.is_file(): @@ -147,7 +155,7 @@ def inspect(root, unmanaged=()): findings.append(f'MISSING-DIRECT-EDGE {name} {ref}') for ref in sorted(mapped_set - depkeys): findings.append(f'MISSING-RECORD {name} {ref}') - if not refs and not local_calls and not unsupported and not mapped_set: + if not refs and not remote_calls and not local_calls and not unsupported and not mapped_set: notes.append(f'NO-USES-MAPPED {name}') for target in local_calls: notes.append(f'LOCAL-REUSABLE {name} -> {target} (callee checked separately)') diff --git a/scripts/lib/lock-closure-receipts.tsv b/scripts/lib/lock-closure-receipts.tsv index 69aedb7d..fe5d5bd4 100644 --- a/scripts/lib/lock-closure-receipts.tsv +++ b/scripts/lib/lock-closure-receipts.tsv @@ -53,3 +53,12 @@ metadatastician/standards 75bd5994470661a099df562d6d5ad2fe744dbd66 webfactory/ss actions/upload-pages-artifact fc324d3547104276b827a68afc52ff2a11cc49c9 actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # Reviewed source: hyperpolymath/cicd-suite/.github/workflows/secret-scanner-reusable.yml@4404c3c0474e17753fe7689f3485c02890aac113 hyperpolymath/cicd-suite 4404c3c0474e17753fe7689f3485c02890aac113 actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + +# Reviewed source: metadatastician/standards/.github/workflows/governance-reusable.yml@ea2f4d8a886585ed442687ae63d4ad7493379ba4 +# Exact Git blob: 8ac530b97d18f7d91972de91c448b7b44c055dfb; source-bound verification is required. +metadatastician/standards ea2f4d8a886585ed442687ae63d4ad7493379ba4 actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 +metadatastician/standards ea2f4d8a886585ed442687ae63d4ad7493379ba4 actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 +metadatastician/standards ea2f4d8a886585ed442687ae63d4ad7493379ba4 denoland/setup-deno@22d081ff2d3a40755e97629de92e3bcbfa7cf2ed +metadatastician/standards ea2f4d8a886585ed442687ae63d4ad7493379ba4 editorconfig-checker/action-editorconfig-checker@51f63319f592f97930c73d9c46184d20bd206393 +metadatastician/standards ea2f4d8a886585ed442687ae63d4ad7493379ba4 erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 +metadatastician/standards ea2f4d8a886585ed442687ae63d4ad7493379ba4 oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 diff --git a/scripts/lib/pinned-reusable-sources.json b/scripts/lib/pinned-reusable-sources.json new file mode 100644 index 00000000..0c18499f --- /dev/null +++ b/scripts/lib/pinned-reusable-sources.json @@ -0,0 +1,827 @@ +{ + "version": 1, + "sources": [ + { + "repository": "metadatastician/standards", + "commit": "ea2f4d8a886585ed442687ae63d4ad7493379ba4", + "path": ".github/workflows/governance-reusable.yml", + "blob": "8ac530b97d18f7d91972de91c448b7b44c055dfb", + "content_sha256": "3b7b753935d0d40f8b854e1b337d7aced884eef60e139292533fdf9f4517577d", + "role": "active", + "callers": [ + { + "workflow": ".github/workflows/governance.yml", + "job": "governance", + "line": 38 + } + ], + "uses": [ + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 51 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 77 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 118 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 138 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 164 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 184 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 207 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 227 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 279 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 343 + }, + { + "kind": "step", + "ref": "erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124", + "line": 349 + }, + { + "kind": "step", + "ref": "actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9", + "line": 376 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 446 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 544 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 590 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 885 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 916 + }, + { + "kind": "step", + "ref": "oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6", + "line": 934 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 962 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1204 + }, + { + "kind": "step", + "ref": "editorconfig-checker/action-editorconfig-checker@51f63319f592f97930c73d9c46184d20bd206393", + "line": 1251 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1278 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1393 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1474 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1487 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1596 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1636 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1655 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1787 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1815 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1832 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1860 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1880 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1904 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1966 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1990 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 2053 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 2073 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 2109 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 2141 + }, + { + "kind": "step", + "ref": "oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6", + "line": 2152 + }, + { + "kind": "step", + "ref": "denoland/setup-deno@22d081ff2d3a40755e97629de92e3bcbfa7cf2ed", + "line": 2157 + } + ], + "source_url": "https://github.com/metadatastician/standards/blob/ea2f4d8a886585ed442687ae63d4ad7493379ba4/.github/workflows/governance-reusable.yml", + "local_file": "C:\\Users\\sudoodle\\OneDrive\\Documents\\ChatGPT\\TYPE THEORY\\echo-reader-compat-20261106\\closure-cycle-20261107\\sources\\metadatastician\\standards\\ea2f4d8a886585ed442687ae63d4ad7493379ba4\\.github\\workflows\\governance-reusable.yml" + }, + { + "repository": "metadatastician/standards", + "commit": "75bd5994470661a099df562d6d5ad2fe744dbd66", + "path": ".github/workflows/hypatia-scan-reusable.yml", + "blob": "0f1c5ba55861c84f6665a103deb6ae184bb89996", + "content_sha256": "eb72f4afd7bf17d2b31b808131ad697bb808a6ae0c8aa07773b5e1dbb4b7df7c", + "role": "active", + "callers": [ + { + "workflow": ".github/workflows/hypatia-scan.yml", + "job": "hypatia", + "line": 34 + } + ], + "uses": [ + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 55 + }, + { + "kind": "step", + "ref": "erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124", + "line": 61 + }, + { + "kind": "step", + "ref": "actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9", + "line": 87 + }, + { + "kind": "step", + "ref": "actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9", + "line": 234 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 389 + }, + { + "kind": "step", + "ref": "github/codeql-action/upload-sarif@b96794f015dfd88f77b49b1c93e0fa7110f94c63", + "line": 439 + }, + { + "kind": "step", + "ref": "actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9", + "line": 445 + } + ], + "source_url": "https://github.com/metadatastician/standards/blob/75bd5994470661a099df562d6d5ad2fe744dbd66/.github/workflows/hypatia-scan-reusable.yml", + "local_file": "C:\\Users\\sudoodle\\OneDrive\\Documents\\ChatGPT\\TYPE THEORY\\echo-reader-compat-20261106\\closure-cycle-20261107\\sources\\metadatastician\\standards\\75bd5994470661a099df562d6d5ad2fe744dbd66\\.github\\workflows\\hypatia-scan-reusable.yml" + }, + { + "repository": "metadatastician/standards", + "commit": "75bd5994470661a099df562d6d5ad2fe744dbd66", + "path": ".github/workflows/mirror-reusable.yml", + "blob": "ec52acc3a975fbd528d9a38741bd868fb55de6e5", + "content_sha256": "bc7443e325e255dd19baf48722583f07f7be1cd0cdce0f1ce229cfbeab3d17a1", + "role": "active", + "callers": [ + { + "workflow": ".github/workflows/mirror.yml", + "job": "mirror", + "line": 18 + } + ], + "uses": [ + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 91 + }, + { + "kind": "step", + "ref": "webfactory/ssh-agent@e83874834305fe9a4a2997156cb26c5de65a8555", + "line": 95 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 149 + }, + { + "kind": "step", + "ref": "webfactory/ssh-agent@e83874834305fe9a4a2997156cb26c5de65a8555", + "line": 153 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 201 + }, + { + "kind": "step", + "ref": "webfactory/ssh-agent@e83874834305fe9a4a2997156cb26c5de65a8555", + "line": 205 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 253 + }, + { + "kind": "step", + "ref": "webfactory/ssh-agent@e83874834305fe9a4a2997156cb26c5de65a8555", + "line": 257 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 305 + }, + { + "kind": "step", + "ref": "webfactory/ssh-agent@e83874834305fe9a4a2997156cb26c5de65a8555", + "line": 309 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 357 + }, + { + "kind": "step", + "ref": "webfactory/ssh-agent@e83874834305fe9a4a2997156cb26c5de65a8555", + "line": 361 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 411 + }, + { + "kind": "step", + "ref": "dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f", + "line": 427 + } + ], + "source_url": "https://github.com/metadatastician/standards/blob/75bd5994470661a099df562d6d5ad2fe744dbd66/.github/workflows/mirror-reusable.yml", + "local_file": "C:\\Users\\sudoodle\\OneDrive\\Documents\\ChatGPT\\TYPE THEORY\\echo-reader-compat-20261106\\closure-cycle-20261107\\sources\\metadatastician\\standards\\75bd5994470661a099df562d6d5ad2fe744dbd66\\.github\\workflows\\mirror-reusable.yml" + }, + { + "repository": "metadatastician/standards", + "commit": "75bd5994470661a099df562d6d5ad2fe744dbd66", + "path": ".github/workflows/scorecard-reusable.yml", + "blob": "4e4fd33a6259b34cdc8c5f08d0d4280a216fb492", + "content_sha256": "db6ae19dcb09329533d1537e63c329d469dc602badcb4cc514af5da7b9b3a9b6", + "role": "active", + "callers": [ + { + "workflow": ".github/workflows/scorecard.yml", + "job": "scorecard", + "line": 21 + } + ], + "uses": [ + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 33 + }, + { + "kind": "step", + "ref": "ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc", + "line": 38 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 55 + }, + { + "kind": "step", + "ref": "github/codeql-action/upload-sarif@b96794f015dfd88f77b49b1c93e0fa7110f94c63", + "line": 102 + }, + { + "kind": "step", + "ref": "actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9", + "line": 107 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 139 + }, + { + "kind": "step", + "ref": "ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc", + "line": 146 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 199 + }, + { + "kind": "step", + "ref": "github/codeql-action/upload-sarif@b96794f015dfd88f77b49b1c93e0fa7110f94c63", + "line": 246 + }, + { + "kind": "step", + "ref": "actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9", + "line": 250 + } + ], + "source_url": "https://github.com/metadatastician/standards/blob/75bd5994470661a099df562d6d5ad2fe744dbd66/.github/workflows/scorecard-reusable.yml", + "local_file": "C:\\Users\\sudoodle\\OneDrive\\Documents\\ChatGPT\\TYPE THEORY\\echo-reader-compat-20261106\\closure-cycle-20261107\\sources\\metadatastician\\standards\\75bd5994470661a099df562d6d5ad2fe744dbd66\\.github\\workflows\\scorecard-reusable.yml" + }, + { + "repository": "hyperpolymath/cicd-suite", + "commit": "4404c3c0474e17753fe7689f3485c02890aac113", + "path": ".github/workflows/secret-scanner-reusable.yml", + "blob": "81b2ba1dfb074f6f8913baa81eb6768b7497cc2a", + "content_sha256": "4745e9318346b1a3eae2e0bf65c6732f73d2ead78ef29de9e20659a71f8057d8", + "role": "active", + "callers": [ + { + "workflow": ".github/workflows/secret-scanner.yml", + "job": "scan", + "line": 39 + } + ], + "uses": [ + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 28 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 50 + } + ], + "source_url": "https://github.com/hyperpolymath/cicd-suite/blob/4404c3c0474e17753fe7689f3485c02890aac113/.github/workflows/secret-scanner-reusable.yml", + "local_file": "C:\\Users\\sudoodle\\OneDrive\\Documents\\ChatGPT\\TYPE THEORY\\echo-reader-compat-20261106\\closure-cycle-20261107\\sources\\hyperpolymath\\cicd-suite\\4404c3c0474e17753fe7689f3485c02890aac113\\.github\\workflows\\secret-scanner-reusable.yml" + }, + { + "repository": "metadatastician/standards", + "commit": "75bd5994470661a099df562d6d5ad2fe744dbd66", + "path": ".github/workflows/governance-reusable.yml", + "blob": "b041be08dc971f789ef3bc4d7849ac336ab30206", + "content_sha256": "2e8bab47c6fcdfa0b94cc866214c337950a20c13a8af4da179f06485bf494f0c", + "role": "retained-review", + "callers": [], + "uses": [ + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 41 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 53 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 94 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 100 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 126 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 132 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 155 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 161 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 199 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 263 + }, + { + "kind": "step", + "ref": "erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124", + "line": 269 + }, + { + "kind": "step", + "ref": "actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9", + "line": 296 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 352 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 435 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 470 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 765 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 782 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 815 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1057 + }, + { + "kind": "step", + "ref": "editorconfig-checker/action-editorconfig-checker@51f63319f592f97930c73d9c46184d20bd206393", + "line": 1104 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1117 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1232 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1298 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1311 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1415 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1452 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1457 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1589 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1603 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1620 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1634 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1654 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1664 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1699 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1709 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1770 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1776 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1811 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 1829 + }, + { + "kind": "step", + "ref": "oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6", + "line": 1840 + }, + { + "kind": "step", + "ref": "denoland/setup-deno@22d081ff2d3a40755e97629de92e3bcbfa7cf2ed", + "line": 1845 + } + ], + "source_url": "https://github.com/metadatastician/standards/blob/75bd5994470661a099df562d6d5ad2fe744dbd66/.github/workflows/governance-reusable.yml", + "local_file": "C:\\Users\\sudoodle\\OneDrive\\Documents\\ChatGPT\\TYPE THEORY\\echo-reader-compat-20261106\\closure-cycle-20261107\\sources\\metadatastician\\standards\\75bd5994470661a099df562d6d5ad2fe744dbd66\\.github\\workflows\\governance-reusable.yml" + }, + { + "repository": "metadatastician/standards", + "commit": "75bd5994470661a099df562d6d5ad2fe744dbd66", + "path": ".github/workflows/secret-scanner-reusable.yml", + "blob": "2eee815b0e4c4c70a777b9c1d8c899b1fb7026c9", + "content_sha256": "0c132f45da9832caf7cdb5dcfa164a727208e397f97e51f6a335a17464f2a0fd", + "role": "retained-review", + "callers": [], + "uses": [ + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 123 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 199 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 275 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 541 + }, + { + "kind": "step", + "ref": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "line": 702 + } + ], + "source_url": "https://github.com/metadatastician/standards/blob/75bd5994470661a099df562d6d5ad2fe744dbd66/.github/workflows/secret-scanner-reusable.yml", + "local_file": "C:\\Users\\sudoodle\\OneDrive\\Documents\\ChatGPT\\TYPE THEORY\\echo-reader-compat-20261106\\closure-cycle-20261107\\sources\\metadatastician\\standards\\75bd5994470661a099df562d6d5ad2fe744dbd66\\.github\\workflows\\secret-scanner-reusable.yml" + } + ], + "dependencies": { + "denoland/setup-deno@22d081ff2d3a40755e97629de92e3bcbfa7cf2ed": { + "ref": "v2.0.5", + "commit": "sha1-22d081ff2d3a40755e97629de92e3bcbfa7cf2ed", + "owner_id": 42048915, + "repo_id": 356423100, + "uses": [], + "original_lock_record": " 'denoland/setup-deno@22d081ff2d3a40755e97629de92e3bcbfa7cf2ed':\n ref: 'v2.0.5'\n commit: 'sha1-22d081ff2d3a40755e97629de92e3bcbfa7cf2ed'\n owner_id: 42048915\n repo_id: 356423100\n" + }, + "dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f": { + "ref": "v1", + "commit": "sha1-e2a55d2ffb04f378e9626c28d38b36d230d1e12f", + "owner_id": 1940490, + "repo_id": 260749683, + "uses": [], + "original_lock_record": " 'dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f':\n ref: 'v1'\n commit: 'sha1-e2a55d2ffb04f378e9626c28d38b36d230d1e12f'\n owner_id: 1940490\n repo_id: 260749683\n" + }, + "editorconfig-checker/action-editorconfig-checker@51f63319f592f97930c73d9c46184d20bd206393": { + "ref": "v3.0.0", + "commit": "sha1-51f63319f592f97930c73d9c46184d20bd206393", + "owner_id": 26415196, + "repo_id": 297874902, + "uses": [], + "original_lock_record": " 'editorconfig-checker/action-editorconfig-checker@51f63319f592f97930c73d9c46184d20bd206393':\n ref: 'v3.0.0'\n commit: 'sha1-51f63319f592f97930c73d9c46184d20bd206393'\n owner_id: 26415196\n repo_id: 297874902\n" + }, + "erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124": { + "ref": "v1.24.1", + "commit": "sha1-54075bcc5e249e4758d363f27d099f55d843f124", + "owner_id": 47606891, + "repo_id": 331103973, + "uses": [], + "original_lock_record": " 'erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124':\n ref: 'v1.24.1'\n commit: 'sha1-54075bcc5e249e4758d363f27d099f55d843f124'\n owner_id: 47606891\n repo_id: 331103973\n" + }, + "github/codeql-action@b96794f015dfd88f77b49b1c93e0fa7110f94c63": { + "ref": "v4.38.0", + "commit": "sha1-b96794f015dfd88f77b49b1c93e0fa7110f94c63", + "owner_id": 9919, + "repo_id": 259445878, + "uses": [], + "original_lock_record": " 'github/codeql-action@b96794f015dfd88f77b49b1c93e0fa7110f94c63':\n ref: 'v4.38.0'\n commit: 'sha1-b96794f015dfd88f77b49b1c93e0fa7110f94c63'\n owner_id: 9919\n repo_id: 259445878\n" + }, + "hyperpolymath/cicd-suite@4404c3c0474e17753fe7689f3485c02890aac113": { + "ref": "4404c3c0474e17753fe7689f3485c02890aac113", + "commit": "sha1-4404c3c0474e17753fe7689f3485c02890aac113", + "owner_id": 6759885, + "repo_id": 1326697643, + "uses": [ + "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1" + ], + "original_lock_record": " 'hyperpolymath/cicd-suite@4404c3c0474e17753fe7689f3485c02890aac113':\n ref: '4404c3c0474e17753fe7689f3485c02890aac113'\n commit: 'sha1-4404c3c0474e17753fe7689f3485c02890aac113'\n owner_id: 6759885\n repo_id: 1326697643\n uses:\n - 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1'\n" + }, + "metadatastician/standards@75bd5994470661a099df562d6d5ad2fe744dbd66": { + "ref": "75bd5994470661a099df562d6d5ad2fe744dbd66", + "commit": "sha1-75bd5994470661a099df562d6d5ad2fe744dbd66", + "owner_id": 297579217, + "repo_id": 1116521501, + "uses": [ + "actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9", + "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9", + "denoland/setup-deno@22d081ff2d3a40755e97629de92e3bcbfa7cf2ed", + "dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f", + "editorconfig-checker/action-editorconfig-checker@51f63319f592f97930c73d9c46184d20bd206393", + "erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124", + "github/codeql-action@b96794f015dfd88f77b49b1c93e0fa7110f94c63", + "ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc", + "oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6", + "webfactory/ssh-agent@e83874834305fe9a4a2997156cb26c5de65a8555" + ], + "original_lock_record": " 'metadatastician/standards@75bd5994470661a099df562d6d5ad2fe744dbd66':\n ref: '75bd5994470661a099df562d6d5ad2fe744dbd66'\n commit: 'sha1-75bd5994470661a099df562d6d5ad2fe744dbd66'\n owner_id: 297579217\n repo_id: 1116521501\n uses:\n - 'actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9'\n - 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1'\n - 'actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9'\n - 'denoland/setup-deno@22d081ff2d3a40755e97629de92e3bcbfa7cf2ed'\n - 'dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f'\n - 'editorconfig-checker/action-editorconfig-checker@51f63319f592f97930c73d9c46184d20bd206393'\n - 'erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124'\n - 'github/codeql-action@b96794f015dfd88f77b49b1c93e0fa7110f94c63'\n - 'ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc'\n - 'oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6'\n - 'webfactory/ssh-agent@e83874834305fe9a4a2997156cb26c5de65a8555'\n" + }, + "ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc": { + "ref": "v2.4.4", + "commit": "sha1-2d1146689b8cda280b9bc96326124645441f03bc", + "owner_id": 67707773, + "repo_id": 421101922, + "uses": [], + "original_lock_record": " 'ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc':\n ref: 'v2.4.4'\n commit: 'sha1-2d1146689b8cda280b9bc96326124645441f03bc'\n owner_id: 67707773\n repo_id: 421101922\n" + }, + "oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6": { + "ref": "v2.2.0", + "commit": "sha1-0c5077e51419868618aeaa5fe8019c62421857d6", + "owner_id": 108928776, + "repo_id": 512644635, + "uses": [], + "original_lock_record": " 'oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6':\n ref: 'v2.2.0'\n commit: 'sha1-0c5077e51419868618aeaa5fe8019c62421857d6'\n owner_id: 108928776\n repo_id: 512644635\n" + }, + "webfactory/ssh-agent@e83874834305fe9a4a2997156cb26c5de65a8555": { + "ref": "v0.10.0", + "commit": "sha1-e83874834305fe9a4a2997156cb26c5de65a8555", + "owner_id": 135788, + "repo_id": 208510314, + "uses": [], + "original_lock_record": " 'webfactory/ssh-agent@e83874834305fe9a4a2997156cb26c5de65a8555':\n ref: 'v0.10.0'\n commit: 'sha1-e83874834305fe9a4a2997156cb26c5de65a8555'\n owner_id: 135788\n repo_id: 208510314\n" + } + } +} diff --git a/scripts/tests/lock-closure-test.sh b/scripts/tests/lock-closure-test.sh index b6c7186e..153eb775 100755 --- a/scripts/tests/lock-closure-test.sh +++ b/scripts/tests/lock-closure-test.sh @@ -23,6 +23,12 @@ set -uo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" AUDIT="$ROOT/scripts/audit-lock-closure.sh" +# Repository-owned source authentication is an offline gate. Bootstrap the +# reviewed bare Git cache separately; absence is not a passing test result. +if [[ -f "$ROOT/scripts/lib/pinned-reusable-sources.json" && -z "${PINNED_SOURCE_CACHE:-}" ]]; then + echo 'lock-closure-test: set PINNED_SOURCE_CACHE to the prepared pinned-source cache' >&2 + exit 64 +fi WORK="$(mktemp -d "${TMPDIR:-/tmp}/lock-closure-test.XXXXXX")" trap 'rm -rf "$WORK"' EXIT diff --git a/scripts/tests/pinned-reusable-sources-test.sh b/scripts/tests/pinned-reusable-sources-test.sh new file mode 100644 index 00000000..3d81474f --- /dev/null +++ b/scripts/tests/pinned-reusable-sources-test.sh @@ -0,0 +1,270 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell +# Offline commit/tree/blob fixtures; no network, host configuration or credentials. +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +HELPER="$ROOT/scripts/audit-pinned-reusable-sources.sh" +WORK="$(mktemp -d "${TMPDIR:-/tmp}/pinned-reusable-test.XXXXXX")" +trap 'rm -rf "$WORK"' EXIT +export GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null +PASS=0; FAIL=0 +for tool in git jq timeout; do command -v "$tool" >/dev/null || { echo "Missing test prerequisite: $tool" >&2; exit 64; }; done + +# Set fixture identities only for this invocation. No signing key or host Git +# configuration is used: the property under test is immutable object identity. +commit_tree() { + local repo="$1" tree="$2" message="$3" + printf '%s\n' "$message" | GIT_AUTHOR_DATE='2026-11-07T00:00:00Z' GIT_COMMITTER_DATE='2026-11-07T00:00:00Z' \ + git -c user.name='Closure fixture' -c user.email='fixture@example.invalid' -c commit.gpgsign=false \ + --git-dir="$repo" commit-tree "$tree" +} +source_object() { + local repo="$1" text="$2" mode="${3:-100644}" index tree + NEW_BLOB="$(printf '%s\n' "$text" | git --git-dir="$repo" hash-object -w --stdin)" + index="$(mktemp "$WORK/index.XXXXXX")"; rm "$index" + GIT_INDEX_FILE="$index" git --git-dir="$repo" read-tree --empty + GIT_INDEX_FILE="$index" git --git-dir="$repo" update-index --add --cacheinfo "$mode,$NEW_BLOB,.github/workflows/reuse.yml" + GIT_INDEX_FILE="$index" git --git-dir="$repo" update-index --add --cacheinfo "100644,$NEW_BLOB,.github/workflows/other.yml" + tree="$(GIT_INDEX_FILE="$index" git --git-dir="$repo" write-tree)" + NEW_COMMIT="$(commit_tree "$repo" "$tree" 'Fixture source')" + git --git-dir="$repo" update-ref refs/heads/fixture "$NEW_COMMIT" + rm "$index" +} +BASE="$WORK/base"; mkdir -p "$BASE/cache/org" "$BASE/repo/.github/workflows" "$BASE/repo/scripts/lib" +git init -q --bare "$BASE/cache/org/reusable.git" +git init -q --bare "$BASE/cache/org/child.git" +CHILD_SHA="$(commit_tree "$BASE/cache/org/child.git" "$(git --git-dir="$BASE/cache/org/child.git" mktree "$BASE/repo/.github/workflows/test.yml" < "$BASE/repo/scripts/lib/pinned-reusable-sources.json" +cat > "$BASE/repo/.github/workflows/actions.lock" < "$BASE/repo/receipts.tsv" +: > "$BASE/repo/baseline.tsv" +case_fixture() { CASE="$WORK/$1"; mkdir "$CASE"; cp -a "$BASE/." "$CASE/"; MANIFEST="$CASE/repo/scripts/lib/pinned-reusable-sources.json"; } +edit_manifest() { jq "$@" "$MANIFEST" > "$CASE/edit.json"; mv "$CASE/edit.json" "$MANIFEST"; } +replace_pin() { + local old="$1" new="$2" file + for file in "$MANIFEST" "$CASE/repo/.github/workflows/test.yml" "$CASE/repo/.github/workflows/actions.lock" "$CASE/repo/receipts.tsv"; do + sed "s/$old/$new/g" "$file" > "$CASE/replaced"; mv "$CASE/replaced" "$file" + done +} +check() { + local label="$1" expected="$2" rc=0 valid=0 + timeout 15s bash "$HELPER" --json --repo-root "$CASE/repo" --manifest "$MANIFEST" \ + --lock "$CASE/repo/.github/workflows/actions.lock" --receipts "$CASE/repo/receipts.tsv" --cache "$CASE/cache" \ + > "$CASE/result.json" 2> "$CASE/stderr" || rc=$? + if jq -e 'type=="object" and (.findings|type=="array") and (.not_run|type=="array") and (.verified_roots|type=="array") and (.verified_receipt_sources|type=="array")' "$CASE/result.json" >/dev/null 2>&1; then + case "$expected" in + clean) [[ "$rc" == 0 ]] && jq -e '(.findings|length)==0 and (.not_run|length)==0 and (.verified_roots|length)==1 and (.verified_receipt_sources|length)>0' "$CASE/result.json" >/dev/null && valid=1 ;; + fatal) [[ "$rc" == 1 || "$rc" == 3 ]] && jq -e 'any(.findings[]; .severity=="FATAL" and (.rule|length)>0 and (.detail|length)>0) and (.verified_roots|length)==0 and (.verified_receipt_sources|length)==0' "$CASE/result.json" >/dev/null && valid=1 ;; + notrun) [[ "$rc" == 2 ]] && jq -e '(.findings|length)==0 and any(.not_run[]; (.rule|length)>0 and (.detail|length)>0) and (.verified_roots|length)==0 and (.verified_receipt_sources|length)==0' "$CASE/result.json" >/dev/null && valid=1 ;; + esac + fi + if [[ "$valid" == 1 ]]; then PASS=$((PASS+1)); printf 'PASS: %s (%s, exit %s)\n' "$label" "$expected" "$rc" + else FAIL=$((FAIL+1)); printf 'FAIL: %s (expected %s; exit %s)\n' "$label" "$expected" "$rc"; cat "$CASE/result.json" "$CASE/stderr"; fi +} + +# Demonstrate old false passes, rather than counting a missing new helper as +# evidence. Run only against the preimplementation candidate; nonzero is red. +if [[ "${1:-}" == --preimplementation-probe ]]; then + gaps=0 + for attack in wrong-blob omitted-active; do + case_fixture "$attack" + if [[ "$attack" == wrong-blob ]]; then edit_manifest '.sources[0].blob="1111111111111111111111111111111111111111"'; else edit_manifest '.sources=[]'; fi + rc=0 + timeout 15s bash "${CLOSURE_AUDIT_UNDER_TEST:-$ROOT/scripts/audit-lock-closure.sh}" --json --repo-root "$CASE/repo" \ + --workflows "$CASE/repo/.github/workflows" --receipts "$CASE/repo/receipts.tsv" --baseline "$CASE/repo/baseline.tsv" \ + > "$CASE/old.json" 2> "$CASE/old.stderr" || rc=$? + if [[ "$rc" == 0 ]] && jq -e '(.findings|length)==0 and (.not_run|length)==0' "$CASE/old.json" >/dev/null; then + printf 'DEMONSTRATED GAP: old audit accepted %s with matching receipt/lock\n' "$attack"; gaps=$((gaps+1)) + else printf 'Probe did not demonstrate %s (exit %s)\n' "$attack" "$rc"; cat "$CASE/old.json" "$CASE/old.stderr"; fi + done + [[ "$gaps" == 2 ]] && exit 1 + exit 64 +fi +[[ $# == 0 ]] || { echo 'Usage: pinned-reusable-sources-test.sh [--preimplementation-probe]' >&2; exit 64; } +[[ -f "$HELPER" ]] || { echo 'Missing helper; this is a prerequisite failure, not a successful rejection control.' >&2; exit 64; } + +case_fixture good; check 'authenticated active source and exact receipt' clean +case_fixture historical; edit_manifest '.sources += [(.sources[0] | .role="retained-review" | .callers=[] | .path=".github/workflows/other.yml")]'; check 'explicit historical review can coexist with active source' clean +case_fixture unrelated-vendor; cat >> "$CASE/repo/.github/workflows/actions.lock" < "$CASE/wf"; mv "$CASE/wf" "$CASE/repo/.github/workflows/test.yml"; check 'active caller cannot drift to a branch' fatal +case_fixture omitted-active; edit_manifest '.sources=[]'; : > "$CASE/repo/receipts.tsv"; check 'omitted active root is fatal' fatal +case_fixture historical-only; edit_manifest '.sources[0].role="retained-review" | .sources[0].callers=[]'; check 'historical source cannot stand in for active root' fatal +case_fixture wrong-job; edit_manifest '.sources[0].callers[0].job="another"'; check 'exact caller job is bound' fatal +case_fixture wrong-workflow; edit_manifest '.sources[0].callers[0].workflow=".github/workflows/other.yml"'; check 'exact caller workflow is bound' fatal +case_fixture duplicate-source; edit_manifest '.sources += [.sources[0]]'; check 'duplicate source declarations are rejected' fatal +case_fixture dropped-source-edge +edit_manifest --arg root "$ROOT_KEY" '.sources[0].uses=[] | .dependencies[$root].uses=[]' +# Keep lock/catalog/receipts mutually consistent: only the authenticated blob +# reveals the omitted edge, so a receipt-to-lock-only checker cannot pass this. +cat > "$CASE/repo/.github/workflows/actions.lock" < "$CASE/repo/receipts.tsv" +check 'authenticated source edge cannot disappear from consistent metadata' fatal +case_fixture extra-source-edge +extra="extra/action@$CHILD_SHA" +edit_manifest --arg root "$ROOT_KEY" --arg extra "$extra" --arg sha "$CHILD_SHA" '.sources[0].uses += [{kind:"step",ref:$extra}] | .dependencies[$root].uses += [$extra] | .dependencies[$extra]={ref:"v1",commit:("sha1-"+$sha)}' +awk -v child=" - '$CHILD_KEY'" -v extra=" - '$extra'" '{print} $0==child {print extra}' "$CASE/repo/.github/workflows/actions.lock" > "$CASE/l"; mv "$CASE/l" "$CASE/repo/.github/workflows/actions.lock" +cat >> "$CASE/repo/.github/workflows/actions.lock" <> "$CASE/repo/receipts.tsv" +check 'consistent metadata cannot invent an edge absent from authenticated blob' fatal +case_fixture wrong-edge-kind; edit_manifest '.sources[0].uses[0].kind="job"'; check 'job and step source references remain distinct' fatal +case_fixture missing-receipt; : > "$CASE/repo/receipts.tsv"; check 'missing receipt union is fatal despite valid source' fatal +case_fixture extra-receipt; printf 'org/reusable\t%s\textra/action@v1\n' "$ROOT_SHA" >> "$CASE/repo/receipts.tsv"; check 'retained receipt union cannot contain an extra edge' fatal +case_fixture wrong-receipt-sha; sed "s/$ROOT_SHA/$CHILD_SHA/" "$CASE/repo/receipts.tsv" > "$CASE/r"; mv "$CASE/r" "$CASE/repo/receipts.tsv"; check 'receipt for another source commit cannot satisfy union' fatal +case_fixture changed-child-pin; edit_manifest --arg child "$CHILD_KEY" '.dependencies[$child].commit="sha1-1111111111111111111111111111111111111111"'; check 'shared vendor immutable pin must match supplement' fatal +case_fixture changed-vendor-ref; sed "s/ref: 'v1'/ref: 'v2'/" "$CASE/repo/.github/workflows/actions.lock" > "$CASE/l"; mv "$CASE/l" "$CASE/repo/.github/workflows/actions.lock"; check 'changed shared vendor reviewed ref is fatal' fatal +case_fixture changed-vendor-edge; cat >> "$CASE/repo/.github/workflows/actions.lock" < "$CASE/l"; mv "$CASE/l" "$CASE/repo/.github/workflows/actions.lock"; check 'omitted supplemental and vendor child is fatal' fatal +case_fixture dependency-cycle; edit_manifest --arg root "$ROOT_KEY" --arg child "$CHILD_KEY" '.dependencies[$child].uses=[$root]'; cat >> "$CASE/repo/.github/workflows/actions.lock" < "$MANIFEST"; check 'malformed catalog is fatal' fatal +case_fixture bad-lock; printf 'version: bad\nworkflows: []\ndependencies: []\n' > "$CASE/repo/.github/workflows/actions.lock"; check 'malformed vendor lock cannot yield source verification' fatal +case_fixture duplicate-lock-field +awk '{print} $0==" ref: '\''v1'\''" {print}' "$CASE/repo/.github/workflows/actions.lock" > "$CASE/l"; mv "$CASE/l" "$CASE/repo/.github/workflows/actions.lock" +check 'duplicate vendor scalar field cannot be certified by dump' fatal +case_fixture replacement; other="$(commit_tree "$CASE/cache/org/reusable.git" "$(git --git-dir="$CASE/cache/org/reusable.git" rev-parse "$ROOT_SHA^{tree}")" 'Replacement')"; git --git-dir="$CASE/cache/org/reusable.git" update-ref "refs/replace/$ROOT_SHA" "$other"; check 'replacement refs cannot authenticate source objects' fatal +case_fixture graft; mkdir -p "$CASE/cache/org/reusable.git/info"; printf '%s %s\n' "$ROOT_SHA" "$CHILD_SHA" > "$CASE/cache/org/reusable.git/info/grafts"; check 'grafts cannot authenticate source objects' fatal +case_fixture corrupt-object; object="$CASE/cache/org/reusable.git/objects/${ROOT_BLOB:0:2}/${ROOT_BLOB:2}"; chmod u+w "$object"; printf 'corrupted loose object\n' > "$object"; check 'tampered Git object bytes are fatal' fatal + +# Preserve metadata shape from the real inventory; it is provenance text, +# never a replacement for the structured dependency record. +case_fixture original-lock-text +awk -v child=" '$CHILD_KEY':" 'seen && $0==child {exit} /^ / {seen=1} seen {print}' "$CASE/repo/.github/workflows/actions.lock" > "$CASE/original-record" +edit_manifest --rawfile record "$CASE/original-record" '.sources[0].original_lock_record=$record' +check 'original_lock_record YAML text remains metadata' clean +case_fixture supplemental-only +awk -v child=" '$CHILD_KEY':" '$0==child {exit} {print}' "$CASE/repo/.github/workflows/actions.lock" > "$CASE/l"; mv "$CASE/l" "$CASE/repo/.github/workflows/actions.lock" +check 'declared child can supplement absent vendor record' clean +case_fixture supplemental-ref-mismatch +awk -v child=" '$CHILD_KEY':" '$0==child {exit} {print}' "$CASE/repo/.github/workflows/actions.lock" > "$CASE/l"; mv "$CASE/l" "$CASE/repo/.github/workflows/actions.lock" +edit_manifest --arg child "$CHILD_KEY" '.dependencies[$child].ref="1111111111111111111111111111111111111111"' +check 'injected full-SHA ref cannot disagree with its commit' fatal +case_fixture case-overlap +sed 's/org\/child@/ORG\/CHILD@/g' "$CASE/repo/.github/workflows/actions.lock" > "$CASE/l"; mv "$CASE/l" "$CASE/repo/.github/workflows/actions.lock" +edit_manifest --arg child "$CHILD_KEY" '.dependencies[$child].ref="v2"' +check 'case-folded vendor overlap still detects disagreement' fatal +case_fixture case-collision +edit_manifest --arg child "$CHILD_KEY" --arg upper "ORG/CHILD@$CHILD_SHA" '.dependencies[$upper]=.dependencies[$child]' +check 'case-normalized supplemental key collision is rejected' fatal +case_fixture duplicate-json-key +sed 's/"version": 1,/"version": 1, "version": 1,/' "$MANIFEST" > "$CASE/duplicate.json"; mv "$CASE/duplicate.json" "$MANIFEST" +check 'duplicate JSON mapping keys are rejected' fatal +case_fixture missing-tree +tree="$(git --git-dir="$CASE/cache/org/reusable.git" rev-parse "$ROOT_SHA^{tree}")" +rm "$CASE/cache/org/reusable.git/objects/${tree:0:2}/${tree:2}" +check 'incomplete cache missing a tree is NOT RUN' notrun +case_fixture missing-blob +rm "$CASE/cache/org/reusable.git/objects/${ROOT_BLOB:0:2}/${ROOT_BLOB:2}" +check 'incomplete cache missing a blob is NOT RUN' notrun +case_fixture corrupt-tree +tree="$(git --git-dir="$CASE/cache/org/reusable.git" rev-parse "$ROOT_SHA^{tree}")" +object="$CASE/cache/org/reusable.git/objects/${tree:0:2}/${tree:2}"; chmod u+w "$object"; printf 'corrupted tree\n' > "$object" +check 'tampered tree bytes are fatal' fatal +case_fixture corrupt-commit +object="$CASE/cache/org/reusable.git/objects/${ROOT_SHA:0:2}/${ROOT_SHA:2}"; chmod u+w "$object"; printf 'corrupted commit\n' > "$object" +check 'tampered commit bytes are fatal' fatal +# Unsupported source constructs must remain explicit unavailable evidence, +# rather than pretending local/docker dependencies were independently closed. +for variant in local-action local-reusable nested-reusable docker block-scalar alias merge flow cyclic-alias tagged-action; do + case_fixture "$variant" + case "$variant" in + local-action) text=$'name: R\non: workflow_call\njobs:\n build:\n steps:\n - uses: ./.github/actions/local'; raw='./.github/actions/local'; kind=step ;; + local-reusable) text=$'name: R\non: workflow_call\njobs:\n nested:\n uses: ./.github/workflows/local.yml'; raw='./.github/workflows/local.yml'; kind=job ;; + nested-reusable) text=$'name: R\non: workflow_call\njobs:\n nested:\n uses: org/child/.github/workflows/nested.yml@'"$CHILD_SHA"; raw="org/child/.github/workflows/nested.yml@$CHILD_SHA"; kind=job ;; + alias) text=$'name: R\non: workflow_call\nshared: &shared\n uses: org/child@'"$CHILD_SHA"$'\njobs:\n build:\n steps:\n - *shared'; raw="$CHILD_KEY"; kind=step ;; + merge) text=$'name: R\non: workflow_call\nshared: &shared\n uses: org/child@'"$CHILD_SHA"$'\njobs:\n build:\n steps:\n - <<: *shared'; raw="$CHILD_KEY"; kind=step ;; + flow) text=$'name: R\non: workflow_call\njobs:\n build:\n steps:\n - {uses: org/child@'"$CHILD_SHA"'}'; raw="$CHILD_KEY"; kind=step ;; + cyclic-alias) text=$'name: R\non: workflow_call\ncycle: &cycle\n self: *cycle\njobs:\n build:\n steps:\n - uses: org/child@'"$CHILD_SHA"; raw="$CHILD_KEY"; kind=step ;; + tagged-action) text=$'name: R\non: workflow_call\njobs:\n build:\n steps:\n - uses: org/child@v1'; raw='org/child@v1'; kind=step ;; + docker) text=$'name: R\non: workflow_call\njobs:\n build:\n steps:\n - uses: docker://alpine:3'; raw='docker://alpine:3'; kind=step ;; + block-scalar) text=$'name: R\non: workflow_call\njobs:\n build:\n steps:\n - uses: >-\n org/child@'"$CHILD_SHA"; raw="$CHILD_KEY"; kind=step ;; + esac + source_object "$CASE/cache/org/reusable.git" "$text"; replace_pin "$ROOT_SHA" "$NEW_COMMIT" + edit_manifest --arg blob "$NEW_BLOB" --arg raw "$raw" --arg kind "$kind" '.sources[0].blob=$blob | .sources[0].uses=[{kind:$kind,ref:$raw}]' + expected=notrun; [[ "$variant" != tagged-action ]] || expected=fatal + check "$variant source evidence cannot claim closure" "$expected" +done +case_fixture symlink; source_object "$CASE/cache/org/reusable.git" 'other.yml' 120000; replace_pin "$ROOT_SHA" "$NEW_COMMIT"; edit_manifest --arg blob "$NEW_BLOB" '.sources[0].blob=$blob'; check 'Git symlink source is not authenticated as a workflow file' fatal +# Exercise the owning entrypoint too: a correct helper cannot protect callers +# if automatic catalog selection or source-cache forwarding is bypassed. +for variant in main-good main-wrong-blob; do + case_fixture "$variant" + [[ "$variant" != main-wrong-blob ]] || edit_manifest '.sources[0].blob="1111111111111111111111111111111111111111"' + rc=0; valid=0 + timeout 15s bash "$ROOT/scripts/audit-lock-closure.sh" --json --repo-root "$CASE/repo" \ + --workflows "$CASE/repo/.github/workflows" --receipts "$CASE/repo/receipts.tsv" \ + --baseline "$CASE/repo/baseline.tsv" --source-cache "$CASE/cache" > "$CASE/main.json" 2> "$CASE/main.stderr" || rc=$? + if [[ "$variant" == main-good && "$rc" == 0 ]]; then + jq -e '(.findings|length)==0 and (.not_run|length)==0' "$CASE/main.json" >/dev/null 2>&1 && valid=1 + elif [[ "$variant" == main-wrong-blob && ( "$rc" == 1 || "$rc" == 3 ) ]]; then + jq -e 'any(.findings[]; .severity=="FATAL" and (.rule|startswith("SOURCE-")))' "$CASE/main.json" >/dev/null 2>&1 && valid=1 + fi + if [[ "$valid" == 1 ]]; then PASS=$((PASS+1)); printf 'PASS: %s entrypoint authenticates catalog (exit %s)\n' "$variant" "$rc" + else FAIL=$((FAIL+1)); printf 'FAIL: %s entrypoint did not enforce source authentication (exit %s)\n' "$variant" "$rc"; cat "$CASE/main.json" "$CASE/main.stderr"; fi +done +printf '\n%d passed, %d failed\n' "$PASS" "$FAIL" +[[ "$FAIL" == 0 ]] \ No newline at end of file diff --git a/scripts/tests/pinned-source-entrypoints-test.sh b/scripts/tests/pinned-source-entrypoints-test.sh new file mode 100644 index 00000000..031afa3f --- /dev/null +++ b/scripts/tests/pinned-source-entrypoints-test.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell +# Exercise the real preflight on isolated proposed trees. Cached Git objects are +# read as evidence; no remote workflow/action code or live canary is executed. +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +: "${PINNED_SOURCE_CACHE:?Set PINNED_SOURCE_CACHE to the authenticated bare Git cache}" +for tool in bash cp jq timeout; do + command -v "$tool" >/dev/null || { echo "Missing test prerequisite: $tool" >&2; exit 64; } +done +WORK="$(mktemp -d "${TMPDIR:-/tmp}/pinned-source-entrypoints.XXXXXX")" +trap 'rm -rf "$WORK"' EXIT +PASS=0; FAIL=0 +MIG='docs/migrations/secret-scanner-cicd-suite' +BASE="$WORK/base" +mkdir -p "$BASE/scripts/lib" "$BASE/$MIG" +cp -R "$ROOT/.github" "$BASE/" +cp "$ROOT/scripts/audit-lock-closure.sh" "$ROOT/scripts/audit-pinned-reusable-sources.sh" "$BASE/scripts/" +for name in lock-closure-receipts.tsv lock-audit-baseline.tsv pinned-reusable-sources.json; do + cp "$ROOT/scripts/lib/$name" "$BASE/scripts/lib/" +done +cp "$ROOT/$MIG/preflight.sh" "$ROOT/$MIG/required-contexts.txt" "$BASE/$MIG/" +cp -R "$ROOT/$MIG/fixtures" "$ROOT/$MIG/patches" "$BASE/$MIG/" +# Deliberately omit the canary directory. Every invocation binds --receipt to +# a nonexistent isolated path, so these static tests cannot authorize rollout. +new_case() { CASE="$WORK/$1"; cp -R "$BASE" "$CASE"; } +run_preflight() { + RC=0 + timeout 120s bash "$CASE/$MIG/preflight.sh" --receipt "$CASE/missing-live-canary.json" "$@" > "$CASE/output" 2>&1 || RC=$? +} +pass() { printf 'PASS: %s\n' "$1"; PASS=$((PASS + 1)); } +fail() { printf 'FAIL: %s (exit %s)\n' "$1" "$RC"; cat "$CASE/output"; FAIL=$((FAIL + 1)); } +expect_finding() { + local label="$1" rule="$2" + run_preflight + if [[ "$RC" == 1 || "$RC" == 3 ]] && grep -q '^== FINDINGS ' "$CASE/output" && grep -q "^${rule}"$'\t' "$CASE/output"; then + pass "$label" + else fail "$label"; fi +} +manifest_edit() { + jq "$1" "$CASE/scripts/lib/pinned-reusable-sources.json" > "$CASE/manifest.tmp" + mv "$CASE/manifest.tmp" "$CASE/scripts/lib/pinned-reusable-sources.json" +} +new_case untouched +run_preflight --quiet +if [[ "$RC" == 2 && ! -s "$CASE/output" ]]; then pass 'untouched candidate remains blocked by missing canary (quiet exit 2)'; else fail 'untouched candidate remains blocked by missing canary'; fi +run_preflight +# Nonquiet offline mode additionally reports ONLINE as an explicit optional +# evidence limit. Neither that limit nor CANARY may become a passing result. +if [[ "$RC" == 2 ]] && ! grep -q '^== FINDINGS ' "$CASE/output" && + grep -q $'^NOT-RUN\tCANARY\t' "$CASE/output" && + ! grep '^NOT-RUN' "$CASE/output" | grep -Ev $'^NOT-RUN\t(CANARY|ONLINE)\t' >/dev/null; then + pass 'untouched diagnostics contain only missing canary and optional offline evidence' +else fail 'untouched diagnostic classification'; fi +new_case wrong-blob +manifest_edit '.sources[0].blob = "0000000000000000000000000000000000000000"' +expect_finding 'altered source blob cannot bypass proposed-tree authentication' LOCK-CLOSURE +new_case omitted-catalog-root +manifest_edit '.sources |= map(select(.repository != "hyperpolymath/cicd-suite"))' +expect_finding 'omitted active suite catalog root remains a finding' LOCK-CLOSURE +new_case removed-manifest +rm "$CASE/scripts/lib/pinned-reusable-sources.json" +expect_finding 'removed source route requires the missing vendor records' LOCK-MISSING +new_case dropped-receipt +awk -F '\t' '$1 != "hyperpolymath/cicd-suite"' "$CASE/scripts/lib/lock-closure-receipts.tsv" > "$CASE/receipts.tmp" +mv "$CASE/receipts.tmp" "$CASE/scripts/lib/lock-closure-receipts.tsv" +expect_finding 'dropped reviewed suite receipt remains a finding' CLOSURE-NO-RECEIPT +new_case altered-receipt +sed 's/actions\/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1/actions\/checkout@0000000000000000000000000000000000000000/' "$CASE/scripts/lib/lock-closure-receipts.tsv" > "$CASE/receipts.tmp" +mv "$CASE/receipts.tmp" "$CASE/scripts/lib/lock-closure-receipts.tsv" +expect_finding 'altered reviewed suite receipt remains a finding' CLOSURE-NO-RECEIPT +new_case fast-pr +sed 's/fast-pr: false/fast-pr: true/' "$CASE/.github/workflows/secret-scanner.yml" > "$CASE/workflow.tmp" +mv "$CASE/workflow.tmp" "$CASE/.github/workflows/secret-scanner.yml" +expect_finding 'fast-pr true remains a finding' WF-FAST-PR +new_case secrets-inherit +printf ' secrets: inherit\n' >> "$CASE/.github/workflows/secret-scanner.yml" +expect_finding 'secret inheritance remains a finding' WF-SECRETS +printf '\n%s passed, %s failed\n' "$PASS" "$FAIL" +[[ "$FAIL" == 0 ]] diff --git a/tests/workflow-lock-map-test.py b/tests/workflow-lock-map-test.py index 51d2d1e6..f13322c9 100644 --- a/tests/workflow-lock-map-test.py +++ b/tests/workflow-lock-map-test.py @@ -62,13 +62,35 @@ def test_missing_direct_ref_even_with_workflow_key(self): self.assertEqual(code, 1, output) self.assertIn('MISSING-DIRECT-EDGE', output) - def test_reusable_missing_edge_is_failure(self): - self.write('main.yml', {'jobs': {'scan': {'uses': 'acme/suite/.github/workflows/scan.yml@abc'}}}) + def test_sha_reusable_empty_map_requires_source_review(self): + self.write('main.yml', {'jobs': {'scan': {'uses': 'acme/suite/.github/workflows/scan.yml@' + 'a' * 40}}}) self.lock['workflows']['.github/workflows/main.yml'] = [] code, output = self.run_audit() - self.assertEqual(code, 1, output) - self.assertIn('MISSING-DIRECT-EDGE', output) + self.assertEqual(code, 0, output) self.assertIn('REMOTE-REUSABLE-SOURCE-REVIEW', output) + self.assertIn('UNSUPPORTED-LOCK-SCOPE', output) + self.assertNotIn('MISSING-DIRECT-EDGE', output) + self.assertNotIn('NO-USES-MAPPED .github/workflows/main.yml', output) + + def test_remote_reusable_floating_and_short_pins_fail(self): + for ref in ('main', 'v1', 'abc1234', 'a' * 39, 'a' * 41, 'g' * 40): + with self.subTest(ref=ref): + self.write('main.yml', {'jobs': {'scan': {'uses': 'acme/suite/.github/workflows/scan.yml@' + ref}}}) + self.lock['workflows']['.github/workflows/main.yml'] = [] + code, output = self.run_audit() + self.assertEqual(code, 1, output) + self.assertIn('REMOTE-REUSABLE-NOT-SHA', output) + + def test_reusable_does_not_exempt_missing_ordinary_action_edge(self): + self.write('main.yml', {'jobs': { + 'scan': {'uses': 'acme/suite/.github/workflows/scan.yml@' + 'a' * 40}, + 'build': {'steps': [{'uses': 'acme/action@v1'}]}, + }}) + self.lock['workflows']['.github/workflows/main.yml'] = [] + code, output = self.run_audit() + self.assertEqual(code, 1, output) + self.assertIn('MISSING-DIRECT-EDGE .github/workflows/main.yml acme/action@v1', output) + self.assertIn('UNSUPPORTED-LOCK-SCOPE', output) def test_missing_nested_record(self): self.lock['dependencies']['acme/action@v1']['uses'] = ['acme/child@v1'] @@ -199,20 +221,19 @@ def test_scanner_triggers_and_access(self): self.assertNotIn('secrets', job) self.assertEqual(job['with'], {'fast-pr': 'false'}) - def test_scanner_pin_and_lock_closure(self): + def test_scanner_source_pin_and_supported_lock_scope(self): sha = '4404c3c0474e17753fe7689f3485c02890aac113' suite = 'hyperpolymath/cicd-suite@' + sha checkout = 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' self.assertEqual(self.workflow['jobs']['scan']['uses'], 'hyperpolymath/cicd-suite/.github/workflows/secret-scanner-reusable.yml@' + sha) - self.assertEqual(self.lock['workflows']['.github/workflows/secret-scanner.yml'], [suite]) - record = self.lock['dependencies'][suite] - self.assertEqual((record['ref'], record['commit']), (sha, 'sha1-' + sha)) - self.assertEqual(record['uses'], [checkout]) + # gh-actions-lock v0.4.1 does not lock external jobs..uses. + # The caller SHA above is checked separately from ordinary action records. + self.assertEqual(self.lock['workflows']['.github/workflows/secret-scanner.yml'], []) + self.assertNotIn(suite, self.lock['dependencies']) self.assertIn(checkout, self.lock['dependencies']) - standards = 'metadatastician/standards@75bd5994470661a099df562d6d5ad2fe744dbd66' - self.assertIn(standards, self.lock['dependencies']) - self.assertTrue(any(standards in edges for edges in self.lock['workflows'].values())) + self.assertEqual(self.lock['dependencies'][checkout]['commit'], + 'sha1-3d3c42e5aac5ba805825da76410c181273ba90b1') if __name__ == '__main__': From db7262023a08253e116318c7d3005bd05d754d91 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Sat, 7 Nov 2026 10:13:25 +0000 Subject: [PATCH 3/5] Fix closure workflow context and address new quality findings --- .github/workflows/e2e.yml | 11 +-- docs/migrations/pinned-source-closure.adoc | 7 +- .../secret-scanner-cicd-suite/preflight.sh | 4 +- scripts/audit-lock-closure.sh | 26 +++--- scripts/audit-pinned-reusable-sources.sh | 86 ++++++++++--------- scripts/tests/pinned-reusable-sources-test.sh | 2 + .../tests/pinned-source-entrypoints-test.sh | 3 +- 7 files changed, 74 insertions(+), 65 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 2e25fe3a..d5b02c52 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -8,12 +8,13 @@ on: branches: [main] pull_request: branches: ['**'] -permissions: - actions: read - contents: read +permissions: {} jobs: e2e: name: E2E tests + permissions: + actions: read + contents: read needs: pinned-source-closure if: ${{ always() }} runs-on: ubuntu-latest @@ -57,10 +58,10 @@ jobs: timeout-minutes: 15 permissions: contents: read - env: - PINNED_SOURCE_CACHE: ${{ runner.temp }}/fleet-pinned-sources steps: - uses: actions/checkout@v7.0.1 + - name: Set source cache + run: printf 'PINNED_SOURCE_CACHE=%s/fleet-pinned-sources\n' "$RUNNER_TEMP" >> "$GITHUB_ENV" - name: Test pinned-source failure modes run: bash scripts/tests/pinned-reusable-sources-test.sh - name: Authenticate pinned reusable sources diff --git a/docs/migrations/pinned-source-closure.adoc b/docs/migrations/pinned-source-closure.adoc index 6e257af4..2600ef71 100644 --- a/docs/migrations/pinned-source-closure.adoc +++ b/docs/migrations/pinned-source-closure.adoc @@ -6,7 +6,7 @@ The local implementation verifies reviewed direct reusable-workflow edges against exact Git source objects and joins their reviewed dependency records to the ordinary vendor lock for closure auditing. Local acceptance checks pass; -fresh hosted checks are pending. This records work based on Fleet draft +hosted results are recorded in Fleet draft #652. This records work based on `48de76a5bcf466f46e4e973a21ca911ebec1b23c`, with main/base `34908580b98cd354caa3bfc5a000bc7935957fce`. It does not certify the entire fleet or declare the secret-scanner migration ready. @@ -118,6 +118,7 @@ The final local acceptance results are: |Explicit HTTPS fetch followed by offline helper |5 verified roots; 11 records |Integrated closure audit |fatal 0; review 0; not_run 0; accepted 3; source_attested_records 11 |Vendor verify-local, full verify and lock sync |Pass +|Workflow expression validation |actionlint 1.7.12 passes; E2E guard checked for success, failure, skipped, cancelled and empty results |=== The source job runs helper fixtures, explicit source fetching, the preserved @@ -160,8 +161,8 @@ inspect source data and fixtures, not fetched executable code. Local Debian `en_GB.UTF-8` report-back runs have three ordering failures on both the base and candidate; this closure change does not claim to repair them or to make the full repository E2E suite pass. Nine governance findings remained at the -last observation. Fresh hosted checks remain pending, so local acceptance is -not a claim that every hosted repository check is green. +last observation. The latest hosted outcomes are recorded in draft PR #652; +local acceptance is not a claim that every hosted repository check is green. No normative policy, standards draft 1419, fleet draft 651, ruleset, mirror or credential is changed by this closure integration. Source caches and raw local diff --git a/docs/migrations/secret-scanner-cicd-suite/preflight.sh b/docs/migrations/secret-scanner-cicd-suite/preflight.sh index c8a05042..0c9cf6fb 100755 --- a/docs/migrations/secret-scanner-cicd-suite/preflight.sh +++ b/docs/migrations/secret-scanner-cicd-suite/preflight.sh @@ -78,7 +78,7 @@ cp "$ROOT/scripts/lib/lock-closure-receipts.tsv" "$TMP/tree/scripts/lib/" 2>/dev cp "$ROOT/scripts/lib/lock-audit-baseline.tsv" "$TMP/tree/scripts/lib/" 2>/dev/null || true # Preserve the independent source proof in the proposed tree when the vendor # lock omits remote reusables. The closure audit below must still verify it. -if [ -f "$ROOT/scripts/lib/pinned-reusable-sources.json" ]; then +if [[ -f "$ROOT/scripts/lib/pinned-reusable-sources.json" ]]; then cp "$ROOT/scripts/lib/pinned-reusable-sources.json" "$TMP/tree/scripts/lib/" || finding TREE "source manifest could not be copied" cp "$ROOT/scripts/audit-pinned-reusable-sources.sh" "$TMP/tree/scripts/" || finding TREE "source verifier could not be copied" fi @@ -96,7 +96,7 @@ if [ -f "$TMP/tree/.github/workflows/secret-scanner.yml" ] && # integrated audit below must authenticate every active binding and edge, # and missing evidence remains nonzero NOT RUN, never a migration pass. count=$(grep -Fc "$source_key" "$TMP/tree/.github/workflows/actions.lock" 2>/dev/null || true) - if [ "$count" -lt 2 ] && [ ! -f "$TMP/tree/scripts/lib/pinned-reusable-sources.json" ]; then + if [[ "$count" -lt 2 ]] && [[ ! -f "$TMP/tree/scripts/lib/pinned-reusable-sources.json" ]]; then finding LOCK-MISSING "applied scanner lacks both pinned suite vendor records and a source-verification route" apply_ok=0 fi diff --git a/scripts/audit-lock-closure.sh b/scripts/audit-lock-closure.sh index cb6ae2b0..69cf9094 100755 --- a/scripts/audit-lock-closure.sh +++ b/scripts/audit-lock-closure.sh @@ -482,7 +482,7 @@ while IFS=$'\t' read -r tag a b c; do ;; DEPFIELD) key="$a"; field="$b"; val="$c" - if [ -n "${DEP_HASFIELD["$key|$field"]:-}" ]; then + if [[ -n "${DEP_HASFIELD["$key|$field"]:-}" ]]; then add_finding FATAL LOCK-DUP-FIELD "$key" "dependency field $field appears more than once" "Keep one unambiguous value per field" fi DEP_HASFIELD["$key|$field"]=1 @@ -492,7 +492,7 @@ while IFS=$'\t' read -r tag a b c; do uses) DEP_USES["$key"]="${DEP_USES["$key"]:-} $val" ;; owner_id|repo_id) [[ "$val" =~ ^[0-9]+$ ]] || add_finding FATAL REF-FIELD "$key" "$field is '$val', not an integer" "Record the numeric GitHub id or omit the field" - if [ "$field" = owner_id ]; then DEP_OWNER_ID["$key"]="$val"; else DEP_REPO_ID["$key"]="$val"; fi + if [[ "$field" = owner_id ]]; then DEP_OWNER_ID["$key"]="$val"; else DEP_REPO_ID["$key"]="$val"; fi ;; *) add_finding REVIEW REF-FIELD "$key" "unknown dependency field '$field'" "Remove it or extend the audit" ;; esac @@ -542,8 +542,8 @@ done # Internal data-only interface used by the independent source verifier. It # exits before source verification, avoiding recursion, and never emits a # successful record dump for malformed lock/receipt input. -if [ "$DUMP_LOCK_RECORDS" -eq 1 ]; then - if [ "${#F_SEV[@]}" -gt 0 ] || [ "${#N_RULE[@]}" -gt 0 ]; then +if [[ "$DUMP_LOCK_RECORDS" -eq 1 ]]; then + if [[ "${#F_SEV[@]}" -gt 0 ]] || [[ "${#N_RULE[@]}" -gt 0 ]]; then printf '%s\n' 'audit-lock-closure: cannot dump invalid or incomplete input' >&2 exit 1 fi @@ -563,15 +563,15 @@ fi # supply additional records only after exact commit/tree/blob, caller/path, # reviewed-edge and receipt checks all succeed. These are source-attested # records, not a claim that gh-actions-lock generated or verified them. -if [ -z "$SOURCE_MANIFEST" ] && { [ -f "$REPO_ROOT/scripts/lib/pinned-reusable-sources.json" ] || [ -f "$REPO_ROOT/scripts/audit-pinned-reusable-sources.sh" ]; }; then +if [[ -z "$SOURCE_MANIFEST" ]] && { [[ -f "$REPO_ROOT/scripts/lib/pinned-reusable-sources.json" ]] || [[ -f "$REPO_ROOT/scripts/audit-pinned-reusable-sources.sh" ]]; }; then SOURCE_MANIFEST="$REPO_ROOT/scripts/lib/pinned-reusable-sources.json" fi SOURCE_ATTESTED_COUNT=0 -if [ -n "$SOURCE_MANIFEST" ]; then +if [[ -n "$SOURCE_MANIFEST" ]]; then helper="$(dirname "${BASH_SOURCE[0]}")/audit-pinned-reusable-sources.sh" args=(--manifest "$SOURCE_MANIFEST" --repo-root "$REPO_ROOT" --lock "$LOCK" --receipts "$RECEIPTS" --json) - [ -z "$SOURCE_CACHE" ] || args+=(--cache "$SOURCE_CACHE") - [ "$FETCH_SOURCES" -eq 0 ] || args+=(--fetch-sources) + [[ -z "$SOURCE_CACHE" ]] || args+=(--cache "$SOURCE_CACHE") + [[ "$FETCH_SOURCES" -eq 0 ]] || args+=(--fetch-sources) source_rc=0 bash "$helper" "${args[@]}" > "$TMP/source-report.json" || source_rc=$? if ! jq -e '(.findings | type == "array") and (.not_run | type == "array") and (.dependencies | type == "object")' "$TMP/source-report.json" >/dev/null 2>&1; then @@ -583,24 +583,24 @@ if [ -n "$SOURCE_MANIFEST" ]; then while IFS= read -r item; do add_notrun "$(jq -r .rule <<< "$item")" "$(jq -r .subject <<< "$item")" "$(jq -r .detail <<< "$item")" "$(jq -r .fix <<< "$item")" done < <(jq -c '.not_run[]' "$TMP/source-report.json") - if [ "$source_rc" -eq 0 ] && jq -e '(.findings | length == 0) and (.not_run | length == 0)' "$TMP/source-report.json" >/dev/null; then + if [[ "$source_rc" -eq 0 ]] && jq -e '(.findings | length == 0) and (.not_run | length == 0)' "$TMP/source-report.json" >/dev/null; then while IFS= read -r item; do key="$(jq -r .key <<< "$item")" f="$(fold_ref "$key")" # Overlaps have already been compared by the verifier. Keep every # existing vendor record untouched, including its spelling. - [ -z "${DEP_IDX[$f]:-}" ] || continue + [[ -z "${DEP_IDX[$f]:-}" ]] || continue DEP_KEYS+=("$key"); DEP_IDX["$f"]="$key"; DISP["$f"]="$key" DEP_REF["$key"]="$(jq -r .value.ref <<< "$item")" commit="$(jq -r .value.commit <<< "$item")" DEP_COMMIT["$key"]="${commit#sha1-}" owner_id="$(jq -r '.value.owner_id//empty' <<< "$item")"; repo_id="$(jq -r '.value.repo_id//empty' <<< "$item")" - [ -z "$owner_id" ] || DEP_OWNER_ID["$key"]="$owner_id" - [ -z "$repo_id" ] || DEP_REPO_ID["$key"]="$repo_id" + [[ -z "$owner_id" ]] || DEP_OWNER_ID["$key"]="$owner_id" + [[ -z "$repo_id" ]] || DEP_REPO_ID["$key"]="$repo_id" DEP_USES["$key"]="$(jq -r '(.value.uses // []) | join(" ")' <<< "$item")" SOURCE_ATTESTED_COUNT=$((SOURCE_ATTESTED_COUNT + 1)) done < <(jq -c '.dependencies | to_entries[]' "$TMP/source-report.json") - elif [ "$(jq '(.findings | length) + (.not_run | length)' "$TMP/source-report.json")" -eq 0 ]; then + elif [[ "$(jq '(.findings | length) + (.not_run | length)' "$TMP/source-report.json")" -eq 0 ]]; then add_notrun SOURCE-REPORT "$SOURCE_MANIFEST" "source verifier exited $source_rc without a diagnostic" "Inspect the verifier; no supplemental record is trusted" fi fi diff --git a/scripts/audit-pinned-reusable-sources.sh b/scripts/audit-pinned-reusable-sources.sh index cd4b944b..3154a906 100644 --- a/scripts/audit-pinned-reusable-sources.sh +++ b/scripts/audit-pinned-reusable-sources.sh @@ -8,7 +8,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" MANIFEST=""; LOCK=""; RECEIPTS=""; CACHE="${PINNED_SOURCE_CACHE:-${XDG_CACHE_HOME:-${HOME:-/tmp}/.cache}/fleet-pinned-sources}" FETCH=0; JSON=0 -while [ "$#" -gt 0 ]; do +while [[ "$#" -gt 0 ]]; do case "$1" in --manifest|--source-manifest) MANIFEST="${2:?manifest file required}"; shift ;; --repo-root) ROOT="${2:?repository root required}"; shift ;; @@ -33,32 +33,34 @@ trap 'rm -rf "$TMP"' EXIT : > "$TMP/findings"; : > "$TMP/notrun"; : > "$TMP/roots"; : > "$TMP/groups" printf '{}\n' > "$TMP/supplement.json" finding() { - jq -nc --arg rule "$1" --arg subject "$2" --arg detail "$3" \ + local rule="$1" subject="$2" detail="$3" + jq -nc --arg rule "$rule" --arg subject "$subject" --arg detail "$detail" \ '{severity:"FATAL",rule:$rule,subject:$subject,detail:$detail,fix:"Review the source-bound manifest and immutable objects; do not waive closure evidence"}' >> "$TMP/findings" } notrun() { - jq -nc --arg rule "$1" --arg subject "$2" --arg detail "$3" \ + local rule="$1" subject="$2" detail="$3" + jq -nc --arg rule "$rule" --arg subject "$subject" --arg detail "$detail" \ '{rule:$rule,subject:$subject,detail:$detail,fix:"Provide authenticated source objects and supported parseable input; fetched source is never executed"}' >> "$TMP/notrun" } finish() { local f n rc=0 f="$(jq -s length "$TMP/findings")"; n="$(jq -s length "$TMP/notrun")" - [ "$f" -eq 0 ] || rc=1 - [ "$n" -eq 0 ] || rc=$((rc + 2)) - if [ "$rc" -ne 0 ]; then + [[ "$f" -eq 0 ]] || rc=1 + [[ "$n" -eq 0 ]] || rc=$((rc + 2)) + if [[ "$rc" -ne 0 ]]; then : > "$TMP/roots"; : > "$TMP/groups"; printf '{}\n' > "$TMP/supplement.json" fi jq -n --slurpfile findings "$TMP/findings" --slurpfile not_run "$TMP/notrun" \ --slurpfile roots "$TMP/roots" --slurpfile groups "$TMP/groups" --slurpfile deps "$TMP/supplement.json" \ '{findings:$findings,not_run:$not_run,verified_roots:$roots,verified_receipt_sources:($groups|unique),dependencies:$deps[0]}' > "$TMP/report.json" - if [ "$JSON" -eq 1 ]; then cat "$TMP/report.json"; else + if [[ "$JSON" -eq 1 ]]; then cat "$TMP/report.json"; else jq -r '.findings[] | "FATAL\t\(.rule)\t\(.subject)\t\(.detail)"' "$TMP/report.json" jq -r '.not_run[] | "NOT-RUN\t\(.rule)\t\(.subject)\t\(.detail)"' "$TMP/report.json" printf 'pinned reusable sources: findings=%s not-run=%s verified-roots=%s (direct source edges; no fetched execution)\n' "$f" "$n" "$(jq '.verified_roots|length' "$TMP/report.json")" fi return "$rc" } -if [ ! -f "$MANIFEST" ]; then notrun SOURCE-MISSING "$MANIFEST" 'No source manifest'; finish; exit $?; fi +if [[ ! -f "$MANIFEST" ]]; then notrun SOURCE-MISSING "$MANIFEST" 'No source manifest'; finish; exit $?; fi # Streaming paths retain duplicate object keys which ordinary jq object # parsing would discard. Container-close events identify their parent path. if ! jq --stream -es 'map(if length==2 then .[0] else .[0][0:-1] end) as $paths | ($paths|length)==($paths|unique|length)' "$MANIFEST" >/dev/null 2>&1; then @@ -98,14 +100,15 @@ jq '.dependencies | with_entries(.value |= with_entries(select(.key=="ref" or .k # Repository names are case-insensitive; ref spelling is not. Reject a # collision before normalization, then compare records in one key space. normalize_records() { + local input="$1" output="$2" jq -e ' def key: capture("^(?[^@]+)@(?[^@]+)$") | ((.repository|ascii_downcase)+"@"+.ref); (keys|map(key)) as $keys | ($keys|length)==($keys|unique|length) - ' "$1" >/dev/null 2>&1 || return 1 + ' "$input" >/dev/null 2>&1 || return 1 jq ' def key: capture("^(?[^@]+)@(?[^@]+)$") | ((.repository|ascii_downcase)+"@"+.ref); with_entries(.key |= key | if (.value|has("uses")) then .value.uses |= map(key) else . end) - ' "$1" > "$2" + ' "$input" > "$output" } for graph in vendor supplement; do if ! normalize_records "$TMP/$graph.json" "$TMP/$graph.normalized.json"; then @@ -126,6 +129,7 @@ normalize_ref() { # Conservative block-YAML reader matching the existing auditor's profile. # A flow/scalar/anchor spelling we cannot read is evidence missing, not green. extract_uses() { + local workflow_file="$1" awk ' function uncomment(s, i,c,q,out) { q="";out="" @@ -155,7 +159,7 @@ extract_uses() { if(line~/(^|[[:space:],{])["\047]?uses["\047]?[[:space:]]*:/)print "UNPARSED\tuses in unsupported YAML shape" s=uncomment(line);if(s~/(^|[[:space:]-])["\047]?[A-Za-z0-9_.-]+["\047]?[[:space:]]*:[[:space:]]*[|>][-+]?[0-9]?[[:space:]]*$/){block=1;blockind=ind} } - ' "$1" + ' "$workflow_file" } # Discover the exact live caller/job/path roots independently of the catalog. @@ -163,10 +167,10 @@ shopt -s nullglob for workflow in "$ROOT"/.github/workflows/*.yml "$ROOT"/.github/workflows/*.yaml; do caller=".github/workflows/$(basename "$workflow")" while IFS=$'\t' read -r tag kind raw job; do - if [ "$tag" = UNPARSED ]; then notrun SOURCE-PARSE "$caller" "$kind"; continue; fi - [ "$kind" = job ] || continue - case "$raw" in ./*) continue ;; esac - if [[ ! "$raw" =~ ^([A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*)/(\.github/workflows/[A-Za-z0-9_.-]+\.ya?ml)@([0-9a-f]{40})$ ]] || [ -z "$job" ]; then + if [[ "$tag" = UNPARSED ]]; then notrun SOURCE-PARSE "$caller" "$kind"; continue; fi + [[ "$kind" = job ]] || continue + if [[ "$raw" == ./* ]]; then continue; fi + if [[ ! "$raw" =~ ^([A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*)/(\.github/workflows/[A-Za-z0-9_.-]+\.ya?ml)@([0-9a-f]{40})$ ]] || [[ -z "$job" ]]; then finding SOURCE-PIN "$caller/$job" "Remote reusable must have an exact workflow path and full lowercase 40-hex source pin: $raw"; continue fi jq -nc --arg workflow "$caller" --arg job "$job" --arg repository "${BASH_REMATCH[1],,}" --arg path "${BASH_REMATCH[2]}" --arg commit "${BASH_REMATCH[3]}" \ @@ -193,11 +197,11 @@ verify_object() { local type="$1" oid="$2" dest="$3" actual git_data cat-file "$type" "$oid" > "$dest" 2>/dev/null || return 1 actual="$(git_data hash-object -t "$type" --stdin < "$dest")" || return 1 - [ "$actual" = "$oid" ] + [[ "$actual" = "$oid" ]] } object_failure() { local type="$1" oid="$2" subject="$3" - if git_data cat-file -e "$oid" 2>/dev/null || [ -e "$BARE/objects/${oid:0:2}/${oid:2}" ]; then + if git_data cat-file -e "$oid" 2>/dev/null || [[ -e "$BARE/objects/${oid:0:2}/${oid:2}" ]]; then finding SOURCE-ORIGIN "$subject" "Cached $type object is present but corrupt, has the wrong type, or fails its raw content hash" else notrun SOURCE-MISSING "$subject" "Offline cache lacks required $type object $oid" @@ -207,25 +211,25 @@ source_bytes() { local repository="$1" commit="$2" path="$3" blob="$4" dest="$5" tree entry mode type oid name part i BARE="$CACHE/$repository.git" for entry in "$CACHE" "$CACHE/${repository%/*}" "$BARE"; do - if [ -L "$entry" ]; then finding SOURCE-ORIGIN "$repository@$commit" 'Symlink in source-cache path'; return 1; fi + if [[ -L "$entry" ]]; then finding SOURCE-ORIGIN "$repository@$commit" 'Symlink in source-cache path'; return 1; fi done - if [ ! -d "$BARE" ] && [ "$FETCH" -eq 1 ]; then + if [[ ! -d "$BARE" ]] && [[ "$FETCH" -eq 1 ]]; then mkdir -p "$(dirname "$BARE")" if ! env GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1 git -c core.hooksPath=/dev/null init --bare --object-format=sha1 "$BARE" >/dev/null 2>&1; then notrun SOURCE-MISSING "$repository@$commit" 'Cannot initialize isolated bare source cache'; return 1 fi fi - if [ ! -d "$BARE" ]; then notrun SOURCE-MISSING "$repository@$commit" 'No authenticated bare cache; bootstrap with --fetch-sources'; return 1; fi - if [ "$(git_data rev-parse --is-bare-repository 2>/dev/null)" != true ]; then finding SOURCE-ORIGIN "$repository@$commit" 'Source cache is not a bare Git repository'; return 1; fi - if [ -e "$BARE/info/grafts" ] || [ -L "$BARE/info/grafts" ] || [ -n "$(git_data for-each-ref --format='%(refname)' refs/replace/ 2>/dev/null)" ] || + if [[ ! -d "$BARE" ]]; then notrun SOURCE-MISSING "$repository@$commit" 'No authenticated bare cache; bootstrap with --fetch-sources'; return 1; fi + if [[ "$(git_data rev-parse --is-bare-repository 2>/dev/null)" != true ]]; then finding SOURCE-ORIGIN "$repository@$commit" 'Source cache is not a bare Git repository'; return 1; fi + if [[ -e "$BARE/info/grafts" ]] || [[ -L "$BARE/info/grafts" ]] || [[ -n "$(git_data for-each-ref --format='%(refname)' refs/replace/ 2>/dev/null)" ]] || git_data config --local --get-regexp '^(include\.|includeif\.|url\.|extensions\.partialclone|remote\..*\.promisor)' >/dev/null 2>&1; then finding SOURCE-ORIGIN "$repository@$commit" 'Cache has grafts, replacement refs or source-rewriting configuration'; return 1 fi if ! verify_object commit "$commit" "$TMP/commit"; then - if git_data cat-file -e "$commit" 2>/dev/null || [ -e "$BARE/objects/${commit:0:2}/${commit:2}" ]; then + if git_data cat-file -e "$commit" 2>/dev/null || [[ -e "$BARE/objects/${commit:0:2}/${commit:2}" ]]; then finding SOURCE-ORIGIN "$repository@$commit" 'Cached commit exists but its raw content/type/hash is corrupt or mismatched'; return 1 fi - if [ "$FETCH" -eq 1 ]; then + if [[ "$FETCH" -eq 1 ]]; then if ! git_data fetch --quiet --no-tags --depth=1 --no-recurse-submodules "https://github.com/$repository.git" "$commit" > "$TMP/fetch.out" 2> "$TMP/fetch.err"; then notrun SOURCE-MISSING "$repository@$commit" 'Exact public HTTPS source fetch failed'; return 1 fi @@ -240,12 +244,12 @@ source_bytes() { if ! verify_object tree "$tree" "$TMP/tree"; then object_failure tree "$tree" "$repository@$commit/$path"; return 1; fi entry="$(git_data ls-tree "$tree" -- "$part")" IFS=$' \t' read -r mode type oid name <<< "$entry" - if [ "$name" != "$part" ]; then finding SOURCE-BLOB "$repository@$commit/$path" 'Requested source path absent from exact commit tree'; return 1; fi - if [ "$i" -lt "$((${#parts[@]}-1))" ]; then - if [ "$mode" != 040000 ] || [ "$type" != tree ]; then finding SOURCE-BLOB "$repository@$commit/$path" 'Non-tree or symlink in source path'; return 1; fi + if [[ "$name" != "$part" ]]; then finding SOURCE-BLOB "$repository@$commit/$path" 'Requested source path absent from exact commit tree'; return 1; fi + if [[ "$i" -lt "$((${#parts[@]}-1))" ]]; then + if [[ "$mode" != 040000 ]] || [[ "$type" != tree ]]; then finding SOURCE-BLOB "$repository@$commit/$path" 'Non-tree or symlink in source path'; return 1; fi tree="$oid" else - if [[ "$mode" != 100644 && "$mode" != 100755 ]] || [ "$type" != blob ] || [ "$oid" != "$blob" ]; then + if [[ "$mode" != 100644 && "$mode" != 100755 ]] || [[ "$type" != blob ]] || [[ "$oid" != "$blob" ]]; then finding SOURCE-BLOB "$repository@$commit/$path" 'Blob identity or regular-file mode differs from manifest'; return 1 fi if ! verify_object blob "$blob" "$dest"; then object_failure blob "$blob" "$repository@$commit/$path"; return 1; fi @@ -263,13 +267,13 @@ while IFS= read -r source; do parse_incomplete=0 : > "$TMP/uses" while IFS=$'\t' read -r tag kind raw job; do - if [ "$tag" = UNPARSED ]; then notrun SOURCE-PARSE "$subject" "$kind"; parse_incomplete=1; continue; fi - if [ "$kind" = job ]; then notrun SOURCE-LOCAL "$subject" 'Nested reusable calls are outside this reviewed direct-action source profile'; parse_incomplete=1; continue; fi + if [[ "$tag" = UNPARSED ]]; then notrun SOURCE-PARSE "$subject" "$kind"; parse_incomplete=1; continue; fi + if [[ "$kind" = job ]]; then notrun SOURCE-LOCAL "$subject" 'Nested reusable calls are outside this reviewed direct-action source profile'; parse_incomplete=1; continue; fi if ! ref="$(normalize_ref "$raw")"; then notrun SOURCE-LOCAL "$subject" "Unsupported local, Docker, dynamic or malformed uses: $raw"; parse_incomplete=1; continue; fi if [[ ! "${ref##*@}" =~ ^[0-9a-f]{40}$ ]]; then finding SOURCE-PIN "$subject" "Authenticated direct remote uses must have a literal full 40-hex source pin: $raw"; continue; fi jq -nc --arg kind "$kind" --arg ref "$raw" '{kind:$kind,ref:$ref}' >> "$TMP/uses" done < <(extract_uses "$bytes") - [ "$parse_incomplete" -eq 0 ] || continue + [[ "$parse_incomplete" -eq 0 ]] || continue jq -s 'unique_by(.kind,.ref)|sort_by(.kind,.ref)' "$TMP/uses" > "$TMP/actual-uses" jq '.uses|map({kind,ref})|unique_by(.kind,.ref)|sort_by(.kind,.ref)' <<< "$source" > "$TMP/expected-uses" if ! cmp -s "$TMP/actual-uses" "$TMP/expected-uses"; then @@ -282,7 +286,7 @@ done < <(jq -c '.sources[]' "$MANIFEST") # Do not turn absent authenticated data into a misleading edge mismatch. # No source-derived records are exported unless the whole evidence pass is # complete, including every retained-review path. -if [ -s "$TMP/findings" ] || [ -s "$TMP/notrun" ]; then finish; exit $?; fi +if [[ -s "$TMP/findings" ]] || [[ -s "$TMP/notrun" ]]; then finish; exit $?; fi # A retained source may extend an existing reviewed union, but cannot create # an unrelated historical dependency population with no current caller. @@ -299,7 +303,7 @@ while IFS= read -r group; do edges="$(jq -Rsc 'split("\n")|map(select(length>0))' "$TMP/group-normalized")" if jq -e --arg k "$rid" 'has($k)' "$TMP/graph.json" >/dev/null; then prior="$(jq -c --arg k "$rid" '.[$k].uses//[]|sort|unique' "$TMP/graph.json")" - if [ "$prior" != "$(jq -c 'sort|unique' <<< "$edges")" ]; then finding SOURCE-DEPENDENCY "$rid" 'Source parent record differs from its authenticated direct-edge union'; fi + if [[ "$prior" != "$(jq -c 'sort|unique' <<< "$edges")" ]]; then finding SOURCE-DEPENDENCY "$rid" 'Source parent record differs from its authenticated direct-edge union'; fi else jq --arg k "$rid" --arg sha "$commit" --argjson edges "$edges" '.+{($k):{ref:$sha,commit:("sha1-"+$sha),uses:$edges}}' "$TMP/supplement.json" > "$TMP/next.json" mv "$TMP/next.json" "$TMP/supplement.json" @@ -312,30 +316,30 @@ done < <(jq -sc 'sort_by(.repository,.commit)|group_by(.repository,.commit)|map( # claiming recursive source authentication beyond the reviewed direct sets. while IFS= read -r entry; do key="$(jq -r .key <<< "$entry")"; normalized="$(normalize_ref "$key")" || normalized="" - if [ "$normalized" != "$key" ] || ! jq -e '.value|(.ref|type=="string" and test("^[^[:space:]@]+$")) and (.commit|type=="string" and test("^sha1-[0-9a-f]{40}$")) and ((.uses//[])|type=="array")' <<< "$entry" >/dev/null; then + if [[ "$normalized" != "$key" ]] || ! jq -e '.value|(.ref|type=="string" and test("^[^[:space:]@]+$")) and (.commit|type=="string" and test("^sha1-[0-9a-f]{40}$")) and ((.uses//[])|type=="array")' <<< "$entry" >/dev/null; then finding SOURCE-DEPENDENCY "$key" 'Malformed supplemental dependency identity'; continue fi ref="$(jq -r .value.ref <<< "$entry")"; digest="$(jq -r .value.commit <<< "$entry")"; named="${key##*@}" - if [[ "$ref" =~ ^[0-9a-f]{40}$ ]] && [ "$digest" != "sha1-$ref" ]; then finding SOURCE-DEPENDENCY "$key" 'Full-SHA ref and resolved commit disagree'; fi + if [[ "$ref" =~ ^[0-9a-f]{40}$ ]] && [[ "$digest" != "sha1-$ref" ]]; then finding SOURCE-DEPENDENCY "$key" 'Full-SHA ref and resolved commit disagree'; fi if [[ "$ref" == sha1-* ]]; then finding SOURCE-DEPENDENCY "$key" 'Dependency ref must not use the commit digest prefix'; fi if [[ "$named" =~ ^[0-9a-f]{40}$ ]]; then - [ "$digest" = "sha1-$named" ] || finding SOURCE-DEPENDENCY "$key" 'SHA key and resolved commit disagree' - elif [ "$named" != "$ref" ]; then finding SOURCE-DEPENDENCY "$key" 'Named ref and dependency ref disagree'; fi + [[ "$digest" = "sha1-$named" ]] || finding SOURCE-DEPENDENCY "$key" 'SHA key and resolved commit disagree' + elif [[ "$named" != "$ref" ]]; then finding SOURCE-DEPENDENCY "$key" 'Named ref and dependency ref disagree'; fi if jq -e --arg k "$key" 'has($k)' "$TMP/vendor.json" >/dev/null; then left="$(jq -cS --arg k "$key" '.[$k]|{ref,commit,uses:(.uses//[]|sort)} + (if has("owner_id") then {owner_id} else {} end) + (if has("repo_id") then {repo_id} else {} end)' "$TMP/vendor.json")" right="$(jq -cS '.value|{ref,commit,uses:(.uses//[]|sort)} + (if has("owner_id") then {owner_id} else {} end) + (if has("repo_id") then {repo_id} else {} end)' <<< "$entry")" - [ "$left" = "$right" ] || finding SOURCE-DEPENDENCY "$key" 'Shared vendor and supplemental records disagree' + [[ "$left" = "$right" ]] || finding SOURCE-DEPENDENCY "$key" 'Shared vendor and supplemental records disagree' fi done < <(jq -c 'to_entries[]' "$TMP/supplement.json") declare -A VISITING=() DONE=() visit() { local key="$1" depth="$2" child - [ -z "${DONE[$key]:-}" ] || return 0 - if [ "$depth" -gt 100 ] || [ -n "${VISITING[$key]:-}" ]; then finding SOURCE-DEPENDENCY "$key" 'Cyclic or excessively deep supplemental graph'; return 0; fi + [[ -z "${DONE[$key]:-}" ]] || return 0 + if [[ "$depth" -gt 100 ]] || [[ -n "${VISITING[$key]:-}" ]]; then finding SOURCE-DEPENDENCY "$key" 'Cyclic or excessively deep supplemental graph'; return 0; fi if ! jq -e --arg k "$key" 'has($k)' "$TMP/graph.json" >/dev/null; then finding SOURCE-DEPENDENCY "$key" 'Required direct/transitive dependency record is missing'; return 0; fi VISITING["$key"]=1 while IFS= read -r child; do - if ! normalized="$(normalize_ref "$child")" || [ "$child" != "$normalized" ]; then finding SOURCE-DEPENDENCY "$key" "Invalid nested edge: $child"; continue; fi + if ! normalized="$(normalize_ref "$child")" || [[ "$child" != "$normalized" ]]; then finding SOURCE-DEPENDENCY "$key" "Invalid nested edge: $child"; continue; fi visit "$child" "$((depth+1))" done < <(jq -r --arg k "$key" '.[$k].uses//[]|.[]' "$TMP/graph.json") unset 'VISITING[$key]'; DONE["$key"]=1 diff --git a/scripts/tests/pinned-reusable-sources-test.sh b/scripts/tests/pinned-reusable-sources-test.sh index 3d81474f..fdb3b514 100644 --- a/scripts/tests/pinned-reusable-sources-test.sh +++ b/scripts/tests/pinned-reusable-sources-test.sh @@ -86,6 +86,7 @@ check() { clean) [[ "$rc" == 0 ]] && jq -e '(.findings|length)==0 and (.not_run|length)==0 and (.verified_roots|length)==1 and (.verified_receipt_sources|length)>0' "$CASE/result.json" >/dev/null && valid=1 ;; fatal) [[ "$rc" == 1 || "$rc" == 3 ]] && jq -e 'any(.findings[]; .severity=="FATAL" and (.rule|length)>0 and (.detail|length)>0) and (.verified_roots|length)==0 and (.verified_receipt_sources|length)==0' "$CASE/result.json" >/dev/null && valid=1 ;; notrun) [[ "$rc" == 2 ]] && jq -e '(.findings|length)==0 and any(.not_run[]; (.rule|length)>0 and (.detail|length)>0) and (.verified_roots|length)==0 and (.verified_receipt_sources|length)==0' "$CASE/result.json" >/dev/null && valid=1 ;; + *) printf 'Unexpected fixture expectation: %s\n' "$expected" >&2; exit 64 ;; esac fi if [[ "$valid" == 1 ]]; then PASS=$((PASS+1)); printf 'PASS: %s (%s, exit %s)\n' "$label" "$expected" "$rc" @@ -242,6 +243,7 @@ for variant in local-action local-reusable nested-reusable docker block-scalar a tagged-action) text=$'name: R\non: workflow_call\njobs:\n build:\n steps:\n - uses: org/child@v1'; raw='org/child@v1'; kind=step ;; docker) text=$'name: R\non: workflow_call\njobs:\n build:\n steps:\n - uses: docker://alpine:3'; raw='docker://alpine:3'; kind=step ;; block-scalar) text=$'name: R\non: workflow_call\njobs:\n build:\n steps:\n - uses: >-\n org/child@'"$CHILD_SHA"; raw="$CHILD_KEY"; kind=step ;; + *) printf 'Unexpected fixture variant: %s\n' "$variant" >&2; exit 64 ;; esac source_object "$CASE/cache/org/reusable.git" "$text"; replace_pin "$ROOT_SHA" "$NEW_COMMIT" edit_manifest --arg blob "$NEW_BLOB" --arg raw "$raw" --arg kind "$kind" '.sources[0].blob=$blob | .sources[0].uses=[{kind:$kind,ref:$raw}]' diff --git a/scripts/tests/pinned-source-entrypoints-test.sh b/scripts/tests/pinned-source-entrypoints-test.sh index 031afa3f..6e83a35a 100644 --- a/scripts/tests/pinned-source-entrypoints-test.sh +++ b/scripts/tests/pinned-source-entrypoints-test.sh @@ -39,7 +39,8 @@ expect_finding() { else fail "$label"; fi } manifest_edit() { - jq "$1" "$CASE/scripts/lib/pinned-reusable-sources.json" > "$CASE/manifest.tmp" + local expression="$1" + jq "$expression" "$CASE/scripts/lib/pinned-reusable-sources.json" > "$CASE/manifest.tmp" mv "$CASE/manifest.tmp" "$CASE/scripts/lib/pinned-reusable-sources.json" } new_case untouched From e911e683e7007a9a456d06268c9ea4698781721b Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Sat, 7 Nov 2026 11:13:40 +0000 Subject: [PATCH 4/5] Resolve governance findings while preserving reviewed source closure --- .github/workflows/actions.lock | 9 + .github/workflows/e2e.yml | 16 +- .github/workflows/inbox-steward.yml | 19 +- .github/workflows/instant-sync.yml | 7 + .github/workflows/push-email-notify.yml | 10 + .github/workflows/supervised-fleet-scan.yml | 6 + docs/migrations/pinned-source-closure.adoc | 2 +- scripts/audit-lock-regeneration.py | 119 ---- scripts/audit-lock-regeneration.sh | 5 + scripts/audit-workflow-lock-map.py | 193 ------ scripts/audit-workflow-lock-map.sh | 5 + tests/lock-regeneration-test.py | 125 ---- tests/workflow-lock-map-test.py | 240 -------- tools/workflow-lock-audit/.gitignore | 2 + tools/workflow-lock-audit/Cargo.lock | 203 +++++++ tools/workflow-lock-audit/Cargo.toml | 15 + tools/workflow-lock-audit/README.adoc | 48 ++ tools/workflow-lock-audit/src/main.rs | 514 ++++++++++++++++ tools/workflow-lock-audit/tests/contracts.rs | 608 +++++++++++++++++++ 19 files changed, 1461 insertions(+), 685 deletions(-) delete mode 100755 scripts/audit-lock-regeneration.py create mode 100644 scripts/audit-lock-regeneration.sh delete mode 100755 scripts/audit-workflow-lock-map.py create mode 100644 scripts/audit-workflow-lock-map.sh delete mode 100644 tests/lock-regeneration-test.py delete mode 100644 tests/workflow-lock-map-test.py create mode 100644 tools/workflow-lock-audit/.gitignore create mode 100644 tools/workflow-lock-audit/Cargo.lock create mode 100644 tools/workflow-lock-audit/Cargo.toml create mode 100644 tools/workflow-lock-audit/README.adoc create mode 100644 tools/workflow-lock-audit/src/main.rs create mode 100644 tools/workflow-lock-audit/tests/contracts.rs diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock index d8e7dbaa..63238b1c 100644 --- a/.github/workflows/actions.lock +++ b/.github/workflows/actions.lock @@ -35,8 +35,10 @@ workflows: '.github/workflows/hypatia-scan.yml': [] '.github/workflows/inbox-steward.yml': - 'actions/checkout@v7.0.1' + - 'step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43' '.github/workflows/instant-sync.yml': - 'peter-evans/repository-dispatch@v4.0.1' + - 'step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43' '.github/workflows/label-triage.yml': [] '.github/workflows/labels.yml': [] '.github/workflows/learning-loop.yml': @@ -55,6 +57,7 @@ workflows: - 'swatinem/rust-cache@v2.9.3' '.github/workflows/push-email-notify.yml': - 'hyperpolymath/smtp-notify-action@v0.5.0' + - 'step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43' '.github/workflows/repo-integrity-guard.yml': - 'actions/checkout@v7.0.1' '.github/workflows/rust.yml': @@ -63,6 +66,7 @@ workflows: '.github/workflows/secret-scanner.yml': [] '.github/workflows/supervised-fleet-scan.yml': - 'actions/checkout@v7.0.1' + - 'step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43' '.github/workflows/vendored-bot-drift.yml': - 'actions/checkout@v7.0.1' dependencies: @@ -163,6 +167,11 @@ dependencies: commit: 'sha1-28959ce8df70de7be546dd1250a005dd32156697' owner_id: 18365890 repo_id: 220359305 + 'step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43': + ref: 'v2.22.3' + commit: 'sha1-46b06cea07f9e64621fdf066ba77ac6915163d43' + owner_id: 88700172 + repo_id: 422287306 'swatinem/rust-cache@v2.9.3': ref: 'v2.9.3' commit: 'sha1-d1527c8f67ef9af357290475d4af4b0fab8f12e0' diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index d5b02c52..625d0f99 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -29,13 +29,19 @@ jobs: exit 1 fi - uses: actions/checkout@v7.0.1 - - name: Install PyYAML (for workflow YAML validation) - run: pip install pyyaml --quiet + - name: Audit workflow lock auditor dependencies + run: | + set -euo pipefail + # Official RustSec binary, immutable content hash from its release asset. + archive="$RUNNER_TEMP/cargo-audit.tgz" + curl -fsSL --retry 3 https://github.com/rustsec/rustsec/releases/download/cargo-audit/v0.22.2/cargo-audit-x86_64-unknown-linux-gnu-v0.22.2.tgz -o "$archive" + printf '%s %s\n' ab28a1bdb54db4d5d8ad5981cf1f959410370b3d28250dbd35f6a44248620e39 "$archive" | sha256sum -c - + tar -xzf "$archive" -C "$RUNNER_TEMP" + "$RUNNER_TEMP/cargo-audit-x86_64-unknown-linux-gnu-v0.22.2/cargo-audit" audit --file tools/workflow-lock-audit/Cargo.lock --deny warnings - name: Test workflow lock-map and regeneration audit run: | - python3 tests/workflow-lock-map-test.py - python3 tests/lock-regeneration-test.py - python3 scripts/audit-workflow-lock-map.py + cargo test --locked --manifest-path tools/workflow-lock-audit/Cargo.toml + bash scripts/audit-workflow-lock-map.sh - name: Verify automation quarantine run: bash tests/automation-quarantine-test.sh - name: Verify fix-outcome verdicts diff --git a/.github/workflows/inbox-steward.yml b/.github/workflows/inbox-steward.yml index 7900afba..94dfea3e 100644 --- a/.github/workflows/inbox-steward.yml +++ b/.github/workflows/inbox-steward.yml @@ -48,8 +48,8 @@ on: # MONITORING; the auto-merge job itself is gated OFF by default (see below). permissions: - contents: write - pull-requests: write + contents: read + pull-requests: read repository-projects: read actions: read @@ -280,6 +280,12 @@ jobs: # Job 3: Auto-merge qualifying PRs auto-merge-prs: name: Auto-merge qualifying PRs + permissions: + contents: write + pull-requests: write + outputs: + merged_count: ${{ steps.merge.outputs.merged_count }} + failed_count: ${{ steps.merge.outputs.failed_count }} runs-on: ubuntu-latest timeout-minutes: 10 needs: validate-prs @@ -379,6 +385,15 @@ jobs: if: always() && (needs.identify-passed-prs.result == 'success' || needs.validate-prs.result == 'success') steps: + - name: Harden credential-bearing runner + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 + with: + egress-policy: block + disable-telemetry: true + allowed-endpoints: >- + github.com:443 + api.github.com:443 + codeload.github.com:443 - name: Checkout gitbot-fleet uses: actions/checkout@v7.0.1 diff --git a/.github/workflows/instant-sync.yml b/.github/workflows/instant-sync.yml index 7fb72702..26e59f2b 100644 --- a/.github/workflows/instant-sync.yml +++ b/.github/workflows/instant-sync.yml @@ -27,6 +27,13 @@ jobs: env: HAS_TOKEN: ${{ secrets.FARM_DISPATCH_TOKEN != '' }} steps: + - name: Harden credential-bearing runner + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 + with: + egress-policy: block + disable-telemetry: true + allowed-endpoints: >- + api.github.com:443 - name: Trigger Propagation if: env.HAS_TOKEN == 'true' uses: peter-evans/repository-dispatch@v4.0.1 diff --git a/.github/workflows/push-email-notify.yml b/.github/workflows/push-email-notify.yml index 3cd69a56..43d89b2b 100644 --- a/.github/workflows/push-email-notify.yml +++ b/.github/workflows/push-email-notify.yml @@ -40,6 +40,16 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: + - name: Harden credential-bearing runner + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 + with: + egress-policy: block + disable-telemetry: true + allowed-endpoints: >- + github.com:443 + api.github.com:443 + release-assets.githubusercontent.com:443 + ${{ secrets.SMTP_HOST }}:${{ secrets.SMTP_PORT || '465' }} - name: Send push notification email uses: hyperpolymath/smtp-notify-action@v0.5.0 with: diff --git a/.github/workflows/supervised-fleet-scan.yml b/.github/workflows/supervised-fleet-scan.yml index cb06008e..740a6c6e 100644 --- a/.github/workflows/supervised-fleet-scan.yml +++ b/.github/workflows/supervised-fleet-scan.yml @@ -34,6 +34,12 @@ jobs: timeout-minutes: 60 steps: + - name: Harden credential-bearing runner + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 + with: + egress-policy: audit + # Scheduled fleet scans contact rule-specific services; audit the + # observed boundary before enabling a reviewed endpoint block list. - name: Checkout gitbot-fleet uses: actions/checkout@v7.0.1 with: diff --git a/docs/migrations/pinned-source-closure.adoc b/docs/migrations/pinned-source-closure.adoc index 2600ef71..3c3cc936 100644 --- a/docs/migrations/pinned-source-closure.adoc +++ b/docs/migrations/pinned-source-closure.adoc @@ -147,7 +147,7 @@ bash scripts/audit-pinned-reusable-sources.sh --json bash scripts/tests/pinned-reusable-sources-test.sh bash scripts/tests/lock-closure-test.sh bash scripts/tests/pinned-source-entrypoints-test.sh -python3 tests/workflow-lock-map-test.py +cargo test --locked --manifest-path tools/workflow-lock-audit/Cargo.toml bash scripts/audit-lock-closure.sh --json ---- diff --git a/scripts/audit-lock-regeneration.py b/scripts/audit-lock-regeneration.py deleted file mode 100755 index c6db5b98..00000000 --- a/scripts/audit-lock-regeneration.py +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MPL-2.0 -"""Compare reviewed closure against regeneration; does not discover remote closure.""" -import re -import sys -import yaml - - -class UniqueLoader(yaml.SafeLoader): - pass - - -def unique_mapping(loader, node, deep=False): - """Construct a YAML mapping, raising ValueError for duplicate keys. - - Pass deep through to PyYAML's key and value construction. YAML construction - errors and TypeError for unhashable keys propagate to the caller. - """ - mapping = {} - for key_node, value_node in node.value: - item = loader.construct_object(key_node, deep=deep) - if item in mapping: - raise ValueError(f'duplicate YAML key: {item!r}') - mapping[item] = loader.construct_object(value_node, deep=deep) - return mapping - - -UniqueLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, unique_mapping) - - -def key(value): - """Normalize an owner/repository@ref key, preserving the ref's case. - - Raise ValueError if the reference is not in the supported form. - """ - if not isinstance(value, str) or not re.fullmatch(r'[^\s/@]+/[^\s/@]+@[^\s@]+', value): - raise ValueError(f'invalid dependency reference: {value!r}') - repo, ref = value.split('@') - return f'{repo.lower()}@{ref}' - - -def load(path): - """Load and validate a lock file as (dependencies, workflow roots). - - Dependency keys and edges are normalized; each dependency value contains - its ref, commit, and set of edges. Raise ValueError for invalid records, - references, or missing targets. File, YAML, and key construction errors - propagate to the caller. - """ - with open(path, encoding='utf-8') as f: - doc = yaml.load(f, Loader=UniqueLoader) - if not isinstance(doc, dict) or not isinstance(doc.get('workflows'), dict) or not isinstance(doc.get('dependencies'), dict): - raise ValueError(f'{path}: expected workflows and dependencies mappings') - deps = {} - for name, record in doc['dependencies'].items(): - k = key(name) - if k in deps or not isinstance(record, dict): - raise ValueError(f'{path}: duplicate or invalid record {name}') - ref, commit = record.get('ref'), record.get('commit') - if not isinstance(ref, str) or not re.fullmatch(r'[^\s@]+', ref) or ref.startswith('sha1-'): - raise ValueError(f'{path}: invalid ref for {name}') - if not isinstance(commit, str) or not re.fullmatch(r'sha1-[0-9a-f]{40}', commit): - raise ValueError(f'{path}: invalid commit for {name}') - named_ref = name.split('@')[1] - if re.fullmatch(r'[0-9a-f]{40}', named_ref): - if commit != 'sha1-' + named_ref: - raise ValueError(f'{path}: SHA key and commit differ for {name}') - elif named_ref != ref: - raise ValueError(f'{path}: tag ref differs from key for {name}') - edges = record.get('uses', []) - if not isinstance(edges, list): - raise ValueError(f'{path}: invalid uses for {name}') - deps[k] = (ref, commit, {key(edge) for edge in edges}) - roots = set() - for workflow, edges in doc['workflows'].items(): - if not isinstance(workflow, str) or not isinstance(edges, list): - raise ValueError(f'{path}: invalid workflow {workflow!r}') - roots.update(key(edge) for edge in edges) - for edge in roots | set().union(*(r[2] for r in deps.values())): - if edge not in deps: - raise ValueError(f'{path}: missing dependency {edge}') - return deps, roots - - -def audit(old_path, new_path): - """Compare reviewed dependencies reachable from proposed workflow roots. - - For roots present in the reviewed lock, check that their reviewed - reachable records, refs, commits, and edges remain in the proposed lock. - Print a success summary; new sources still require independent review. - Raise ValueError on a mismatch and propagate lock loading errors. - """ - old, _ = load(old_path) - new, roots = load(new_path) - queue = list(roots & old.keys()) - seen = set() - while queue: - k = queue.pop() - if k in seen: - continue - seen.add(k) - if k not in new: - raise ValueError(f'dropped reviewed dependency {k}') - if old[k][:2] != new[k][:2]: - raise ValueError(f'changed reviewed ref/commit {k}') - missing = old[k][2] - new[k][2] - if missing: - raise ValueError(f'dropped reviewed edges from {k}: {sorted(missing)}') - queue.extend(old[k][2]) - print(f'Preserved reviewed reachable closure ({len(seen)} records); new sources require independent review.') - - -if __name__ == '__main__': - if len(sys.argv) != 3: - sys.exit(f'usage: {sys.argv[0]} REVIEWED_LOCK PROPOSED_LOCK') - try: - audit(*sys.argv[1:]) - except (OSError, ValueError, yaml.YAMLError) as exc: - sys.exit(f'lock regeneration audit: {exc}') diff --git a/scripts/audit-lock-regeneration.sh b/scripts/audit-lock-regeneration.sh new file mode 100644 index 00000000..5af00523 --- /dev/null +++ b/scripts/audit-lock-regeneration.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +set -euo pipefail +root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +exec cargo run --quiet --locked --manifest-path "$root/tools/workflow-lock-audit/Cargo.toml" -- regeneration "$@" diff --git a/scripts/audit-workflow-lock-map.py b/scripts/audit-workflow-lock-map.py deleted file mode 100755 index f2fa46e3..00000000 --- a/scripts/audit-workflow-lock-map.py +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MPL-2.0 -"""Inventory workflow lock-map coverage, not remote source closure or provenance.""" -import argparse -import pathlib -import re -import sys -import yaml - - -class UniqueLoader(yaml.SafeLoader): - pass - - -def unique_mapping(loader, node, deep=False): - """Construct a YAML mapping, raising ValueError for duplicate keys. - - Pass deep through to PyYAML's key and value construction. YAML construction - errors and TypeError for unhashable keys propagate to the caller. - """ - result = {} - for key, value in node.value: - k = loader.construct_object(key, deep=deep) - if k in result: - raise ValueError(f'duplicate YAML key: {k!r}') - result[k] = loader.construct_object(value, deep=deep) - return result - - -UniqueLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, unique_mapping) - - -def load(path): - """Read UTF-8 YAML with duplicate keys rejected. - - File, YAML, duplicate-key, and key construction errors propagate. - """ - with path.open(encoding='utf-8') as f: - return yaml.load(f, Loader=UniqueLoader) - - -def ref_key(value): - """Normalize a remote uses reference to owner/repository@ref. - - Lowercase the owner and repository, discard any subpath, and preserve the ref's case. Return None for local - and docker references; raise ValueError for unsupported or invalid input. - """ - if not isinstance(value, str) or not value or re.search(r'\s', value): - raise ValueError(f'invalid uses: {value!r}') - if value.startswith('./'): - return None - if value.startswith('docker://'): - return None - m = re.fullmatch(r'([^/@]+)/([^/@]+)(?:/[^@]+)?@([^@]+)', value) - if not m: - raise ValueError(f'unsupported or invalid uses: {value!r}') - return f'{m[1].lower()}/{m[2].lower()}@{m[3]}' - - -def inspect(root, unmanaged=()): - """Return (findings, notes) for the workflow lock map under root. - - Unmanaged contains repository-relative workflow paths to treat as explicit - exceptions; their dependency closure remains unverified. The inventory - does not verify remote source closure or provenance. Invalid structures or - references raise ValueError; file, YAML, and construction errors propagate. - """ - root = pathlib.Path(root).resolve() - wfdir = root / '.github/workflows' - lock = load(wfdir / 'actions.lock') - if not isinstance(lock, dict) or not isinstance(lock.get('workflows'), dict) or not isinstance(lock.get('dependencies'), dict): - raise ValueError('invalid workflows or dependencies mapping') - workflows, deps = lock['workflows'], lock['dependencies'] - files = {p.relative_to(root).as_posix(): p for p in sorted(wfdir.glob('*')) if p.suffix in ('.yml', '.yaml') and p.is_file()} - unmanaged = set(unmanaged) - findings, notes = [], [] - for path in sorted(unmanaged - files.keys()): - findings.append(f'UNMANAGED-UNKNOWN {path}') - for path in sorted(workflows.keys() - files.keys()): - findings.append(f'STALE-WORKFLOW {path}') - depkeys = set() - for dep in deps: - norm = ref_key(dep) - if norm is None or norm in depkeys: - raise ValueError(f'invalid or duplicate dependency key: {dep!r}') - depkeys.add(norm) - for dep, record in deps.items(): - if not isinstance(record, dict) or not isinstance(record.get('uses', []), list): - raise ValueError(f'invalid dependency record: {dep!r}') - for edge in record.get('uses', []): - normalized = ref_key(edge) - if normalized is None: - raise ValueError(f'invalid dependency edge: {edge!r}') - if normalized not in depkeys: - findings.append(f'MISSING-TRANSITIVE-RECORD {dep} {normalized}') - for name, file in files.items(): - doc = load(file) - if not isinstance(doc, dict) or not isinstance(doc.get('jobs'), dict): - raise ValueError(f'{name}: expected jobs mapping') - refs, remote_calls, local_calls, unsupported = set(), set(), [], [] - for job in doc['jobs'].values(): - if not isinstance(job, dict): - raise ValueError(f'{name}: invalid job') - entries = [] - if 'uses' in job: - entries.append((job['uses'], True)) - steps = job.get('steps', []) - if not isinstance(steps, list): - raise ValueError(f'{name}: invalid steps') - for step in steps: - if not isinstance(step, dict): - raise ValueError(f'{name}: invalid step') - if 'uses' in step: - entries.append((step['uses'], False)) - for raw, is_job in entries: - normalized = ref_key(raw) - if normalized: - if is_job: - # Official gh-actions-lock v0.4.1 excludes external - # jobs..uses. Check their source pins separately; - # do not infer reusable dependency closure from the lock. - remote_calls.add(normalized) - if not re.fullmatch(r'[0-9a-fA-F]{40}', normalized.rsplit('@', 1)[1]): - findings.append(f'REMOTE-REUSABLE-NOT-SHA {name} {normalized}') - notes.append(f'REMOTE-REUSABLE-SOURCE-REVIEW {name} {normalized}') - notes.append(f'UNSUPPORTED-LOCK-SCOPE {name} {normalized} (external reusable dependencies not locked)') - else: - refs.add(normalized) - elif raw.startswith('./') and is_job: - local = (root / raw).resolve() - if not local.is_relative_to(wfdir) or local.suffix not in ('.yml', '.yaml') or not local.is_file(): - findings.append(f'INVALID-LOCAL-WORKFLOW {name} {raw}') - else: - local_calls.append(local.relative_to(root).as_posix()) - else: - unsupported.append(raw) - for raw in unsupported: - notes.append(f'NOT-LOCKED-SOURCE-REVIEW {name} {raw}') - if name not in workflows: - if name in unmanaged: - notes.append(f'INTENTIONALLY-UNMANAGED {name} (dependency closure not verified)') - else: - findings.append(f'MISSING-WORKFLOW {name}') - continue - if name in unmanaged: - findings.append(f'UNMANAGED-BUT-MAPPED {name}') - mapped = workflows[name] - if not isinstance(mapped, list): - raise ValueError(f'{name}: expected lock list') - normalized_mapped = [ref_key(item) for item in mapped] - if None in normalized_mapped or len(set(normalized_mapped)) != len(normalized_mapped): - raise ValueError(f'{name}: invalid or duplicate lock reference') - mapped_set = set(normalized_mapped) - for ref in sorted(refs - mapped_set): - findings.append(f'MISSING-DIRECT-EDGE {name} {ref}') - for ref in sorted(mapped_set - depkeys): - findings.append(f'MISSING-RECORD {name} {ref}') - if not refs and not remote_calls and not local_calls and not unsupported and not mapped_set: - notes.append(f'NO-USES-MAPPED {name}') - for target in local_calls: - notes.append(f'LOCAL-REUSABLE {name} -> {target} (callee checked separately)') - for ref in sorted(mapped_set - refs): - notes.append(f'EXTRA-OR-TRANSITIVE-EDGE {name} {ref} (source review needed)') - return findings, notes - - -def main(): - """Print the inventory and return its command-line status. - - Return 1 for findings, 2 for caught audit errors or unmanaged exceptions, - and 0 otherwise. Caught errors go to stderr; argument parsing may exit. - """ - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('root', nargs='?', default='.') - parser.add_argument('--unmanaged', action='append', default=[], help='explicit workflow path excluded by an owner; still requires manual closure review') - args = parser.parse_args() - try: - findings, notes = inspect(args.root, args.unmanaged) - except (OSError, ValueError, TypeError, yaml.YAMLError) as exc: - print(f'NOT-RUN: {exc}', file=sys.stderr) - return 2 - for item in notes: - print('NOTE:', item) - for item in findings: - print('FINDING:', item) - print(f'workflow-map findings={len(findings)}; remote source closure and provenance NOT VERIFIED') - if findings: - return 1 - return 2 if args.unmanaged else 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/scripts/audit-workflow-lock-map.sh b/scripts/audit-workflow-lock-map.sh new file mode 100644 index 00000000..41a02695 --- /dev/null +++ b/scripts/audit-workflow-lock-map.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +set -euo pipefail +root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +exec cargo run --quiet --locked --manifest-path "$root/tools/workflow-lock-audit/Cargo.toml" -- map "$@" diff --git a/tests/lock-regeneration-test.py b/tests/lock-regeneration-test.py deleted file mode 100644 index 6a8ade32..00000000 --- a/tests/lock-regeneration-test.py +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MPL-2.0 -import copy -import pathlib -import subprocess -import sys -import tempfile -import unittest -import yaml - -script = pathlib.Path(__file__).resolve().parents[1] / 'scripts/audit-lock-regeneration.py' -a, b, c = 'a' * 40, 'b' * 40, 'c' * 40 -base = {'workflows': {'.github/workflows/test.yml': ['org/reusable@v1']}, 'dependencies': { - 'org/reusable@v1': {'ref': 'v1', 'commit': 'sha1-' + a, 'uses': ['org/composite@' + b]}, - 'org/composite@' + b: {'ref': 'v2', 'commit': 'sha1-' + b, 'uses': ['org/child@v3']}, - 'org/child@v3': {'ref': 'v3', 'commit': 'sha1-' + c}}} -with tempfile.TemporaryDirectory() as d: - old, new = pathlib.Path(d) / 'old.yml', pathlib.Path(d) / 'new.yml' - old.write_text(yaml.safe_dump(base)) - def check(label, doc, expected): - new.write_text(yaml.safe_dump(doc)) - result = subprocess.run([sys.executable, script, old, new], capture_output=True, text=True) - assert (result.returncode == 0) == expected, (label, result.stdout, result.stderr) - print('ok:', label) - check('unchanged closure', copy.deepcopy(base), True) - doc = copy.deepcopy(base); doc['dependencies']['org/reusable@v1'].pop('uses'); del doc['dependencies']['org/composite@' + b]; del doc['dependencies']['org/child@v3'] - check('dropped reusable closure without dangling edge', doc, False) - doc = copy.deepcopy(base); doc['dependencies']['org/composite@' + b].pop('uses'); del doc['dependencies']['org/child@v3'] - check('dropped composite closure without dangling edge', doc, False) - doc = copy.deepcopy(base); del doc['dependencies']['org/child@v3']; check('dangling edge', doc, False) - doc = copy.deepcopy(base); doc['dependencies']['org/composite@' + b]['ref'] = 'sha1-' + b; check('lost reviewed tag', doc, False) - doc = copy.deepcopy(base); doc['dependencies']['org/child@v3']['commit'] = c; check('malformed commit', doc, False) - doc = copy.deepcopy(base); doc['dependencies']['org/child@v3']['ref'] = 'v4'; check('wrong tag', doc, False) - doc = copy.deepcopy(base); doc['dependencies']['org/composite@' + b]['commit'] = 'sha1-' + c; check('SHA mismatch', doc, False) - - -class RegenerationAuditTests(unittest.TestCase): - def setUp(self): - self.tmp = tempfile.TemporaryDirectory() - self.addCleanup(self.tmp.cleanup) - self.old = pathlib.Path(self.tmp.name) / 'old.yml' - self.new = pathlib.Path(self.tmp.name) / 'new.yml' - - def run_audit(self, reviewed, proposed): - for path, doc in ((self.old, reviewed), (self.new, proposed)): - path.write_text(doc if isinstance(doc, str) else yaml.safe_dump(doc), encoding='utf-8') - return subprocess.run([sys.executable, script, self.old, self.new], capture_output=True, text=True) - - def test_changed_reviewed_tag_resolution_is_rejected(self): - doc = copy.deepcopy(base) - doc['dependencies']['org/child@v3']['commit'] = 'sha1-' + a - result = self.run_audit(base, doc) - self.assertEqual(result.returncode, 1, result.stderr) - self.assertIn('changed reviewed ref/commit org/child@v3', result.stderr) - - def test_changed_reviewed_sha_record_ref_is_rejected(self): - doc = copy.deepcopy(base) - doc['dependencies']['org/composite@' + b]['ref'] = 'v4' - result = self.run_audit(base, doc) - self.assertEqual(result.returncode, 1, result.stderr) - self.assertIn('changed reviewed ref/commit org/composite@' + b, result.stderr) - - def test_added_edges_and_new_roots_are_allowed_but_require_review(self): - doc = copy.deepcopy(base) - doc['dependencies']['org/child@v3']['uses'] = ['new/action@v1'] - doc['dependencies']['new/action@v1'] = {'ref': 'v1', 'commit': 'sha1-' + a} - doc['workflows']['new.yml'] = ['new/action@v1'] - result = self.run_audit(base, doc) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn('Preserved reviewed reachable closure (3 records)', result.stdout) - self.assertIn('new sources require independent review', result.stdout) - - def test_removed_old_root_does_not_force_unused_closure(self): - proposed = {'workflows': {}, 'dependencies': {}} - result = self.run_audit(base, proposed) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn('closure (0 records)', result.stdout) - - def test_cycles_and_case_insensitive_repository_names(self): - reviewed = copy.deepcopy(base) - reviewed['dependencies']['org/child@v3']['uses'] = ['ORG/Reusable@v1'] - proposed = copy.deepcopy(reviewed) - proposed['workflows']['.github/workflows/test.yml'] = ['ORG/REUSABLE@v1'] - result = self.run_audit(reviewed, proposed) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn('closure (3 records)', result.stdout) - - def test_dropped_edge_is_identified_even_when_record_remains(self): - doc = copy.deepcopy(base) - doc['dependencies']['org/reusable@v1']['uses'] = [] - result = self.run_audit(base, doc) - self.assertEqual(result.returncode, 1, result.stderr) - self.assertIn('dropped reviewed edges from org/reusable@v1', result.stderr) - self.assertIn('org/composite@' + b, result.stderr) - - def test_invalid_documents_fail_on_either_input(self): - invalid = [ - ('workflows: {}\nworkflows: {}\ndependencies: {}\n', 'duplicate YAML key'), - ({'workflows': [], 'dependencies': {}}, 'expected workflows and dependencies mappings'), - ({'workflows': {'test.yml': ['org/missing@v1']}, 'dependencies': {}}, 'missing dependency'), - ({'workflows': {'test.yml': 'org/action@v1'}, 'dependencies': {}}, 'invalid workflow'), - ({'workflows': {}, 'dependencies': {'org/action@v1': []}}, 'invalid record'), - ({'workflows': {}, 'dependencies': {'org/action@v1': {'ref': 'v1', 'commit': 'sha1-' + a, 'uses': 'org/child@v1'}}}, 'invalid uses'), - ({'workflows': {}, 'dependencies': {'org/action/sub@v1': {}}}, 'invalid dependency reference'), - ({'workflows': {}, 'dependencies': {'org/action@v1': {'ref': 'v1', 'commit': 'sha1-' + a}, 'ORG/ACTION@v1': {'ref': 'v1', 'commit': 'sha1-' + a}}}, 'duplicate or invalid record'), - ] - for doc, message in invalid: - for old_is_invalid in (True, False): - with self.subTest(message=message, old_is_invalid=old_is_invalid): - result = self.run_audit(doc if old_is_invalid else base, base if old_is_invalid else doc) - self.assertEqual(result.returncode, 1, result.stderr) - self.assertIn(message, result.stderr) - - def test_cli_requires_two_paths_and_handles_missing_file(self): - result = subprocess.run([sys.executable, script], capture_output=True, text=True) - self.assertEqual(result.returncode, 1) - self.assertIn('usage:', result.stderr) - self.old.write_text(yaml.safe_dump(base), encoding='utf-8') - result = subprocess.run([sys.executable, script, self.old, self.new], capture_output=True, text=True) - self.assertEqual(result.returncode, 1) - self.assertIn('lock regeneration audit:', result.stderr) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/workflow-lock-map-test.py b/tests/workflow-lock-map-test.py deleted file mode 100644 index f13322c9..00000000 --- a/tests/workflow-lock-map-test.py +++ /dev/null @@ -1,240 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MPL-2.0 -import pathlib -import subprocess -import sys -import tempfile -import unittest -import yaml - -SCRIPT = pathlib.Path(__file__).resolve().parents[1] / 'scripts/audit-workflow-lock-map.py' - - -class WorkflowMapTests(unittest.TestCase): - def setUp(self): - self.tmp = tempfile.TemporaryDirectory() - self.addCleanup(self.tmp.cleanup) - self.root = pathlib.Path(self.tmp.name) - self.dir = self.root / '.github/workflows' - self.dir.mkdir(parents=True) - self.lock = {'workflows': {'.github/workflows/main.yml': ['acme/action@v1'], - '.github/workflows/empty.yml': []}, - 'dependencies': {'acme/action@v1': {'ref': 'v1', 'commit': 'sha1-' + 'a' * 40}}} - self.write('main.yml', {'jobs': {'build': {'steps': [{'uses': 'Acme/action/subpath@v1'}]}}}) - self.write('empty.yml', {'jobs': {'build': {'steps': [{'run': 'echo hello'}]}}}) - - def write(self, name, data): - (self.dir / name).write_text(yaml.safe_dump(data), encoding='utf-8') - - def run_audit(self, *args): - (self.dir / 'actions.lock').write_text(yaml.safe_dump(self.lock), encoding='utf-8') - result = subprocess.run([sys.executable, SCRIPT, self.root, *args], text=True, capture_output=True) - return result.returncode, result.stdout + result.stderr - - def test_mapped_no_uses_is_not_missing_or_unmanaged(self): - code, output = self.run_audit() - self.assertEqual(code, 0, output) - self.assertIn('NO-USES-MAPPED .github/workflows/empty.yml', output) - self.assertNotIn('INTENTIONALLY-UNMANAGED', output) - self.assertIn('remote source closure and provenance NOT VERIFIED', output) - - def test_missing_workflow_is_not_silently_exempted(self): - del self.lock['workflows']['.github/workflows/empty.yml'] - code, output = self.run_audit() - self.assertEqual(code, 1, output) - self.assertIn('MISSING-WORKFLOW .github/workflows/empty.yml', output) - - def test_explicit_unmanaged_is_not_a_closure_pass(self): - del self.lock['workflows']['.github/workflows/empty.yml'] - code, output = self.run_audit('--unmanaged', '.github/workflows/empty.yml') - self.assertEqual(code, 2, output) - self.assertIn('INTENTIONALLY-UNMANAGED', output) - self.assertIn('closure not verified', output) - - def test_invalid_unmanaged_exception_fails(self): - code, output = self.run_audit('--unmanaged', '.github/workflows/typo.yml') - self.assertEqual(code, 1, output) - self.assertIn('UNMANAGED-UNKNOWN', output) - - def test_missing_direct_ref_even_with_workflow_key(self): - self.lock['workflows']['.github/workflows/main.yml'] = [] - code, output = self.run_audit() - self.assertEqual(code, 1, output) - self.assertIn('MISSING-DIRECT-EDGE', output) - - def test_sha_reusable_empty_map_requires_source_review(self): - self.write('main.yml', {'jobs': {'scan': {'uses': 'acme/suite/.github/workflows/scan.yml@' + 'a' * 40}}}) - self.lock['workflows']['.github/workflows/main.yml'] = [] - code, output = self.run_audit() - self.assertEqual(code, 0, output) - self.assertIn('REMOTE-REUSABLE-SOURCE-REVIEW', output) - self.assertIn('UNSUPPORTED-LOCK-SCOPE', output) - self.assertNotIn('MISSING-DIRECT-EDGE', output) - self.assertNotIn('NO-USES-MAPPED .github/workflows/main.yml', output) - - def test_remote_reusable_floating_and_short_pins_fail(self): - for ref in ('main', 'v1', 'abc1234', 'a' * 39, 'a' * 41, 'g' * 40): - with self.subTest(ref=ref): - self.write('main.yml', {'jobs': {'scan': {'uses': 'acme/suite/.github/workflows/scan.yml@' + ref}}}) - self.lock['workflows']['.github/workflows/main.yml'] = [] - code, output = self.run_audit() - self.assertEqual(code, 1, output) - self.assertIn('REMOTE-REUSABLE-NOT-SHA', output) - - def test_reusable_does_not_exempt_missing_ordinary_action_edge(self): - self.write('main.yml', {'jobs': { - 'scan': {'uses': 'acme/suite/.github/workflows/scan.yml@' + 'a' * 40}, - 'build': {'steps': [{'uses': 'acme/action@v1'}]}, - }}) - self.lock['workflows']['.github/workflows/main.yml'] = [] - code, output = self.run_audit() - self.assertEqual(code, 1, output) - self.assertIn('MISSING-DIRECT-EDGE .github/workflows/main.yml acme/action@v1', output) - self.assertIn('UNSUPPORTED-LOCK-SCOPE', output) - - def test_missing_nested_record(self): - self.lock['dependencies']['acme/action@v1']['uses'] = ['acme/child@v1'] - code, output = self.run_audit() - self.assertEqual(code, 1, output) - self.assertIn('MISSING-TRANSITIVE-RECORD', output) - - def test_local_reusable_is_checked_at_its_own_path(self): - self.write('main.yml', {'jobs': {'call': {'uses': './.github/workflows/callee.yml'}}}) - self.lock['workflows']['.github/workflows/main.yml'] = [] - self.write('callee.yml', {'jobs': {'run': {'steps': [{'uses': 'acme/action@v1'}]}}}) - self.lock['workflows']['.github/workflows/callee.yml'] = [] - code, output = self.run_audit() - self.assertEqual(code, 1, output) - self.assertIn('LOCAL-REUSABLE', output) - self.assertIn('MISSING-DIRECT-EDGE .github/workflows/callee.yml', output) - - def test_malformed_and_duplicate_yaml_fail_closed(self): - self.write('main.yml', {'jobs': {'run': {'steps': [{'uses': 'no-at-sign'}]}}}) - code, output = self.run_audit() - self.assertEqual(code, 2, output) - self.assertIn('NOT-RUN', output) - (self.dir / 'main.yml').write_text('jobs: {}\njobs: {}\n', encoding='utf-8') - code, output = self.run_audit() - self.assertEqual(code, 2, output) - self.assertIn('duplicate YAML key', output) - - def test_extra_edge_is_review_not_proof_of_failure(self): - self.lock['workflows']['.github/workflows/empty.yml'] = ['acme/action@v1'] - code, output = self.run_audit() - self.assertEqual(code, 0, output) - self.assertIn('EXTRA-OR-TRANSITIVE-EDGE', output) - - def test_mapped_ref_without_record_and_stale_workflow(self): - self.lock['workflows']['.github/workflows/main.yml'].append('acme/missing@v2') - self.lock['workflows']['.github/workflows/deleted.yml'] = [] - code, output = self.run_audit() - self.assertEqual(code, 1, output) - self.assertIn('MISSING-RECORD .github/workflows/main.yml acme/missing@v2', output) - self.assertIn('STALE-WORKFLOW .github/workflows/deleted.yml', output) - - def test_mapped_workflow_cannot_also_be_unmanaged(self): - code, output = self.run_audit('--unmanaged', '.github/workflows/main.yml') - self.assertEqual(code, 1, output) - self.assertIn('UNMANAGED-BUT-MAPPED .github/workflows/main.yml', output) - - def test_invalid_local_workflow_paths(self): - for path in ('./.github/workflows/missing.yml', './outside.yml', './.github/workflows/actions.lock'): - with self.subTest(path=path): - self.write('main.yml', {'jobs': {'call': {'uses': path}}}) - self.lock['workflows']['.github/workflows/main.yml'] = [] - code, output = self.run_audit() - self.assertEqual(code, 1, output) - self.assertIn('INVALID-LOCAL-WORKFLOW .github/workflows/main.yml ' + path, output) - - def test_local_actions_and_docker_are_reported_for_review(self): - self.write('main.yml', {'jobs': {'build': {'steps': [ - {'uses': './my-action'}, {'uses': 'docker://example/image:1'}]}}}) - self.lock['workflows']['.github/workflows/main.yml'] = [] - code, output = self.run_audit() - self.assertEqual(code, 0, output) - self.assertIn('NOT-LOCKED-SOURCE-REVIEW .github/workflows/main.yml ./my-action', output) - self.assertIn('NOT-LOCKED-SOURCE-REVIEW .github/workflows/main.yml docker://example/image:1', output) - - def test_normalizes_owner_repo_and_subpath_but_preserves_ref_case(self): - self.write('main.yml', {'jobs': {'build': {'steps': [{'uses': 'ACME/ACTION/sub@V1'}]}}}) - code, output = self.run_audit() - self.assertEqual(code, 1, output) - self.assertIn('MISSING-DIRECT-EDGE .github/workflows/main.yml acme/action@V1', output) - self.lock['workflows']['.github/workflows/main.yml'] = ['acme/action@V1'] - self.lock['dependencies'] = {'Acme/Action@V1': {}} - code, output = self.run_audit() - self.assertEqual(code, 0, output) - - def test_duplicate_normalized_refs_and_bad_records_fail_closed(self): - cases = [ - ('duplicate dependency', lambda: self.lock['dependencies'].update({'ACME/ACTION@v1': {}})), - ('duplicate map edge', lambda: self.lock['workflows']['.github/workflows/main.yml'].append('ACME/ACTION/sub@v1')), - ('invalid record', lambda: self.lock['dependencies'].update({'acme/action@v1': []})), - ('invalid transitive edge', lambda: self.lock['dependencies']['acme/action@v1'].update({'uses': ['./local']})), - ('invalid workflow map', lambda: self.lock['workflows'].update({'.github/workflows/main.yml': 'acme/action@v1'})), - ] - original = yaml.safe_dump(self.lock) - for label, mutate in cases: - with self.subTest(label=label): - self.lock = yaml.safe_load(original) - mutate() - code, output = self.run_audit() - self.assertEqual(code, 2, output) - self.assertIn('NOT-RUN:', output) - - def test_invalid_workflow_shapes_fail_closed(self): - cases = [ - {'jobs': []}, - {'jobs': {'build': None}}, - {'jobs': {'build': {'steps': {}}}}, - {'jobs': {'build': {'steps': [None]}}}, - {'jobs': {'build': {'steps': [{'uses': 'acme/action@bad ref'}]}}}, - ] - for doc in cases: - with self.subTest(doc=doc): - self.write('main.yml', doc) - code, output = self.run_audit() - self.assertEqual(code, 2, output) - self.assertIn('NOT-RUN:', output) - - def test_yaml_extension_is_inventoried(self): - self.write('extra.yaml', {'jobs': {'run': {'steps': []}}}) - code, output = self.run_audit() - self.assertEqual(code, 1, output) - self.assertIn('MISSING-WORKFLOW .github/workflows/extra.yaml', output) - - -class ScannerCallerTests(unittest.TestCase): - @classmethod - def setUpClass(cls): - root = pathlib.Path(__file__).resolve().parents[1] / '.github/workflows' - # BaseLoader preserves GitHub's `on` key and the spelling of scalar values. - cls.workflow = yaml.load((root / 'secret-scanner.yml').read_text(encoding='utf-8'), Loader=yaml.BaseLoader) - cls.lock = yaml.safe_load((root / 'actions.lock').read_text(encoding='utf-8')) - - def test_scanner_triggers_and_access(self): - self.assertEqual(self.workflow['on'], {'pull_request': '', 'push': {'branches': ['main']}}) - self.assertEqual(self.workflow['permissions'], {'contents': 'read'}) - self.assertEqual(set(self.workflow['jobs']), {'scan'}) - job = self.workflow['jobs']['scan'] - self.assertEqual(job['permissions'], {'contents': 'read'}) - self.assertNotIn('secrets', job) - self.assertEqual(job['with'], {'fast-pr': 'false'}) - - def test_scanner_source_pin_and_supported_lock_scope(self): - sha = '4404c3c0474e17753fe7689f3485c02890aac113' - suite = 'hyperpolymath/cicd-suite@' + sha - checkout = 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' - self.assertEqual(self.workflow['jobs']['scan']['uses'], - 'hyperpolymath/cicd-suite/.github/workflows/secret-scanner-reusable.yml@' + sha) - # gh-actions-lock v0.4.1 does not lock external jobs..uses. - # The caller SHA above is checked separately from ordinary action records. - self.assertEqual(self.lock['workflows']['.github/workflows/secret-scanner.yml'], []) - self.assertNotIn(suite, self.lock['dependencies']) - self.assertIn(checkout, self.lock['dependencies']) - self.assertEqual(self.lock['dependencies'][checkout]['commit'], - 'sha1-3d3c42e5aac5ba805825da76410c181273ba90b1') - - -if __name__ == '__main__': - unittest.main() diff --git a/tools/workflow-lock-audit/.gitignore b/tools/workflow-lock-audit/.gitignore new file mode 100644 index 00000000..a3c49614 --- /dev/null +++ b/tools/workflow-lock-audit/.gitignore @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: MPL-2.0 +/target/ diff --git a/tools/workflow-lock-audit/Cargo.lock b/tools/workflow-lock-audit/Cargo.lock new file mode 100644 index 00000000..0fe976b1 --- /dev/null +++ b/tools/workflow-lock-audit/Cargo.lock @@ -0,0 +1,203 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "fleet-workflow-lock-audit" +version = "0.1.0" +dependencies = [ + "saphyr", + "saphyr-parser", + "serde_json", +] + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashlink" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "ordered-float" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e" +dependencies = [ + "num-traits", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "saphyr" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3aa37a62b2a28868277e83b6f651dc9581751db73d1cb51132c73c0b4ec7f70a" +dependencies = [ + "hashlink", + "ordered-float", + "saphyr-parser", + "thiserror", +] + +[[package]] +name = "saphyr-parser" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48cf5a82d41247a625f040af08e91008fec945bce3b0bffba992c44af092caa2" +dependencies = [ + "arraydeque", + "thiserror", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/tools/workflow-lock-audit/Cargo.toml b/tools/workflow-lock-audit/Cargo.toml new file mode 100644 index 00000000..74996968 --- /dev/null +++ b/tools/workflow-lock-audit/Cargo.toml @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: MPL-2.0 +[package] +name = "fleet-workflow-lock-audit" +version = "0.1.0" +edition = "2021" +rust-version = "1.85" +license = "MPL-2.0" +publish = false + +[dependencies] +saphyr = { version = "=0.3.3", default-features = false } +saphyr-parser = "=0.3.3" + +[dev-dependencies] +serde_json = "=1.0.149" diff --git a/tools/workflow-lock-audit/README.adoc b/tools/workflow-lock-audit/README.adoc new file mode 100644 index 00000000..9371118b --- /dev/null +++ b/tools/workflow-lock-audit/README.adoc @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 += Workflow lock audit + +This Rust CLI replaces the retired Python lock-map and regeneration auditors. +Run it with Rust/Cargo 1.85 or later; the committed Cargo.lock is required. + +[source,bash] +---- +cargo test --locked --manifest-path tools/workflow-lock-audit/Cargo.toml +bash scripts/audit-workflow-lock-map.sh . +bash scripts/audit-lock-regeneration.sh /path/to/reviewed.lock .github/workflows/actions.lock +---- + +The map inventory checks workflow coverage, direct action edges, transitive +record presence and reusable caller source-pin shape. External reusable +workflow dependencies are outside gh-actions-lock v0.4.1's map scope; their +source-review notes are not evidence of provenance. Map exit statuses are +0 (no map findings), 1 (findings), and 2 (invalid input, usage or explicit +unmanaged scope). The existing explicit unmanaged option still returns 2 +when no map finding exists; it does not confer a closure pass. + +Regeneration comparison preserves the reviewed reachable records, refs, +commits and edges beneath proposed roots shared with the reviewed graph. +It exits 0 on preservation and 1 on invalid input or a changed reviewed +closure. Removed roots no longer force unused closure, and new roots/edges +still require independent review. Use the pinned-source closure verifier +and receipts separately; neither of these commands discovers remote closure. + +The parser is saphyr/saphyr-parser 0.3.3, with duplicate detection enabled, +a 100,000-node alias-expansion budget and a second duplicate check after +scalar resolution. Input must contain exactly one document. Complex mapping +keys, explicit tags and YAML merge keys are rejected. Legacy YAML 1.1 +unquoted yes/no/on/off remain booleans, so a bare boolean cannot become a +valid ref after migration. Ambiguous plain timestamps, sexagesimal values +and overflowing numbers fail closed; quote intended strings explicitly. +These bounded, explicit restrictions avoid silently changing input meaning. + +The 41 CLI tests retain all 22 prior map/scanner controls and 16 prior +regeneration controls, plus strict parsing, symlink containment and legacy +scalar tests. Test fixtures use serde_json solely to write inputs; production +uses the saphyr tree directly. ordered-float is locked to 5.3.0 to retain +Rust 1.85 compatibility. Regenerate dependencies deliberately and rerun tests +with that minimum toolchain. + +E2E audits this crate's Cargo.lock with cargo-audit 0.22.2 against the live +RustSec advisory database and denies all warnings, including unmaintained, +unsound and yanked dependencies. Its official Linux x86_64 release archive +is SHA-256 checked before extraction. No advisory exclusion is configured. diff --git a/tools/workflow-lock-audit/src/main.rs b/tools/workflow-lock-audit/src/main.rs new file mode 100644 index 00000000..6bcdf365 --- /dev/null +++ b/tools/workflow-lock-audit/src/main.rs @@ -0,0 +1,514 @@ +// SPDX-License-Identifier: MPL-2.0 +//! Lock-map inventory and reviewed-graph comparison, not source provenance. +use saphyr::{MappingOwned as Mapping, ScalarOwned, ScalarStyle, YamlLoader, YamlOwned as Value}; +use saphyr_parser::Parser; +use std::{ + collections::{BTreeMap, BTreeSet}, + env, fs, + path::{Path, PathBuf}, + process, +}; +type Result = std::result::Result; +type Set = BTreeSet; + +fn mapping<'a>(value: &'a Value, context: &str) -> Result<&'a Mapping> { + value + .as_mapping() + .ok_or_else(|| format!("{context}: expected mapping")) +} +fn list<'a>(value: &'a Value, context: &str) -> Result<&'a Vec> { + value + .as_sequence() + .ok_or_else(|| format!("{context}: expected list")) +} +fn string<'a>(value: &'a Value, context: &str) -> Result<&'a str> { + value + .as_str() + .ok_or_else(|| format!("{context}: expected string")) +} +fn get<'a>(map: &'a Mapping, key: &str) -> Option<&'a Value> { + map.get(&Value::Value(ScalarOwned::String(key.into()))) +} +fn read_yaml(path: &Path) -> Result { + let data = fs::read_to_string(path).map_err(|e| format!("{}: {e}", path.display()))?; + // Keep spelling/style until schema conversion: this preserves the old + // SafeLoader boolean type checks without losing quoted ref strings. + let mut loader = YamlLoader::::default(); + loader.early_parse(false); + loader.allow_duplicate_keys(false); + loader.alias_limit(100_000); + let mut parser = Parser::new_from_str(&data); + parser + .load(&mut loader, true) + .map_err(|e| format!("{}: {e}", path.display()))?; + if let Some(error) = loader.error() { + let error = error.to_string(); + return Err(if error.contains("duplicated key") { + format!("{}: duplicate YAML key: {error}", path.display()) + } else { + format!("{}: {error}", path.display()) + }); + } + let mut docs = loader.into_documents(); + if docs.len() != 1 { + return Err("expected exactly one YAML document".into()); + } + resolve_yaml(docs.pop().unwrap()) +} +fn resolve_yaml(value: Value) -> Result { + match value { + Value::Representation(raw, style, tag) => { + // Explicit tags and merge keys are outside this audit's supported + // schema. Refuse them instead of auditing a different document. + if tag.is_some() { + return Err("YAML tags are unsupported".into()); + } + if style == ScalarStyle::Plain { + let bytes = raw.as_bytes(); + let timestamp = bytes.len() >= 10 + && bytes[4] == b'-' + && bytes[7] == b'-' + && bytes[..4] + .iter() + .chain(bytes[5..7].iter()) + .chain(bytes[8..10].iter()) + .all(u8::is_ascii_digit); + let sexagesimal = raw.contains(':') + && raw + .chars() + .all(|c| c.is_ascii_digit() || matches!(c, ':' | '+' | '-' | '.' | '_')); + if timestamp || sexagesimal { + return Err("ambiguous YAML 1.1 typed scalar; quote strings explicitly".into()); + } + } + let scalar = if style == ScalarStyle::Plain { + match raw.as_str() { + "yes" | "Yes" | "YES" | "on" | "On" | "ON" => ScalarOwned::Boolean(true), + "no" | "No" | "NO" | "off" | "Off" | "OFF" => ScalarOwned::Boolean(false), + _ => ScalarOwned::parse_from_cow_and_metadata(raw.clone().into(), style, None) + .ok_or("invalid YAML scalar")?, + } + } else { + ScalarOwned::String(raw.clone()) + }; + // The YAML 1.2 parser bounds integers to i64; the old loader did + // not. Never turn an overflowing bare number into an accepted ref. + if style == ScalarStyle::Plain + && matches!(scalar, ScalarOwned::String(_)) + && raw.parse::().is_ok() + { + return Err( + "out-of-range plain numeric YAML scalar; quote strings explicitly".into(), + ); + } + Ok(Value::Value(scalar)) + } + Value::Mapping(map) => { + let mut resolved = Mapping::new(); + for (key, value) in map { + let key = resolve_yaml(key)?; + if matches!(key, Value::Mapping(_) | Value::Sequence(_)) { + return Err("complex YAML mapping keys are unsupported".into()); + } + if key.as_str() == Some("<<") { + return Err("YAML merge keys are unsupported".into()); + } + let value = resolve_yaml(value)?; + // Differently quoted spellings and YAML 1.1 boolean aliases + // must not bypass duplicate detection during normalization. + if resolved.insert(key, value).is_some() { + return Err("duplicate YAML key after scalar resolution".into()); + } + } + Ok(Value::Mapping(resolved)) + } + Value::Sequence(items) => Ok(Value::Sequence( + items.into_iter().map(resolve_yaml).collect::>()?, + )), + Value::Tagged(_, _) => Err("YAML tags are unsupported".into()), + Value::Alias(_) | Value::BadValue => Err("unresolved or invalid YAML node".into()), + Value::Value(_) => Ok(value), + } +} +fn reference(raw: &str, dependency: bool) -> Result> { + if raw.is_empty() || raw.chars().any(char::is_whitespace) { + return Err(format!("invalid uses: {raw:?}")); + } + if !dependency && (raw.starts_with("./") || raw.starts_with("docker://")) { + return Ok(None); + } + let (repo, pin) = raw + .split_once('@') + .ok_or_else(|| format!("invalid dependency reference: {raw:?}"))?; + let parts: Vec<_> = repo.split('/').collect(); + if pin.is_empty() + || pin.contains('@') + || parts.len() < 2 + || parts.iter().any(|p| p.is_empty()) + || (dependency && parts.len() != 2) + { + return Err(format!("invalid dependency reference: {raw:?}")); + } + Ok(Some(format!( + "{}/{}@{}", + parts[0].to_lowercase(), + parts[1].to_lowercase(), + pin + ))) +} +fn key(value: &Value, strict: bool) -> Result { + reference(string(value, "dependency reference")?, strict)? + .ok_or_else(|| "invalid dependency reference: local/docker reference".into()) +} +fn sha(pin: &str) -> bool { + pin.len() == 40 && pin.bytes().all(|b| b.is_ascii_hexdigit()) +} +fn lock_parts(value: &Value) -> Result<(&Mapping, &Mapping)> { + let doc = mapping(value, "lock")?; + let w = get(doc, "workflows").and_then(Value::as_mapping); + let d = get(doc, "dependencies").and_then(Value::as_mapping); + w.zip(d) + .ok_or_else(|| "expected workflows and dependencies mappings".into()) +} +#[derive(PartialEq)] +struct Record { + reference: String, + commit: String, + edges: Set, +} +fn load_lock(path: &Path) -> Result<(BTreeMap, Set)> { + let value = read_yaml(path)?; + let (workflows, dependencies) = lock_parts(&value)?; + let mut deps = BTreeMap::new(); + for (name, record) in dependencies { + let k = key(name, true)?; + let rec = record + .as_mapping() + .ok_or_else(|| format!("duplicate or invalid record {k}"))?; + if deps.contains_key(&k) { + return Err(format!("duplicate or invalid record {k}")); + } + let reference = get(rec, "ref") + .and_then(Value::as_str) + .ok_or_else(|| format!("invalid ref for {k}"))?; + if reference.is_empty() + || reference.chars().any(char::is_whitespace) + || reference.contains('@') + || reference.starts_with("sha1-") + { + return Err(format!("invalid ref for {k}")); + } + let commit = get(rec, "commit") + .and_then(Value::as_str) + .ok_or_else(|| format!("invalid commit for {k}"))?; + if !commit + .strip_prefix("sha1-") + .is_some_and(|s| sha(s) && s.bytes().all(|b| !b.is_ascii_uppercase())) + { + return Err(format!("invalid commit for {k}")); + } + let named = k.rsplit_once('@').unwrap().1; + if sha(named) && named.bytes().all(|b| !b.is_ascii_uppercase()) { + if commit != format!("sha1-{named}") { + return Err(format!("SHA key and commit differ for {k}")); + } + } else if named != reference { + return Err(format!("tag ref differs from key for {k}")); + } + let mut edges = Set::new(); + if let Some(uses) = get(rec, "uses") { + for edge in list(uses, &format!("invalid uses for {k}"))? { + edges.insert(key(edge, true)?); + } + } + deps.insert( + k, + Record { + reference: reference.into(), + commit: commit.into(), + edges, + }, + ); + } + let mut roots = Set::new(); + for (workflow, edges) in workflows { + string(workflow, "invalid workflow")?; + for edge in list(edges, "invalid workflow")? { + roots.insert(key(edge, true)?); + } + } + for edge in roots + .iter() + .chain(deps.values().flat_map(|r| r.edges.iter())) + { + if !deps.contains_key(edge) { + return Err(format!("missing dependency {edge}")); + } + } + Ok((deps, roots)) +} +fn regeneration(old_path: &Path, new_path: &Path) -> Result<()> { + let (old, _) = load_lock(old_path)?; + let (new, roots) = load_lock(new_path)?; + let mut queue: Vec<_> = roots.into_iter().filter(|k| old.contains_key(k)).collect(); + let mut seen = Set::new(); + while let Some(k) = queue.pop() { + if !seen.insert(k.clone()) { + continue; + } + let reviewed = &old[&k]; + let proposed = new + .get(&k) + .ok_or_else(|| format!("dropped reviewed dependency {k}"))?; + if reviewed.reference != proposed.reference || reviewed.commit != proposed.commit { + return Err(format!("changed reviewed ref/commit {k}")); + } + let missing: Vec<_> = reviewed.edges.difference(&proposed.edges).collect(); + if !missing.is_empty() { + return Err(format!("dropped reviewed edges from {k}: {missing:?}")); + } + queue.extend(reviewed.edges.iter().cloned()); + } + println!("Preserved reviewed reachable closure ({} records); new sources require independent review.", seen.len()); + Ok(()) +} +fn inventory(root: &Path, unmanaged: &Set) -> Result<(Vec, Vec)> { + let root = fs::canonicalize(root).map_err(|e| e.to_string())?; + let wfdir = root.join(".github/workflows"); + let canonical_wfdir = fs::canonicalize(&wfdir).map_err(|e| e.to_string())?; + let lock = read_yaml(&wfdir.join("actions.lock"))?; + let (workflows, deps) = lock_parts(&lock)?; + let mut files = BTreeMap::new(); + for entry in fs::read_dir(&wfdir).map_err(|e| e.to_string())? { + let path = entry.map_err(|e| e.to_string())?.path(); + if path.is_file() + && matches!( + path.extension().and_then(|s| s.to_str()), + Some("yml" | "yaml") + ) + { + files.insert( + format!( + ".github/workflows/{}", + path.file_name() + .and_then(|s| s.to_str()) + .ok_or("non-UTF8 workflow name")? + ), + path, + ); + } + } + let mut findings = Vec::new(); + let mut notes = Vec::new(); + for path in unmanaged { + if !files.contains_key(path) { + findings.push(format!("UNMANAGED-UNKNOWN {path}")); + } + } + for name in workflows.keys() { + let name = string(name, "workflow path")?; + if !files.contains_key(name) { + findings.push(format!("STALE-WORKFLOW {name}")); + } + } + let mut depkeys = Set::new(); + for dep in deps.keys() { + if !depkeys.insert(key(dep, false)?) { + return Err("invalid or duplicate dependency key".into()); + } + } + for (dep, record) in deps { + let dep = string(dep, "dependency key")?; + let record = mapping(record, "invalid dependency record")?; + if let Some(edges) = get(record, "uses") { + for edge in list(edges, "dependency uses")? { + let normalized = key(edge, false)?; + if !depkeys.contains(&normalized) { + findings.push(format!("MISSING-TRANSITIVE-RECORD {dep} {normalized}")); + } + } + } + } + for (name, file) in &files { + let doc = read_yaml(file)?; + let doc = mapping(&doc, "workflow")?; + let jobs = get(doc, "jobs") + .and_then(Value::as_mapping) + .ok_or_else(|| format!("{name}: expected jobs mapping"))?; + let mut refs = Set::new(); + let mut remote_calls = Set::new(); + let mut local_calls = Vec::new(); + let mut unsupported = Vec::new(); + for job in jobs.values() { + let job = mapping(job, "invalid job")?; + let mut entries = Vec::new(); + if let Some(uses) = get(job, "uses") { + entries.push((uses, true)); + } + if let Some(steps) = get(job, "steps") { + for step in list(steps, "invalid steps")? { + let step = mapping(step, "invalid step")?; + if let Some(uses) = get(step, "uses") { + entries.push((uses, false)); + } + } + } + for (raw, is_job) in entries { + let raw = string(raw, "uses")?; + match reference(raw, false)? { + Some(normalized) if is_job => { + if !sha(normalized.rsplit_once('@').unwrap().1) { + findings.push(format!("REMOTE-REUSABLE-NOT-SHA {name} {normalized}")); + } + notes.push(format!("REMOTE-REUSABLE-SOURCE-REVIEW {name} {normalized}")); + notes.push(format!("UNSUPPORTED-LOCK-SCOPE {name} {normalized} (external reusable dependencies not locked)")); + remote_calls.insert(normalized); + } + Some(normalized) => { + refs.insert(normalized); + } + None if raw.starts_with("./") && is_job => { + let local = fs::canonicalize(root.join(raw)); + match local { + Ok(local) + if local.starts_with(&canonical_wfdir) + && local.is_file() + && matches!( + local.extension().and_then(|s| s.to_str()), + Some("yml" | "yaml") + ) => + { + local_calls.push( + local + .strip_prefix(&root) + .map_err(|e| e.to_string())? + .to_string_lossy() + .into_owned(), + ); + } + _ => findings.push(format!("INVALID-LOCAL-WORKFLOW {name} {raw}")), + } + } + None => unsupported.push(raw.to_owned()), + } + } + } + for raw in &unsupported { + notes.push(format!("NOT-LOCKED-SOURCE-REVIEW {name} {raw}")); + } + let Some(mapped) = get(workflows, name) else { + if unmanaged.contains(name) { + notes.push(format!( + "INTENTIONALLY-UNMANAGED {name} (dependency closure not verified)" + )); + } else { + findings.push(format!("MISSING-WORKFLOW {name}")); + } + continue; + }; + if unmanaged.contains(name) { + findings.push(format!("UNMANAGED-BUT-MAPPED {name}")); + } + let mut mapped_set = Set::new(); + for item in list(mapped, "workflow lock list")? { + if !mapped_set.insert(key(item, false)?) { + return Err(format!("{name}: invalid or duplicate lock reference")); + } + } + for r in refs.difference(&mapped_set) { + findings.push(format!("MISSING-DIRECT-EDGE {name} {r}")); + } + for r in mapped_set.difference(&depkeys) { + findings.push(format!("MISSING-RECORD {name} {r}")); + } + if refs.is_empty() + && remote_calls.is_empty() + && local_calls.is_empty() + && unsupported.is_empty() + && mapped_set.is_empty() + { + notes.push(format!("NO-USES-MAPPED {name}")); + } + for target in local_calls { + notes.push(format!( + "LOCAL-REUSABLE {name} -> {target} (callee checked separately)" + )); + } + for r in mapped_set.difference(&refs) { + notes.push(format!( + "EXTRA-OR-TRANSITIVE-EDGE {name} {r} (source review needed)" + )); + } + } + Ok((findings, notes)) +} +fn main() { + let args: Vec<_> = env::args().skip(1).collect(); + let code = match args.first().map(String::as_str) { + Some("regeneration") if args.len() == 3 => { + match regeneration(Path::new(&args[1]), Path::new(&args[2])) { + Ok(()) => 0, + Err(e) => { + eprintln!("lock regeneration audit: {e}"); + 1 + } + } + } + Some("regeneration") => { + eprintln!("usage: audit-lock-regeneration.sh REVIEWED_LOCK PROPOSED_LOCK"); + 1 + } + Some("map") => { + let mut root = PathBuf::from("."); + let mut has_root = false; + let mut unmanaged = Set::new(); + let mut rest = args.iter().skip(1); + let mut invalid = false; + while let Some(arg) = rest.next() { + if arg == "--unmanaged" { + if let Some(path) = rest.next() { + unmanaged.insert(path.clone()); + } else { + invalid = true; + } + } else if arg.starts_with('-') || has_root { + invalid = true; + } else { + root = PathBuf::from(arg); + has_root = true; + } + } + if invalid { + eprintln!("usage: audit-workflow-lock-map.sh [ROOT] [--unmanaged WORKFLOW]"); + process::exit(2); + } + match inventory(&root, &unmanaged) { + Err(e) => { + eprintln!("NOT-RUN: {e}"); + 2 + } + Ok((findings, notes)) => { + for note in notes { + println!("NOTE: {note}"); + } + for finding in &findings { + println!("FINDING: {finding}"); + } + println!("workflow-map findings={}; remote source closure and provenance NOT VERIFIED", findings.len()); + if !findings.is_empty() { + 1 + } else if !unmanaged.is_empty() { + 2 + } else { + 0 + } + } + } + } + _ => { + eprintln!("usage: fleet-workflow-lock-audit map|regeneration ..."); + 2 + } + }; + process::exit(code); +} diff --git a/tools/workflow-lock-audit/tests/contracts.rs b/tools/workflow-lock-audit/tests/contracts.rs new file mode 100644 index 00000000..b6a39eda --- /dev/null +++ b/tools/workflow-lock-audit/tests/contracts.rs @@ -0,0 +1,608 @@ +// SPDX-License-Identifier: MPL-2.0 +//! CLI-level fixtures: assertions inspect status and evidence, independently +//! of implementation functions. Every invocation parses files from disk. +use saphyr::{LoadableYamlNode, ScalarOwned, YamlOwned}; +use serde_json::Value; +use std::{ + fs, + path::{Path, PathBuf}, + process::Command, + sync::atomic::{AtomicU64, Ordering}, +}; +fn parse_yaml(text: &str) -> Result { + fn convert(y: YamlOwned) -> Result { + Ok(match y { + YamlOwned::Value(ScalarOwned::Null) => Value::Null, + YamlOwned::Value(ScalarOwned::Boolean(v)) => Value::Bool(v), + YamlOwned::Value(ScalarOwned::Integer(v)) => Value::Number(v.into()), + YamlOwned::Value(ScalarOwned::FloatingPoint(v)) => serde_json::json!(v.0), + YamlOwned::Value(ScalarOwned::String(v)) => Value::String(v), + YamlOwned::Sequence(v) => { + Value::Array(v.into_iter().map(convert).collect::>()?) + } + YamlOwned::Mapping(v) => Value::Object( + v.into_iter() + .map(|(k, v)| Ok((k.as_str().ok_or("fixture key")?.to_owned(), convert(v)?))) + .collect::>()?, + ), + _ => return Err("unsupported fixture YAML".into()), + }) + } + let mut docs = YamlOwned::load_from_str(text).map_err(|e| e.to_string())?; + if docs.len() != 1 { + return Err("fixture document count".into()); + } + convert(docs.pop().unwrap()) +} +fn value(text: &str) -> Value { + parse_yaml(text).unwrap() +} +fn text(s: &str) -> Value { + Value::String(s.into()) +} +fn remove(doc: &mut Value, key: &str) { + doc.as_object_mut().unwrap().remove(key); +} +static NEXT: AtomicU64 = AtomicU64::new(0); +struct Fixture { + root: PathBuf, + lock: Value, +} +impl Drop for Fixture { + fn drop(&mut self) { + fs::remove_dir_all(&self.root).unwrap(); + } +} +impl Fixture { + fn new() -> Self { + let root = std::env::temp_dir().join(format!( + "fleet-lock-audit-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir_all(root.join(".github/workflows")).unwrap(); + let f = Self { root, lock: value("workflows:\n .github/workflows/main.yml: [acme/action@v1]\n .github/workflows/empty.yml: []\ndependencies:\n acme/action@v1: {ref: v1, commit: sha1-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa}\n") }; + f.write( + "main.yml", + "jobs: {build: {steps: [{uses: Acme/action/subpath@v1}]}}", + ); + f.write("empty.yml", "jobs: {build: {steps: [{run: echo hello}]}}"); + f + } + fn write(&self, name: &str, data: &str) { + fs::write(self.root.join(".github/workflows").join(name), data).unwrap(); + } + fn audit(&self, args: &[&str]) -> (i32, String) { + self.write("actions.lock", &serde_json::to_string(&self.lock).unwrap()); + run(&[vec!["map", self.root.to_str().unwrap()], args.to_vec()].concat()) + } +} +fn run(args: &[&str]) -> (i32, String) { + let result = Command::new(env!("CARGO_BIN_EXE_fleet-workflow-lock-audit")) + .args(args) + .output() + .unwrap(); + ( + result.status.code().unwrap(), + String::from_utf8(result.stdout).unwrap() + &String::from_utf8(result.stderr).unwrap(), + ) +} +fn expect(result: (i32, String), code: i32, evidence: &str) -> String { + assert_eq!(result.0, code, "{}", result.1); + assert!( + result.1.contains(evidence), + "expected {evidence:?}: {}", + result.1 + ); + result.1 +} +#[test] +fn mapped_no_uses_is_not_missing_or_unmanaged() { + let out = expect( + Fixture::new().audit(&[]), + 0, + "NO-USES-MAPPED .github/workflows/empty.yml", + ); + assert!(!out.contains("INTENTIONALLY-UNMANAGED")); + assert!(out.contains("remote source closure and provenance NOT VERIFIED")); +} +#[test] +fn missing_workflow_is_not_silently_exempted() { + let mut f = Fixture::new(); + remove(&mut f.lock["workflows"], ".github/workflows/empty.yml"); + expect( + f.audit(&[]), + 1, + "MISSING-WORKFLOW .github/workflows/empty.yml", + ); +} +#[test] +fn explicit_unmanaged_is_not_a_closure_pass() { + let mut f = Fixture::new(); + remove(&mut f.lock["workflows"], ".github/workflows/empty.yml"); + let out = expect( + f.audit(&["--unmanaged", ".github/workflows/empty.yml"]), + 2, + "INTENTIONALLY-UNMANAGED", + ); + assert!(out.contains("closure not verified")); +} +#[test] +fn invalid_unmanaged_exception_fails() { + expect( + Fixture::new().audit(&["--unmanaged", ".github/workflows/typo.yml"]), + 1, + "UNMANAGED-UNKNOWN", + ); +} +#[test] +fn missing_direct_ref_even_with_workflow_key() { + let mut f = Fixture::new(); + f.lock["workflows"][".github/workflows/main.yml"] = value("[]"); + expect(f.audit(&[]), 1, "MISSING-DIRECT-EDGE"); +} +#[test] +fn sha_reusable_empty_map_requires_source_review() { + let mut f = Fixture::new(); + f.write( + "main.yml", + &format!( + "jobs: {{scan: {{uses: acme/suite/.github/workflows/scan.yml@{}}}}}", + "a".repeat(40) + ), + ); + f.lock["workflows"][".github/workflows/main.yml"] = value("[]"); + let out = expect(f.audit(&[]), 0, "REMOTE-REUSABLE-SOURCE-REVIEW"); + assert!(out.contains("UNSUPPORTED-LOCK-SCOPE")); + assert!(!out.contains("MISSING-DIRECT-EDGE")); + assert!(!out.contains("NO-USES-MAPPED .github/workflows/main.yml")); +} +#[test] +fn remote_reusable_floating_and_short_pins_fail() { + for pin in [ + "main".into(), + "v1".into(), + "abc1234".into(), + "a".repeat(39), + "a".repeat(41), + "g".repeat(40), + ] { + let mut f = Fixture::new(); + f.write( + "main.yml", + &format!("jobs: {{scan: {{uses: acme/suite/.github/workflows/scan.yml@{pin}}}}}"), + ); + f.lock["workflows"][".github/workflows/main.yml"] = value("[]"); + expect(f.audit(&[]), 1, "REMOTE-REUSABLE-NOT-SHA"); + } +} +#[test] +fn reusable_does_not_exempt_missing_ordinary_action_edge() { + let mut f = Fixture::new(); + f.write("main.yml", &format!("jobs:\n scan: {{uses: acme/suite/.github/workflows/scan.yml@{}}}\n build: {{steps: [{{uses: acme/action@v1}}]}}", "a".repeat(40))); + f.lock["workflows"][".github/workflows/main.yml"] = value("[]"); + let out = expect( + f.audit(&[]), + 1, + "MISSING-DIRECT-EDGE .github/workflows/main.yml acme/action@v1", + ); + assert!(out.contains("UNSUPPORTED-LOCK-SCOPE")); +} +#[test] +fn missing_nested_record() { + let mut f = Fixture::new(); + f.lock["dependencies"]["acme/action@v1"]["uses"] = value("[acme/child@v1]"); + expect(f.audit(&[]), 1, "MISSING-TRANSITIVE-RECORD"); +} +#[test] +fn local_reusable_is_checked_at_its_own_path() { + let mut f = Fixture::new(); + f.write( + "main.yml", + "jobs: {call: {uses: ./.github/workflows/callee.yml}}", + ); + f.write( + "callee.yml", + "jobs: {run: {steps: [{uses: acme/action@v1}]}}", + ); + f.lock["workflows"][".github/workflows/main.yml"] = value("[]"); + f.lock["workflows"][".github/workflows/callee.yml"] = value("[]"); + let out = expect( + f.audit(&[]), + 1, + "MISSING-DIRECT-EDGE .github/workflows/callee.yml", + ); + assert!(out.contains("LOCAL-REUSABLE")); +} +#[test] +fn malformed_and_duplicate_yaml_fail_closed() { + let f = Fixture::new(); + f.write("main.yml", "jobs: {run: {steps: [{uses: no-at-sign}]}}"); + expect(f.audit(&[]), 2, "NOT-RUN"); + f.write("main.yml", "jobs: {}\njobs: {}\n"); + expect(f.audit(&[]), 2, "duplicate YAML key"); +} +#[test] +fn extra_edge_is_review_not_proof_of_failure() { + let mut f = Fixture::new(); + f.lock["workflows"][".github/workflows/empty.yml"] = value("[acme/action@v1]"); + expect(f.audit(&[]), 0, "EXTRA-OR-TRANSITIVE-EDGE"); +} +#[test] +fn mapped_ref_without_record_and_stale_workflow() { + let mut f = Fixture::new(); + f.lock["workflows"][".github/workflows/main.yml"] = value("[acme/action@v1, acme/missing@v2]"); + f.lock["workflows"][".github/workflows/deleted.yml"] = value("[]"); + let out = expect( + f.audit(&[]), + 1, + "MISSING-RECORD .github/workflows/main.yml acme/missing@v2", + ); + assert!(out.contains("STALE-WORKFLOW .github/workflows/deleted.yml")); +} +#[test] +fn mapped_workflow_cannot_also_be_unmanaged() { + expect( + Fixture::new().audit(&["--unmanaged", ".github/workflows/main.yml"]), + 1, + "UNMANAGED-BUT-MAPPED", + ); +} +#[test] +fn invalid_local_workflow_paths() { + for path in [ + "./.github/workflows/missing.yml", + "./outside.yml", + "./.github/workflows/actions.lock", + ] { + let mut f = Fixture::new(); + f.write("main.yml", &format!("jobs: {{call: {{uses: {path}}}}}")); + f.lock["workflows"][".github/workflows/main.yml"] = value("[]"); + expect( + f.audit(&[]), + 1, + &format!("INVALID-LOCAL-WORKFLOW .github/workflows/main.yml {path}"), + ); + } +} +#[test] +fn local_actions_and_docker_are_reported_for_review() { + let mut f = Fixture::new(); + f.write( + "main.yml", + "jobs: {build: {steps: [{uses: ./my-action}, {uses: 'docker://example/image:1'}]}}", + ); + f.lock["workflows"][".github/workflows/main.yml"] = value("[]"); + let out = expect( + f.audit(&[]), + 0, + "NOT-LOCKED-SOURCE-REVIEW .github/workflows/main.yml ./my-action", + ); + assert!(out + .contains("NOT-LOCKED-SOURCE-REVIEW .github/workflows/main.yml docker://example/image:1")); +} +#[test] +fn normalizes_owner_repo_and_subpath_but_preserves_ref_case() { + let mut f = Fixture::new(); + f.write( + "main.yml", + "jobs: {build: {steps: [{uses: ACME/ACTION/sub@V1}]}}", + ); + expect( + f.audit(&[]), + 1, + "MISSING-DIRECT-EDGE .github/workflows/main.yml acme/action@V1", + ); + f.lock["workflows"][".github/workflows/main.yml"] = value("[acme/action@V1]"); + f.lock["dependencies"] = value("{Acme/Action@V1: {}}"); + expect(f.audit(&[]), 0, "findings=0"); +} +#[test] +fn duplicate_normalized_refs_and_bad_records_fail_closed() { + for n in 0..5 { + let mut f = Fixture::new(); + match n { + 0 => f.lock["dependencies"]["ACME/ACTION@v1"] = value("{}"), + 1 => { + f.lock["workflows"][".github/workflows/main.yml"] = + value("[acme/action@v1, ACME/ACTION/sub@v1]") + } + 2 => f.lock["dependencies"]["acme/action@v1"] = value("[]"), + 3 => f.lock["dependencies"]["acme/action@v1"]["uses"] = value("[./local]"), + _ => f.lock["workflows"][".github/workflows/main.yml"] = text("acme/action@v1"), + } + expect(f.audit(&[]), 2, "NOT-RUN:"); + } +} +#[test] +fn invalid_workflow_shapes_fail_closed() { + for doc in [ + "jobs: []", + "jobs: {build: null}", + "jobs: {build: {steps: {}}}", + "jobs: {build: {steps: [null]}}", + "jobs: {build: {steps: [{uses: 'acme/action@bad ref'}]}}", + ] { + let f = Fixture::new(); + f.write("main.yml", doc); + expect(f.audit(&[]), 2, "NOT-RUN:"); + } +} +#[test] +fn yaml_extension_is_inventoried() { + let f = Fixture::new(); + f.write("extra.yaml", "jobs: {run: {steps: []}}"); + expect( + f.audit(&[]), + 1, + "MISSING-WORKFLOW .github/workflows/extra.yaml", + ); +} +fn repo() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../..") +} +#[test] +fn scanner_triggers_and_access() { + let workflow: Value = parse_yaml( + &fs::read_to_string(repo().join(".github/workflows/secret-scanner.yml")).unwrap(), + ) + .unwrap(); + assert_eq!( + workflow["on"], + value("{pull_request: null, push: {branches: [main]}}") + ); + assert_eq!(workflow["permissions"], value("{contents: read}")); + assert_eq!(workflow["jobs"].as_object().unwrap().len(), 1); + let job = &workflow["jobs"]["scan"]; + assert_eq!(job["permissions"], value("{contents: read}")); + assert!(job.get("secrets").is_none()); + assert_eq!(job["with"], value("{fast-pr: false}")); +} +#[test] +fn scanner_source_pin_and_supported_lock_scope() { + let workflow: Value = parse_yaml( + &fs::read_to_string(repo().join(".github/workflows/secret-scanner.yml")).unwrap(), + ) + .unwrap(); + let lock: Value = + parse_yaml(&fs::read_to_string(repo().join(".github/workflows/actions.lock")).unwrap()) + .unwrap(); + let pin = "4404c3c0474e17753fe7689f3485c02890aac113"; + assert_eq!( + workflow["jobs"]["scan"]["uses"], + text(&format!( + "hyperpolymath/cicd-suite/.github/workflows/secret-scanner-reusable.yml@{pin}" + )) + ); + assert_eq!( + lock["workflows"][".github/workflows/secret-scanner.yml"], + value("[]") + ); + assert!(lock["dependencies"] + .get(format!("hyperpolymath/cicd-suite@{pin}")) + .is_none()); + assert_eq!( + lock["dependencies"]["actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1"]["commit"], + text("sha1-3d3c42e5aac5ba805825da76410c181273ba90b1") + ); +} +fn base() -> Value { + value("workflows:\n .github/workflows/test.yml: [org/reusable@v1]\ndependencies:\n org/reusable@v1: {ref: v1, commit: sha1-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, uses: [org/composite@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb]}\n org/composite@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb: {ref: v2, commit: sha1-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb, uses: [org/child@v3]}\n org/child@v3: {ref: v3, commit: sha1-cccccccccccccccccccccccccccccccccccccccc}\n") +} +const COMPOSITE: &str = "org/composite@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +fn regen_raw(old: &str, new: &str) -> (i32, String) { + let f = Fixture::new(); + let a = f.root.join("old.yml"); + let b = f.root.join("new.yml"); + fs::write(&a, old).unwrap(); + fs::write(&b, new).unwrap(); + run(&["regeneration", a.to_str().unwrap(), b.to_str().unwrap()]) +} +fn regen(old: &Value, new: &Value) -> (i32, String) { + regen_raw( + &serde_json::to_string(old).unwrap(), + &serde_json::to_string(new).unwrap(), + ) +} +#[test] +fn unchanged_closure() { + expect(regen(&base(), &base()), 0, "closure (3 records)"); +} +#[test] +fn dropped_reusable_closure_without_dangling_edge() { + let mut d = base(); + remove(&mut d["dependencies"]["org/reusable@v1"], "uses"); + remove(&mut d["dependencies"], COMPOSITE); + remove(&mut d["dependencies"], "org/child@v3"); + expect(regen(&base(), &d), 1, "dropped reviewed edges"); +} +#[test] +fn dropped_composite_closure_without_dangling_edge() { + let mut d = base(); + remove(&mut d["dependencies"][COMPOSITE], "uses"); + remove(&mut d["dependencies"], "org/child@v3"); + expect(regen(&base(), &d), 1, "dropped reviewed edges"); +} +#[test] +fn dangling_edge() { + let mut d = base(); + remove(&mut d["dependencies"], "org/child@v3"); + expect(regen(&base(), &d), 1, "missing dependency"); +} +#[test] +fn lost_reviewed_tag() { + let mut d = base(); + d["dependencies"][COMPOSITE]["ref"] = text("sha1-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); + expect(regen(&base(), &d), 1, "invalid ref"); +} +#[test] +fn malformed_commit() { + let mut d = base(); + d["dependencies"]["org/child@v3"]["commit"] = text(&"c".repeat(40)); + expect(regen(&base(), &d), 1, "invalid commit"); +} +#[test] +fn wrong_tag() { + let mut d = base(); + d["dependencies"]["org/child@v3"]["ref"] = text("v4"); + expect(regen(&base(), &d), 1, "tag ref differs"); +} +#[test] +fn sha_mismatch() { + let mut d = base(); + d["dependencies"][COMPOSITE]["commit"] = text(&format!("sha1-{}", "c".repeat(40))); + expect(regen(&base(), &d), 1, "SHA key and commit differ"); +} +#[test] +fn changed_reviewed_tag_resolution_is_rejected() { + let mut d = base(); + d["dependencies"]["org/child@v3"]["commit"] = text(&format!("sha1-{}", "a".repeat(40))); + expect( + regen(&base(), &d), + 1, + "changed reviewed ref/commit org/child@v3", + ); +} +#[test] +fn changed_reviewed_sha_record_ref_is_rejected() { + let mut d = base(); + d["dependencies"][COMPOSITE]["ref"] = text("v4"); + expect( + regen(&base(), &d), + 1, + &format!("changed reviewed ref/commit {COMPOSITE}"), + ); +} +#[test] +fn added_edges_and_new_roots_are_allowed_but_require_review() { + let mut d = base(); + d["dependencies"]["org/child@v3"]["uses"] = value("[new/action@v1]"); + d["dependencies"]["new/action@v1"] = + value("{ref: v1, commit: sha1-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa}"); + d["workflows"]["new.yml"] = value("[new/action@v1]"); + let out = expect(regen(&base(), &d), 0, "closure (3 records)"); + assert!(out.contains("new sources require independent review")); +} +#[test] +fn removed_old_root_does_not_force_unused_closure() { + expect( + regen(&base(), &value("{workflows: {}, dependencies: {}}")), + 0, + "closure (0 records)", + ); +} +#[test] +fn cycles_and_case_insensitive_repository_names() { + let mut d = base(); + d["dependencies"]["org/child@v3"]["uses"] = value("[ORG/Reusable@v1]"); + let mut proposed = d.clone(); + proposed["workflows"][".github/workflows/test.yml"] = value("[ORG/REUSABLE@v1]"); + expect(regen(&d, &proposed), 0, "closure (3 records)"); +} +#[test] +fn dropped_edge_is_identified_even_when_record_remains() { + let mut d = base(); + d["dependencies"]["org/reusable@v1"]["uses"] = value("[]"); + let out = expect( + regen(&base(), &d), + 1, + "dropped reviewed edges from org/reusable@v1", + ); + assert!(out.contains(COMPOSITE)); +} +#[test] +fn invalid_documents_fail_on_either_input() { + for (doc, message) in [ + ("workflows: {}\nworkflows: {}\ndependencies: {}", "duplicate YAML key"), + ("{workflows: [], dependencies: {}}", "expected workflows and dependencies mappings"), + ("{workflows: {test.yml: [org/missing@v1]}, dependencies: {}}", "missing dependency"), + ("{workflows: {test.yml: org/action@v1}, dependencies: {}}", "invalid workflow"), + ("{workflows: {}, dependencies: {org/action@v1: []}}", "invalid record"), + ("{workflows: {}, dependencies: {org/action@v1: {ref: v1, commit: sha1-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, uses: org/child@v1}}}", "invalid uses"), + ("{workflows: {}, dependencies: {org/action/sub@v1: {}}}", "invalid dependency reference"), + ("{workflows: {}, dependencies: {org/action@v1: {ref: v1, commit: sha1-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa}, ORG/ACTION@v1: {ref: v1, commit: sha1-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa}}}", "duplicate or invalid record"), + ] { + let good = serde_json::to_string(&base()).unwrap(); + expect(regen_raw(doc, &good), 1, message); expect(regen_raw(&good, doc), 1, message); + } +} +#[test] +fn cli_requires_two_paths_and_handles_missing_file() { + expect(run(&["regeneration"]), 1, "usage:"); + let f = Fixture::new(); + let a = f.root.join("old.yml"); + fs::write(&a, serde_json::to_string(&base()).unwrap()).unwrap(); + expect( + run(&[ + "regeneration", + a.to_str().unwrap(), + f.root.join("absent.yml").to_str().unwrap(), + ]), + 1, + "lock regeneration audit:", + ); + expect(run(&["map", "--unmanaged"]), 2, "usage:"); +} +#[test] +fn strict_yaml_framing_keys_tags_and_nested_duplicates() { + for invalid in [ + "jobs: {}\n---\njobs: {}", + "jobs: !Unsupported {}", + "jobs: {build: {steps: [], steps: []}}", + "jobs: {[complex, key]: {}}", + "jobs: {build: {<<: {steps: []}}}", + "jobs: [", + ] { + let f = Fixture::new(); + f.write("main.yml", invalid); + expect(f.audit(&[]), 2, "NOT-RUN:"); + } + let good = serde_json::to_string(&base()).unwrap(); + for bad in [ + "workflows: {}\ndependencies: {org/action@v1: {ref: v1, ref: v2}}", + "workflows: {}\ndependencies: {}\n---\nworkflows: {}", + "workflows: {}\ndependencies: !Unsupported {}", + ] { + expect(regen_raw(bad, &good), 1, "lock regeneration audit:"); + expect(regen_raw(&good, bad), 1, "lock regeneration audit:"); + } +} +#[cfg(unix)] +#[test] +fn local_callee_symlink_cannot_escape_workflow_directory() { + let mut f = Fixture::new(); + let outside = f.root.join("outside.yml"); + fs::write(&outside, "jobs: {}").unwrap(); + std::os::unix::fs::symlink(&outside, f.root.join(".github/workflows/escape.yml")).unwrap(); + f.write( + "main.yml", + "jobs: {call: {uses: ./.github/workflows/escape.yml}}", + ); + f.lock["workflows"][".github/workflows/main.yml"] = value("[]"); + f.lock["workflows"][".github/workflows/escape.yml"] = value("[]"); + expect(f.audit(&[]), 1, "INVALID-LOCAL-WORKFLOW"); +} + +#[test] +fn legacy_yaml_boolean_and_timestamp_refs_do_not_become_strings() { + for bare in [ + "yes", + "No", + "ON", + "off", + "2026-11-07", + "1:20", + "11111111111111111111111111111111111111111", + ] { + let good = format!("workflows: {{test.yml: [org/action@{bare}]}}\ndependencies: {{org/action@{bare}: {{ref: '{bare}', commit: sha1-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa}}}}"); + expect(regen_raw(&good, &good), 0, "closure (1 records)"); + let bad = good.replace(&format!("ref: '{bare}'"), &format!("ref: {bare}")); + expect(regen_raw(&bad, &good), 1, "lock regeneration audit:"); + expect(regen_raw(&good, &bad), 1, "lock regeneration audit:"); + } + let duplicate = "workflows: {}\ndependencies: {}\n'workflows': {}"; + expect(regen_raw(duplicate, duplicate), 1, "duplicate YAML key"); + let f = Fixture::new(); + f.write("main.yml", "jobs: {}\njobs: {}\n"); + expect(f.audit(&[]), 2, "duplicate YAML key"); +} From a070e1e7b70ab3b2464c4a661d65575ff44213bc Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Sat, 7 Nov 2026 11:38:11 +0000 Subject: [PATCH 5/5] Reject ambiguous numeric refs and address audit quality findings --- .github/workflows/e2e.yml | 2 +- .github/workflows/inbox-steward.yml | 18 +- tools/workflow-lock-audit/README.adoc | 4 +- tools/workflow-lock-audit/src/main.rs | 533 +++++++++++-------- tools/workflow-lock-audit/tests/contracts.rs | 23 + 5 files changed, 358 insertions(+), 222 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 625d0f99..296264ef 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -34,7 +34,7 @@ jobs: set -euo pipefail # Official RustSec binary, immutable content hash from its release asset. archive="$RUNNER_TEMP/cargo-audit.tgz" - curl -fsSL --retry 3 https://github.com/rustsec/rustsec/releases/download/cargo-audit/v0.22.2/cargo-audit-x86_64-unknown-linux-gnu-v0.22.2.tgz -o "$archive" + curl --proto '=https' --proto-redir '=https' -fsSL --retry 3 https://github.com/rustsec/rustsec/releases/download/cargo-audit/v0.22.2/cargo-audit-x86_64-unknown-linux-gnu-v0.22.2.tgz -o "$archive" printf '%s %s\n' ab28a1bdb54db4d5d8ad5981cf1f959410370b3d28250dbd35f6a44248620e39 "$archive" | sha256sum -c - tar -xzf "$archive" -C "$RUNNER_TEMP" "$RUNNER_TEMP/cargo-audit-x86_64-unknown-linux-gnu-v0.22.2/cargo-audit" audit --file tools/workflow-lock-audit/Cargo.lock --deny warnings diff --git a/.github/workflows/inbox-steward.yml b/.github/workflows/inbox-steward.yml index 94dfea3e..83de4dc9 100644 --- a/.github/workflows/inbox-steward.yml +++ b/.github/workflows/inbox-steward.yml @@ -47,11 +47,7 @@ on: # unattended. It still runs on PR/review/check events and workflow_dispatch for # MONITORING; the auto-merge job itself is gated OFF by default (see below). -permissions: - contents: read - pull-requests: read - repository-projects: read - actions: read +permissions: {} concurrency: group: ${{ github.workflow }}-${{ github.repository }} @@ -61,6 +57,11 @@ jobs: # Job 1: Identify PRs that have passed all required checks identify-passed-prs: name: Identify PRs passed CICD + permissions: + contents: read + pull-requests: read + repository-projects: read + actions: read runs-on: ubuntu-latest timeout-minutes: 10 outputs: @@ -154,6 +155,11 @@ jobs: # Job 2: Validate each PR against merge criteria validate-prs: name: Validate PRs against criteria + permissions: + contents: read + pull-requests: read + repository-projects: read + actions: read runs-on: ubuntu-latest timeout-minutes: 15 needs: identify-passed-prs @@ -379,6 +385,8 @@ jobs: # Job 4: Dispatch lessons to Hypatia dispatch-to-hypatia: name: Dispatch lessons to Hypatia + permissions: + contents: read runs-on: ubuntu-latest timeout-minutes: 5 needs: [identify-passed-prs, validate-prs, auto-merge-prs] diff --git a/tools/workflow-lock-audit/README.adoc b/tools/workflow-lock-audit/README.adoc index 9371118b..0104492a 100644 --- a/tools/workflow-lock-audit/README.adoc +++ b/tools/workflow-lock-audit/README.adoc @@ -32,10 +32,10 @@ scalar resolution. Input must contain exactly one document. Complex mapping keys, explicit tags and YAML merge keys are rejected. Legacy YAML 1.1 unquoted yes/no/on/off remain booleans, so a bare boolean cannot become a valid ref after migration. Ambiguous plain timestamps, sexagesimal values -and overflowing numbers fail closed; quote intended strings explicitly. +and overflowing or underscore-separated numbers fail closed; quote intended strings explicitly. These bounded, explicit restrictions avoid silently changing input meaning. -The 41 CLI tests retain all 22 prior map/scanner controls and 16 prior +The 42 CLI tests retain all 22 prior map/scanner controls and 16 prior regeneration controls, plus strict parsing, symlink containment and legacy scalar tests. Test fixtures use serde_json solely to write inputs; production uses the saphyr tree directly. ordered-float is locked to 5.3.0 to retain diff --git a/tools/workflow-lock-audit/src/main.rs b/tools/workflow-lock-audit/src/main.rs index 6bcdf365..a9927edc 100644 --- a/tools/workflow-lock-audit/src/main.rs +++ b/tools/workflow-lock-audit/src/main.rs @@ -63,44 +63,7 @@ fn resolve_yaml(value: Value) -> Result { if tag.is_some() { return Err("YAML tags are unsupported".into()); } - if style == ScalarStyle::Plain { - let bytes = raw.as_bytes(); - let timestamp = bytes.len() >= 10 - && bytes[4] == b'-' - && bytes[7] == b'-' - && bytes[..4] - .iter() - .chain(bytes[5..7].iter()) - .chain(bytes[8..10].iter()) - .all(u8::is_ascii_digit); - let sexagesimal = raw.contains(':') - && raw - .chars() - .all(|c| c.is_ascii_digit() || matches!(c, ':' | '+' | '-' | '.' | '_')); - if timestamp || sexagesimal { - return Err("ambiguous YAML 1.1 typed scalar; quote strings explicitly".into()); - } - } - let scalar = if style == ScalarStyle::Plain { - match raw.as_str() { - "yes" | "Yes" | "YES" | "on" | "On" | "ON" => ScalarOwned::Boolean(true), - "no" | "No" | "NO" | "off" | "Off" | "OFF" => ScalarOwned::Boolean(false), - _ => ScalarOwned::parse_from_cow_and_metadata(raw.clone().into(), style, None) - .ok_or("invalid YAML scalar")?, - } - } else { - ScalarOwned::String(raw.clone()) - }; - // The YAML 1.2 parser bounds integers to i64; the old loader did - // not. Never turn an overflowing bare number into an accepted ref. - if style == ScalarStyle::Plain - && matches!(scalar, ScalarOwned::String(_)) - && raw.parse::().is_ok() - { - return Err( - "out-of-range plain numeric YAML scalar; quote strings explicitly".into(), - ); - } + let scalar = resolve_scalar(raw, style)?; Ok(Value::Value(scalar)) } Value::Mapping(map) => { @@ -130,6 +93,67 @@ fn resolve_yaml(value: Value) -> Result { Value::Value(_) => Ok(value), } } +fn resolve_scalar(raw: String, style: ScalarStyle) -> Result { + if style == ScalarStyle::Plain { + let bytes = raw.as_bytes(); + let timestamp = bytes.len() >= 10 + && bytes[4] == b'-' + && bytes[7] == b'-' + && bytes[..4] + .iter() + .chain(bytes[5..7].iter()) + .chain(bytes[8..10].iter()) + .all(u8::is_ascii_digit); + let sexagesimal = raw.contains(':') + && raw + .chars() + .all(|c| c.is_ascii_digit() || matches!(c, ':' | '+' | '-' | '.' | '_')); + if timestamp || sexagesimal { + return Err("ambiguous YAML 1.1 typed scalar; quote strings explicitly".into()); + } + } + let scalar = if style == ScalarStyle::Plain { + match raw.as_str() { + "yes" | "Yes" | "YES" | "on" | "On" | "ON" => ScalarOwned::Boolean(true), + "no" | "No" | "NO" | "off" | "Off" | "OFF" => ScalarOwned::Boolean(false), + _ => ScalarOwned::parse_from_cow_and_metadata(raw.clone().into(), style, None) + .ok_or("invalid YAML scalar")?, + } + } else { + ScalarOwned::String(raw.clone()) + }; + // The YAML 1.2 parser bounds integers to i64; the old loader did + // not. Never turn a bare number (including separator spellings) into an accepted ref. + if style == ScalarStyle::Plain + && matches!(scalar, ScalarOwned::String(_)) + && is_legacy_number(&raw) + { + return Err("numeric YAML scalar cannot be a string; quote strings explicitly".into()); + } + Ok(scalar) +} +/// Recognize numeric spellings that a bounded YAML 1.2 scalar parser may +/// leave as strings. YAML 1.1 allows underscore separators and arbitrary +/// precision integers; neither may silently become a valid string ref. +fn is_legacy_number(raw: &str) -> bool { + let unsigned = raw.strip_prefix(['+', '-']).unwrap_or(raw); + // A nonempty underscore-only radix tail matched the old numeric resolver + // but failed conversion. Keep it an error, rather than accepting a ref. + let radix_tail = |tail: &str, radix: u32| { + !tail.is_empty() && tail.chars().all(|c| c == '_' || c.is_digit(radix)) + }; + if let Some(binary) = unsigned.strip_prefix("0b") { + return radix_tail(binary, 2); + } + if let Some(hexadecimal) = unsigned.strip_prefix("0x") { + return radix_tail(hexadecimal, 16); + } + let compact = unsigned.replace('_', ""); + let digits = |text: &str| !text.is_empty() && text.chars().all(|c| c.is_ascii_digit()); + // The unseparated form also covers octal integers; only the scalar's + // non-string classification matters here, not its numeric value. + digits(&compact) || compact.parse::().is_ok() +} fn reference(raw: &str, dependency: bool) -> Result> { if raw.is_empty() || raw.chars().any(char::is_whitespace) { return Err(format!("invalid uses: {raw:?}")); @@ -272,48 +296,50 @@ fn regeneration(old_path: &Path, new_path: &Path) -> Result<()> { println!("Preserved reviewed reachable closure ({} records); new sources require independent review.", seen.len()); Ok(()) } -fn inventory(root: &Path, unmanaged: &Set) -> Result<(Vec, Vec)> { - let root = fs::canonicalize(root).map_err(|e| e.to_string())?; - let wfdir = root.join(".github/workflows"); - let canonical_wfdir = fs::canonicalize(&wfdir).map_err(|e| e.to_string())?; - let lock = read_yaml(&wfdir.join("actions.lock"))?; - let (workflows, deps) = lock_parts(&lock)?; +#[derive(Default)] +struct InventoryReport { + findings: Vec, + notes: Vec, +} +#[derive(Default)] +struct WorkflowUses { + refs: Set, + remote_calls: Set, + local_calls: Vec, + unsupported: Vec, +} +impl WorkflowUses { + fn is_empty(&self) -> bool { + self.refs.is_empty() + && self.remote_calls.is_empty() + && self.local_calls.is_empty() + && self.unsupported.is_empty() + } +} +fn workflow_files(wfdir: &Path) -> Result> { let mut files = BTreeMap::new(); - for entry in fs::read_dir(&wfdir).map_err(|e| e.to_string())? { + for entry in fs::read_dir(wfdir).map_err(|e| e.to_string())? { let path = entry.map_err(|e| e.to_string())?.path(); - if path.is_file() - && matches!( - path.extension().and_then(|s| s.to_str()), - Some("yml" | "yaml") - ) - { - files.insert( - format!( - ".github/workflows/{}", - path.file_name() - .and_then(|s| s.to_str()) - .ok_or("non-UTF8 workflow name")? - ), - path, - ); + if path.is_file() && is_workflow_path(&path) { + let filename = path + .file_name() + .and_then(|s| s.to_str()) + .ok_or("non-UTF8 workflow name")?; + files.insert(format!(".github/workflows/{filename}"), path); } } - let mut findings = Vec::new(); - let mut notes = Vec::new(); - for path in unmanaged { - if !files.contains_key(path) { - findings.push(format!("UNMANAGED-UNKNOWN {path}")); - } - } - for name in workflows.keys() { - let name = string(name, "workflow path")?; - if !files.contains_key(name) { - findings.push(format!("STALE-WORKFLOW {name}")); - } - } - let mut depkeys = Set::new(); + Ok(files) +} +fn is_workflow_path(path: &Path) -> bool { + matches!( + path.extension().and_then(|s| s.to_str()), + Some("yml" | "yaml") + ) +} +fn dependency_keys(deps: &Mapping, report: &mut InventoryReport) -> Result { + let mut keys = Set::new(); for dep in deps.keys() { - if !depkeys.insert(key(dep, false)?) { + if !keys.insert(key(dep, false)?) { return Err("invalid or duplicate dependency key".into()); } } @@ -323,124 +349,249 @@ fn inventory(root: &Path, unmanaged: &Set) -> Result<(Vec, Vec)> if let Some(edges) = get(record, "uses") { for edge in list(edges, "dependency uses")? { let normalized = key(edge, false)?; - if !depkeys.contains(&normalized) { - findings.push(format!("MISSING-TRANSITIVE-RECORD {dep} {normalized}")); + if !keys.contains(&normalized) { + report + .findings + .push(format!("MISSING-TRANSITIVE-RECORD {dep} {normalized}")); } } } } - for (name, file) in &files { - let doc = read_yaml(file)?; - let doc = mapping(&doc, "workflow")?; - let jobs = get(doc, "jobs") - .and_then(Value::as_mapping) - .ok_or_else(|| format!("{name}: expected jobs mapping"))?; - let mut refs = Set::new(); - let mut remote_calls = Set::new(); - let mut local_calls = Vec::new(); - let mut unsupported = Vec::new(); + Ok(keys) +} +fn job_entries(job: &Value) -> Result> { + let job = mapping(job, "invalid job")?; + let mut entries = Vec::new(); + if let Some(uses) = get(job, "uses") { + entries.push((uses, true)); + } + if let Some(steps) = get(job, "steps") { + for step in list(steps, "invalid steps")? { + let step = mapping(step, "invalid step")?; + if let Some(uses) = get(step, "uses") { + entries.push((uses, false)); + } + } + } + Ok(entries) +} +struct InventoryContext<'a> { + root: &'a Path, + wfdir: &'a Path, + workflows: &'a Mapping, + depkeys: Set, + unmanaged: &'a Set, +} +impl InventoryContext<'_> { + fn local_target(&self, raw: &str) -> Result> { + let Ok(local) = fs::canonicalize(self.root.join(raw)) else { + return Ok(None); + }; + if !local.starts_with(self.wfdir) || !local.is_file() || !is_workflow_path(&local) { + return Ok(None); + } + Ok(Some( + local + .strip_prefix(self.root) + .map_err(|e| e.to_string())? + .to_string_lossy() + .into_owned(), + )) + } + fn inspect_uses( + &self, + name: &str, + jobs: &Mapping, + report: &mut InventoryReport, + ) -> Result { + let mut uses = WorkflowUses::default(); for job in jobs.values() { - let job = mapping(job, "invalid job")?; - let mut entries = Vec::new(); - if let Some(uses) = get(job, "uses") { - entries.push((uses, true)); + for (raw, is_job) in job_entries(job)? { + self.inspect_entry(name, string(raw, "uses")?, is_job, &mut uses, report)?; } - if let Some(steps) = get(job, "steps") { - for step in list(steps, "invalid steps")? { - let step = mapping(step, "invalid step")?; - if let Some(uses) = get(step, "uses") { - entries.push((uses, false)); - } + } + for raw in &uses.unsupported { + report + .notes + .push(format!("NOT-LOCKED-SOURCE-REVIEW {name} {raw}")); + } + Ok(uses) + } + fn inspect_entry( + &self, + name: &str, + raw: &str, + is_job: bool, + uses: &mut WorkflowUses, + report: &mut InventoryReport, + ) -> Result<()> { + match reference(raw, false)? { + Some(normalized) if is_job => { + if !sha(normalized.rsplit_once('@').unwrap().1) { + report + .findings + .push(format!("REMOTE-REUSABLE-NOT-SHA {name} {normalized}")); } + report + .notes + .push(format!("REMOTE-REUSABLE-SOURCE-REVIEW {name} {normalized}")); + report.notes.push(format!("UNSUPPORTED-LOCK-SCOPE {name} {normalized} (external reusable dependencies not locked)")); + uses.remote_calls.insert(normalized); } - for (raw, is_job) in entries { - let raw = string(raw, "uses")?; - match reference(raw, false)? { - Some(normalized) if is_job => { - if !sha(normalized.rsplit_once('@').unwrap().1) { - findings.push(format!("REMOTE-REUSABLE-NOT-SHA {name} {normalized}")); - } - notes.push(format!("REMOTE-REUSABLE-SOURCE-REVIEW {name} {normalized}")); - notes.push(format!("UNSUPPORTED-LOCK-SCOPE {name} {normalized} (external reusable dependencies not locked)")); - remote_calls.insert(normalized); - } - Some(normalized) => { - refs.insert(normalized); - } - None if raw.starts_with("./") && is_job => { - let local = fs::canonicalize(root.join(raw)); - match local { - Ok(local) - if local.starts_with(&canonical_wfdir) - && local.is_file() - && matches!( - local.extension().and_then(|s| s.to_str()), - Some("yml" | "yaml") - ) => - { - local_calls.push( - local - .strip_prefix(&root) - .map_err(|e| e.to_string())? - .to_string_lossy() - .into_owned(), - ); - } - _ => findings.push(format!("INVALID-LOCAL-WORKFLOW {name} {raw}")), - } - } - None => unsupported.push(raw.to_owned()), - } + Some(normalized) => { + uses.refs.insert(normalized); } + None if raw.starts_with("./") && is_job => match self.local_target(raw)? { + Some(local) => uses.local_calls.push(local), + None => report + .findings + .push(format!("INVALID-LOCAL-WORKFLOW {name} {raw}")), + }, + None => uses.unsupported.push(raw.to_owned()), } - for raw in &unsupported { - notes.push(format!("NOT-LOCKED-SOURCE-REVIEW {name} {raw}")); - } - let Some(mapped) = get(workflows, name) else { - if unmanaged.contains(name) { - notes.push(format!( + Ok(()) + } + fn compare_map( + &self, + name: &str, + uses: WorkflowUses, + report: &mut InventoryReport, + ) -> Result<()> { + let Some(mapped) = get(self.workflows, name) else { + if self.unmanaged.contains(name) { + report.notes.push(format!( "INTENTIONALLY-UNMANAGED {name} (dependency closure not verified)" )); } else { - findings.push(format!("MISSING-WORKFLOW {name}")); + report.findings.push(format!("MISSING-WORKFLOW {name}")); } - continue; + return Ok(()); }; - if unmanaged.contains(name) { - findings.push(format!("UNMANAGED-BUT-MAPPED {name}")); + if self.unmanaged.contains(name) { + report.findings.push(format!("UNMANAGED-BUT-MAPPED {name}")); } - let mut mapped_set = Set::new(); - for item in list(mapped, "workflow lock list")? { - if !mapped_set.insert(key(item, false)?) { - return Err(format!("{name}: invalid or duplicate lock reference")); - } - } - for r in refs.difference(&mapped_set) { - findings.push(format!("MISSING-DIRECT-EDGE {name} {r}")); + let mapped_set = mapped_references(name, mapped)?; + for r in uses.refs.difference(&mapped_set) { + report + .findings + .push(format!("MISSING-DIRECT-EDGE {name} {r}")); } - for r in mapped_set.difference(&depkeys) { - findings.push(format!("MISSING-RECORD {name} {r}")); + for r in mapped_set.difference(&self.depkeys) { + report.findings.push(format!("MISSING-RECORD {name} {r}")); } - if refs.is_empty() - && remote_calls.is_empty() - && local_calls.is_empty() - && unsupported.is_empty() - && mapped_set.is_empty() - { - notes.push(format!("NO-USES-MAPPED {name}")); + if uses.is_empty() && mapped_set.is_empty() { + report.notes.push(format!("NO-USES-MAPPED {name}")); } - for target in local_calls { - notes.push(format!( + for target in uses.local_calls { + report.notes.push(format!( "LOCAL-REUSABLE {name} -> {target} (callee checked separately)" )); } - for r in mapped_set.difference(&refs) { - notes.push(format!( + for r in mapped_set.difference(&uses.refs) { + report.notes.push(format!( "EXTRA-OR-TRANSITIVE-EDGE {name} {r} (source review needed)" )); } + Ok(()) + } + fn inspect_file(&self, name: &str, file: &Path, report: &mut InventoryReport) -> Result<()> { + let doc = read_yaml(file)?; + let doc = mapping(&doc, "workflow")?; + let jobs = get(doc, "jobs") + .and_then(Value::as_mapping) + .ok_or_else(|| format!("{name}: expected jobs mapping"))?; + let uses = self.inspect_uses(name, jobs, report)?; + self.compare_map(name, uses, report) + } +} +fn mapped_references(name: &str, mapped: &Value) -> Result { + let mut refs = Set::new(); + for item in list(mapped, "workflow lock list")? { + if !refs.insert(key(item, false)?) { + return Err(format!("{name}: invalid or duplicate lock reference")); + } + } + Ok(refs) +} +fn inventory(root: &Path, unmanaged: &Set) -> Result<(Vec, Vec)> { + let root = fs::canonicalize(root).map_err(|e| e.to_string())?; + let wfdir = root.join(".github/workflows"); + let canonical_wfdir = fs::canonicalize(&wfdir).map_err(|e| e.to_string())?; + let lock = read_yaml(&wfdir.join("actions.lock"))?; + let (workflows, deps) = lock_parts(&lock)?; + let files = workflow_files(&wfdir)?; + let mut report = InventoryReport::default(); + for path in unmanaged { + if !files.contains_key(path) { + report.findings.push(format!("UNMANAGED-UNKNOWN {path}")); + } + } + for name in workflows.keys() { + let name = string(name, "workflow path")?; + if !files.contains_key(name) { + report.findings.push(format!("STALE-WORKFLOW {name}")); + } + } + let depkeys = dependency_keys(deps, &mut report)?; + let context = InventoryContext { + root: &root, + wfdir: &canonical_wfdir, + workflows, + depkeys, + unmanaged, + }; + for (name, file) in &files { + context.inspect_file(name, file, &mut report)?; + } + Ok((report.findings, report.notes)) +} +fn parse_map_arguments(args: &[String]) -> Result<(PathBuf, Set)> { + let mut root = PathBuf::from("."); + let mut has_root = false; + let mut unmanaged = Set::new(); + let mut rest = args.iter(); + while let Some(arg) = rest.next() { + if arg == "--unmanaged" { + unmanaged.insert(rest.next().ok_or("missing unmanaged workflow")?.clone()); + } else if arg.starts_with('-') || has_root { + return Err("unexpected map argument".into()); + } else { + root = PathBuf::from(arg); + has_root = true; + } + } + Ok((root, unmanaged)) +} +fn map_command(args: &[String]) -> i32 { + let Ok((root, unmanaged)) = parse_map_arguments(args) else { + eprintln!("usage: audit-workflow-lock-map.sh [ROOT] [--unmanaged WORKFLOW]"); + return 2; + }; + match inventory(&root, &unmanaged) { + Err(e) => { + eprintln!("NOT-RUN: {e}"); + 2 + } + Ok((findings, notes)) => { + for note in notes { + println!("NOTE: {note}"); + } + for finding in &findings { + println!("FINDING: {finding}"); + } + println!( + "workflow-map findings={}; remote source closure and provenance NOT VERIFIED", + findings.len() + ); + if !findings.is_empty() { + 1 + } else if !unmanaged.is_empty() { + 2 + } else { + 0 + } + } } - Ok((findings, notes)) } fn main() { let args: Vec<_> = env::args().skip(1).collect(); @@ -458,53 +609,7 @@ fn main() { eprintln!("usage: audit-lock-regeneration.sh REVIEWED_LOCK PROPOSED_LOCK"); 1 } - Some("map") => { - let mut root = PathBuf::from("."); - let mut has_root = false; - let mut unmanaged = Set::new(); - let mut rest = args.iter().skip(1); - let mut invalid = false; - while let Some(arg) = rest.next() { - if arg == "--unmanaged" { - if let Some(path) = rest.next() { - unmanaged.insert(path.clone()); - } else { - invalid = true; - } - } else if arg.starts_with('-') || has_root { - invalid = true; - } else { - root = PathBuf::from(arg); - has_root = true; - } - } - if invalid { - eprintln!("usage: audit-workflow-lock-map.sh [ROOT] [--unmanaged WORKFLOW]"); - process::exit(2); - } - match inventory(&root, &unmanaged) { - Err(e) => { - eprintln!("NOT-RUN: {e}"); - 2 - } - Ok((findings, notes)) => { - for note in notes { - println!("NOTE: {note}"); - } - for finding in &findings { - println!("FINDING: {finding}"); - } - println!("workflow-map findings={}; remote source closure and provenance NOT VERIFIED", findings.len()); - if !findings.is_empty() { - 1 - } else if !unmanaged.is_empty() { - 2 - } else { - 0 - } - } - } - } + Some("map") => map_command(&args[1..]), _ => { eprintln!("usage: fleet-workflow-lock-audit map|regeneration ..."); 2 diff --git a/tools/workflow-lock-audit/tests/contracts.rs b/tools/workflow-lock-audit/tests/contracts.rs index b6a39eda..9d6e5c79 100644 --- a/tools/workflow-lock-audit/tests/contracts.rs +++ b/tools/workflow-lock-audit/tests/contracts.rs @@ -606,3 +606,26 @@ fn legacy_yaml_boolean_and_timestamp_refs_do_not_become_strings() { f.write("main.yml", "jobs: {}\njobs: {}\n"); expect(f.audit(&[]), 2, "duplicate YAML key"); } + +#[test] +fn unquoted_legacy_numeric_refs_fail_closed_and_quoted_refs_remain_strings() { + for raw in [ + "1_000", + "0xFF_FF", + "0xFFFFFFFFFFFFFFFFFFFFFFFF", + "0b1_0", + "+1_000", + "1_0.0", + "0b_", + "0x_", + "0b__", + "0x__", + "+0b_", + ] { + let quoted = format!("workflows: {{test.yml: [org/action@{raw}]}}\ndependencies: {{org/action@{raw}: {{ref: '{raw}', commit: sha1-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa}}}}"); + expect(regen_raw("ed, "ed), 0, "closure (1 records)"); + let bare = quoted.replace(&format!("ref: '{raw}'"), &format!("ref: {raw}")); + expect(regen_raw(&bare, "ed), 1, "lock regeneration audit:"); + expect(regen_raw("ed, &bare), 1, "lock regeneration audit:"); + } +}