diff --git a/.github/actions/setup-bazel/action.yml b/.github/actions/setup-bazel/action.yml index 009d721800..bcbbecae30 100644 --- a/.github/actions/setup-bazel/action.yml +++ b/.github/actions/setup-bazel/action.yml @@ -18,6 +18,9 @@ inputs: prefix misses. Use when two jobs share rustc flags and runner image so Clippy can reuse Test Linux action outputs. default: "" + cache-save: + description: Whether this setup may save Bazelisk and repository caches. + default: "true" tailscale-client-id: description: OIDC client for trusted Linux CI; empty keeps the hosted-only build. default: "" @@ -79,7 +82,7 @@ runs: disk-cache: false bazelrc: >- build --disk_cache=${{ steps.disk-cache-config.outputs.path }} - cache-save: ${{ github.ref == 'refs/heads/master' }} + cache-save: ${{ github.ref == 'refs/heads/master' && inputs.cache-save == 'true' }} # Windows build-script runfiles nest each vendored crate's sources # under its repository name twice; a short output base (default # D:/_bazel) keeps MSVC's header paths under MAX_PATH. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b360ec5809..a42967eb2c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -211,6 +211,7 @@ jobs: python3 scripts/test-package-release-archive.py python3 scripts/test-plan-release-recovery.py python3 scripts/test-check-release-artifacts.py + python3 scripts/test-generate-release-inventory.py python3 scripts/test-update-release-checksums.py python3 scripts/test-neutralize-release-pr-closing-keywords.py bash tests/macos_stable_codesign_test.sh diff --git a/.github/workflows/release-beta.yml b/.github/workflows/release-beta.yml index 8dba727562..4484c42866 100644 --- a/.github/workflows/release-beta.yml +++ b/.github/workflows/release-beta.yml @@ -38,6 +38,7 @@ jobs: source_epoch: ${{ steps.source.outputs.source_epoch }} matrix: ${{ steps.source.outputs.matrix }} build_required: ${{ steps.source.outputs.build_required }} + package_build_required: ${{ steps.source.outputs.package_build_required }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -45,6 +46,17 @@ jobs: fetch-depth: 0 persist-credentials: false + - name: Checkout release recovery planner + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.workflow_sha }} + path: .release-automation + persist-credentials: false + sparse-checkout-cone-mode: false + sparse-checkout: | + scripts/plan-release-recovery.py + scripts/verify-retained-release-assets.sh + - name: Validate prerelease env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -75,7 +87,7 @@ jobs: echo "source_epoch=${source_epoch}" >> "$GITHUB_OUTPUT" gh api "repos/${{ github.repository }}/releases/tags/${RELEASE_TAG}" \ --jq '.assets[].name' > remote-asset-names - python3 scripts/plan-release-recovery.py \ + python3 .release-automation/scripts/plan-release-recovery.py \ --manifest .github/release-targets.json \ --tag "$RELEASE_TAG" \ --profile beta \ @@ -83,7 +95,7 @@ jobs: --retained-output retained-asset-names \ --github-output "$GITHUB_OUTPUT" mkdir retained-assets - scripts/verify-retained-release-assets.sh \ + .release-automation/scripts/verify-retained-release-assets.sh \ --tag "$RELEASE_TAG" \ --repo "${{ github.repository }}" \ --signer-workflow "${{ github.repository }}/.github/workflows/release-beta.yml" \ @@ -126,8 +138,36 @@ jobs: with: python-version: "3.15.0" + - name: Install the locked-inventory YAML reader + run: python3 -m pip install pyyaml==6.0.2 + + # Recovery automation is pinned to this workflow revision, not the release + # source being inventoried. + - name: Checkout release packaging helpers + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.workflow_sha }} + path: .release-automation + persist-credentials: false + sparse-checkout: | + .github/actions/package-release + .github/actions/setup-bazel + scripts/generate-release-inventory.py + + - name: Set up Bazel for inventory-only recovery + if: matrix.inventory_only == true + uses: ./.release-automation/.github/actions/setup-bazel + with: + cache-key: release-${{ matrix.target }} + cache-save: "false" + + - name: Build dashboard bundle for inventory-only recovery + if: matrix.inventory_only == true + run: bazel build --config=dist --config=ci //dashboard:app_dist + # Bazel builds the dashboard bundle the binary embeds (//dashboard:app_dist). - name: Build release binary for packaging + if: matrix.inventory_only != true id: cli uses: ./.github/actions/build-cli with: @@ -135,18 +175,8 @@ jobs: dest: target/${{ matrix.target }}/release cache-key: release-${{ matrix.target }} - # Packaging orchestration comes from this immutable workflow revision, - # so recovery also works for tags predating the shared helper. The CLI - # and its packager inputs still come from the validated release source. - - name: Checkout release packaging helpers - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.workflow_sha }} - path: .release-automation - persist-credentials: false - sparse-checkout: .github/actions/package-release - - name: Package and verify native archive and MCPB + if: matrix.inventory_only != true uses: ./.release-automation/.github/actions/package-release with: binary: ${{ steps.cli.outputs.path }} @@ -156,7 +186,29 @@ jobs: archive: ${{ matrix.archive }} epoch: ${{ needs.validate.outputs.source_epoch }} + - name: Record shipping component scope + shell: bash + env: + PLATFORM: ${{ matrix.name }} + SOURCE_SHA: ${{ needs.validate.outputs.source_sha }} + run: | + set -euo pipefail + python3 .release-automation/scripts/generate-release-inventory.py scope \ + --source-root . \ + --platform "$PLATFORM" \ + --source-sha "$SOURCE_SHA" \ + --dashboard-dist "$(bazel info --config=dist --config=ci bazel-bin)/dashboard/app-dist" \ + --output "inventory-scope-${PLATFORM}.json" + + - name: Upload component scope + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: inventory-scope-${{ matrix.name }} + path: inventory-scope-${{ matrix.name }}.json + if-no-files-found: error + - name: Upload MCPB artifact + if: matrix.inventory_only != true uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: mcpb-beta-${{ matrix.name }} @@ -164,6 +216,7 @@ jobs: if-no-files-found: error - name: Upload binary archive artifact + if: matrix.inventory_only != true uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: beta-${{ matrix.name }} @@ -193,17 +246,23 @@ jobs: ref: ${{ needs.validate.outputs.source_sha }} persist-credentials: false - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - if: needs.validate.outputs.build_required == 'true' + if: needs.validate.outputs.package_build_required == 'true' with: pattern: beta-* path: new-binaries merge-multiple: true - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - if: needs.validate.outputs.build_required == 'true' + if: needs.validate.outputs.package_build_required == 'true' with: pattern: mcpb-beta-* path: new-mcpbs merge-multiple: true + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + if: needs.validate.outputs.build_required == 'true' + with: + pattern: inventory-scope-* + path: new-inventory-scopes + merge-multiple: true # Use the helpers from the running workflow revision for older-tag recovery. - name: Checkout release recovery helpers @@ -215,7 +274,20 @@ jobs: sparse-checkout-cone-mode: false sparse-checkout: | scripts/assemble-release-artifacts.sh + scripts/check-release-artifacts.py + scripts/generate-release-inventory.py + scripts/plan-release-recovery.py scripts/update-release-checksums.py + scripts/verify-retained-release-assets.sh + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + if: needs.validate.outputs.build_required == 'true' + with: + python-version: "3.15.0" + + - name: Install the locked-inventory YAML reader + if: needs.validate.outputs.build_required == 'true' + run: python3 -m pip install pyyaml==6.0.2 - name: Assemble retained and newly built prerelease assets env: @@ -239,8 +311,10 @@ jobs: shopt -s nullglob archives=(new-binaries/tracedecay-beta-*) bundles=(new-mcpbs/*.mcpb) + inventories=(new-inventories/*.spdx.json) echo "archives=${#archives[@]}" >>"$GITHUB_OUTPUT" echo "bundles=${#bundles[@]}" >>"$GITHUB_OUTPUT" + echo "inventories=${#inventories[@]}" >>"$GITHUB_OUTPUT" - name: Attest newly built prerelease archives if: steps.built.outputs.archives != '0' @@ -254,6 +328,12 @@ jobs: with: subject-path: new-mcpbs/*.mcpb + - name: Attest newly generated component inventories + if: steps.built.outputs.inventories != '0' + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4 + with: + subject-path: new-inventories/*.spdx.json + - name: Upload missing prerelease assets env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -267,16 +347,22 @@ jobs: cd mcpbs sha256sum -- *.mcpb ) >> SHA256SUMS + ( + cd inventories + sha256sum -- *.spdx.json + ) >> SHA256SUMS mapfile -t archives < <(find binaries -maxdepth 1 -type f -name 'tracedecay-beta-*' -print | sort) test "${#archives[@]}" -gt 0 mapfile -t bundles < <(find mcpbs -maxdepth 1 -type f -name '*.mcpb' -print | sort) test "${#bundles[@]}" -gt 0 + mapfile -t inventories < <(find inventories -maxdepth 1 -type f -name '*.spdx.json' -print | sort) + test "${#inventories[@]}" -gt 0 # `install.sh` ships with the prerelease as it does with a stable # release. README tells users to curl the installer from a release, # and only release.yml published one, so the hardened script on # master reached nobody following the beta line. The checkout is # pinned to the tag's source sha, so this is that tag's installer. - release_assets=("${archives[@]}" "${bundles[@]}" SHA256SUMS install.sh) + release_assets=("${archives[@]}" "${bundles[@]}" "${inventories[@]}" SHA256SUMS install.sh) gh api "repos/${{ github.repository }}/releases/tags/${RELEASE_TAG}" \ --jq '.assets[].name' > remote-asset-names for release_asset in "${release_assets[@]}"; do @@ -294,7 +380,7 @@ jobs: # Verify by name, not by glob: a partial publish may have no .zip # (Windows) or no .tar.gz yet, and `gh release download --pattern` # fails on an empty match. - mkdir verified-binaries verified-mcpbs + mkdir verified-binaries verified-mcpbs verified-inventories gh api "repos/${{ github.repository }}/releases/tags/${RELEASE_TAG}" \ --jq '.assets[].name' > remote-asset-names while IFS= read -r name; do @@ -302,22 +388,36 @@ jobs: tracedecay-beta-"${RELEASE_TAG}"-*.mcpb) gh release download "$RELEASE_TAG" --repo "${{ github.repository }}" \ --pattern "$name" --dir verified-mcpbs ;; + tracedecay-beta-"${RELEASE_TAG}"-*.spdx.json) + gh release download "$RELEASE_TAG" --repo "${{ github.repository }}" \ + --pattern "$name" --dir verified-inventories ;; tracedecay-beta-"${RELEASE_TAG}"-*.tar.gz | tracedecay-beta-"${RELEASE_TAG}"-*.zip) gh release download "$RELEASE_TAG" --repo "${{ github.repository }}" \ --pattern "$name" --dir verified-binaries ;; esac done < remote-asset-names - python3 scripts/check-release-artifacts.py \ + python3 .release-automation/scripts/check-release-artifacts.py \ --manifest .github/release-targets.json \ --profile beta \ --tag "$RELEASE_TAG" \ --binaries verified-binaries \ - --mcpbs verified-mcpbs - scripts/verify-retained-release-assets.sh \ + --mcpbs verified-mcpbs \ + --inventories verified-inventories + for inventory in verified-inventories/*.spdx.json; do + platform="${inventory#verified-inventories/tracedecay-beta-${RELEASE_TAG}-}" + platform="${platform%.spdx.json}" + archive_format="$(jq -r --arg platform "$platform" \ + '.include[] | select(.name == $platform) | .archive' .github/release-targets.json)" + python3 .release-automation/scripts/generate-release-inventory.py verify \ + --inventory "$inventory" \ + --archive "verified-binaries/tracedecay-beta-${RELEASE_TAG}-${platform}.${archive_format}" \ + --mcpb "verified-mcpbs/tracedecay-beta-${RELEASE_TAG}-${platform}.mcpb" + done + .release-automation/scripts/verify-retained-release-assets.sh \ --tag "$RELEASE_TAG" \ --repo "${{ github.repository }}" \ --signer-workflow "${{ github.repository }}/.github/workflows/release-beta.yml" \ --source-digest "${{ needs.validate.outputs.source_sha }}" \ --signer-ref "refs/tags/${RELEASE_TAG}" \ --signer-ref "refs/heads/master" \ - --files verified-binaries/* verified-mcpbs/* + --files verified-binaries/* verified-mcpbs/* verified-inventories/* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6272427c18..e81fbe31d7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -40,6 +40,7 @@ jobs: prerelease: ${{ steps.release.outputs.prerelease }} matrix: ${{ steps.release.outputs.matrix }} build_required: ${{ steps.release.outputs.build_required }} + package_build_required: ${{ steps.release.outputs.package_build_required }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -47,6 +48,17 @@ jobs: ref: ${{ env.RELEASE_TAG }} persist-credentials: false + - name: Checkout release recovery planner + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.workflow_sha }} + path: .release-automation + persist-credentials: false + sparse-checkout-cone-mode: false + sparse-checkout: | + scripts/plan-release-recovery.py + scripts/verify-retained-release-assets.sh + - name: Validate stable release env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -85,7 +97,7 @@ jobs: } >> "$GITHUB_OUTPUT" gh api "repos/${{ github.repository }}/releases/tags/${tag}" \ --jq '.assets[].name' > remote-asset-names - python3 scripts/plan-release-recovery.py \ + python3 .release-automation/scripts/plan-release-recovery.py \ --manifest .github/release-targets.json \ --tag "$tag" \ --profile stable \ @@ -93,7 +105,7 @@ jobs: --retained-output retained-asset-names \ --github-output "$GITHUB_OUTPUT" mkdir retained-assets - scripts/verify-retained-release-assets.sh \ + .release-automation/scripts/verify-retained-release-assets.sh \ --tag "$tag" \ --repo "${{ github.repository }}" \ --signer-workflow "${{ github.repository }}/.github/workflows/release.yml" \ @@ -123,9 +135,41 @@ jobs: with: ref: ${{ needs.validate-release.outputs.source_sha }} + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.15.0" + + - name: Install the locked-inventory YAML reader + run: python3 -m pip install pyyaml==6.0.2 + + # Recovery automation is pinned to this workflow revision, not the release + # source being inventoried. + - name: Checkout release packaging helpers + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.workflow_sha }} + path: .release-automation + persist-credentials: false + sparse-checkout: | + .github/actions/package-release + .github/actions/setup-bazel + scripts/generate-release-inventory.py + + - name: Set up Bazel for inventory-only recovery + if: matrix.inventory_only == true + uses: ./.release-automation/.github/actions/setup-bazel + with: + cache-key: release-${{ matrix.target }} + cache-save: "false" + + - name: Build dashboard bundle for inventory-only recovery + if: matrix.inventory_only == true + run: bazel build --config=dist --config=ci //dashboard:app_dist + # Bazel builds and validates //dashboard:app_dist, and the CLI build # script refuses to embed it unless it matches //dashboard:bundle_sha256. - name: Build release binary for packaging + if: matrix.inventory_only != true id: cli uses: ./.github/actions/build-cli with: @@ -135,6 +179,7 @@ jobs: # verify-release requires every target to embed the same bundle. - name: Record the embedded dashboard digest + if: matrix.inventory_only != true shell: bash run: | set -euo pipefail @@ -144,24 +189,15 @@ jobs: "dashboard-sha256-${{ matrix.name }}.txt" - name: Upload embedded dashboard digest + if: matrix.inventory_only != true uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: dashboard-sha256-${{ matrix.name }} path: dashboard-sha256-${{ matrix.name }}.txt if-no-files-found: error - # Packaging orchestration comes from this immutable workflow revision, - # so recovery also works for tags predating the shared helper. The CLI - # and its packager inputs still come from the validated release source. - - name: Checkout release packaging helpers - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.workflow_sha }} - path: .release-automation - persist-credentials: false - sparse-checkout: .github/actions/package-release - - name: Package and verify native archive and MCPB + if: matrix.inventory_only != true uses: ./.release-automation/.github/actions/package-release with: binary: ${{ steps.cli.outputs.path }} @@ -171,7 +207,29 @@ jobs: archive: ${{ matrix.archive }} epoch: ${{ needs.validate-release.outputs.source_epoch }} + - name: Record shipping component scope + shell: bash + env: + PLATFORM: ${{ matrix.name }} + SOURCE_SHA: ${{ needs.validate-release.outputs.source_sha }} + run: | + set -euo pipefail + python3 .release-automation/scripts/generate-release-inventory.py scope \ + --source-root . \ + --platform "$PLATFORM" \ + --source-sha "$SOURCE_SHA" \ + --dashboard-dist "$(bazel info --config=dist --config=ci bazel-bin)/dashboard/app-dist" \ + --output "inventory-scope-${PLATFORM}.json" + + - name: Upload component scope + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: inventory-scope-${{ matrix.name }} + path: inventory-scope-${{ matrix.name }}.json + if-no-files-found: error + - name: Upload MCPB artifact + if: matrix.inventory_only != true uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: mcpb-${{ matrix.name }} @@ -179,6 +237,7 @@ jobs: if-no-files-found: error - name: Upload binary archive artifact + if: matrix.inventory_only != true uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: binary-${{ matrix.name }} @@ -214,7 +273,7 @@ jobs: merge-multiple: true - name: Require one embedded dashboard across targets - if: needs.validate-release.outputs.build_required == 'true' + if: needs.validate-release.outputs.package_build_required == 'true' run: scripts/check-dashboard-digests.sh dashboard-digests - uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable @@ -264,8 +323,17 @@ jobs: ref: ${{ needs.validate-release.outputs.source_sha }} persist-credentials: false - - name: Download newly built binary artifacts + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 if: needs.validate-release.outputs.build_required == 'true' + with: + python-version: "3.15.0" + + - name: Install the locked-inventory YAML reader + if: needs.validate-release.outputs.build_required == 'true' + run: python3 -m pip install pyyaml==6.0.2 + + - name: Download newly built binary artifacts + if: needs.validate-release.outputs.package_build_required == 'true' uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: binary-* @@ -273,13 +341,21 @@ jobs: merge-multiple: true - name: Download newly built MCPB artifacts - if: needs.validate-release.outputs.build_required == 'true' + if: needs.validate-release.outputs.package_build_required == 'true' uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: mcpb-* path: new-mcpbs merge-multiple: true + - name: Download shipping component scopes + if: needs.validate-release.outputs.build_required == 'true' + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: inventory-scope-* + path: new-inventory-scopes + merge-multiple: true + # Use the helpers from the running workflow revision for older-tag recovery. - name: Checkout release recovery helpers uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -290,7 +366,11 @@ jobs: sparse-checkout-cone-mode: false sparse-checkout: | scripts/assemble-release-artifacts.sh + scripts/check-release-artifacts.py + scripts/generate-release-inventory.py + scripts/plan-release-recovery.py scripts/update-release-checksums.py + scripts/verify-retained-release-assets.sh - name: Assemble exact retained and newly built artifacts env: @@ -313,6 +393,8 @@ jobs: bundles=(new-mcpbs/*.mcpb) echo "archives=${#archives[@]}" >>"$GITHUB_OUTPUT" echo "bundles=${#bundles[@]}" >>"$GITHUB_OUTPUT" + inventories=(new-inventories/*.spdx.json) + echo "inventories=${#inventories[@]}" >>"$GITHUB_OUTPUT" - name: Attest newly built binary archives if: steps.built.outputs.archives != '0' @@ -326,6 +408,12 @@ jobs: with: subject-path: new-mcpbs/*.mcpb + - name: Attest newly generated component inventories + if: steps.built.outputs.inventories != '0' + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4 + with: + subject-path: new-inventories/*.spdx.json + - name: Upload validated artifacts to release env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -334,7 +422,8 @@ jobs: shopt -s nullglob binaries=(binaries/tracedecay-*) bundles=(mcpbs/*.mcpb) - assets=("${binaries[@]}" "${bundles[@]}") + inventories=(inventories/*.spdx.json) + assets=("${binaries[@]}" "${bundles[@]}" "${inventories[@]}") ( cd binaries sha256sum tracedecay-* @@ -343,6 +432,10 @@ jobs: cd mcpbs sha256sum -- *.mcpb ) >> SHA256SUMS + ( + cd inventories + sha256sum -- *.spdx.json + ) >> SHA256SUMS assets+=(SHA256SUMS install.sh) TAG="${{ needs.validate-release.outputs.tag }}" SOURCE_SHA="${{ needs.validate-release.outputs.source_sha }}" @@ -388,7 +481,7 @@ jobs: # Verify by name, not by glob: a partial publish may have no .zip # or no .tar.gz yet, and `gh release download --pattern` fails on # an empty match. - mkdir -p remote-assets/{binaries,mcpbs} + mkdir -p remote-assets/{binaries,mcpbs,inventories} gh api "repos/${{ github.repository }}/releases/tags/${TAG}" \ --jq '.assets[].name' > remote-asset-names while IFS= read -r name; do @@ -396,24 +489,38 @@ jobs: tracedecay-"${TAG}"-*.mcpb) gh release download "$TAG" --repo "${{ github.repository }}" \ --pattern "$name" --dir remote-assets/mcpbs ;; + tracedecay-"${TAG}"-*.spdx.json) + gh release download "$TAG" --repo "${{ github.repository }}" \ + --pattern "$name" --dir remote-assets/inventories ;; tracedecay-"${TAG}"-*.tar.gz | tracedecay-"${TAG}"-*.zip) gh release download "$TAG" --repo "${{ github.repository }}" \ --pattern "$name" --dir remote-assets/binaries ;; esac done < remote-asset-names - python3 scripts/check-release-artifacts.py \ + python3 .release-automation/scripts/check-release-artifacts.py \ --manifest .github/release-targets.json \ --tag "$TAG" \ --binaries remote-assets/binaries \ - --mcpbs remote-assets/mcpbs - scripts/verify-retained-release-assets.sh \ + --mcpbs remote-assets/mcpbs \ + --inventories remote-assets/inventories + for inventory in remote-assets/inventories/*.spdx.json; do + platform="${inventory#remote-assets/inventories/tracedecay-${TAG}-}" + platform="${platform%.spdx.json}" + archive_format="$(jq -r --arg platform "$platform" \ + '.include[] | select(.name == $platform) | .archive' .github/release-targets.json)" + python3 .release-automation/scripts/generate-release-inventory.py verify \ + --inventory "$inventory" \ + --archive "remote-assets/binaries/tracedecay-${TAG}-${platform}.${archive_format}" \ + --mcpb "remote-assets/mcpbs/tracedecay-${TAG}-${platform}.mcpb" + done + .release-automation/scripts/verify-retained-release-assets.sh \ --tag "$TAG" \ --repo "${{ github.repository }}" \ --signer-workflow "${{ github.repository }}/.github/workflows/release.yml" \ --source-digest "$SOURCE_SHA" \ --signer-ref "refs/tags/${TAG}" \ --signer-ref "refs/heads/master" \ - --files remote-assets/binaries/* remote-assets/mcpbs/* + --files remote-assets/binaries/* remote-assets/mcpbs/* remote-assets/inventories/* - name: Mark stable release as latest if: needs.validate-release.outputs.prerelease == 'false' diff --git a/SECURITY.md b/SECURITY.md index 80992d8d4b..8e71886f33 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -142,7 +142,20 @@ consistency, not independent publisher authenticity. Other distribution/update paths have their own verification; do not assume installer checks apply to all paths. -**Windows:** Authenticode code signing via the [SignPath.io Foundation](https://signpath.io/foundation) program is being rolled out so Windows binaries are signed as part of the release workflow (addresses the Smart App Control block reported in #79). Until that lands in a published release, Windows binaries remain unsigned. +**Release component inventory:** Each native archive and MCPB is accompanied by a +platform-specific SPDX 2.3 inventory asset. The inventory names and SHA-256 hashes +both final package files; `SHA256SUMS` and the release build-provenance attestation +cover the downloaded inventory asset itself. `NOASSERTION` licenses or unknown +sources are explicitly annotated for review. The inventory is a traceability report, +not a license scan or a claim that all components have been legally reviewed. + +**Windows:** Windows release binaries are currently unsigned. The repository contains a +SignPath artifact-configuration example, but no release-workflow signing integration or +verified SignPath project/certificate provisioning. Signing is not enforced by the release +workflow; consumers must not assume an Authenticode signature or Smart App Control +acceptance. Enabling signing remains blocked on external signing-service and certificate +provisioning, followed by workflow and published-artifact verification. The previously cited +PR #79 was unrelated and does not provide evidence for a Smart App Control report. ### Opt-in background daemon diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 5f568bcca2..0b7f4885c7 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -109,6 +109,8 @@ exports_files( "test-package-release-archive.py", "test-plan-release-recovery.py", "test-check-release-artifacts.py", + "generate-release-inventory.py", + "test-generate-release-inventory.py", "test-update-release-checksums.py", "bazel/gen_builds.py", "bazel/rust_test_environment_probe.txt", diff --git a/scripts/assemble-release-artifacts.sh b/scripts/assemble-release-artifacts.sh index 5afefa5521..f52853bfad 100644 --- a/scripts/assemble-release-artifacts.sh +++ b/scripts/assemble-release-artifacts.sh @@ -13,14 +13,15 @@ esac retained_dir="$(mktemp -d)" trap 'rm -rf -- "$retained_dir"' EXIT mkdir binaries mcpbs +mkdir inventories new-inventories gh api "repos/${repo}/releases/tags/${tag}" --jq '.assets[].name' > "$retained_dir/remote-names" -python3 scripts/plan-release-recovery.py \ +python3 .release-automation/scripts/plan-release-recovery.py \ --manifest .github/release-targets.json --tag "$tag" --profile "$profile" \ --asset-names "$retained_dir/remote-names" \ --retained-output "$retained_dir/names" --github-output "$retained_dir/planner-output" mkdir "$retained_dir/assets" -scripts/verify-retained-release-assets.sh \ +.release-automation/scripts/verify-retained-release-assets.sh \ --tag "$tag" --repo "$repo" --signer-workflow "$signer_workflow" \ --source-digest "$source_sha" \ --signer-ref "refs/tags/${tag}" --signer-ref refs/heads/master \ @@ -29,6 +30,8 @@ while IFS= read -r asset; do [[ -z "$asset" ]] && continue if [[ "$asset" == *.mcpb ]]; then cp "$retained_dir/assets/$asset" "mcpbs/$asset" + elif [[ "$asset" == *.spdx.json ]]; then + cp "$retained_dir/assets/$asset" "inventories/$asset" else cp "$retained_dir/assets/$asset" "binaries/$asset" fi @@ -43,13 +46,46 @@ if [[ "$build_required" == true ]]; then destination="mcpbs/$(basename "$artifact")" [[ -e "$destination" ]] || cp "$artifact" "$destination" done + for scope in new-inventory-scopes/*.json; do + [[ -f "$scope" ]] || continue + platform="$(jq -r '.platform' "$scope")" + archive_format="$(jq -r --arg platform "$platform" \ + '.include[] | select(.name == $platform) | .archive' .github/release-targets.json)" + archive="${prefix}-${tag}-${platform}.${archive_format}" + mcpb="${prefix}-${tag}-${platform}.mcpb" + inventory="${prefix}-${tag}-${platform}.spdx.json" + if [[ -e "inventories/$inventory" ]]; then + continue + fi + python3 .release-automation/scripts/generate-release-inventory.py document \ + --source-root . \ + --scope "$scope" \ + --archive "binaries/$archive" \ + --mcpb "mcpbs/$mcpb" \ + --tag "$tag" \ + --source-sha "$source_sha" \ + --source-epoch "$(git show -s --format=%ct "$source_sha")" \ + --repository "$repo" \ + --output "new-inventories/$inventory" + cp "new-inventories/$inventory" "inventories/$inventory" + done fi -python3 scripts/check-release-artifacts.py \ +python3 .release-automation/scripts/check-release-artifacts.py \ --manifest .github/release-targets.json --tag "$tag" --profile "$profile" \ - --binaries binaries --mcpbs mcpbs + --binaries binaries --mcpbs mcpbs --inventories inventories for bundle in mcpbs/*.mcpb; do platform="${bundle#mcpbs/"${prefix}-${tag}-"}" platform="${platform%.mcpb}" python3 scripts/build-mcpb.py verify \ --bundle "$bundle" --version "${tag#v}" --platform "$platform" done +for inventory in inventories/*.spdx.json; do + platform="${inventory#inventories/${prefix}-${tag}-}" + platform="${platform%.spdx.json}" + archive_format="$(jq -r --arg platform "$platform" \ + '.include[] | select(.name == $platform) | .archive' .github/release-targets.json)" + python3 .release-automation/scripts/generate-release-inventory.py verify \ + --inventory "$inventory" \ + --archive "binaries/${prefix}-${tag}-${platform}.${archive_format}" \ + --mcpb "mcpbs/${prefix}-${tag}-${platform}.mcpb" +done diff --git a/scripts/check-release-artifacts.py b/scripts/check-release-artifacts.py index abc8b15f23..dd4d7424c1 100644 --- a/scripts/check-release-artifacts.py +++ b/scripts/check-release-artifacts.py @@ -57,6 +57,7 @@ def main() -> int: parser.add_argument("--binaries", type=Path, required=True) parser.add_argument("--profile", choices=("stable", "beta"), default="stable") parser.add_argument("--mcpbs", type=Path) + parser.add_argument("--inventories", type=Path) arguments = parser.parse_args() targets = target_matrix(arguments.manifest) prefix = "tracedecay-beta" if arguments.profile == "beta" else "tracedecay" @@ -68,6 +69,10 @@ def main() -> int: target["name"]: f"{prefix}-{arguments.tag}-{target['name']}.mcpb" for target in targets } + inventory_name = { + target["name"]: f"{prefix}-{arguments.tag}-{target['name']}.spdx.json" + for target in targets + } if arguments.mcpbs is None: raise SystemExit("release validation requires an MCPB directory") binaries = files(arguments.binaries) @@ -75,6 +80,9 @@ def main() -> int: require_exact("binary", binaries, set(binary_name.values())) require_exact("MCPB", mcpbs, set(mcpb_name.values())) + if arguments.inventories is not None: + inventories = files(arguments.inventories) + require_exact("component inventory", inventories, set(inventory_name.values())) print("release artifact coverage matches target manifest") return 0 diff --git a/scripts/generate-release-inventory.py b/scripts/generate-release-inventory.py new file mode 100644 index 0000000000..75e8656faa --- /dev/null +++ b/scripts/generate-release-inventory.py @@ -0,0 +1,956 @@ +#!/usr/bin/env python3 +"""Generate a deterministic SPDX inventory for a shipped CLI release target.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import subprocess +import tarfile +import tomllib +import zipfile +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import quote + + +TARGET = "//crates/tracedecay-cli:tracedecay" +FONT_LICENSES = { + "@fontsource-variable/archivo": "OFL-1.1", + "@fontsource-variable/ibm-plex-sans": "OFL-1.1", + "@fontsource/ibm-plex-mono": "OFL-1.1", +} + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for block in iter(lambda: source.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def canonical_json(value: object) -> bytes: + return (json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n").encode() + + +def source_tree_digest(path: Path) -> str: + digest = hashlib.sha256() + for file_path in sorted(item for item in path.rglob("*") if item.is_file()): + relative = file_path.relative_to(path).as_posix().encode() + digest.update(len(relative).to_bytes(8, "big")) + digest.update(relative) + digest.update(bytes.fromhex(sha256(file_path))) + return digest.hexdigest() + + +def embedded_source_components( + source_root: Path, rust_components: list[dict[str, object]] +) -> list[dict[str, object]]: + local_owners = { + str(component["local_path"]) + for component in rust_components + if str(component.get("local_path", "")).startswith("crates/") + } + runtime_names = {str(component["name"]) for component in rust_components} + components: list[dict[str, object]] = [] + for owner in sorted(local_owners): + for provenance in sorted((source_root / owner).rglob("vendor/**/PROVENANCE.md")): + vendored = provenance.parent + text = provenance.read_text(encoding="utf-8") + repository = re.search( + r"(?im)^\|\s*upstream (?:repository|URL)\s*\|\s*<([^>]+)>", + text, + ) + license_match = re.search( + r"(?im)^\|\s*licen[cs]e\s*\|\s*([^|]+)\|", text + ) + license_value = ( + license_match.group(1).split("—", 1)[0].strip() + if license_match + else "NOASSERTION" + ) + version_match = re.search( + r"(?im)^\|\s*upstream release current at retrieval\s*\|\s*`?([^|`]+)", + text, + ) + if version_match is None: + version_match = re.search( + r"(?i)crates\.io package `[^`]+`\s+([0-9][^ )|]*)", text + ) + version = version_match.group(1).strip() if version_match else "NOASSERTION" + package_checksum = re.search( + r"(?im)^\|\s*package sha256\s*\|\s*`?([0-9a-f]{64})", text + ) + source_digest = source_tree_digest(vendored) + name = vendored.name + purl = f"pkg:generic/{quote(name, safe='')}" + if version != "NOASSERTION": + purl += f"@{quote(version, safe='')}" + components.append( + { + "name": name, + "version": version, + "source": repository.group(1) if repository else "NOASSERTION", + "downloadLocation": repository.group(1) if repository else "NOASSERTION", + "sourceInfo": ( + f"Vendored at {vendored.relative_to(source_root).as_posix()}; " + f"provenance {provenance.relative_to(source_root).as_posix()}; " + f"source tree SHA256 {source_digest}" + ), + "license": license_value, + "license_evidence": provenance.relative_to(source_root).as_posix(), + "checksum": ( + { + "algorithm": "SHA256", + "checksumValue": package_checksum.group(1), + } + if package_checksum + else { + "algorithm": "SHA256", + "checksumValue": source_digest, + } + ), + "purl": purl, + } + ) + + notice_path = source_root / "NOTICE" + memory_path = source_root / "crates/tracedecay-session-memory/src/memory/diff.rs" + if ( + "tracedecay-session-memory" in runtime_names + and notice_path.is_file() + and memory_path.is_file() + ): + notice = notice_path.read_text(encoding="utf-8") + source_url = re.search(r"(?im)^mnemon\s+\((https?://[^)]+)\)", notice) + license_name = ( + "Apache-2.0" + if re.search(r"(?i)Apache License, Version 2\.0", notice) + else "NOASSERTION" + ) + purl = "pkg:generic/mnemon-adapted-content-cues" + components.append( + { + "name": "mnemon adapted content cues", + "version": "NOASSERTION", + "source": source_url.group(1) if source_url else "NOASSERTION", + "downloadLocation": source_url.group(1) if source_url else "NOASSERTION", + "sourceInfo": ( + "Adaptation identified in NOTICE at " + "crates/tracedecay-session-memory/src/memory/diff.rs; " + "the upstream revision is not recorded." + ), + "license": license_name, + "license_evidence": "NOTICE", + "checksum": { + "algorithm": "SHA256", + "checksumValue": sha256(memory_path), + }, + "purl": purl, + } + ) + return components + + +def label_name(label: str) -> str: + label = label.split(" (", 1)[0] + return label.strip('"') + + +def cquery(source_root: Path) -> tuple[str, str]: + env = os.environ.copy() + graph = subprocess.run( + [ + "bazel", + "cquery", + "--config=dist", + "--config=ci", + f"deps({TARGET})", + "--output=graph", + ], + cwd=source_root, + env=env, + check=True, + capture_output=True, + text=True, + ) + kinds = subprocess.run( + [ + "bazel", + "cquery", + "--config=dist", + "--config=ci", + f"deps({TARGET})", + "--output=label_kind", + ], + cwd=source_root, + env=env, + check=True, + capture_output=True, + text=True, + ) + return graph.stdout, kinds.stdout + + +def parse_graph(graph: str, kinds: str) -> list[dict[str, str]]: + target_kinds: dict[str, str] = {} + for line in kinds.splitlines(): + match = re.match(r"(.+?) rule (.+)$", line.strip()) + if match: + target_kinds[label_name(match.group(2))] = match.group(1) + + edges = [ + (label_name(parent), label_name(child)) + for parent, child in re.findall(r'"([^"]+)"\s*->\s*"([^"]+)"', graph) + ] + adjacency: dict[str, list[str]] = {} + for parent, child in edges: + adjacency.setdefault(parent, []).append(child) + + root = label_name(TARGET) + pending = [root] + visited: set[str] = set() + included: set[str] = set() + while pending: + current = pending.pop() + if current in visited: + continue + visited.add(current) + kind = target_kinds.get(current, "") + if kind in {"cargo_build_script", "rust_proc_macro", "rust_test"}: + continue + if kind == "rust_library": + if current.startswith("@"): + included.add(current) + elif current.startswith("//crates/") or current.startswith("//patches/"): + included.add(current) + pending.extend(adjacency.get(current, ())) + + if root not in visited or not included: + raise ValueError("Bazel query did not expose the configured CLI Rust dependency graph") + return [{"label": label, "kind": target_kinds.get(label, "")} for label in sorted(included)] + + +def lock_package_index(source_root: Path) -> tuple[dict[str, dict[str, object]], dict[str, dict[str, object]]]: + lock = tomllib.loads((source_root / "Cargo.lock").read_text(encoding="utf-8")) + by_repo: dict[str, dict[str, object]] = {} + by_name: dict[str, dict[str, object]] = {} + for package in lock.get("package", []): + if package.get("source"): + by_repo[f"c__{package['name']}-{package['version']}"] = package + else: + by_name[package["name"]] = package + return by_repo, by_name + + +def cargo_package( + source_root: Path, + output_base: Path, + label: str, + lock_packages: tuple[dict[str, dict[str, object]], dict[str, dict[str, object]]], +) -> dict[str, object] | None: + by_repo, by_name = lock_packages + if label.startswith("@"): + repository = label.split("//", 1)[0].lstrip("@") + package = by_repo.get(repository) + matched_repository = repository if package is not None else "" + if package is None: + # Canonical repository labels may include a module-version suffix. + matches = [ + (key, value) + for key, value in by_repo.items() + if repository.endswith(key) + or repository.endswith(key.removeprefix("c__")) + ] + if matches: + matched_repository, package = max(matches, key=lambda item: len(item[0])) + if package is None: + unknown_name = repository.split("+")[-1].removeprefix("c__") + return { + "name": unknown_name, + "version": "NOASSERTION", + "source": "NOASSERTION", + "downloadLocation": "NOASSERTION", + "license": "NOASSERTION", + "checksum": None, + "local_path": None, + "purl": f"pkg:cargo/{quote(unknown_name, safe='')}@NOASSERTION", + } + manifest = output_base / "external" / repository / "Cargo.toml" + if not manifest.is_file(): + manifest = output_base / "external" / matched_repository / "Cargo.toml" + source = str(package.get("source", "NOASSERTION")) + return package_record(package, manifest, source_root, source) + + package_path = label.split("//", 1)[-1].split(":", 1)[0] + if not package_path: + return None + manifest = source_root / package_path / "Cargo.toml" + if not manifest.is_file(): + return None + with manifest.open("rb") as source_file: + metadata = tomllib.load(source_file).get("package", {}) + package_name = str(metadata.get("name", "")) + package = by_name.get(package_name, {}) + source = f"git+https://github.com/ScriptedAlchemy/tracedecay#source-revision" + local_path = package_path + return package_record( + { + "name": package_name, + # Cargo.lock resolves workspace-inherited package versions to strings. + "version": package.get("version", "NOASSERTION"), + "checksum": package.get("checksum"), + }, + manifest, + source_root, + source, + local_path=local_path, + ) + + +def package_record( + package: dict[str, object], + manifest: Path, + source_root: Path, + source: str, + *, + local_path: str | None = None, +) -> dict[str, object]: + metadata: dict[str, object] = {} + if manifest.is_file(): + with manifest.open("rb") as source_file: + metadata = tomllib.load(source_file).get("package", {}) + license_value = metadata.get("license") + name = str(package.get("name", metadata.get("name", "unknown"))) + version = str(package.get("version", metadata.get("version", "NOASSERTION"))) + source_url = source.removeprefix("git+") + if source.startswith("registry+"): + source_url = f"https://crates.io/crates/{quote(name, safe='')}/{quote(version, safe='')}" + record: dict[str, object] = { + "name": name, + "version": version, + "source": source, + "downloadLocation": source_url or "NOASSERTION", + "license": license_value if isinstance(license_value, str) and license_value else "NOASSERTION", + "checksum": package.get("checksum"), + "purl": f"pkg:cargo/{quote(name, safe='')}@{quote(version, safe='')}", + } + if local_path: + record["local_path"] = local_path + record["source"] = f"{local_path} at the release source revision" + record["downloadLocation"] = "NOASSERTION" + record["source_digest"] = source_tree_digest(source_root / local_path) + return record + + +def component_id(purl: str, source: object) -> str: + identity = f"{purl}\0{source or ''}" + return "SPDXRef-" + hashlib.sha256(identity.encode()).hexdigest()[:24] + + +def checksum_from_sri(integrity: str) -> dict[str, str] | None: + for item in integrity.split(): + algorithm, separator, encoded = item.partition("-") + if not separator or algorithm.lower() not in {"sha1", "sha256", "sha384", "sha512"}: + continue + import base64 + + try: + digest = base64.b64decode(encoded, validate=True).hex() + except ValueError: + continue + return {"algorithm": algorithm.upper().replace("SHA", "SHA"), "checksumValue": digest} + return None + + +def npm_component_key(name: str, version: str, package_keys: set[str]) -> str | None: + exact = f"{name}@{version.split('(', 1)[0]}" + return exact if exact in package_keys else None + + +def npm_snapshot_key(name: str, version: str, snapshot_keys: set[str]) -> str | None: + exact = f"{name}@{version}" + if exact in snapshot_keys: + return exact + base = f"{name}@{version.split('(', 1)[0]}" + candidates = sorted(key for key in snapshot_keys if key.startswith(base + "(")) + return candidates[0] if candidates else (base if base in snapshot_keys else None) + + +def npm_package_identity(key: str) -> tuple[str, str]: + delimiter = key.find("@", 1 if key.startswith("@") else 0) + if delimiter < 0: + raise ValueError(f"invalid pnpm package key: {key}") + name = key[:delimiter] + version = key[delimiter + 1 :].split("(", 1)[0] + if not name or not version: + raise ValueError(f"invalid pnpm package key: {key}") + return name, version + + +def npm_components(source_root: Path, platform: str) -> list[dict[str, object]]: + try: + import yaml + except ImportError as error: + raise RuntimeError("PyYAML is required to read the locked dashboard package graph") from error + + documents = yaml.safe_load_all( + (source_root / "pnpm-lock.yaml").read_text(encoding="utf-8") + ) + lock = next( + ( + document + for document in reversed(list(documents)) + if "dashboard" in document.get("importers", {}) + ), + None, + ) + if lock is None: + raise ValueError("pnpm lockfile has no dashboard importer") + return npm_components_from_lock(lock, platform) + + +def npm_components_from_lock(lock: dict[str, object], platform: str) -> list[dict[str, object]]: + dashboard = lock.get("importers", {}).get("dashboard", {}) + roots = dashboard.get("dependencies", {}) + packages = lock.get("packages", {}) + snapshots = lock.get("snapshots", {}) + package_keys = set(packages) + snapshot_keys = set(snapshots) + os_name, arch = { + "aarch64-macos": ("darwin", "arm64"), + "x86_64-linux": ("linux", "x64"), + "aarch64-linux": ("linux", "arm64"), + "x86_64-windows": ("win32", "x64"), + }[platform] + + pending: list[tuple[str, str]] = [] + for name, record in roots.items(): + version = record.get("version") + if version: + package_key = npm_component_key(name, str(version), package_keys) + snapshot_key = npm_snapshot_key(name, str(version), snapshot_keys) + if package_key and snapshot_key: + pending.append((package_key, snapshot_key)) + + visited_packages: set[str] = set() + visited_snapshots: set[str] = set() + result: list[dict[str, object]] = [] + while pending: + package_key, snapshot_key = pending.pop() + if snapshot_key in visited_snapshots: + continue + visited_snapshots.add(snapshot_key) + metadata = packages.get(package_key, {}) + name, version = npm_package_identity(package_key) + resolution = metadata.get("resolution", {}) + source = resolution.get("tarball") or "NOASSERTION" + package = { + "name": name, + "version": version, + "source": source, + "downloadLocation": source, + "sourceInfo": ( + f"pnpm-lock.yaml package {package_key}; " + f"integrity {resolution.get('integrity', 'NOASSERTION')}; " + "source URL is not recorded in the lockfile" + ), + "license": FONT_LICENSES.get(name, "NOASSERTION"), + "license_evidence": ( + "dashboard/src/theme/fonts.css" if name in FONT_LICENSES else None + ), + "checksum": checksum_from_sri( + str(resolution.get("integrity", "")) + ), + "purl": f"pkg:npm/{quote(name, safe='@/')}@{quote(version, safe='')}", + } + if package_key not in visited_packages: + result.append(package) + visited_packages.add(package_key) + node = snapshots.get(snapshot_key, {}) + for group in ("dependencies", "optionalDependencies"): + for dependency, child_version in node.get(group, {}).items(): + child_package = npm_component_key( + dependency, str(child_version), package_keys + ) + child_snapshot = npm_snapshot_key( + dependency, str(child_version), snapshot_keys + ) + if child_package is None or child_snapshot is None: + continue + constraints = packages.get(child_package, {}) + allowed_os = constraints.get("os", []) + if allowed_os and ( + os_name in {item[1:] for item in allowed_os if item.startswith("!")} + or ( + any(not item.startswith("!") for item in allowed_os) + and os_name not in allowed_os + ) + ): + continue + allowed_cpu = constraints.get("cpu", []) + if allowed_cpu and ( + arch in {item[1:] for item in allowed_cpu if item.startswith("!")} + or ( + any(not item.startswith("!") for item in allowed_cpu) + and arch not in allowed_cpu + ) + ): + continue + pending.append((child_package, child_snapshot)) + return sorted(result, key=lambda component: str(component["purl"])) + + +def scope_inventory( + source_root: Path, + output: Path, + platform: str, + source_sha: str, + dashboard_dist: Path, + graph_file: Path | None, + kinds_file: Path | None, + output_base_arg: Path | None, +) -> None: + if platform not in { + "aarch64-macos", + "x86_64-linux", + "aarch64-linux", + "x86_64-windows", + }: + raise ValueError(f"unsupported release platform: {platform}") + if graph_file and kinds_file: + graph = graph_file.read_text(encoding="utf-8") + kinds = kinds_file.read_text(encoding="utf-8") + else: + graph, kinds = cquery(source_root) + labels = parse_graph(graph, kinds) + output_base = output_base_arg + if output_base is None: + output_base = Path( + subprocess.run( + ["bazel", "info", "--config=dist", "--config=ci", "output_base"], + cwd=source_root, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + ) + cargo_lock = lock_package_index(source_root) + rust_components = { + str(record["purl"]): record + for item in labels + if ( + record := cargo_package(source_root, output_base, item["label"], cargo_lock) + ) + is not None + } + assets = [] + if not dashboard_dist.is_dir(): + raise FileNotFoundError(f"built dashboard asset directory is missing: {dashboard_dist}") + for path in sorted(item for item in dashboard_dist.rglob("*") if item.is_file()): + assets.append( + { + "path": path.relative_to(dashboard_dist).as_posix(), + "sha256": sha256(path), + "size": path.stat().st_size, + } + ) + adapted_sources = [] + module_file = source_root / "MODULE.bazel" + if module_file.is_file(): + module_text = module_file.read_text(encoding="utf-8") + patch_paths = sorted(set(re.findall(r'"//scripts:([^"]+\.patch)"', module_text))) + git_packages = { + str(component["name"]) + for component in rust_components.values() + if str(component.get("source", "")).startswith("git+") + } + for relative in patch_paths: + if not any(Path(relative).name.startswith(f"{name}-") for name in git_packages): + continue + path = source_root / "scripts" / relative + if path.is_file(): + adapted_sources.append( + { + "path": path.relative_to(source_root).as_posix(), + "sha256": sha256(path), + } + ) + scope = { + "format": "tracedecay-release-component-scope-v1", + "platform": platform, + "bazel_target": TARGET, + "source_sha": source_sha, + "rust_components": [rust_components[key] for key in sorted(rust_components)], + "embedded_sources": embedded_source_components( + source_root, list(rust_components.values()) + ), + "dashboard_assets": assets, + "adapted_sources": adapted_sources, + } + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(canonical_json(scope)) + + +def artifact_binary(archive: Path, platform: str) -> bytes: + member = "tracedecay.exe" if platform == "x86_64-windows" else "tracedecay" + if archive.name.endswith(".zip"): + with zipfile.ZipFile(archive) as packaged: + return packaged.read(member) + with tarfile.open(archive, "r:gz") as packaged: + content = packaged.extractfile(member) + if content is None: + raise ValueError(f"archive does not contain {member}") + return content.read() + + +def mcpb_contents(bundle: Path, platform: str) -> tuple[bytes, bytes]: + member = "tracedecay.exe" if platform == "x86_64-windows" else "tracedecay" + with zipfile.ZipFile(bundle) as packaged: + expected = {"manifest.json", f"server/{member}"} + if set(packaged.namelist()) != expected: + raise ValueError(f"MCPB does not contain the expected entries: {bundle.name}") + return packaged.read("manifest.json"), packaged.read(f"server/{member}") + + +def spdx_package(record: dict[str, object]) -> dict[str, object]: + purl = str(record["purl"]) + source = record.get("source") + license_value = str(record.get("license") or "NOASSERTION") + package: dict[str, object] = { + "name": record["name"], + "SPDXID": component_id(purl, source), + "versionInfo": str(record.get("version") or "NOASSERTION"), + "downloadLocation": str(record.get("downloadLocation") or record.get("source") or "NOASSERTION"), + "filesAnalyzed": False, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": license_value, + "copyrightText": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": purl, + } + ], + } + checksum = record.get("checksum") + if isinstance(checksum, dict): + package["checksums"] = [checksum] + elif isinstance(checksum, str) and checksum: + package["checksums"] = [{"algorithm": "SHA256", "checksumValue": checksum}] + source_info = record.get("sourceInfo") or record.get("source") + if source_info and source_info != package["downloadLocation"]: + package["sourceInfo"] = str(source_info) + if record.get("source_digest"): + package["sourceInfo"] = ( + str(package.get("sourceInfo", "")) + + f"; local source tree SHA256 {record['source_digest']}" + ).lstrip("; ") + if record.get("license_evidence"): + package["sourceInfo"] = ( + str(package.get("sourceInfo", "")) + + f"; license metadata recorded in {record['license_evidence']}" + ).lstrip("; ") + return package + + +def release_inventory( + source_root: Path, + scope_path: Path, + archive: Path, + mcpb: Path, + tag: str, + source_sha: str, + source_epoch: int, + repository: str, + output: Path, +) -> None: + scope = json.loads(scope_path.read_text(encoding="utf-8")) + platform = str(scope["platform"]) + if scope.get("format") != "tracedecay-release-component-scope-v1": + raise ValueError("component scope has an unsupported format") + if scope.get("source_sha") != source_sha: + raise ValueError("component scope does not match the immutable release source") + if not archive.is_file() or not mcpb.is_file(): + raise FileNotFoundError("final archive and MCPB are required to generate an inventory") + binary_bytes = artifact_binary(archive, platform) + manifest_bytes, bundled_binary = mcpb_contents(mcpb, platform) + if binary_bytes != bundled_binary: + raise ValueError("native archive and MCPB do not contain identical CLI binaries") + npm = npm_components(source_root, platform) + components = [*scope["rust_components"], *scope.get("embedded_sources", []), *npm] + packages = [spdx_package(item) for item in components] + package_ids = {str(package["SPDXID"]) for package in packages} + archive_package_id = "SPDXRef-Archive" + mcpb_package_id = "SPDXRef-MCPB" + binary_id = "SPDXRef-CLI-Binary" + with (source_root / "crates/tracedecay-cli/Cargo.toml").open("rb") as source_file: + product_manifest = tomllib.load(source_file).get("package", {}) + asset_package = lambda name, path, identifier: { + "name": name, + "SPDXID": identifier, + "versionInfo": tag.removeprefix("v"), + "downloadLocation": f"https://github.com/{repository}/releases/download/{quote(tag, safe='')}/{quote(path.name, safe='.-_')}", + "filesAnalyzed": False, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": product_manifest.get("license") or "NOASSERTION", + "copyrightText": "NOASSERTION", + "sourceInfo": ( + f"Root LICENSE SHA256 {sha256(source_root / 'LICENSE')}; " + f"root NOTICE SHA256 {sha256(source_root / 'NOTICE')}" + if (source_root / "LICENSE").is_file() + and (source_root / "NOTICE").is_file() + else "Root LICENSE or NOTICE file is unavailable" + ), + "checksums": [{"algorithm": "SHA256", "checksumValue": sha256(path)}], + } + packages.extend( + [ + asset_package(archive.name, archive, archive_package_id), + asset_package(mcpb.name, mcpb, mcpb_package_id), + { + "name": "tracedecay CLI binary", + "SPDXID": binary_id, + "versionInfo": tag.removeprefix("v"), + "downloadLocation": f"https://github.com/{repository}", + "filesAnalyzed": False, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": product_manifest.get("license") or "NOASSERTION", + "copyrightText": "NOASSERTION", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": hashlib.sha256(binary_bytes).hexdigest(), + } + ], + }, + ] + ) + file_entries = [] + for item in scope["dashboard_assets"]: + path = str(item["path"]) + file_id = "SPDXRef-Asset-" + hashlib.sha256(path.encode()).hexdigest()[:20] + font_package = next( + ( + name + for name in FONT_LICENSES + if path.startswith(name.split("/", 1)[-1] + "-") + or name.split("/", 1)[-1] in path + ), + None, + ) + file_entries.append( + { + "fileName": f"dashboard/app-dist/{path}", + "SPDXID": file_id, + "checksums": [{"algorithm": "SHA256", "checksumValue": item["sha256"]}], + "licenseConcluded": "NOASSERTION", + "licenseInfoInFiles": [FONT_LICENSES[font_package]] if font_package else ["NOASSERTION"], + "copyrightText": "NOASSERTION", + } + ) + + manifest_file_id = "SPDXRef-MCPB-Manifest" + manifest_entry = { + "fileName": "mcpb/manifest.json", + "SPDXID": manifest_file_id, + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": hashlib.sha256(manifest_bytes).hexdigest(), + } + ], + "licenseConcluded": "NOASSERTION", + "licenseInfoInFiles": ["NOASSERTION"], + "copyrightText": "NOASSERTION", + } + file_entries.append(manifest_entry) + patch_file_entries = [ + { + "fileName": str(item["path"]), + "SPDXID": "SPDXRef-Adapted-" + hashlib.sha256(str(item["path"]).encode()).hexdigest()[:20], + "checksums": [{"algorithm": "SHA256", "checksumValue": item["sha256"]}], + "licenseConcluded": "NOASSERTION", + "licenseInfoInFiles": ["NOASSERTION"], + "copyrightText": "NOASSERTION", + } + for item in scope.get("adapted_sources", []) + ] + file_entries.extend(patch_file_entries) + relationships = [ + {"spdxElementId": archive_package_id, "relationshipType": "CONTAINS", "relatedSpdxElement": binary_id}, + {"spdxElementId": mcpb_package_id, "relationshipType": "CONTAINS", "relatedSpdxElement": binary_id}, + ] + for identifier in sorted(package_ids): + relationships.extend( + [ + {"spdxElementId": binary_id, "relationshipType": "DEPENDS_ON", "relatedSpdxElement": identifier}, + {"spdxElementId": archive_package_id, "relationshipType": "CONTAINS", "relatedSpdxElement": identifier}, + {"spdxElementId": mcpb_package_id, "relationshipType": "CONTAINS", "relatedSpdxElement": identifier}, + ] + ) + for item in file_entries: + if item in patch_file_entries: + relationships.append( + { + "spdxElementId": binary_id, + "relationshipType": "GENERATED_FROM", + "relatedSpdxElement": item["SPDXID"], + } + ) + continue + parents = ( + (mcpb_package_id,) + if item["SPDXID"] == manifest_file_id + else (binary_id, archive_package_id, mcpb_package_id) + ) + for parent in parents: + relationships.append( + { + "spdxElementId": parent, + "relationshipType": "CONTAINS", + "relatedSpdxElement": item["SPDXID"], + } + ) + review_items = sorted( + f"{package['name']}@{package.get('versionInfo', 'NOASSERTION')}" + for package in packages + if package.get("licenseDeclared") == "NOASSERTION" + or package.get("downloadLocation") == "NOASSERTION" + ) + review_items.extend( + str(item["fileName"]) + for item in file_entries + if "NOASSERTION" in item.get("licenseInfoInFiles", []) + ) + review_items.sort() + annotations = [ + { + "annotationDate": datetime.fromtimestamp( + source_epoch, timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ"), + "annotationType": "OTHER", + "annotator": "Tool: tracedecay-release-inventory", + "comment": f"Review required: license or source metadata is unknown for {item}.", + } + for item in review_items + ] + document_name = f"TraceDecay {tag} {platform} component inventory" + namespace = ( + "https://spdx.org/spdxdocs/" + + quote(document_name.replace(" ", "-"), safe="-_.") + + "/" + + hashlib.sha256((source_sha + sha256(archive) + sha256(mcpb)).encode()).hexdigest() + ) + document = { + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": document_name, + "documentNamespace": namespace, + "creationInfo": { + "creators": ["Tool: tracedecay-release-inventory"], + "created": datetime.fromtimestamp( + source_epoch, timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ"), + "comment": f"Source revision {source_sha}; Bazel target {scope['bazel_target']}. Unknown metadata is called out in annotations.", + }, + "packages": sorted(packages, key=lambda package: str(package["SPDXID"])), + "files": sorted(file_entries, key=lambda item: str(item["SPDXID"])), + "relationships": sorted( + relationships, + key=lambda item: ( + item["spdxElementId"], + item["relationshipType"], + item["relatedSpdxElement"], + ), + ), + "annotations": annotations, + } + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(canonical_json(document)) + + +def verify_inventory(inventory: Path, archive: Path, mcpb: Path) -> None: + document = json.loads(inventory.read_text(encoding="utf-8")) + by_id = {package["SPDXID"]: package for package in document.get("packages", [])} + expected = { + "SPDXRef-Archive": archive, + "SPDXRef-MCPB": mcpb, + } + for identifier, path in expected.items(): + package = by_id.get(identifier) + if package is None or package.get("name") != path.name: + raise ValueError(f"inventory does not identify the expected artifact: {path.name}") + checksums = { + checksum.get("algorithm"): checksum.get("checksumValue") + for checksum in package.get("checksums", []) + } + if checksums.get("SHA256") != sha256(path): + raise ValueError(f"inventory digest does not match the final artifact: {path.name}") + platform = "x86_64-windows" if archive.name.endswith(".zip") else "x86_64-linux" + if artifact_binary(archive, platform) != mcpb_contents(mcpb, platform)[1]: + raise ValueError("native archive and MCPB do not contain identical CLI binaries") + + +def main() -> None: + parser = argparse.ArgumentParser() + commands = parser.add_subparsers(dest="command", required=True) + scope = commands.add_parser("scope") + scope.add_argument("--source-root", type=Path, default=Path(".")) + scope.add_argument("--platform", required=True) + scope.add_argument("--source-sha", required=True) + scope.add_argument("--dashboard-dist", type=Path, required=True) + scope.add_argument("--graph", type=Path) + scope.add_argument("--kinds", type=Path) + scope.add_argument("--output-base", type=Path) + scope.add_argument("--output", type=Path, required=True) + document = commands.add_parser("document") + document.add_argument("--source-root", type=Path, default=Path(".")) + document.add_argument("--scope", type=Path, required=True) + document.add_argument("--archive", type=Path, required=True) + document.add_argument("--mcpb", type=Path, required=True) + document.add_argument("--tag", required=True) + document.add_argument("--source-sha", required=True) + document.add_argument("--source-epoch", type=int, required=True) + document.add_argument("--repository", required=True) + document.add_argument("--output", type=Path, required=True) + verify = commands.add_parser("verify") + verify.add_argument("--inventory", type=Path, required=True) + verify.add_argument("--archive", type=Path, required=True) + verify.add_argument("--mcpb", type=Path, required=True) + args = parser.parse_args() + if args.command == "scope": + scope_inventory( + args.source_root, + args.output, + args.platform, + args.source_sha, + args.dashboard_dist, + args.graph, + args.kinds, + args.output_base, + ) + elif args.command == "document": + release_inventory( + args.source_root, + args.scope, + args.archive, + args.mcpb, + args.tag, + args.source_sha, + args.source_epoch, + args.repository, + args.output, + ) + else: + verify_inventory(args.inventory, args.archive, args.mcpb) + + +if __name__ == "__main__": + main() diff --git a/scripts/plan-release-recovery.py b/scripts/plan-release-recovery.py index ea88c4bb92..3b9bfd18c4 100644 --- a/scripts/plan-release-recovery.py +++ b/scripts/plan-release-recovery.py @@ -21,14 +21,11 @@ def target_assets( ) -> tuple[str, ...]: name = target["name"] archive = target["archive"] - if profile == "beta": - return ( - f"tracedecay-beta-{tag}-{name}.{archive}", - f"tracedecay-beta-{tag}-{name}.mcpb", - ) + prefix = "tracedecay-beta" if profile == "beta" else "tracedecay" return ( - f"tracedecay-{tag}-{name}.{archive}", - f"tracedecay-{tag}-{name}.mcpb", + f"{prefix}-{tag}-{name}.{archive}", + f"{prefix}-{tag}-{name}.mcpb", + f"{prefix}-{tag}-{name}.spdx.json", ) @@ -55,13 +52,28 @@ def plan( "unexpected existing release assets: " + ", ".join(unexpected) ) + missing_packages = { + target["name"]: any( + asset not in existing + for asset in target_assets(target, tag, profile)[:2] + ) + for target in targets + } missing_targets = [ - target + { + **target, + **( + {"inventory_only": True} + if not missing_packages[target["name"]] + else {} + ), + } for target in targets - if any(asset not in existing for asset in target_assets(target, tag, profile)) + if missing_packages[target["name"]] + or target_assets(target, tag, profile)[2] not in existing ] finalized = sorted(existing & fixed) - if finalized and missing_targets: + if finalized and any(missing_packages[target["name"]] for target in missing_targets): raise SystemExit( "final release metadata exists before all immutable artifacts " f"({', '.join(finalized)}); refusing destructive recovery" @@ -92,9 +104,15 @@ def main() -> None: encoding="utf-8", ) matrix = json.dumps({"include": missing}, separators=(",", ":")) + package_build_required = any( + not target.get("inventory_only", False) for target in missing + ) with arguments.github_output.open("a", encoding="utf-8") as output: output.write(f"matrix={matrix}\n") output.write(f"build_required={'true' if missing else 'false'}\n") + output.write( + f"package_build_required={'true' if package_build_required else 'false'}\n" + ) if __name__ == "__main__": diff --git a/scripts/test-check-release-artifacts.py b/scripts/test-check-release-artifacts.py index 706e10bf82..af28c0dd24 100755 --- a/scripts/test-check-release-artifacts.py +++ b/scripts/test-check-release-artifacts.py @@ -28,6 +28,8 @@ def run(root: Path, profile: str = "stable") -> subprocess.CompletedProcess[str] str(root / "binaries"), "--mcpbs", str(root / "mcpbs"), + "--inventories", + str(root / "inventories"), ], capture_output=True, text=True, @@ -67,7 +69,7 @@ def main() -> int: ] } (root / "targets.json").write_text(json.dumps(manifest), encoding="utf-8") - for child in ("binaries", "mcpbs"): + for child in ("binaries", "mcpbs", "inventories"): (root / child).mkdir() expected = { "binaries": ( @@ -78,11 +80,22 @@ def main() -> int: "tracedecay-v1.2.3-linux.mcpb", "tracedecay-v1.2.3-windows.mcpb", ), + "inventories": ( + "tracedecay-v1.2.3-linux.spdx.json", + "tracedecay-v1.2.3-windows.spdx.json", + ), } for child, names in expected.items(): for name in names: (root / child / name).write_bytes(b"artifact") expect(root, 0, "release artifact coverage matches target manifest") + (root / "inventories" / expected["inventories"][0]).unlink() + expect( + root, + 1, + "component inventory coverage mismatch: missing tracedecay-v1.2.3-linux.spdx.json", + ) + (root / "inventories" / expected["inventories"][0]).write_bytes(b"inventory") (root / "mcpbs" / expected["mcpbs"][0]).unlink() expect(root, 1, "MCPB coverage mismatch: missing tracedecay-v1.2.3-linux.mcpb") (root / "mcpbs" / expected["mcpbs"][0]).write_bytes(b"artifact") @@ -92,6 +105,8 @@ def main() -> int: item.unlink() for item in (root / "mcpbs").iterdir(): item.unlink() + for item in (root / "inventories").iterdir(): + item.unlink() for target in manifest["include"]: (root / "binaries" / ( f"tracedecay-beta-v1.2.3-{target['name']}.{target['archive']}" @@ -99,6 +114,9 @@ def main() -> int: (root / "mcpbs" / ( f"tracedecay-beta-v1.2.3-{target['name']}.mcpb" )).write_bytes(b"artifact") + (root / "inventories" / ( + f"tracedecay-beta-v1.2.3-{target['name']}.spdx.json" + )).write_bytes(b"inventory") expect(root, 0, "release artifact coverage matches target manifest", profile="beta") (root / "binaries" / "tracedecay-beta-v1.2.3-linux.tar.gz").unlink() expect( @@ -118,7 +136,7 @@ def main() -> int: ) # An MCPB that leaked into the binaries directory (a `tracedecay-beta-*` # artifact glob did this) is foreign there. - for child in ("binaries", "mcpbs"): + for child in ("binaries", "mcpbs", "inventories"): for item in (root / child).iterdir(): item.unlink() for target in manifest["include"]: @@ -128,6 +146,9 @@ def main() -> int: (root / "mcpbs" / ( f"tracedecay-beta-v1.2.3-{target['name']}.mcpb" )).write_bytes(b"artifact") + (root / "inventories" / ( + f"tracedecay-beta-v1.2.3-{target['name']}.spdx.json" + )).write_bytes(b"inventory") (root / "binaries" / "tracedecay-beta-v1.2.3-linux.mcpb").write_bytes(b"artifact") expect( root, diff --git a/scripts/test-generate-release-inventory.py b/scripts/test-generate-release-inventory.py new file mode 100644 index 0000000000..704f91a92d --- /dev/null +++ b/scripts/test-generate-release-inventory.py @@ -0,0 +1,353 @@ +#!/usr/bin/env python3 +"""Behavioral tests for release component inventory scope and artifact digests.""" + +from __future__ import annotations + +import importlib.util +import json +import tarfile +import tempfile +import zipfile +from pathlib import Path + +SPEC = importlib.util.spec_from_file_location( + "release_inventory", Path(__file__).with_name("generate-release-inventory.py") +) +assert SPEC is not None and SPEC.loader is not None +INVENTORY = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(INVENTORY) + + +def write_package(path: Path, name: str, version: str, license_value: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + f'[package]\nname = "{name}"\nversion = "{version}"\nlicense = "{license_value}"\n', + encoding="utf-8", + ) + + +def make_archive(path: Path, payload: bytes) -> None: + with tarfile.open(path, "w:gz") as archive: + entry = tarfile.TarInfo("tracedecay") + entry.size = len(payload) + archive.addfile(entry, __import__("io").BytesIO(payload)) + + +def main() -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "source" + output_base = root / "bazel" + dashboard = root / "app-dist" + dashboard.mkdir(parents=True) + (dashboard / "index.js").write_bytes(b"bundled npm") + (dashboard / "archivo-latin.woff2").write_bytes(b"font bytes") + (source / "crates/tracedecay-cli").mkdir(parents=True) + (source / "crates/tracedecay/vendor/sample-vendor").mkdir(parents=True) + (source / "crates/tracedecay-session-memory/src/memory").mkdir(parents=True) + (source / "patches/tokensave-medium-treesitters").mkdir(parents=True) + (source / "scripts/bazel/patches").mkdir(parents=True) + (source / "LICENSE").write_text("MIT License\n", encoding="utf-8") + (source / "NOTICE").write_text( + "mnemon (https://github.com/mnemon-dev/mnemon)\n" + "Licensed under the Apache License, Version 2.0\n" + "Adapted from crates/tracedecay-session-memory/src/memory/diff.rs\n", + encoding="utf-8", + ) + (source / "crates/tracedecay-session-memory/src/memory/diff.rs").write_text( + "adapted cues\n", encoding="utf-8" + ) + (source / "crates/tracedecay/vendor/sample-vendor/PROVENANCE.md").write_text( + "| upstream repository | |\n" + "| licence | MIT — see LICENSE |\n" + "| upstream release current at retrieval | `v2.3.4` |\n", + encoding="utf-8", + ) + (source / "crates/tracedecay/vendor/sample-vendor/LICENSE").write_text( + "MIT License\n", encoding="utf-8" + ) + (source / "scripts/bazel/patches/asupersync-test.patch").write_text("adaptation\n") + (source / "MODULE.bazel").write_text( + 'patches = ["//scripts:bazel/patches/asupersync-test.patch"]\n', encoding="utf-8" + ) + (source / "Cargo.toml").write_text( + '[workspace]\nmembers = ["crates/*"]\n\n' + '[workspace.package]\nversion = "1.2.3"\n', + encoding="utf-8", + ) + write_package( + source / "crates/tracedecay-cli/Cargo.toml", + "tracedecay-cli", + "1.2.3", + "MIT", + ) + (source / "crates/tracedecay/Cargo.toml").write_text( + '[package]\nname = "tracedecay"\nversion.workspace = true\nlicense = "MIT"\n', + encoding="utf-8", + ) + write_package( + source / "crates/tracedecay-session-memory/Cargo.toml", + "tracedecay-session-memory", + "1.2.3", + "MIT", + ) + write_package( + source / "patches/tokensave-medium-treesitters/Cargo.toml", + "tokensave-medium-treesitters", + "0.1.0", + "NOASSERTION", + ) + (source / "Cargo.lock").write_text( + """version = 4 + +[[package]] +name = "tracedecay-cli" +version = "1.2.3" +dependencies = ["tracedecay"] + +[[package]] +name = "tracedecay" +version = "1.2.3" +dependencies = ["asupersync 0.6.0 (git+https://github.com/vendor/asupersync?rev=abc#abc)", "serde 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "tracedecay-session-memory"] + +[[package]] +name = "tracedecay-session-memory" +version = "1.2.3" + +[[package]] +name = "asupersync" +version = "0.6.0" +source = "git+https://github.com/vendor/asupersync?rev=abc#abc" + +[[package]] +name = "serde" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0123456789abcdef" + +[[package]] +name = "tokensave-medium-treesitters" +version = "0.1.0" +""", + encoding="utf-8", + ) + (output_base / "external/c__asupersync-0.6.0").mkdir(parents=True) + (output_base / "external/c__serde-1.0.0").mkdir(parents=True) + write_package( + output_base / "external/c__asupersync-0.6.0/Cargo.toml", + "asupersync", + "0.6.0", + "MIT OR Apache-2.0", + ) + write_package( + output_base / "external/c__serde-1.0.0/Cargo.toml", + "serde", + "1.0.0", + "MIT OR Apache-2.0", + ) + graph = root / "graph.dot" + graph.write_text( + '\n'.join( + ( + '"//crates/tracedecay-cli:tracedecay" -> "//crates/tracedecay:tracedecay";', + '"//crates/tracedecay:tracedecay" -> "@@c__serde-1.0.0//:serde";', + '"//crates/tracedecay:tracedecay" -> "@@c__asupersync-0.6.0//:asupersync";', + '"//crates/tracedecay:tracedecay" -> "//crates/tracedecay-session-memory:tracedecay-session-memory";', + '"//crates/tracedecay:tracedecay" -> "//patches/tokensave-medium-treesitters:tokensave-medium-treesitters";', + '"//crates/tracedecay:tracedecay" -> "//scripts:build_script_build";', + '"//scripts:build_script_build" -> "@@c__build-tool-1.0.0//:build_tool";', + ) + ), + encoding="utf-8", + ) + kinds = root / "kinds.txt" + kinds.write_text( + """rust_binary rule //crates/tracedecay-cli:tracedecay +rust_library rule //crates/tracedecay:tracedecay +rust_library rule //crates/tracedecay-session-memory:tracedecay-session-memory +rust_library rule @@c__serde-1.0.0//:serde +rust_library rule @@c__asupersync-0.6.0//:asupersync +rust_library rule //patches/tokensave-medium-treesitters:tokensave-medium-treesitters +cargo_build_script rule //scripts:build_script_build +rust_library rule @@c__build-tool-1.0.0//:build_tool +""", + encoding="utf-8", + ) + scope = root / "scope.json" + INVENTORY.scope_inventory( + source, + scope, + "x86_64-linux", + "a" * 40, + dashboard, + graph, + kinds, + output_base, + ) + scope_bytes = scope.read_bytes() + assert scope_bytes == INVENTORY.canonical_json(json.loads(scope_bytes)) + scoped = json.loads(scope_bytes) + rust = {component["name"]: component for component in scoped["rust_components"]} + assert {"tracedecay", "serde", "asupersync", "tokensave-medium-treesitters"} <= set(rust) + assert "build-tool" not in rust + assert rust["tracedecay"]["version"] == "1.2.3" + assert rust["tracedecay"]["purl"] == "pkg:cargo/tracedecay@1.2.3" + assert "https://github.com/vendor/asupersync" in rust["asupersync"]["downloadLocation"] + embedded = {component["name"]: component for component in scoped["embedded_sources"]} + assert embedded["sample-vendor"]["version"] == "v2.3.4" + assert embedded["sample-vendor"]["license"] == "MIT" + assert embedded["mnemon adapted content cues"]["license"] == "Apache-2.0" + assert embedded["mnemon adapted content cues"]["version"] == "NOASSERTION" + assert scoped["adapted_sources"][0]["path"] == ( + "scripts/bazel/patches/asupersync-test.patch" + ) + + npm_lock = { + "importers": { + "dashboard": { + "dependencies": { + "@fontsource-variable/archivo": {"version": "5.3.0"}, + "react": {"version": "19.0.0"}, + "react-dom": {"version": "19.0.0(react@19.0.0)"}, + }, + "devDependencies": {"playwright": {"version": "1.0.0"}}, + } + }, + "packages": { + "@fontsource-variable/archivo@5.3.0": { + "resolution": {"integrity": "sha512-YWJj"} + }, + "react@19.0.0": { + "resolution": {"integrity": "sha512-cmVhY3Q="} + }, + "react-dom@19.0.0": { + "resolution": {"integrity": "sha512-cmVhY3Qy" + } + }, + "scheduler@0.23.2": { + "resolution": {"integrity": "sha512-c2NoZWR1bGVy"} + }, + "playwright@1.0.0": { + "resolution": {"integrity": "sha512-cGxheXdyaWdodA=="} + }, + }, + "snapshots": { + "@fontsource-variable/archivo@5.3.0": {}, + "react@19.0.0": {"dependencies": {"scheduler": "0.23.2"}}, + "react-dom@19.0.0(react@19.0.0)": { + "dependencies": {"react": "19.0.0"} + }, + "scheduler@0.23.2": {}, + "playwright@1.0.0": {}, + }, + } + binary = b"release-binary" + archive = root / "tracedecay-v1.2.3-x86_64-linux.tar.gz" + make_archive(archive, binary) + mcpb = root / "tracedecay-v1.2.3-x86_64-linux.mcpb" + with zipfile.ZipFile(mcpb, "w") as bundle: + bundle.writestr("manifest.json", "{}") + bundle.writestr("server/tracedecay", binary) + inventory = root / "inventory.spdx.json" + arguments = ( + source, + scope, + archive, + mcpb, + "v1.2.3", + "a" * 40, + 1_700_000_000, + "ScriptedAlchemy/tracedecay", + inventory, + ) + npm_components = INVENTORY.npm_components + INVENTORY.npm_components = lambda _source_root, platform: ( + INVENTORY.npm_components_from_lock(npm_lock, platform) + ) + npm_roots = INVENTORY.npm_components_from_lock(npm_lock, "x86_64-linux") + npm_lock["importers"]["dashboard"]["devDependencies"]["playwright"] = { + "version": "1.0.0" + } + npm_lock["packages"]["dev-only@2.0.0"] = { + "resolution": {"integrity": "sha512-ZGV2"} + } + npm_lock["snapshots"]["dev-only@2.0.0"] = {} + npm_lock["importers"]["dashboard"]["devDependencies"]["dev-only"] = { + "version": "2.0.0" + } + assert INVENTORY.npm_components_from_lock(npm_lock, "x86_64-linux") == npm_roots + INVENTORY.release_inventory(*arguments) + first = inventory.read_bytes() + INVENTORY.release_inventory(*arguments) + assert inventory.read_bytes() == first + + document = json.loads(first) + assert document["spdxVersion"] == "SPDX-2.3" + packages = {package["name"]: package for package in document["packages"]} + assert "react" in packages + assert "react-dom" in packages + assert "scheduler" in packages + assert "@fontsource-variable/archivo" in packages + assert "playwright" not in packages + assert packages["tracedecay"]["versionInfo"] == "1.2.3" + assert packages["tracedecay"]["externalRefs"][0]["referenceLocator"] == ( + "pkg:cargo/tracedecay@1.2.3" + ) + assert packages["asupersync"]["sourceInfo"].startswith("git+https://github.com/vendor/") + assert packages["tokensave-medium-treesitters"]["licenseDeclared"] == "NOASSERTION" + assert packages["sample-vendor"]["licenseDeclared"] == "MIT" + assert packages["mnemon adapted content cues"]["licenseDeclared"] == "Apache-2.0" + assert any("tokensave-medium-treesitters" in item["comment"] for item in document["annotations"]) + assert any(item["fileName"].endswith("archivo-latin.woff2") for item in document["files"]) + assert any( + "dashboard/app-dist/index.js" in item["comment"] + for item in document["annotations"] + ) + artifact_checksums = { + package["name"]: package["checksums"][0]["checksumValue"] + for package in document["packages"] + if package["SPDXID"] in {"SPDXRef-Archive", "SPDXRef-MCPB"} + } + assert artifact_checksums[archive.name] == INVENTORY.sha256(archive) + assert artifact_checksums[mcpb.name] == INVENTORY.sha256(mcpb) + INVENTORY.verify_inventory(inventory, archive, mcpb) + npm_lock["packages"]["react@19.0.0"]["resolution"]["integrity"] = ( + "sha512-cmVhY3Qx" + ) + INVENTORY.release_inventory(*arguments) + INVENTORY.npm_components = npm_components + assert inventory.read_bytes() != first + with tempfile.TemporaryDirectory() as tampered_dir: + tampered_archive = Path(tampered_dir) / archive.name + tampered_archive.write_bytes(archive.read_bytes() + b"changed") + try: + INVENTORY.verify_inventory(inventory, tampered_archive, mcpb) + except ValueError as error: + assert "digest does not match" in str(error) + else: + raise AssertionError("inventory accepted a different package digest") + + (dashboard / "index.js").write_bytes(b"changed bundled npm") + (source / "Cargo.lock").write_text( + (source / "Cargo.lock").read_text(encoding="utf-8").replace( + "rev=abc#abc", "rev=def#def" + ), + encoding="utf-8", + ) + INVENTORY.scope_inventory( + source, + scope, + "x86_64-linux", + "a" * 40, + dashboard, + graph, + kinds, + output_base, + ) + assert scope.read_bytes() != scope_bytes + + print("release component inventory tests passed") + + +if __name__ == "__main__": + main() diff --git a/scripts/test-plan-release-recovery.py b/scripts/test-plan-release-recovery.py index f8d6a255a7..e33dfe0e7b 100755 --- a/scripts/test-plan-release-recovery.py +++ b/scripts/test-plan-release-recovery.py @@ -106,29 +106,60 @@ def main() -> None: build_required, matrix, retained = run(root, (linux_binary, linux_mcpb)) assert build_required == "true" - assert matrix == {"include": [TARGETS["include"][1]]} + assert matrix == { + "include": [ + {**TARGETS["include"][0], "inventory_only": True}, + TARGETS["include"][1], + ] + } assert retained == sorted((linux_binary, linux_mcpb)) stable_assets = ( linux_binary, linux_mcpb, + "tracedecay-v1.2.3-linux.spdx.json", "tracedecay-v1.2.3-windows.zip", "tracedecay-v1.2.3-windows.mcpb", + "tracedecay-v1.2.3-windows.spdx.json", "SHA256SUMS", "install.sh", ) build_required, matrix, retained = run(root, stable_assets) assert build_required == "false" assert matrix == {"include": []} - assert retained == [ - linux_mcpb, - linux_binary, - "tracedecay-v1.2.3-windows.mcpb", - "tracedecay-v1.2.3-windows.zip", - ] + assert retained == sorted( + ( + linux_mcpb, + linux_binary, + "tracedecay-v1.2.3-linux.spdx.json", + "tracedecay-v1.2.3-windows.mcpb", + "tracedecay-v1.2.3-windows.zip", + "tracedecay-v1.2.3-windows.spdx.json", + ) + ) refuses(root, (linux_binary, "SHA256SUMS"), premature_metadata("SHA256SUMS")) refuses(root, (linux_binary, "install.sh"), premature_metadata("install.sh")) + build_required, matrix, retained = run( + root, + ( + linux_binary, + linux_mcpb, + "tracedecay-v1.2.3-windows.zip", + "tracedecay-v1.2.3-windows.mcpb", + "SHA256SUMS", + "install.sh", + ), + ) + assert build_required == "true" + assert matrix == { + "include": [ + {**TARGETS["include"][0], "inventory_only": True}, + {**TARGETS["include"][1], "inventory_only": True}, + ] + } + assert retained == sorted((linux_binary, linux_mcpb, + "tracedecay-v1.2.3-windows.zip", "tracedecay-v1.2.3-windows.mcpb")) refuses( root, ("unexpected.tar.gz",), "unexpected existing release assets: unexpected.tar.gz" ) @@ -141,26 +172,37 @@ def main() -> None: profile="beta", ) assert build_required == "true" - assert matrix == {"include": [TARGETS["include"][1]]} + assert matrix == { + "include": [ + {**TARGETS["include"][0], "inventory_only": True}, + TARGETS["include"][1], + ] + } assert retained == sorted((beta_linux, beta_linux_mcpb)) beta_assets = ( beta_linux, beta_linux_mcpb, + "tracedecay-beta-v1.2.3-linux.spdx.json", "tracedecay-beta-v1.2.3-windows.zip", "tracedecay-beta-v1.2.3-windows.mcpb", + "tracedecay-beta-v1.2.3-windows.spdx.json", "SHA256SUMS", "install.sh", ) build_required, matrix, retained = run(root, beta_assets, profile="beta") assert build_required == "false" assert matrix == {"include": []} - assert retained == [ - beta_linux_mcpb, - beta_linux, - "tracedecay-beta-v1.2.3-windows.mcpb", - "tracedecay-beta-v1.2.3-windows.zip", - ] + assert retained == sorted( + ( + beta_linux_mcpb, + beta_linux, + "tracedecay-beta-v1.2.3-linux.spdx.json", + "tracedecay-beta-v1.2.3-windows.mcpb", + "tracedecay-beta-v1.2.3-windows.zip", + "tracedecay-beta-v1.2.3-windows.spdx.json", + ) + ) refuses(root, (beta_linux, "install.sh"), premature_metadata("install.sh"), profile="beta")