release: make SHA256SUMS cover every payload asset

.github/workflows/release-artifacts.yml

@@ -408,6 +408,10 @@ jobs:
       - name: Generate SHA256SUMS
         # Covers the POSIX tarballs AND the win32 zip: winget, Chocolatey,
         # and the npx launcher all verify against this one checksum file.
+        # Those are the only payload assets that exist this early - the snap,
+        # .deb/.rpm/.AppImage and SBOM assets are built by later jobs, so
+        # publish-release widens this file over the whole draft asset set
+        # (issue #913) before the release is published.
         run: |
           cd dist
           sha256sum libredb-studio-standalone-*.tar.gz libredb-studio-standalone-*.zip > SHA256SUMS
@@ -1121,6 +1125,37 @@ jobs:
     permissions:
       contents: write
     steps:
+      # The widening step below runs a repo script, so the job needs the tree.
+      - name: Checkout code
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6
+
+      # Close the checksum gap (issue #913). The publish job can only cover the
+      # tarballs and the win32 zip, because nothing else exists when it runs;
+      # by now every payload asset is on the draft, so rebuild the one combined
+      # file over the whole set - the two snaps and the CycloneDX SBOM were the
+      # assets left with no checksum of any kind. This has to happen before the
+      # publish below: from there the asset set is frozen (issue #154), and
+      # dist/SHA256SUMS is the file the widening rewrites. The script reads
+      # each asset's recorded sha256 from the releases API and refuses to emit
+      # a file in which an entry the release already carried has changed, so
+      # the npx launcher, Homebrew and Chocolatey/winget lookups stay valid -
+      # a rebuilt file only ever widens.
+      - name: Widen SHA256SUMS to cover every payload asset
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.repository }}
+          TAG: ${{ needs.guard.outputs.version }}
+        run: |
+          set -euo pipefail
+          gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${TAG}" > /tmp/release.json
+          mkdir -p dist
+          gh release download "$TAG" --pattern SHA256SUMS --dir dist --clobber --repo "$GITHUB_REPOSITORY"
+          node scripts/release-sums.mjs /tmp/release.json \
+            --existing dist/SHA256SUMS --allow-download > dist/SHA256SUMS.new
+          mv dist/SHA256SUMS.new dist/SHA256SUMS
+          cat dist/SHA256SUMS
+          gh release upload "$TAG" dist/SHA256SUMS --clobber --repo "$GITHUB_REPOSITORY"
+
       - name: Verify the draft carries the full asset set
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

docs/DISTRIBUTION.md

@@ -196,15 +196,19 @@ Release tags carry **no `v` prefix** (tag `0.9.41` == package.json version). Eac
 |---|---|---|
 | Standalone server tarball | `libredb-studio-standalone-<version>-<os>-<arch>.tar.gz` | `linux-x64`, `linux-arm64`, `darwin-x64`, `darwin-arm64` |
 | Standalone server zip (Windows) | `libredb-studio-standalone-<version>-win32-x64.zip` | `win32-x64` (bundled Node runtime + `libredb-studio.exe` launcher) |
-| Checksums | `SHA256SUMS` | covers all standalone tarballs and the win32 zip |
+| Checksums | `SHA256SUMS` | covers every payload asset except the `.sha256` sidecars |
 | Debian package | `libredb-studio_<version>_<arch>.deb` (+ `.sha256` sidecar) | `amd64`, `arm64` |
 | RPM package | `libredb-studio-<version>.<arch>.rpm` (+ `.sha256` sidecar) | `x86_64`, `aarch64` |
 | Snap | `libredb-studio_<version>_<arch>.snap` | `amd64`, `arm64` (also published to the Snap Store) |
 | Desktop AppImage | `libredb-studio-desktop-<version>-linux-<arch>.AppImage` (+ `.sha256` sidecar) | `x64`, `arm64` (also the artifact the in-repo Flatpak manifest repacks) |
 | Desktop Debian package | `libredb-studio-desktop_<version>_<arch>.deb` (+ `.sha256` sidecar) | `amd64`, `arm64` (from 0.9.62; the artifact FlatPark pins as extra-data) |
 
-`SHA256SUMS` covers the standalone tarballs and the win32 zip; each `.deb`/`.rpm`/`.AppImage`
-ships its own per-file `<artifact>.sha256` sidecar instead (those are built in separate jobs).
+`SHA256SUMS` covers every payload asset in the release — the standalone tarballs, the win32 zip,
+both `.snap` files and the CycloneDX SBOM — so one file answers "what is the hash of what I just
+downloaded?", whichever artifact that was. The `.deb`/`.rpm` packages and the desktop AppImage
+additionally ship a per-file `<artifact>.sha256` sidecar, written by the job that builds them; the
+combined file covers them too, so a reader no longer has to know which of the two mechanisms
+applies to the artifact they picked.
 
 **Two different `.deb`s ship per release and they are not interchangeable.**
 `libredb-studio_<version>_<arch>.deb` is the headless server: it installs a systemd unit and is
@@ -1281,11 +1285,13 @@ All channels have now had their first live run (the Snap publish completed its f
 
 ### Artifact provenance roadmap
 
-Standalone tarballs and `.deb`/`.rpm` packages are checksum-verified against `SHA256SUMS` /
-per-package `.sha256` sidecars (see [Release artifact naming](#release-artifact-naming)) — both
-from the same GitHub release. That pairing detects corruption, not substitution: whoever can
-replace an asset can replace its checksum line too. The `.snap` release asset ships no sidecar
-(`--dangerous` installs skip the Snap Store's own verification).
+Every payload asset is checksum-verified against `SHA256SUMS`, and the `.deb`/`.rpm` packages and
+the desktop AppImage ship their own `<artifact>.sha256` sidecar as well
+(see [Release artifact naming](#release-artifact-naming)) — all from the same GitHub release. That
+pairing detects corruption, not substitution: whoever can replace an asset can replace its
+checksum line too, which is what the signed provenance below is for. The `.snap` assets are covered
+by `SHA256SUMS` like every other payload asset, which matters for `snap install --dangerous`: the
+Snap Store's own verification is skipped there, so the release checksum is the only one a user has.
 
 Signed provenance moves the trust root out of the release — the signer is the workflow's GitHub
 OIDC identity (repo + workflow + commit), recorded in a public transparency log. All three steps of

scripts/release-sums.mjs

@@ -0,0 +1,188 @@
+#!/usr/bin/env node
+/**
+ * Rebuild the release's combined SHA256SUMS so it covers every payload asset
+ * (issue #913).
+ *
+ * Why not in the `publish` job: it runs before the snap, SBOM and desktop jobs
+ * have attached anything, so the file it writes can only cover the standalone
+ * tarballs and the win32 zip. The .deb/.rpm/.AppImage assets were given
+ * per-file `<name>.sha256` sidecars as a workaround, and the two snaps and the
+ * CycloneDX SBOM ended up with neither - so a reader who downloads a .snap
+ * from the release page has nothing to check it against.
+ *
+ * This runs in `publish-release`, immediately before the draft is published,
+ * when every payload asset exists and the asset set can still be amended.
+ *
+ *   - every payload asset is covered (anything that is not SHA256SUMS itself
+ *     and not a `.sha256` sidecar);
+ *   - the digests come from the releases API, which records the sha256 of the
+ *     bytes as stored - hashing a fresh download would be a second opinion on
+ *     the same bytes, at the cost of pulling ~1.5 GB in the last job of the
+ *     release chain. An asset with no digest recorded is downloaded and hashed
+ *     locally instead;
+ *   - every entry the file already carried must survive with the same hash.
+ *     The npx launcher, the Homebrew formula and the Chocolatey/winget
+ *     renderers all read this file, so the rebuild may only widen it;
+ *   - the `sha256sum` output format is preserved (hash, two spaces, bare file
+ *     name, newline) and names are sorted, which is the order the `publish`
+ *     job's glob already produced.
+ *
+ * Usage:
+ *   gh api "repos/<owner>/<repo>/releases/tags/<version>" > release.json
+ *   gh release download <version> --pattern SHA256SUMS --dir dist
+ *   node scripts/release-sums.mjs release.json --existing dist/SHA256SUMS > SHA256SUMS
+ *
+ *   --existing <path>  the SHA256SUMS already on the release; its entries are
+ *                      asserted to survive unchanged
+ *   --allow-download   hash an asset locally when the API records no digest for
+ *                      it (without this flag that is an error)
+ */
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { Readable } from "node:stream";
+import { fileURLToPath } from "node:url";
+
+const SUMS_NAME = "SHA256SUMS";
+const SIDECAR_SUFFIX = ".sha256";
+const DIGEST_PREFIX = "sha256:";
+const HEX64 = /^[0-9a-f]{64}$/;
+const SUMS_LINE = /^([0-9a-f]{64}) {2}(.+)$/;
+
+/** Payload assets are everything a user downloads and runs. */
+export function isPayloadAsset(name) {
+  return name !== SUMS_NAME && !name.endsWith(SIDECAR_SUFFIX);
+}
+
+/** The sha256 the releases API records for an asset, or null when it has none. */
+export function assetSha256(asset) {
+  const digest = typeof asset?.digest === "string" ? asset.digest : "";
+  if (!digest.startsWith(DIGEST_PREFIX)) return null;
+  const hex = digest.slice(DIGEST_PREFIX.length);
+  return HEX64.test(hex) ? hex : null;
+}
+
+/** One `sha256sum` output line. Bare file name: consumers look entries up by name. */
+export function formatSum(hash, name) {
+  return `${hash}  ${name}\n`;
+}
+
+/** Payload assets, sorted by name - the order the release's own glob produces. */
+export function payloadAssets(assets) {
+  const payload = (assets ?? []).filter((asset) => isPayloadAsset(asset?.name));
+  const names = new Set();
+  for (const asset of payload) {
+    if (names.has(asset.name)) throw new Error(`Release asset list carries '${asset.name}' twice`);
+    names.add(asset.name);
+  }
+  return payload.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+}
+
+export function parseSums(text) {
+  const entries = new Map();
+  for (const line of (text ?? "").split("\n")) {
+    if (line.trim() === "") continue;
+    const match = SUMS_LINE.exec(line);
+    if (!match) throw new Error(`Malformed SHA256SUMS line: ${JSON.stringify(line)}`);
+    entries.set(match[2], match[1]);
+  }
+  return entries;
+}
+
+/** Hash a fetch Response body. */
+export async function hashResponse(response) {
+  if (!response.ok) throw new Error(`Cannot download the asset: HTTP ${response.status}`);
+  const hash = crypto.createHash("sha256");
+  for await (const chunk of Readable.fromWeb(response.body)) hash.update(chunk);
+  return hash.digest("hex");
+}
+
+export async function hashUrl(url, fetchImpl = fetch) {
+  return hashResponse(await fetchImpl(url));
+}
+
+/**
+ * Rebuild the combined checksum file from a release's asset list.
+ * Throws rather than emitting a partial file: this runs on a draft that is
+ * about to become immutable, so a wrong file is permanent.
+ */
+export async function buildSums(assets, { allowDownload = false, fetchImpl = fetch } = {}) {
+  const payload = payloadAssets(assets);
+  if (payload.length === 0) throw new Error("Release asset list has no payload assets");
+
+  let sums = "";
+  for (const asset of payload) {
+    let hash = assetSha256(asset);
+    if (hash === null) {
+      if (!allowDownload) {
+        throw new Error(
+          `Release asset '${asset.name}' has no sha256 digest recorded - re-run with --allow-download to hash it locally`,
+        );
+      }
+      if (!asset.browser_download_url) {
+        throw new Error(`Release asset '${asset.name}' has no digest and no download URL`);
+      }
+      hash = await hashUrl(asset.browser_download_url, fetchImpl);
+    }
+    sums += formatSum(hash, asset.name);
+  }
+  return sums;
+}
+
+/** Every entry the previous file carried must survive, unchanged. */
+export function assertSuperset(existingText, rebuiltText) {
+  const existing = parseSums(existingText);
+  const rebuilt = parseSums(rebuiltText);
+  for (const [name, hash] of existing) {
+    const current = rebuilt.get(name);
+    if (current === undefined) {
+      throw new Error(`Rebuilt SHA256SUMS dropped the existing entry for '${name}'`);
+    }
+    if (current !== hash) {
+      throw new Error(`Rebuilt SHA256SUMS changed the hash for '${name}' (was ${hash}, now ${current})`);
+    }
+  }
+}
+
+function parseArgs(argv) {
+  const args = { releasePath: "", existingPath: "", allowDownload: false };
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    if (arg === "--allow-download") {
+      args.allowDownload = true;
+    } else if (arg === "--existing") {
+      args.existingPath = argv[i + 1] ?? "";
+      if (args.existingPath === "") throw new Error("--existing needs a path");
+      i += 1;
+    } else if (arg.startsWith("--")) {
+      throw new Error(`Unknown argument: ${arg}`);
+    } else if (args.releasePath === "") {
+      args.releasePath = arg;
+    } else {
+      throw new Error(`Unexpected argument: ${arg}`);
+    }
+  }
+  if (args.releasePath === "") {
+    throw new Error("Usage: node scripts/release-sums.mjs <release.json> [--existing <SHA256SUMS>] [--allow-download]");
+  }
+  return args;
+}
+
+async function main(argv) {
+  const args = parseArgs(argv);
+  const release = JSON.parse(fs.readFileSync(args.releasePath, "utf8"));
+  const rebuilt = await buildSums(release.assets, { allowDownload: args.allowDownload });
+  if (args.existingPath !== "") {
+    assertSuperset(fs.readFileSync(args.existingPath, "utf8"), rebuilt);
+  }
+  process.stdout.write(rebuilt);
+  console.error(`Rebuilt SHA256SUMS over ${rebuilt.trimEnd().split("\n").length} payload assets`);
+}
+
+// CLI entry only when executed directly (the unit test imports this module).
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  main(process.argv.slice(2)).catch((error) => {
+    console.error(`release-sums: ${error.message}`);
+    process.exit(1);
+  });
+}