diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 05a7a48..ac455b6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -402,6 +402,26 @@ jobs: if (-not (Test-Path $p)) { throw "Missing release artifact: $n" } } + # The bare hub archive (docs/RELEASES.md): the flat hub prefix + # scripts/build-local.ps1 writes, built as THIS release, then archived and + # gated by scripts/package_bare_hub.py (the archived hub must run and report + # this version). A second build of the hub, in its own tree, so the + # installer above is untouched. + - name: Package bare hub archive + shell: pwsh + run: | + ./scripts/build-local.ps1 -ReleaseVersion "${{ needs.version.outputs.version }}" ` + -Out (Join-Path $PWD "out\flat") -Build (Join-Path $PWD "build-flat") + if ($LASTEXITCODE -ne 0) { throw "build-local.ps1 failed" } + python scripts/package_bare_hub.py ` + --version "${{ needs.version.outputs.version }}" ` + --platform windows-x86_64 ` + --commit "$env:GITHUB_SHA" ` + --prefix (Join-Path $PWD "out\flat") ` + --out dist-hub + if ($LASTEXITCODE -ne 0) { throw "package_bare_hub.py failed" } + Get-ChildItem dist-hub + - uses: actions/upload-artifact@v7 with: name: windows-x64 @@ -410,6 +430,13 @@ jobs: dist/Retro-Launcher-windows-x64-setup.exe retention-days: 5 + - uses: actions/upload-artifact@v7 + with: + name: windows-x86_64-hub + path: dist-hub/* + if-no-files-found: error + retention-days: 5 + macos: needs: version strategy: @@ -501,6 +528,21 @@ jobs: exit 1 fi + # The bare hub archive (docs/RELEASES.md): the flat hub prefix + # scripts/build-local.sh writes, built as THIS release, archived and gated + # by scripts/package_bare_hub.py. + - name: Package bare hub archive + run: | + ./scripts/build-local.sh --release-version "${{ needs.version.outputs.version }}" \ + --out "$PWD/out/flat" --build "$PWD/build-flat" + python3 scripts/package_bare_hub.py \ + --version "${{ needs.version.outputs.version }}" \ + --platform "macos-${{ matrix.arch }}" \ + --commit "${GITHUB_SHA}" \ + --prefix "$PWD/out/flat" \ + --out dist-hub + ls -lah dist-hub/ + - uses: actions/upload-artifact@v7 with: name: macos-${{ matrix.arch }} @@ -508,6 +550,13 @@ jobs: dist/Retro-Launcher-macos-${{ matrix.arch }}.dmg retention-days: 5 + - uses: actions/upload-artifact@v7 + with: + name: macos-${{ matrix.arch }}-hub + path: dist-hub/* + if-no-files-found: error + retention-days: 5 + android: needs: version uses: ./.github/workflows/android.yml @@ -550,18 +599,31 @@ jobs: - name: Bare hub archive + manifest run: | set -euo pipefail - HUB=artifacts/linux-x86_64-hub - cp -v "${HUB}"/retro-hub-*.tar.gz "${HUB}"/retro-hub-*.tar.gz.sha256 dist/ + # One bare archive per platform: linux-x86_64, windows-x86_64 and + # macos-{arm64,x86_64}. hub_manifest.py refuses a release in which they + # disagree on version, commit or the Direct-mode contract. + PLATFORMS="linux-x86_64 windows-x86_64 macos-arm64 macos-x86_64" + ENTRIES=() + for p in ${PLATFORMS}; do + HUB="artifacts/${p}-hub" + test -d "${HUB}" || { echo "::error::no bare hub archive artifact for ${p}" >&2; exit 1; } + cp -v "${HUB}"/retro-hub-*.tar.gz "${HUB}"/retro-hub-*.zip dist/ 2>/dev/null || true + cp -v "${HUB}"/retro-hub-*.sha256 dist/ + ENTRIES+=("${HUB}"/retro-hub-*.json) + done python3 scripts/hub_manifest.py \ --version "${{ needs.version.outputs.version }}" \ --tag "${{ needs.version.outputs.tag }}" \ --repo "${GITHUB_REPOSITORY}" \ - --expect linux-x86_64 \ + --expect "${PLATFORMS// /,}" \ --out dist \ - "${HUB}"/retro-hub-*.json + "${ENTRIES[@]}" # Every archive must still match the hash its build recorded. (cd dist && sha256sum -c SHA256SUMS) test -f "dist/retro-hub-${{ needs.version.outputs.version }}-linux-x86_64.tar.gz" + test -f "dist/retro-hub-${{ needs.version.outputs.version }}-windows-x86_64.zip" + test -f "dist/retro-hub-${{ needs.version.outputs.version }}-macos-arm64.tar.gz" + test -f "dist/retro-hub-${{ needs.version.outputs.version }}-macos-x86_64.tar.gz" test -f dist/hub-manifest.json - name: Ensure release tag exists diff --git a/docs/RELEASES.md b/docs/RELEASES.md index 1e57c37..6a16c5e 100644 --- a/docs/RELEASES.md +++ b/docs/RELEASES.md @@ -339,7 +339,7 @@ module ABI, or a package that records none, is not second-guessed. | Game | the newest release of title.json `update.github` | nothing: the game package is generated from the player's ROM and is never published. A newer release is reported with a link to it, to rebuild the app from. | -- | | Core | n64lle's `n64lle-release-manifest.json` ([n64lle docs/RELEASES.md](https://github.com/RetroPortingToolKit/n64lle/blob/main/docs/RELEASES.md) §5), for a title whose core is n64lle's generic core | `/core//`: the core library, its `.rcore.toml` and `installed.json` (module ABI), taken from the release prefix | after a restart | | Runner | Retro-Runtime's `runtime-manifest.json` (the runtime updater, unchanged rules) | `/runtime//` | from the next Play | -| Hub | this repo's `hub-manifest.json` (below), Linux only for now | `/hub//`, the bare hub archive | after a restart | +| Hub | this repo's `hub-manifest.json` (below) | `/hub//`, the bare hub archive | after a restart | Every manifest is the **newest release's, pre-releases included**, found from `https://github.com//releases.atom` (n64lle docs/RELEASES.md §2), with the @@ -385,9 +385,21 @@ which picks the newest of everything once more. ## The bare hub archive -For a tool that runs the hub in Direct mode, such as n64lle's new-project -scaffolder in "release" mode. So far only the Linux job packages one; on -Windows and macOS the hub (with Direct mode) ships inside the installers. +For a tool that runs the hub in Direct mode, or makes a title app, such as +n64lle's new-project scaffolder in "release" mode and its `tools/build_app.sh +--platform`. One archive per platform: `retro-hub--linux-x86_64.tar.gz`, +`-windows-x86_64.zip`, `-macos-arm64.tar.gz` and `-macos-x86_64.tar.gz`; each has +an entry in `hub-manifest.json`. The Linux one is built by +`scripts/package_hub_archive.sh`; Windows and macOS by `scripts/package_bare_hub.py` +from the flat hub prefix `scripts/build-local.{sh,ps1} --release-version ` +writes, with the same gates in spirit (the archived hub is extracted to a clean +directory and must run and report this release). The tables below describe the +Linux archive; on Windows the hub's DLLs (SDL3, curl, zlib) and +`retcomm-portable.exe` (the kit's stub) sit beside `retro-hub.exe`, and on macOS +the hub keeps its dylib install names (the title-app kit's `bundle_dylibs.sh` +copies them into the app). `requires` and `system_libraries` are empty off +Linux. **Windows and macOS archives: written 2026-10-13, run only on Linux +against a stand-in prefix; the first release is their first real test.** Everything is flat at the archive root, so it extracts straight into one directory: @@ -467,9 +479,9 @@ Its shape follows Retro-Runtime's `runtime-manifest.json`: "system_libraries": ["libGLX.so.0", "libOpenGL.so.0", "libc.so.6", "libcurl.so.4", "…"] }, "linux-arm64": { "unavailable": "not built yet" }, - "windows-x86_64": { "unavailable": "no bare archive is built for this platform yet; …" }, - "macos-arm64": { "unavailable": "no bare archive is built for this platform yet; …" }, - "macos-x86_64": { "unavailable": "no bare archive is built for this platform yet; …" } + "windows-x86_64": { "archive": "retro-hub--windows-x86_64.zip", "executable": "retro-hub.exe", "…": "…" }, + "macos-arm64": { "archive": "retro-hub--macos-arm64.tar.gz", "…": "…" }, + "macos-x86_64": { "archive": "retro-hub--macos-x86_64.tar.gz", "…": "…" } } } ``` diff --git a/scripts/build-local.ps1 b/scripts/build-local.ps1 index f8b0004..e3a6dce 100644 --- a/scripts/build-local.ps1 +++ b/scripts/build-local.ps1 @@ -18,13 +18,14 @@ # RETRO_HUB= # # usage: scripts/build-local.ps1 [-Debug] [-Out DIR] [-Build DIR] -# [-Runtime ] [-Jobs N] [-Help] +# [-Runtime ] [-Jobs N] [-ReleaseVersion V] [-Help] param( [switch]$Debug, [string]$Out = "", [string]$Build = "", [string]$Runtime = "", [int]$Jobs = 0, + [string]$ReleaseVersion = "", [switch]$Help ) @@ -48,6 +49,10 @@ this machine, and print its path last, as RETRO_HUB=. scripts\build-local.ps1 when it has one) instead of the submodule's -Jobs N parallel build jobs (default: all CPUs) + -ReleaseVersion V + build as release V (-DRETCOMM_VERSION, -DRETCOMM_RELEASE_BUILD): + the hub's --version says V. What scripts/package_bare_hub.py + archives in the release workflow. -Help this text "@ | Write-Output exit 0 @@ -114,6 +119,7 @@ $cmakeArgs = @("-S", $Root, "-B", $Build, "-G", $generator, "-DVCPKG_TARGET_TRIPLET=$Triplet", "-DCMAKE_BUILD_TYPE=$Config", "-DRETCOMM_COMMIT=$commit") +if ($ReleaseVersion) { $cmakeArgs += @("-DRETCOMM_VERSION=$ReleaseVersion", "-DRETCOMM_RELEASE_BUILD=ON") } if ($generator -like "Visual Studio*") { $cmakeArgs += @("-A", $(if ($Arch -eq "arm64") { "ARM64" } else { "x64" })) } Say "$Platform, $Config, build $Build" Invoke-Checked cmake $cmakeArgs diff --git a/scripts/build-local.sh b/scripts/build-local.sh index 3110df0..56e615e 100755 --- a/scripts/build-local.sh +++ b/scripts/build-local.sh @@ -44,6 +44,10 @@ machine, and print its path last, as RETRO_HUB=. scripts/build-local.sh when it has one) instead of the submodule's --jobs N parallel build jobs (default: all CPUs) + --release-version V + build as release V (-DRETCOMM_VERSION, -DRETCOMM_RELEASE_BUILD): + the hub's --version says V. What scripts/package_bare_hub.py + archives in the release workflow. --help this text is linux-x86_64, linux-arm64, macos-x86_64 or macos-arm64. @@ -51,7 +55,7 @@ Relative paths are taken against the current directory. EOF } -CONFIG=Release OUT="" BUILD="" RUNTIME="" JOBS="" +CONFIG=Release OUT="" BUILD="" RUNTIME="" JOBS="" RELEASE_VERSION="" while [[ $# -gt 0 ]]; do case "$1" in --debug) CONFIG=Debug; shift ;; @@ -59,6 +63,7 @@ while [[ $# -gt 0 ]]; do --build) [[ $# -ge 2 ]] || die "--build needs a directory"; BUILD="$2"; shift 2 ;; --runtime) [[ $# -ge 2 ]] || die "--runtime needs a directory"; RUNTIME="$2"; shift 2 ;; --jobs) [[ $# -ge 2 ]] || die "--jobs needs a number"; JOBS="$2"; shift 2 ;; + --release-version) [[ $# -ge 2 ]] || die "--release-version needs a version"; RELEASE_VERSION="$2"; shift 2 ;; -h|--help) usage; exit 0 ;; *) die "unknown argument '$1' (see --help)" ;; esac @@ -131,6 +136,7 @@ cmake_args=( "-DCMAKE_BUILD_TYPE=${CONFIG}" "-DRETCOMM_COMMIT=${COMMIT}" ) +[[ -z "${RELEASE_VERSION}" ]] || cmake_args+=("-DRETCOMM_VERSION=${RELEASE_VERSION}" -DRETCOMM_RELEASE_BUILD=ON) command -v ninja >/dev/null && cmake_args+=(-G Ninja) [[ -z "${CMAKE_PREFIX_PATH:-}" ]] || cmake_args+=("-DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}") # The runner finds SDL3 beside itself in the flat prefix (the hub already diff --git a/scripts/hub_manifest.py b/scripts/hub_manifest.py index d105032..2506e8e 100755 --- a/scripts/hub_manifest.py +++ b/scripts/hub_manifest.py @@ -29,13 +29,10 @@ # Platforms a tool may ask for that this release cannot serve, and why. A tool # reads the reason instead of finding the platform silently absent. -_INSTALLER_ONLY = ("no bare archive is built for this platform yet; the installer and DMG " - "assets carry retro-hub with Direct mode") +# (Windows and macOS had only the installer and DMG until their bare archives, +# scripts/package_bare_hub.py; an older release's manifest still says so.) UNAVAILABLE = { "linux-arm64": "not built yet", - "windows-x86_64": _INSTALLER_ONLY, - "macos-arm64": _INSTALLER_ONLY, - "macos-x86_64": _INSTALLER_ONLY, } # Fields every platform of one release must agree on. diff --git a/scripts/package_bare_hub.py b/scripts/package_bare_hub.py new file mode 100644 index 0000000..7b502fe --- /dev/null +++ b/scripts/package_bare_hub.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +"""Package the bare (non-installer) retro-hub archive for Windows or macOS. + +The Linux archive is scripts/package_hub_archive.sh (ELF-specific gates). This is +the same contract (docs/RELEASES.md, "The bare hub archive") for the other two +OSes, from the FLAT HUB PREFIX scripts/build-local.sh / build-local.ps1 write -- +the layout n64lle's tools and the title-app kit already run from -- built with +`--release-version ` so the hub reports this release: + + scripts/build-local.sh --release-version 0.9.0 --out out/flat # macOS + scripts/build-local.ps1 -ReleaseVersion 0.9.0 -Out out\\flat # Windows + scripts/package_bare_hub.py --version 0.9.0 --platform macos-arm64 \\ + --prefix out/flat --out dist-hub + +Output in --out, named as the Linux job's: + retro-hub--.zip (windows-*) + retro-hub--.tar.gz (macos-*) + .sha256 + retro-hub--.json this build's manifest entry, merged by + scripts/hub_manifest.py + +Gates, each failing the package rather than warning: + * the prefix holds the hub, its title-app kit, the fonts and the art, and (on + Windows) retcomm-portable.exe, which the kit needs; + * the archive never holds retro-core-runner (a tool takes that from + Retro-Runtime's release), no symlink, and nothing outside the prefix; + * the hub INSIDE the archive, extracted to a clean directory, runs --version + and reports this release's version (and commit when given), Direct mode, the + runner lookup the contract names, a title-app mode and an Update page, and + refuses --run-core without --rom with exit 2. + +UNVERIFIED ON WINDOWS AND macOS: written 2026-10-13 on Linux, where its +packaging, gates and manifest entry were run against a stand-in prefix. The +first release run on each OS is its first real test. +""" +import argparse +import hashlib +import json +import os +import pathlib +import re +import shutil +import stat +import subprocess +import sys +import tarfile +import tempfile +import zipfile + +EXPECT_RUNNER_LOOKUP = "RETRO_CORE_RUNNER exe_dir/retro-core-runner data_dir/runtime//retro-core-runner" +PLATFORMS = { + "windows-x86_64": ("zip", ".exe"), + "macos-arm64": ("tar.gz", ""), + "macos-x86_64": ("tar.gz", ""), +} +# Never in the archive, wherever they sit in the prefix. +EXCLUDE_NAMES = {"retro-core-runner", "retro-core-runner.exe"} + + +def die(msg): + sys.exit("package_bare_hub: " + msg) + + +def stage(prefix, exe_suffix): + """The allowlist: every regular file of the flat prefix but the runner.""" + hub = prefix / ("retro-hub" + exe_suffix) + if not hub.is_file(): + die("%s: not in the prefix (build it with build-local --release-version)" % hub) + files = [] + for root, dirs, names in os.walk(prefix): + dirs.sort() + for n in sorted(names): + p = pathlib.Path(root) / n + rel = p.relative_to(prefix).as_posix() + if p.is_symlink(): + die("%s is a symlink; the archive holds real files only" % rel) + if n in EXCLUDE_NAMES: + continue + files.append(rel) + have = set(files) + for need in ("retcomm.png", "fonts/LatoLatin-Regular.ttf", "fonts/NOTICE.md", "LICENSE"): + if need not in have: + die("the prefix has no %s" % need) + if not any(f.startswith("platforms/") for f in files): + die("platform art missing (platforms/)") + kit = "packaging/title/build-title-app." + ("ps1" if exe_suffix else "sh") + if kit not in have: + die("the title-app kit is missing (%s)" % kit) + if exe_suffix and ("retcomm-portable.exe" not in have or not any(f.lower().endswith(".dll") for f in files)): + die("the Windows prefix needs retcomm-portable.exe (the kit's stub) and the hub's DLLs beside the exe") + return sorted(files) + + +def write_archive(prefix, files, archive, kind): + tmp = archive.with_name(archive.name + ".part") + if kind == "zip": + with zipfile.ZipFile(tmp, "w", zipfile.ZIP_DEFLATED, compresslevel=9) as z: + for rel in files: + zi = zipfile.ZipInfo.from_file(prefix / rel, rel) # forward slashes, mode kept + zi.compress_type = zipfile.ZIP_DEFLATED + with open(prefix / rel, "rb") as f: + z.writestr(zi, f.read()) + else: + with tarfile.open(tmp, "w:gz") as t: + for rel in files: + ti = t.gettarinfo(str(prefix / rel), arcname=rel) + ti.uid = ti.gid = 0 + ti.uname = ti.gname = "" + with open(prefix / rel, "rb") as f: + t.addfile(ti, f) + os.replace(tmp, archive) + + +def extract(archive, kind, dest): + if kind == "zip": + with zipfile.ZipFile(archive) as z: + for info in z.infolist(): + z.extract(info, dest) + else: + with tarfile.open(archive) as t: + for m in t.getmembers(): + if not m.isfile() and not m.isdir(): + die("%s: member %s is not a regular file" % (archive.name, m.name)) + t.extractall(dest) + for p in pathlib.Path(dest).rglob("*"): # a zip drops the mode bits; the hub must run + if p.is_file() and p.name in ("retro-hub", "retro-hub.exe"): + p.chmod(p.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + +def run_hub(hub, *args): + env = {k: v for k, v in os.environ.items() if k not in ("LD_LIBRARY_PATH", "DYLD_LIBRARY_PATH")} + r = subprocess.run([str(hub), *args], capture_output=True, text=True, env=env, + cwd=str(hub.parent), timeout=120) + return r.returncode, r.stdout, r.stderr + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--version", required=True) + ap.add_argument("--platform", required=True, choices=sorted(PLATFORMS)) + ap.add_argument("--prefix", required=True, help="the flat hub prefix (build-local's --out)") + ap.add_argument("--commit", help="the commit the hub must report (default: not checked)") + ap.add_argument("--out", default="dist-hub") + a = ap.parse_args() + if not re.match(r"^[0-9]+\.[0-9]+\.[0-9]+([.+-][A-Za-z0-9.+-]*)?$", a.version): + die("--version %s: expected semver without a leading v" % a.version) + kind, exe_suffix = PLATFORMS[a.platform] + prefix = pathlib.Path(a.prefix).resolve() + if not prefix.is_dir(): + die("--prefix %s: not a directory" % prefix) + out = pathlib.Path(a.out).resolve() + out.mkdir(parents=True, exist_ok=True) + + name = "retro-hub-%s-%s" % (a.version, a.platform) + archive = out / ("%s.%s" % (name, kind)) + files = stage(prefix, exe_suffix) + write_archive(prefix, files, archive, kind) + + # ---- read the claims back out of the archive, in a clean directory ---- + with tempfile.TemporaryDirectory() as clean: + clean = pathlib.Path(clean) + extract(archive, kind, clean) + got = sorted(p.relative_to(clean).as_posix() for p in clean.rglob("*") if p.is_file()) + if got != files: + die("the extracted archive differs from the allowlist") + if any(n in EXCLUDE_NAMES for n in (p.name for p in clean.rglob("*"))): + die("retro-core-runner must not be in the hub archive (it comes from Retro-Runtime's release)") + hub = clean / ("retro-hub" + exe_suffix) + rc, out_text, err = run_hub(hub, "--version") + if rc != 0: + die("the archived hub failed --version (exit %d): %s" % (rc, err.strip())) + fields = {} + for line in out_text.splitlines(): + k, _, v = line.partition(" ") + fields[k] = v.strip() + if fields.get("version") != a.version: + die("the archived hub says version %r, this release is %s" % (fields.get("version"), a.version)) + if a.commit and fields.get("commit") != a.commit: + die("the archived hub says commit %r, this release is %s" % (fields.get("commit"), a.commit)) + if not re.match(r"^[1-9][0-9]*$", fields.get("direct_mode", "")): + die("the archived hub was built without Direct mode") + if fields.get("runner_lookup") != EXPECT_RUNNER_LOOKUP: + die("the archived hub looks for the runner at %r" % fields.get("runner_lookup")) + if not re.match(r"^[1-9][0-9]*$", fields.get("title_app", "")): + die("the archived hub has no title-app mode") + if not re.match(r"^[1-9][0-9]*$", fields.get("updates", "")): + die("the archived hub has no Update page") + rc, _, err = run_hub(hub, "--run-core", str(clean / "none")) + if rc != 2 or "--run-core needs --rom" not in err: + die("the archived hub did not refuse --run-core without --rom (exit %d: %s)" % (rc, err.strip())) + if exe_suffix == "": + r = subprocess.run(["bash", str(clean / "packaging/title/build-title-app.sh"), "--help"], + capture_output=True, text=True) + if r.returncode != 0: + die("the archived title-app kit does not run: " + r.stderr.strip()) + + sha = hashlib.sha256(archive.read_bytes()).hexdigest() + pathlib.Path(str(archive) + ".sha256").write_text("%s %s\n" % (sha, archive.name)) + link = fields["link_protocol"].split(".") + entry = { + "platform": a.platform, + "serves": [a.platform], + "version": a.version, + "commit": fields["commit"], + "archive": archive.name, + "sha256": sha, + "size": archive.stat().st_size, + "executable": "retro-hub" + exe_suffix, + "files": files, + # No glibc floor off Linux; the OS's own libraries are not listed. + "requires": {}, + "system_libraries": [], + "link_protocol": {"major": int(link[0]), "minor": int(link[1])}, + "rcore_abi": {"major": int(fields["rcore_abi_major"]), + "draft_revision": int(fields["rcore_draft_revision"])}, + "direct_mode": {"cli_revision": int(fields["direct_mode"]), + "updates": int(fields["updates"]), + "flags": fields["direct_mode_flags"].split(), + "runner_lookup": fields["runner_lookup"]}, + } + (out / (name + ".json")).write_text(json.dumps(entry, indent=2) + "\n") + print("package_bare_hub: %s" % archive) + print(" sha256 %s, %d bytes, link %s" % (sha, entry["size"], fields["link_protocol"])) + + +if __name__ == "__main__": + main()