diff --git a/.github/release-notes.md.template b/.github/release-notes.md.template new file mode 100644 index 0000000000..533b9e50e7 --- /dev/null +++ b/.github/release-notes.md.template @@ -0,0 +1,4 @@ +## Release resources + +- [Release website](https://mboworks.github.io/mbo/site/tag/@TAG@/) +- [Coverage report](https://mboworks.github.io/mbo/coverage/tag/@VERSION@/) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8849d56964..63f706886b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -31,6 +31,14 @@ jobs: with: python-version: "3.13" - run: python3 -m unittest discover -s tools -p release_site_test.py + - run: python3 -m unittest discover -s tools -p release_notes_test.py + - name: Verify configured documentation and generated links + env: + GH_TOKEN: ${{ github.token }} + run: | + # This is a disposable build on the runner, never a Pages publication. + python3 tools/release_site.py . "${RUNNER_TEMP}/release-site-check" \ + --repository "${GITHUB_REPOSITORY}" --tag 0.0.0-verification --latest "" trunk: runs-on: ubuntu-latest diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 0cce3fc2f7..3bd9185ec1 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -14,6 +14,11 @@ on: type: string required: false + config_path: + description: Optional config file on main for backfill (empty uses the tag config) + type: string + required: false + permissions: {} # Share the lock and retained branch with coverage: every deployment includes both. @@ -92,9 +97,22 @@ jobs: GH_TOKEN: ${{ github.token }} RELEASE_TAG: ${{ steps.release.outputs.tag }} LATEST_TAG: ${{ steps.release.outputs.latest }} + CONFIG_PATH: ${{ inputs.config_path }} run: | + set -euo pipefail + config_args=() + if [[ -n "${CONFIG_PATH}" ]]; then + config_file="$(realpath --canonicalize-existing "source/${CONFIG_PATH}")" + if [[ "${config_file}" != "${GITHUB_WORKSPACE}/source/"* ]]; then + echo "Configuration must be a tracked file inside the main checkout" >&2 + exit 1 + fi + git -C source ls-files --error-unmatch -- "${CONFIG_PATH}" >/dev/null + config_args=(--config "${config_file}") + fi python3 source/tools/release_site.py release site \ - --repository "${GITHUB_REPOSITORY}" --tag "${RELEASE_TAG}" --latest "${LATEST_TAG}" + --repository "${GITHUB_REPOSITORY}" --tag "${RELEASE_TAG}" --latest "${LATEST_TAG}" \ + "${config_args[@]}" - name: Retain and stage complete Pages tree env: RELEASE_TAG: ${{ steps.release.outputs.tag }} diff --git a/.github/workflows/release_prep.sh b/.github/workflows/release_prep.sh index 54bad7898d..d6adf9d6dc 100755 --- a/.github/workflows/release_prep.sh +++ b/.github/workflows/release_prep.sh @@ -131,3 +131,6 @@ Copy [dev.MODULE.bazel](https://github.com/mboworks/${PACKAGE_NAME}/blob/main/ba include("//:dev.MODULE.bazel") \`\`\` EOF + +printf '\n' +bash tools/release_notes.sh "${TAG}" diff --git a/README.md b/README.md index 3bdbee8d4c..005410f0b1 100644 --- a/README.md +++ b/README.md @@ -403,6 +403,10 @@ library's only NOTICE entry is courtesy). ## Release website +Release notes use `.github/release-notes.md.template`, rendered by +`tools/release_notes.sh TAG`, to link to that tag's versioned website and related +release resources. The existing changelog and installation notes remain included. + The [website](https://mboworks.github.io/mbo/) forwards to the latest published stable release at `site/tag//`, preserving the exact Git tag name. Each release keeps its converted HTML, images, and configured files. Retrying @@ -433,12 +437,18 @@ site directory. For example: Use existing source files in the actual configuration. `pages` converts Markdown; optional `files` copies other files unchanged. `README.md` must map to `index.html`. -The generated `documents.html`, `release.json`, and `assets/` paths are reserved. +The generated `documents.html`, `release.json`, `release-site.json`, and `assets/` +paths are reserved. Destination paths cannot have hidden components (names starting +with a dot), because the Pages artifact uploader excludes them. Hidden source +paths remain valid; for example, `.github/workflows/README.md` maps to +`workflows/index.html`. Navigation links support `{owner}`, `{repo}`, `{tag}`, `{version}`, and `{commit}`. `{version}` omits a leading `v` for compatibility with coverage report paths. -The configuration and content come from the release tag. Links to configured -pages follow their destination mappings; other local source links use the exact -release commit. Embedded images are copied, including remote badges. Markdown +By default, the configuration and content come from the release tag. Every linked +local Markdown page (including directory README links) must have a `pages` mapping. +Publication fails for an omitted mapping, a missing generated file, or a broken +anchor within the snapshot. Links to configured pages follow their destination +mappings; other local source links use the exact release commit. Embedded images are copied, including remote badges. Markdown conversion uses the [GitHub Markdown API](https://docs.github.com/en/rest/markdown/markdown) at publication time; browsing the result requires no Markdown renderer or CDN. @@ -447,11 +457,45 @@ on `coverage-pages` and deploys the complete Pages tree. Coverage and site publication share a concurrency group to preserve both trees. GitHub's latest stable release selects the root redirect; backfilling an older release does not make it latest. The workflow can also be dispatched with a published tag to retry -publication (the tag must contain `release-site.json`). Enable GitHub Pages with +publication. Enable GitHub Pages with **GitHub Actions** as its source, and set the repository's About website to `https://mboworks.github.io/mbo/`. +### Backfill a historical release + +No new release or tag change is needed. Manually dispatch `Publish release site` +with `tag` set to the historical release and `config_path` set to a tracked JSON +file on `main`. For `0.15.0`, use the current `release-site.json`: + +```sh +gh workflow run pages.yml --repo mboworks/mbo --ref main \ + -f tag=0.15.0 -f config_path=release-site.json +``` + +The override changes only the publication layout; all Markdown and copied files +still come from the selected tag. A configuration can serve multiple historical +tags when its sources exist in each tag. For a different historical layout, add +another configuration on `main` and select its repository-relative path. Missing +sources or links fail publication instead of falling back to newer content. +Automatic release publication continues to use the configuration from the tag. + +Each new snapshot retains the exact configuration as `release-site.json` and +records its SHA-256, whether it came from the tag or an override, and the source +commit in `release.json`. Retrying a published tag does not replace its HTML or +configuration, even if the selected override has changed since publication. + +To verify a backfill locally without deploying, check out the historical tag in +`/tmp/mbo-0.15.0` and run from the current publisher checkout: + +```sh +python3 tools/release_site.py /tmp/mbo-0.15.0 /tmp/mbo-site-preview \ + --repository mboworks/mbo --tag 0.15.0 --latest 0.15.0 \ + --config release-site.json +``` + Local regression tests: `python3 -m unittest discover -s tools -p release_site_test.py`. +CI also converts the configured documentation and checks the generated links in +a disposable runner directory. It never commits, retains, or deploys that preview. Release coverage links select `https://mboworks.github.io/mbo/coverage/tag//`, matching the coverage publisher rather than the moving main-branch report. diff --git a/release-site.json b/release-site.json index 1b4d604ffd..6899c2b258 100644 --- a/release-site.json +++ b/release-site.json @@ -10,7 +10,18 @@ "mbo/file/README.md": "mbo/file/README.html", "mbo/hash/README.md": "mbo/hash/README.html", "mbo/hash/measurements/README.md": "mbo/hash/measurements/README.html", - "mbo/types/README.md": "mbo/types/README.html" + "mbo/types/README.md": "mbo/types/README.html", + ".github/workflows/README.md": "workflows/index.html", + "AGENTS.md": "AGENTS.html", + "CLAUDE.md": "CLAUDE.html", + "CODE_OF_CONDUCT.md": "CODE_OF_CONDUCT.html", + "GIT_RULES.md": "GIT_RULES.html", + "RULES.md": "RULES.html", + "STYLE_CPP.md": "STYLE_CPP.html", + "STYLE_SH.md": "STYLE_SH.html", + "TODO.md": "TODO.html", + "mbo/diff/TODO.md": "mbo/diff/TODO.html", + "mbo/types/REFLECTION.md": "mbo/types/REFLECTION.html" }, "links": [ { diff --git a/tools/release_notes.sh b/tools/release_notes.sh new file mode 100755 index 0000000000..b29ecbd2c6 --- /dev/null +++ b/tools/release_notes.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Render release-note links without building, publishing, or changing repository state. +set -euo pipefail + +TAG="${1:?Usage: release_notes.sh TAG}" +if [[ ! "${TAG}" =~ ^v?[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then + echo "Invalid release tag: ${TAG}" >&2 + exit 1 +fi +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +sed -e "s|@TAG@|${TAG}|g" -e "s|@VERSION@|${TAG#v}|g" \ + "${ROOT}/.github/release-notes.md.template" diff --git a/tools/release_notes_test.py b/tools/release_notes_test.py new file mode 100644 index 0000000000..071d792f92 --- /dev/null +++ b/tools/release_notes_test.py @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors +# SPDX-License-Identifier: Apache-2.0 +"""Exercise the release-note renderer with the actual repository template.""" + +from pathlib import Path +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +REPO = "mbo" + + +class ReleaseNotesTest(unittest.TestCase): + def render(self, tag): + with tempfile.TemporaryDirectory() as cwd: + return subprocess.run( + ["bash", str(ROOT / "tools/release_notes.sh"), tag], + cwd=cwd, text=True, capture_output=True, check=False) + + def test_release_resources(self): + for tag in ("1.2.3", "v1.2.3", "v1.2.3-rc.1"): + with self.subTest(tag=tag): + result = self.render(tag) + self.assertEqual(result.returncode, 0, result.stderr) + notes = result.stdout + base = f"https://mboworks.github.io/{REPO}" + self.assertIn(f"{base}/site/tag/{tag}/", notes) + self.assertNotIn("@TAG@", notes) + self.assertNotIn("@VERSION@", notes) + version = tag.removeprefix("v") + if REPO in ("mbo", "xff", "carve"): + self.assertIn(f"{base}/coverage/tag/{version}/", notes) + else: + self.assertNotIn("/coverage/", notes) + if REPO == "xff": + self.assertIn(f"{base}/releases/{version}/)", notes) + self.assertIn(f"{base}/releases/{version}/XFF.md", notes) + if REPO == "coderef": + self.assertIn(f"/blob/{tag}/CHANGELOG.md", notes) + self.assertIn(f"{base}/site/tag/{tag}/schema/v1.json", notes) + + def test_reject_invalid_tags_without_partial_notes(self): + for tag in ("", "main", "../1.2.3", "v1.2.3|bad", "1.2.3\nmain"): + with self.subTest(tag=tag): + result = self.render(tag) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, "") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/release_site.py b/tools/release_site.py index fd0cb7a2e9..f5634cd41c 100644 --- a/tools/release_site.py +++ b/tools/release_site.py @@ -48,8 +48,9 @@ def version(tag): return tag.removeprefix("v") -def configuration(source): - config = json.loads((source / "release-site.json").read_text()) +def configuration(source, override=None): + data = (override if override is not None else source / "release-site.json").read_bytes() + config = json.loads(data) pages = config["pages"] if not pages or pages.get("README.md") != "index.html": raise ValueError("README.md must map to index.html") @@ -64,19 +65,25 @@ def configuration(source): raise ValueError("Page mappings must convert .md sources to .html destinations") if dst in destinations or dst == "documents.html" or dst.startswith("assets/"): raise ValueError(f"Duplicate or reserved destination: {dst}") + if any(part.startswith(".") for part in Path(dst).parts): + raise ValueError(f"Pages excludes hidden destinations: {dst}") destinations.add(dst) for src, dst in config.get("files", {}).items(): if src in pages: raise ValueError(f"Source is mapped as both a page and a file: {src}") + if src.lower().endswith(".md"): + raise ValueError(f"Markdown must be converted through pages: {src}") for path in (src, dst): if (not isinstance(path, str) or path.startswith("/") or ".." in Path(path).parts or str(Path(path)) != path or any(char in path for char in "\\?#")): raise ValueError(f"Unsafe asset path: {path!r}") - if dst in destinations or dst in ("documents.html", "release.json") or dst.startswith("assets/"): + if dst in destinations or dst in ("documents.html", "release.json", "release-site.json") or dst.startswith("assets/"): raise ValueError(f"Duplicate or reserved destination: {dst}") + if any(part.startswith(".") for part in Path(dst).parts): + raise ValueError(f"Pages excludes hidden destinations: {dst}") destinations.add(dst) - return config + return config, data def headings(body): @@ -137,13 +144,16 @@ def rewrite(self, value, image=False): if local is not None: if local == ".." or local.startswith("../"): raise ValueError(f"Link escapes repository: {value}") - if local not in self.documents and f"{local.rstrip('/')}/README.md" in self.documents: - local = f"{local.rstrip('/')}/README.md" + readme = f"{local.rstrip('/')}/README.md" + if local not in self.documents and (readme in self.documents or (self.source / readme).is_file()): + local = readme if local == ".": local = "README.md" if local in self.documents and not image: target = posixpath.relpath(self.documents[local], posixpath.dirname(self.documents[self.document]) or ".") return urlunsplit(("", "", quote(target), parsed.query, parsed.fragment)) + if not image and local.lower().endswith(".md"): + raise ValueError(f"{self.document}: linked Markdown has no page mapping: {local}") path = self.source / local if image: if not path.is_file() or path.resolve() != path.absolute(): @@ -198,7 +208,66 @@ def handle_charref(self, name): self.parts.append(f"&#{name};") -def build(source, retained, repository, tag, renderer=render): +class PageReferences(HTMLParser): + """Collect browser-visible anchors and links from final HTML.""" + + def __init__(self, text): + super().__init__() + self.anchors = set() + self.links = [] + self.feed(text) + + def handle_starttag(self, tag, attrs): + for key, value in attrs: + if value is None: + continue + if key == "id" or (tag == "a" and key == "name"): + self.anchors.add(value) + if key in ("href", "src"): + self.links.append(value) + + handle_startendtag = handle_starttag + + +def validate_site(output, repository, tag): + """Reject broken links inside the snapshot before retaining or deploying it.""" + owner, repo = repository.split("/") + base = f"/{repo}/site/tag/{tag}/" + host = f"{owner}.github.io" + pages = {path.relative_to(output).as_posix(): PageReferences(path.read_text()) + for path in output.rglob("*.html")} + for document, page in pages.items(): + for link in page.links: + parsed = urlsplit(link) + if parsed.scheme or parsed.netloc: + if parsed.scheme not in ("http", "https") or parsed.netloc != host: + continue + if not parsed.path.startswith(base): + continue + path = unquote(parsed.path) + if path.startswith("/"): + if not path.startswith(base): + continue # Coverage and other explicitly separate Pages trees. + target = posixpath.normpath(path[len(base):]) + elif path: + target = posixpath.normpath(posixpath.join(posixpath.dirname(document), path)) + else: + target = document + if target == ".." or target.startswith("../"): + raise ValueError(f"{document}: link escapes release snapshot: {link}") + destination = output / target + if destination.is_dir(): + target = posixpath.join(target, "index.html") + destination = output / target + target = posixpath.normpath(target) + if not destination.is_file(): + raise ValueError(f"{document}: missing generated link target: {link}") + fragment = unquote(parsed.fragment) + if fragment and target in pages and fragment not in pages[target].anchors: + raise ValueError(f"{document}: missing generated anchor: {link}") + + +def build(source, retained, repository, tag, renderer=render, config_path=None): source = source.resolve() release_version = version(tag) destination = retained / "site" / "tag" / tag @@ -208,7 +277,7 @@ def build(source, retained, repository, tag, renderer=render): if metadata["commit"] != sha or metadata["tag"] != tag: raise ValueError("A retained release cannot be replaced by a different commit or tag") return - config = configuration(source) + config, config_data = configuration(source, config_path) documents = config["pages"] files = config.get("files", {}) tracked = set(git(source, "ls-files", "-z").split("\0")) @@ -251,7 +320,13 @@ def page(title, body): for path in sorted(documents) ) + "" (output / "documents.html").write_text(page("Documentation", index)) - (output / "release.json").write_text(json.dumps({"tag": tag, "commit": sha}) + "\n") + (output / "release-site.json").write_bytes(config_data) + (output / "release.json").write_text(json.dumps({ + "tag": tag, "commit": sha, + "configuration": {"origin": "override" if config_path is not None else "tag", + "sha256": hashlib.sha256(config_data).hexdigest()}, + }) + "\n") + validate_site(output, repository, tag) # Rename only after every document and image has been converted successfully. output.rename(destination) @@ -276,8 +351,9 @@ def main(): parser.add_argument("--repository", required=True) parser.add_argument("--tag", required=True) parser.add_argument("--latest", required=True) + parser.add_argument("--config", type=Path, help="Explicit configuration override for historical tags") args = parser.parse_args() - build(args.source, args.retained, args.repository, args.tag) + build(args.source, args.retained, args.repository, args.tag, config_path=args.config) if args.latest: redirect(args.retained, args.latest) diff --git a/tools/release_site_test.py b/tools/release_site_test.py index 189dfe212e..07553d7453 100644 --- a/tools/release_site_test.py +++ b/tools/release_site_test.py @@ -161,6 +161,104 @@ def test_one_source_cannot_have_page_and_file_destinations(self): with self.assertRaisesRegex(ValueError, "both a page and a file"): self.build() + def test_backfill_uses_old_source_and_retains_exact_override(self): + override = self.root / "backfill.json" + data = json.dumps(self.config, indent=4).encode() + b"\n" + override.write_bytes(data) + subprocess.run(["git", "-C", str(self.source), "rm", "-q", "release-site.json"], check=True) + subprocess.run(["git", "-C", str(self.source), "-c", "user.name=Test", + "-c", "user.email=test@example.com", "-c", "commit.gpgsign=false", + "commit", "-qm", "Historical source without a site config"], check=True) + site.build(self.source, self.retained, "mboworks/mbo", "v1.2.3", + self.render, config_path=override) + output = self.retained / "site/tag/v1.2.3" + self.assertEqual((output / "release-site.json").read_bytes(), data) + metadata = json.loads((output / "release.json").read_text()) + self.assertEqual(metadata["commit"], site.git(self.source, "rev-parse", "HEAD")) + self.assertEqual(metadata["configuration"]["origin"], "override") + self.assertEqual(metadata["configuration"]["sha256"], site.hashlib.sha256(data).hexdigest()) + self.assertFalse((self.source / "release-site.json").exists()) + before = (output / "index.html").read_bytes() + override.write_text("invalid replacement config") + site.build(self.source, self.retained, "mboworks/mbo", "v1.2.3", + mock.Mock(side_effect=AssertionError("must not render")), config_path=override) + self.assertEqual((output / "index.html").read_bytes(), before) + self.assertEqual((output / "release-site.json").read_bytes(), data) + + def test_override_never_reads_missing_content_from_publisher_checkout(self): + override = self.root / "backfill.json" + self.config["pages"]["newer.md"] = "newer.html" + override.write_text(json.dumps(self.config)) + (self.root / "newer.md").write_text("content only present alongside publisher config") + with self.assertRaisesRegex(ValueError, "Missing.*newer.md"): + site.build(self.source, self.retained, "mboworks/mbo", "v1.2.3", + mock.Mock(side_effect=AssertionError("must not render")), config_path=override) + + def test_tag_config_remains_default_and_is_retained(self): + output = self.build() + metadata = json.loads((output / "release.json").read_text()) + self.assertEqual(metadata["configuration"]["origin"], "tag") + self.assertEqual((output / "release-site.json").read_bytes(), + (self.source / "release-site.json").read_bytes()) + + def test_unmapped_markdown_links_fail_publication(self): + del self.config["pages"]["docs/guide.md"] + self.write("release-site.json", json.dumps(self.config)) + for link in ("docs/guide.md", "/docs/guide.md", + "https://github.com/mboworks/mbo/blob/main/docs/guide.md"): + with self.subTest(link=link), self.assertRaisesRegex(ValueError, "no page mapping"): + self.build(renderer=lambda *_: f'Guide') + self.assertFalse((self.retained / "site/tag/v1.2.3").exists()) + + def test_directory_readme_requires_a_mapping(self): + self.write("docs/README.md", "directory guide") + with self.assertRaisesRegex(ValueError, "no page mapping"): + self.build(renderer=lambda *_: 'Guide') + + def test_missing_generated_anchor_aborts_without_retaining_snapshot(self): + with self.assertRaisesRegex(ValueError, "missing generated anchor"): + self.build(renderer=lambda *_: 'Guide') + self.assertFalse((self.retained / "site/tag/v1.2.3").exists()) + + def test_generated_audit_checks_files_anchors_and_snapshot_boundaries(self): + output = self.root / "html" + output.mkdir() + (output / "guide.html").write_text('

Guide

') + for link, error in (("missing.html", "missing generated link target"), + ("guide.html#absent", "missing generated anchor"), + ("../outside.html", "escapes release snapshot"), + ("/mbo/site/tag/1.2.3/missing.html", "missing generated link target"), + ("https://mboworks.github.io/mbo/site/tag/1.2.3/guide.html#absent", + "missing generated anchor")): + with self.subTest(link=link): + (output / "index.html").write_text(f'Target') + with self.assertRaisesRegex(ValueError, error): + site.validate_site(output, "mboworks/mbo", "1.2.3") + (output / "index.html").write_text( + 'GuideSelf' + 'Coverage' + 'External') + site.validate_site(output, "mboworks/mbo", "1.2.3") + + def test_reserved_config_destination_cannot_be_overwritten(self): + self.config["files"] = {"image.svg": "release-site.json"} + self.write("release-site.json", json.dumps(self.config)) + with self.assertRaisesRegex(ValueError, "reserved destination"): + self.build() + + def test_pages_packaging_exclusions_are_rejected_before_rendering(self): + for destination in (".github/guide.html", "guide/.hidden.html", ".hidden/guide.html"): + with self.subTest(destination=destination): + self.config["pages"]["docs/guide.md"] = destination + self.write("release-site.json", json.dumps(self.config)) + with self.assertRaisesRegex(ValueError, "excludes hidden"): + self.build(renderer=mock.Mock(side_effect=AssertionError("must not render"))) + self.config["pages"]["docs/guide.md"] = "guide.html" + self.config["files"] = {"image.svg": "assets-custom/.hidden.svg"} + self.write("release-site.json", json.dumps(self.config)) + with self.assertRaisesRegex(ValueError, "excludes hidden"): + self.build() + def test_invalid_tag_cannot_escape_site_directory(self): for tag in ("../bad", "v1.2.3/evil", "1.2.3\nextra", "main"): with self.subTest(tag=tag), self.assertRaises(ValueError):