ci: publish a Python documentation preview for each pull request

.github/scripts/noindex-preview.py

@@ -0,0 +1,73 @@
+"""Mark every page of a built site as noindex, in place.
+
+Pull request previews live under posit-dev.github.io/commons/pr-<number>/.
+Crawlers read robots.txt only at the origin root, and this repository does not
+own posit-dev.github.io, so a robots.txt shipped with the site is never
+consulted. A meta tag travels with the page instead.
+
+Usage: noindex-preview.py <directory>
+"""
+
+from __future__ import annotations
+
+import pathlib
+import re
+import sys
+
+TAG = '<meta name="robots" content="noindex, nofollow">'
+HEAD = re.compile(r"<head[^>]*>", re.IGNORECASE)
+HTML = re.compile(r"<html[^>]*>", re.IGNORECASE)
+# Anchored, and tolerant of a leading byte order mark, because a doctype
+# counts as one only when nothing but that precedes it. The anchor is also
+# what keeps a fragment that merely quotes a doctype from passing as a
+# document. Written as an escape: a literal mark here would be invisible.
+DOCTYPE = re.compile("\\A\\ufeff?\\s*<!doctype[^>]*>", re.IGNORECASE)
+
+
+def mark(text: str) -> str | None:
+    """Return `text` with the tag added, or None if it needs no change.
+
+    A file with no head, no html tag, and no leading doctype is a fragment
+    rather than a page. Nothing indexes it on its own, and giving it a head
+    would only corrupt whatever embeds it.
+    """
+    if 'name="robots"' in text:
+        return None
+
+    patched, count = HEAD.subn(lambda m: m.group(0) + TAG, text, count=1)
+    if count:
+        return patched
+
+    # The html tag is tried before the doctype because a head belongs inside
+    # html. Taking whichever appears first in the text would put the head
+    # between the two on every page that has both.
+    head = f"<head>{TAG}</head>"
+    for pattern in (HTML, DOCTYPE):
+        patched, count = pattern.subn(lambda m: m.group(0) + head, text, count=1)
+        if count:
+            return patched
+    return None
+
+
+def main(root: str) -> int:
+    directory = pathlib.Path(root)
+    if not directory.is_dir():
+        print(f"{root} is not a directory", file=sys.stderr)
+        return 1
+
+    marked = skipped = 0
+    for path in sorted(directory.rglob("*.html")):
+        text = path.read_text(encoding="utf-8", errors="surrogateescape")
+        patched = mark(text)
+        if patched is None:
+            skipped += 1
+            continue
+        path.write_text(patched, encoding="utf-8", errors="surrogateescape")
+        marked += 1
+
+    print(f"Marked {marked} page(s) noindex, skipped {skipped}.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1] if len(sys.argv) > 1 else "."))

.github/scripts/test_noindex_preview.py

@@ -0,0 +1,82 @@
+"""Tests for noindex-preview.py. Run: python3 test_noindex_preview.py"""
+
+import importlib.util
+import pathlib
+import unittest
+
+spec = importlib.util.spec_from_file_location(
+    "noindex_preview", pathlib.Path(__file__).parent / "noindex-preview.py"
+)
+assert spec and spec.loader
+noindex = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(noindex)
+
+mark = noindex.mark
+TAG = noindex.TAG
+
+
+class MarkTest(unittest.TestCase):
+    def test_adds_the_tag_inside_an_existing_head(self):
+        out = mark("<!DOCTYPE html>\n<html><head><title>a</title></head></html>")
+        assert out is not None
+        self.assertIn(f"<head>{TAG}", out)
+
+    def test_matches_head_case_insensitively_and_with_attributes(self):
+        out = mark('<html><HEAD lang="en"><title>b</title></HEAD></html>')
+        assert out is not None
+        self.assertIn(f'<HEAD lang="en">{TAG}', out)
+
+    def test_leaves_a_page_that_already_declares_robots(self):
+        page = '<html><head><meta name="robots" content="noindex"></head></html>'
+        self.assertIsNone(mark(page))
+
+    def test_synthesizes_a_head_after_the_html_tag(self):
+        out = mark('<html lang="en"><body>no head</body></html>')
+        assert out is not None
+        self.assertTrue(out.startswith(f'<html lang="en"><head>{TAG}</head>'))
+
+    # The next three are the regressions this script actually shipped: a
+    # doctype has to stay first, or the browser renders the preview in quirks
+    # mode and it no longer resembles the page it previews.
+    def test_keeps_a_doctype_first_when_there_is_no_html_tag(self):
+        out = mark("<!DOCTYPE html>\n<body>x</body>")
+        assert out is not None
+        self.assertTrue(out.startswith("<!DOCTYPE html>"))
+        self.assertIn(TAG, out)
+
+    def test_keeps_a_lowercase_doctype_first(self):
+        out = mark("<!doctype HTML>\n<body>x</body>")
+        assert out is not None
+        self.assertTrue(out.startswith("<!doctype HTML>"))
+
+    def test_keeps_a_byte_order_mark_and_doctype_first(self):
+        out = mark("<!DOCTYPE html>\n<body>x</body>")
+        assert out is not None
+        self.assertTrue(out.startswith("<!DOCTYPE html>"))
+        self.assertIn(TAG, out)
+
+    def test_leaves_a_fragment_alone(self):
+        self.assertIsNone(mark("<div>bare fragment</div>\n"))
+
+    def test_leaves_a_fragment_that_merely_quotes_a_doctype(self):
+        # Only a doctype at the very start makes the file a document. An
+        # unanchored match treated this snippet as one and rewrote it.
+        self.assertIsNone(mark("<p>Start a page with &lt;!DOCTYPE html&gt;</p>"))
+        self.assertIsNone(mark("<pre>example: <!DOCTYPE html></pre>"))
+
+    def test_puts_the_head_inside_html_when_a_doctype_precedes_it(self):
+        # The head belongs inside html. Matching whichever token came first
+        # put it between the doctype and the html tag.
+        out = mark("<!DOCTYPE html>\n<html lang=\"en\"><body>x</body></html>")
+        assert out is not None
+        self.assertIn(f'<html lang="en"><head>{TAG}</head>', out)
+        self.assertTrue(out.startswith("<!DOCTYPE html>"))
+
+    def test_is_idempotent(self):
+        once = mark("<html><head></head></html>")
+        assert once is not None
+        self.assertIsNone(mark(once))
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)

.github/workflows/pkgdown.yaml

@@ -20,10 +20,11 @@ jobs:
     defaults:
       run:
         working-directory: pkg-r
-    # Shared with the sibling site's workflow: both deploy to gh-pages, and
-    # two pushes to one branch race.
+    # Only restrict concurrency for non-PR jobs: those share gh-pages, and a
+    # PR build deploys nothing. Races against quartodoc.yaml's deploy are
+    # handled at the push instead, by the deploy action's rebase.
     concurrency:
-      group: gh-pages-${{ github.event_name != 'pull_request' || github.run_id }}
+      group: pkgdown-${{ github.event_name != 'pull_request' || github.run_id }}
     env:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
     permissions:
@@ -52,3 +53,8 @@ jobs:
           clean: false
           branch: gh-pages
           folder: docs
+          # Both sites and the Python preview deploys push to this one
+          # branch, so a deploy has to rebase onto whatever landed while it
+          # was building. Forcing would silently drop that other deploy.
+          force: false
+          attempt-limit: 10

.github/workflows/preview-cleanup.yml

@@ -0,0 +1,124 @@
+# Removes the pr-<number>/ preview folders that quartodoc.yaml publishes to
+# gh-pages.
+#
+# Closing a pull request handles the ordinary case immediately. The schedule
+# is what makes the branch converge: it does not care whether a close event
+# ever fired, whether this workflow existed when the pull request was opened,
+# or whether an earlier run failed, because it derives the whole answer from
+# the branch and the API each time.
+#
+# workflow_dispatch exists because a schedule only ever runs the copy of this
+# file on the default branch, so this job cannot run at all until it merges.
+# Its dry run defaults to true so the first manual run reports rather than
+# deletes.
+name: preview-cleanup.yml
+
+on:
+  pull_request:
+    types: [closed]
+  schedule:
+    # Weekly. Stale previews cost nothing but repository size, so this only
+    # needs to be faster than the rate at which they accumulate.
+    - cron: "17 4 * * 1"
+  workflow_dispatch:
+    inputs:
+      dry-run:
+        description: List what would be removed without removing it
+        type: boolean
+        default: true
+
+permissions: read-all
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    # Only against other cleanup runs. This job cannot be serialized against
+    # the deploys: Actions keeps one pending run per group, so sharing theirs
+    # would cancel a queued preview. It rebases onto a concurrent deploy
+    # below, and the schedule collects anything the two still interleave.
+    concurrency:
+      group: preview-cleanup
+    permissions:
+      contents: write
+      pull-requests: read
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          ref: gh-pages
+          # A shallow clone cannot be pushed back.
+          fetch-depth: 0
+
+      - name: Remove preview folders
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          EVENT: ${{ github.event_name }}
+          CLOSED_PR: ${{ github.event.number }}
+          DRY_RUN: ${{ inputs.dry-run }}
+        run: |
+          set -euo pipefail
+
+          if [ "$EVENT" = "pull_request" ]; then
+            targets="pr-$CLOSED_PR"
+          else
+            # Ask once for the open pull requests, rather than asking about
+            # each folder. Per-folder queries have to decide what a failed
+            # query means, and the safe answer ("leave it") is the same as
+            # the answer for an open pull request, so the query earns
+            # nothing. One list also fails loudly: a rate limit or a bad
+            # token aborts the run here, where deleting nothing is correct,
+            # instead of reading as "no pull request is open" and collecting
+            # every live preview.
+            open_prs=$(gh api "repos/$REPO/pulls?state=open&per_page=100" --paginate --jq '.[].number')
+
+            targets=""
+            # `-printf` would be shorter, but it is GNU-only, and a portable
+            # listing is one that can be exercised outside the runner.
+            for dir in $(find . -maxdepth 1 -type d -name 'pr-*' | sed 's|^\./||' | sort); do
+              number="${dir#pr-}"
+              if ! printf '%s' "$number" | grep -qE '^[0-9]+$'; then
+                echo "Skipping $dir: not a pull request number"
+                continue
+              fi
+              if printf '%s\n' "$open_prs" | grep -qx "$number"; then
+                echo "Keeping $dir: pull request is open"
+              else
+                echo "Collecting $dir: pull request is not open"
+                targets="$targets $dir"
+              fi
+            done
+          fi
+
+          removed=""
+          for dir in $targets; do
+            [ -d "$dir" ] || continue
+            removed="$removed $dir"
+          done
+
+          if [ -z "$removed" ]; then
+            echo "Nothing to remove."
+            exit 0
+          fi
+
+          if [ "${DRY_RUN:-false}" = "true" ]; then
+            echo "Dry run. Would remove:$removed"
+            exit 0
+          fi
+
+          git rm -r --quiet $removed
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git commit -m "Remove preview folders:$removed"
+
+          # A deploy may have landed on gh-pages while this job ran. Rebase
+          # onto it and try again rather than forcing, which would drop it.
+          for attempt in 1 2 3 4 5; do
+            if git push origin gh-pages; then
+              exit 0
+            fi
+            echo "Push rejected, rebasing (attempt $attempt)"
+            git pull --rebase origin gh-pages
+          done
+
+          echo "Could not push after 5 attempts." >&2
+          exit 1

.github/workflows/preview-fork-notice.yml

@@ -0,0 +1,34 @@
+# Tells a fork's author why their pull request has no documentation preview.
+#
+# quartodoc.yaml skips the preview deploy for forks, and a fork's
+# GITHUB_TOKEN is read-only, so that workflow cannot post this itself.
+# pull_request_target runs with the base repository's token, which can. This workflow never checks out the pull request, and asks for no
+# permission beyond writing the comment, so none of what makes
+# pull_request_target dangerous applies: it reads no fork code and runs none.
+name: preview-fork-notice.yml
+
+on:
+  pull_request_target:
+    types: [opened, reopened]
+
+permissions: {}
+
+jobs:
+  notice:
+    if: github.event.pull_request.head.repo.fork == true
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Post the fork preview notice
+        uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5
+        with:
+          header: preview-fork
+          message: |
+            **No documentation preview for this pull request.**
+
+            Previews are published to this repository's own GitHub Pages site,
+            so building one from a fork would serve unreviewed HTML from
+            `posit-dev.github.io`. A maintainer who wants to see the rendered
+            docs can push this branch to the repository, which produces a
+            preview automatically.