From e6378cc480f0fb5501246573b82a4bd2a44d88ae Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 16 Oct 2026 08:48:00 -0700 Subject: [PATCH] ci: validate repository links against the proposed checkout [skip ci] Build on the portable native Lychee hook to remap canonical main source links to the checkout or exact CI commit. Preserve source URLs, check missing targets, and retain the existing Linux/Windows hook smoke coverage and rendered-fragment policy. Signed-off-by: Ralf W. Grosse-Kunstleve --- .github/actions/check-doc-links/action.yml | 7 ++ .github/workflows/pre-commit.yml | 10 ++ ci/README.md | 38 ++++++++ ci/tools/run_lychee.py | 13 ++- ci/tools/test_lychee_precommit.py | 95 ++++++++++++++++++ ci/tools/tests/test_repo_link_check_action.py | 96 +++++++++++++++++++ ci/tools/tests/test_run_lychee.py | 49 ++++++++++ 7 files changed, 307 insertions(+), 1 deletion(-) create mode 100644 ci/tools/test_lychee_precommit.py create mode 100644 ci/tools/tests/test_repo_link_check_action.py create mode 100644 ci/tools/tests/test_run_lychee.py diff --git a/.github/actions/check-doc-links/action.yml b/.github/actions/check-doc-links/action.yml index b835c744423..8b7f37ca819 100644 --- a/.github/actions/check-doc-links/action.yml +++ b/.github/actions/check-doc-links/action.yml @@ -51,6 +51,13 @@ runs: echo "prefix=${prefix}" >> "${GITHUB_OUTPUT}" echo "key=${prefix}${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "${GITHUB_OUTPUT}" echo "version=${LYCHEE_VERSION}" >> "${GITHUB_OUTPUT}" + # Validate main-branch source links against the actual checkout. Keep + # GitHub's file/directory views so rendered checks retain UI fragments. + checked_commit="$(git rev-parse HEAD)" + for view in blob tree; do + LYCHEE_ARGS+=" --remap \"^https://github[.]com/NVIDIA/cuda-python/${view}/main/ ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/${view}/${checked_commit}/\"" + done + echo "Checking repository source links at ${checked_commit}" echo "args=${LYCHEE_ARGS}" >> "${GITHUB_OUTPUT}" - name: Restore successful checks from the nightly baseline diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 322cc8e127d..8b64306ba35 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -43,6 +43,7 @@ jobs: env: # Full-CI docs builds and nightly check links. The installation reminder # is for developer machines; CI does not install a Git commit hook. + # A separate bounded check below exercises Lychee with local fixtures. SKIP: lychee,check-precommit-installed # Minor version used by setup-python and as the cache restore-key prefix. PYTHON_VERSION: "3.14" @@ -140,6 +141,15 @@ jobs: - name: Run pre-commit run: pre-commit run --all-files --show-diff-on-failure + # Full link scans generate external network traffic and need cache and + # retry handling; documentation CI provides that. These local fixtures + # keep per-PR coverage focused on the installed hook on each platform. + - name: Exercise the configured Lychee hook with local fixtures + env: + SKIP: check-precommit-installed + PYTHONUTF8: "1" + run: python ci/tools/test_lychee_precommit.py + - name: Save pre-commit cache # Only the default branch saves, so per-PR merge refs don't store their # own duplicate copies. !cancelled() keeps a push that fails lint from diff --git a/ci/README.md b/ci/README.md index be94dfb035d..6fdc6ac6b28 100644 --- a/ci/README.md +++ b/ci/README.md @@ -24,6 +24,44 @@ The pinned checkout action's [`README`](https://github.com/actions/checkout/blob/3d3c42e5aac5ba805825da76410c181273ba90b1/README.md#fetch-all-history-for-all-tags-and-branches) documents `fetch-depth: 0` for fetching all history, branches, and tags. +## Repository Links During Main Development + +For development targeting `main`, keep links to the latest repository sources +on `main`, even when their targets are added or moved by the same pull request. +The link checker treats canonical +`https://github.com/NVIDIA/cuda-python/blob/main/` and `tree/main/` URLs as +references to the proposed main tree. Lychee remaps them for validation only; +source files and published documentation keep their URLs. The remaps match only +at the start of a URL, leaving URLs embedded in another URL unchanged. Relative +links and links to tags, commit SHAs, other branches, and other repositories +are checked as written. + +CI remaps these links to the CI repository at the actual checked-out commit +using `--remap`, retaining GitHub's file/directory views and fragment behavior. +The initial check and all retries use the same remaps. Missing targets in that +commit still fail. + +Local pre-commit runs the Python hook `ci/tools/run_lychee.py`, which computes +the checkout's file URI and passes anchored native `--remap` rules to Lychee. +This checks paths in the working tree, including uncommitted and unpushed +targets. The hook installs the official PyPI `lychee-bin==0.24.2` package in +its own Python environment and uses the platform's native wheel. Local checks +retain their existing policy of not checking fragments. + +The Linux and Windows pre-commit CI jobs run a bounded smoke check of the +installed hook against local Markdown and reST fixtures. They check valid +files and directories and missing targets in paths containing spaces, and +verify that the source files are unchanged. This smoke check does not scan all +external documentation links; full authored and rendered checks run in +documentation CI. + +The `check-doc-links` action is not currently present on `12.9.x`, `13.4.x`, +`12.2.x`, or `11.8.x`; merging changes into `main` does not update those +branches' workflows. A maintenance branch's `/main/` URLs may intentionally +refer to a different tree from its checkout. Before adopting this hook or +action there, choose an explicit validation policy for those links instead of +assuming that they refer to the maintenance checkout. + ## Temporary Pixi Lockfile Maintenance Pause Pixi lockfile freshness enforcement and automated refreshes are temporarily diff --git a/ci/tools/run_lychee.py b/ci/tools/run_lychee.py index 0d8be24d06d..7e9f2a598fa 100644 --- a/ci/tools/run_lychee.py +++ b/ci/tools/run_lychee.py @@ -18,6 +18,17 @@ import sysconfig from pathlib import Path +ROOT = Path(__file__).resolve().parents[2] + + +def repo_link_remaps(root: Path = ROOT) -> list[str]: + """Apply the same URL-prefix boundaries as the dedicated CI link check.""" + local_prefix = root.resolve().as_uri() + "/" + arguments = [] + for view in ("blob", "tree"): + arguments.extend(["--remap", f"^https://github[.]com/NVIDIA/cuda-python/{view}/main/ {local_prefix}"]) + return arguments + def main(arguments: list[str] | None = None) -> int: # Resolve the executable from this hook's environment to honor the pin. @@ -27,7 +38,7 @@ def main(arguments: list[str] | None = None) -> int: f"lychee was not found at {executable}; run this script through the configured pre-commit hook" ) arguments = sys.argv[1:] if arguments is None else arguments - return subprocess.run([str(executable), *arguments], check=False).returncode # noqa: S603 + return subprocess.run([str(executable), *repo_link_remaps(), *arguments], check=False).returncode # noqa: S603 if __name__ == "__main__": diff --git a/ci/tools/test_lychee_precommit.py b/ci/tools/test_lychee_precommit.py new file mode 100644 index 00000000000..50f73d0c30f --- /dev/null +++ b/ci/tools/test_lychee_precommit.py @@ -0,0 +1,95 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Exercise the configured Lychee pre-commit hook without scanning external links.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path +from tempfile import TemporaryDirectory +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[2] + + +def run_hook(sources: dict[Path, bytes], missing: Path | None = None) -> None: + result = subprocess.run( # noqa: S603 + [ + sys.executable, + "-m", + "pre_commit", + "run", + "lychee", + "--verbose", + "--color", + "never", + "--files", + *(source.relative_to(ROOT).as_posix() for source in sources), + ], + cwd=ROOT, + env=dict(os.environ, SKIP=""), + capture_output=True, + encoding="utf-8", + timeout=120, + ) + output = result.stdout + result.stderr + print(output, end="", flush=True) + for source, original in sources.items(): + if source.read_bytes() != original: + raise RuntimeError(f"The Lychee hook changed {source}") + if "- hook id: lychee" not in output or "Skipped" in output: + raise RuntimeError("The configured Lychee hook did not run") + if missing is None: + if result.returncode != 0: + raise RuntimeError("The Lychee hook rejected existing checkout targets") + elif ( + result.returncode == 0 + or "File not found" not in output + or missing.as_uri() not in output + or "- exit code: 2" not in output + ): + raise RuntimeError(f"Expected Lychee to report the missing local target {missing}") + + +def main() -> None: + # Generate sources inside the checkout so pre-commit applies the real file filters. + with TemporaryDirectory(prefix="lychee hook smoke ", dir=ROOT) as temporary: + fixture = Path(temporary) + directory = fixture / "new directory" + directory.mkdir() + target = fixture / "new source #$.py" + target.write_text("value = 1\n", encoding="utf-8") + relative = fixture / "relative guide.rst" + relative.write_text("Relative target\n", encoding="utf-8") + prefix = "https://github.com/NVIDIA/cuda-python" + file_url = f"{prefix}/blob/main/{quote(target.relative_to(ROOT).as_posix())}" + directory_url = f"{prefix}/tree/main/{quote(directory.relative_to(ROOT).as_posix())}/" + sources = { + fixture / "source guide.md": ( + f"[File]({file_url})\r\n[Directory]({directory_url})\r\n[Relative](relative%20guide.rst)\r\n" + ).encode(), + fixture / "source guide.rst": ( + f"`File <{file_url}>`_\n`Directory <{directory_url}>`_\n`Relative `_\n" + ).encode(), + } + for source, content in sources.items(): + source.write_bytes(content) + + print("Checking existing local file, directory, and relative targets", flush=True) + run_hook(sources) + target.unlink() + print("Checking that a previously valid file is rejected after deletion", flush=True) + run_hook(sources, missing=target) + target.write_text("value = 1\n", encoding="utf-8") + directory.rmdir() + print("Checking that a previously valid directory is rejected after deletion", flush=True) + run_hook(sources, missing=directory) + + print("Configured Lychee hook checks passed; documentation sources were unchanged.") + + +if __name__ == "__main__": + main() diff --git a/ci/tools/tests/test_repo_link_check_action.py b/ci/tools/tests/test_repo_link_check_action.py new file mode 100644 index 00000000000..37f5b1be55a --- /dev/null +++ b/ci/tools/tests/test_repo_link_check_action.py @@ -0,0 +1,96 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import os +import re +import shlex +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[3] + + +@pytest.mark.agent_authored(model="gpt-6.1-sol") +def test_action_prepares_remaps_for_actual_checkout_commit(tmp_path): + action = yaml.safe_load((ROOT / ".github/actions/check-doc-links/action.yml").read_text(encoding="utf-8")) + prepare = next(step for step in action["runs"]["steps"] if step.get("id") == "inputs") + checkout = tmp_path / "checkout with spaces" + helper = checkout / "ci/tools/prepare_lychee_inputs.py" + helper.parent.mkdir(parents=True) + shutil.copyfile(ROOT / "ci/tools/prepare_lychee_inputs.py", helper) + readme = checkout / "README.md" + readme.write_text("# Smoke test\n", encoding="utf-8") + for arguments in ( + ["init", "--quiet"], + ["add", "README.md"], + [ + "-c", + "user.name=Test", + "-c", + "user.email=test@example.com", + "-c", + "core.hooksPath=/dev/null", + "commit", + "--quiet", + "--no-gpg-sign", + "-m", + "Smoke test", + ], + ): + subprocess.run(["git", *arguments], cwd=checkout, check=True) # noqa: S603, S607 + head = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=checkout, text=True).strip() # noqa: S607 + output = tmp_path / "action-output" + input_list = checkout / "lychee-authored-files.txt" + env = dict( + os.environ, + PATH=f"{Path(sys.executable).parent}{os.pathsep}{os.environ['PATH']}", + KIND="authored", + GITHUB_WORKSPACE=str(checkout), + RUNNER_TEMP=str(tmp_path / "runner temp"), + LYCHEE_VERSION="v0.24.2", + POLICY_HASH="test-policy", + GITHUB_OUTPUT=str(output), + LYCHEE_ARGS=f'--files-from "{input_list}" --cache', + GITHUB_RUN_ID="123", + GITHUB_RUN_ATTEMPT="1", + GITHUB_SHA="0" * 40, + GITHUB_SERVER_URL="https://github.com", + GITHUB_REPOSITORY="example-org/cuda-python-mirror", + ) + + subprocess.run( # noqa: S603 + ["bash", "--noprofile", "--norc", "-euo", "pipefail", "-c", prepare["run"]], # noqa: S607 + cwd=checkout, + env=env, + check=True, + capture_output=True, + text=True, + ) + + outputs = dict(line.split("=", 1) for line in output.read_text(encoding="utf-8").splitlines()) + args = shlex.split(outputs["args"]) + assert args[:3] == ["--files-from", str(input_list), "--cache"] + remaps = [args[index + 1] for index, argument in enumerate(args) if argument == "--remap"] + assert remaps == [ + f"^https://github[.]com/NVIDIA/cuda-python/{view}/main/ " + f"https://github.com/example-org/cuda-python-mirror/{view}/{head}/" + for view in ("blob", "tree") + ] + assert input_list.read_text(encoding="utf-8") == f"{readme}\n" + for remap in remaps: + pattern, replacement = remap.split() + view = "blob" if "/blob/" in pattern else "tree" + url = f"https://github.com/NVIDIA/cuda-python/{view}/main/new-directory" + assert re.sub(pattern, replacement, url) == ( + f"https://github.com/example-org/cuda-python-mirror/{view}/{head}/new-directory" + ) + assert re.search(pattern, url.replace("github.com", "githubXcom")) is None + assert re.search(pattern, url.replace("/main/", "/mainly/")) is None + assert re.search(pattern, "https://example.com/" + url) is None diff --git a/ci/tools/tests/test_run_lychee.py b/ci/tools/tests/test_run_lychee.py new file mode 100644 index 00000000000..2d15f14f49e --- /dev/null +++ b/ci/tools/tests/test_run_lychee.py @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent)) +from run_lychee import repo_link_remaps + + +@pytest.mark.agent_authored(model="gpt-6.1-sol") +def test_remaps_encode_checkout_path_and_preserve_url_suffix(tmp_path): + root = tmp_path / "checkout space$#" + arguments = repo_link_remaps(root) + for index, view in enumerate(("blob", "tree")): + assert arguments[2 * index] == "--remap" + pattern, replacement = arguments[2 * index + 1].split() + url = f"https://github.com/NVIDIA/cuda-python/{view}/main/docs/a%20guide.md?plain=1#some-heading" + assert re.sub(pattern, replacement, url) == f"{root.as_uri()}/docs/a%20guide.md?plain=1#some-heading" + + +@pytest.mark.parametrize( + "url", + [ + "https://github.com/NVIDIA/cuda-python/blob/12.9.x/README.md", + "https://github.com/NVIDIA/cuda-python/tree/v13.0.0/cuda_bindings", + "https://github.com/NVIDIA/cuda-python/blob/0123456789abcdef/README.md", + "https://github.com/NVIDIA/cuda-python/blob/mainly/README.md", + "https://github.com/NVIDIA/cuda-python/tree/main", + "https://github.com/rwgk/cuda-python/blob/main/README.md", + "https://github.com/NVIDIA/other-repository/tree/main/docs", + "http://github.com/NVIDIA/cuda-python/blob/main/README.md", + "https://raw.githubusercontent.com/NVIDIA/cuda-python/main/README.md", + "https://nvidia.github.io/cuda-python/latest/", + "https://example.com/redirect?url=https://github.com/NVIDIA/cuda-python/blob/main/README.md", + "https://example.com/https://github.com/NVIDIA/cuda-python/tree/main/docs", + ], +) +@pytest.mark.agent_authored(model="gpt-6.1-sol") +def test_remaps_leave_other_targets_and_nested_urls_unchanged(tmp_path, url): + arguments = repo_link_remaps(tmp_path) + for index in (1, 3): + pattern, replacement = arguments[index].split() + assert re.sub(pattern, replacement, url) == url