From 142700c660a1e13821b11e2b3ae4d1798e3a2761 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 12 Oct 2026 12:30:36 -0500 Subject: [PATCH] Add private candidate builds and controller-bound run metadata Keep release outputs and their evidence immutable and scoped to a frozen train instead of relying on ordinary CI publication or selecting the newest artifacts. Add private content bundles, train membership records, build-input identity and reuse, and prerequisite selection through isolated Conda and wheel views. Pin build implementations and RAPIDS CMake, apply repository-specific locks and recipe constraints, and stage candidate documentation privately. Preserve the standard catalog action and upload contract alongside the candidate-specific storage path, including matrix and CUDA compatibility metadata. Resolve train metadata from exact controller run bindings so standard project callers need no new inputs. Reject mismatched identities and unenrolled custom jobs, bound missing-record retries, and add schema, storage, dependency and run-binding checks. --- .gitignore | 2 + .pre-commit-config.yaml | 22 + pyproject.toml | 5 +- release-candidate-context/action.yml | 42 + release-candidate-context/resolver.py | 83 ++ release-candidate-dependencies/action.yml | 70 ++ release-candidate-dependencies/cmake.sh | 17 + .../patch_recipe.py | 218 ++++++ release-candidate-dependencies/platform.sh | 25 + release-candidate-dependencies/select_lock.py | 54 ++ .../select_wheel.py | 65 ++ release-candidate-dependencies/setup.sh | 728 ++++++++++++++++++ release-candidate-docs/action.yml | 45 ++ release-candidate-docs/stage.sh | 35 + release-catalog-dispatch/action.yml | 100 +++ release-catalog/README.md | 53 ++ release-catalog/entries.schema.json | 14 +- release-catalog/materialize.sh | 62 +- release-catalog/upload-candidate-s3.sh | 340 ++++++++ tests/release_catalog_discovery_test.sh | 4 +- tests/release_catalog_package_parsing_test.sh | 7 +- tests/release_catalog_upload_layout_test.sh | 250 ++++++ tests/test_candidate_context.py | 107 +++ tests/test_release_candidate_dependencies.py | 387 ++++++++++ 24 files changed, 2720 insertions(+), 15 deletions(-) create mode 100644 release-candidate-context/action.yml create mode 100644 release-candidate-context/resolver.py create mode 100644 release-candidate-dependencies/action.yml create mode 100644 release-candidate-dependencies/cmake.sh create mode 100644 release-candidate-dependencies/patch_recipe.py create mode 100644 release-candidate-dependencies/platform.sh create mode 100644 release-candidate-dependencies/select_lock.py create mode 100644 release-candidate-dependencies/select_wheel.py create mode 100755 release-candidate-dependencies/setup.sh create mode 100644 release-candidate-docs/action.yml create mode 100644 release-candidate-docs/stage.sh create mode 100644 release-catalog-dispatch/action.yml create mode 100755 release-catalog/upload-candidate-s3.sh create mode 100755 tests/release_catalog_upload_layout_test.sh create mode 100644 tests/test_candidate_context.py create mode 100644 tests/test_release_candidate_dependencies.py diff --git a/.gitignore b/.gitignore index 9c92e4ec..850a2c2e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ *.swp .venv +__pycache__/ +*.py[cod] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 641716fc..6ea3a749 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,6 +35,19 @@ repos: args: ["--fix", "--config=pyproject.toml"] - id: ruff-format args: ["--config=pyproject.toml"] + - repo: https://github.com/PyCQA/bandit + rev: 1.8.6 + hooks: + - id: bandit + files: ^(release-candidate-context|release-candidate-dependencies|release-catalog)/.*\.py$ + args: ["--quiet"] + - repo: https://github.com/PyCQA/flake8 + rev: 7.3.0 + hooks: + - id: flake8 + files: ^(release-candidate-context|release-candidate-dependencies|release-catalog)/.*\.py$ + additional_dependencies: [flake8-bugbear==25.10.21] + args: ["--select=B"] - repo: https://github.com/rapidsai/pre-commit-hooks rev: v1.7.0 hooks: @@ -81,3 +94,12 @@ repos: additional_dependencies: [jsonschema==4.25.1] pass_filenames: false files: ^(release-catalog/(config|entries)\.schema\.json|release-catalog/examples/cuvs-java/release-catalog-entries\.json|tests/release-catalog-config/.*\.json|ci/test_release_catalog_schemas\.py)$ + - repo: local + hooks: + - id: candidate-context-tests + name: test candidate run bindings + entry: python -m pytest -q tests/test_candidate_context.py + language: python + additional_dependencies: [pytest] + pass_filenames: false + files: ^(release-candidate-context/|tests/test_candidate_context.py) diff --git a/pyproject.toml b/pyproject.toml index 23c81e9f..cc18e9d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,12 @@ -# Copyright (c) 2024-2026, NVIDIA CORPORATION. +# Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. [tool.ruff] line-length = 120 target-version = "py310" +[tool.ruff.lint.per-file-ignores] +"tests/test_candidate_context.py" = ["S101"] + [tool.ruff.lint] select = [ # flake8-bugbear diff --git a/release-candidate-context/action.yml b/release-candidate-context/action.yml new file mode 100644 index 00000000..de4dba2a --- /dev/null +++ b/release-candidate-context/action.yml @@ -0,0 +1,42 @@ +name: Resolve release-candidate run context +description: Resolve an immutable controller binding without adding project workflow inputs. +inputs: + source-sha: + description: Full source SHA checked out for this build. + required: true + script: + description: Custom job script requiring a centrally reviewed policy; empty for standard builds. + default: '' +outputs: + train-sha256: + description: Exact release train bound to this workflow run. + value: ${{ steps.resolve.outputs.train-sha256 }} + custom-mode: + description: Build, test, or intermediate artifact policy. + value: ${{ steps.resolve.outputs.custom-mode }} +runs: + using: composite + steps: + - name: Locate context resolver in the job container + id: implementation + shell: bash + env: + ACTION_PATH: ${{ github.action_path }} + HOST_WORKSPACE: ${{ github.workspace }} + run: | + host_root="$(dirname "$(dirname "${HOST_WORKSPACE}")")" + container_root="$(dirname "$(dirname "${PWD}")")" + action_path="${ACTION_PATH}" + if [[ ! -d "${action_path}" && "${action_path}" == "${host_root}/"* ]]; then + action_path="${container_root}/${action_path#"${host_root}/"}" + fi + test -f "${action_path}/resolver.py" + echo "path=${action_path}" >> "${GITHUB_OUTPUT}" + - name: Resolve exact run binding + id: resolve + shell: bash + env: + IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} + SOURCE_SHA: ${{ inputs.source-sha }} + CUSTOM_SCRIPT: ${{ inputs.script }} + run: python "${IMPLEMENTATION_PATH}/resolver.py" --source-sha "${SOURCE_SHA}" --script "${CUSTOM_SCRIPT}" diff --git a/release-candidate-context/resolver.py b/release-candidate-context/resolver.py new file mode 100644 index 00000000..bf756b7a --- /dev/null +++ b/release-candidate-context/resolver.py @@ -0,0 +1,83 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Resolve candidate metadata for unchanged caller workflows from an exact run binding.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess # nosec B404 - invokes AWS CLI with an argument list, never a shell. +import time +from collections.abc import Mapping +from pathlib import Path + + +def validate_binding(binding: Mapping, environment: Mapping[str, str], source_sha: str, script: str) -> dict[str, str]: + """Reject unrelated runs/sources and custom jobs not enrolled by the controller.""" + expected = { + "schema_version": 1, + "repository": environment["GITHUB_REPOSITORY"], + "run_id": environment["GITHUB_RUN_ID"], + "source_sha": source_sha, + } + if any(binding.get(key) != value for key, value in expected.items()): + raise ValueError("candidate binding does not match this repository, run, or source SHA") + if not re.fullmatch(r"[0-9a-f]{40}|[0-9a-f]{64}", source_sha): + raise ValueError("candidate source SHA must be a full object ID") + if binding.get("workflow") != "build.yaml": + raise ValueError("candidate binding has an unsupported workflow") + workflow_ref = environment["GITHUB_WORKFLOW_REF"] + expected_prefix = f"{binding['repository']}/.github/workflows/build.yaml@refs/heads/" + if workflow_ref != expected_prefix + binding.get("workflow_ref", ""): + raise ValueError("candidate binding does not match the caller workflow ref") + if environment["GITHUB_EVENT_NAME"] != "workflow_dispatch": + raise ValueError("candidate binding requires a workflow dispatch") + digest = binding.get("train_sha256", "") + if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest): + raise ValueError("candidate binding has no valid train digest") + mode = "build" + if script: + mode = binding.get("custom_jobs", {}).get(script) + if mode not in {"test", "artifact"}: + raise ValueError(f"custom candidate job has no reviewed policy: {script}") + return {"train-sha256": digest, "custom-mode": mode} + + +def read_binding(bucket: str, key: str, run=subprocess.run, sleep=time.sleep, attempts: int = 12) -> Mapping: + """Wait at most 55 seconds for the controller's post-dispatch binding write.""" + for attempt in range(attempts): + response = run(("aws", "s3", "cp", f"s3://{bucket}/{key}", "-"), text=True, capture_output=True, check=False) + if response.returncode == 0: + binding = json.loads(response.stdout) + if not isinstance(binding, dict): + raise ValueError("candidate binding must be a JSON object") + return binding + if "404" not in response.stderr and "NoSuchKey" not in response.stderr: + raise ValueError("unable to read candidate binding; check candidate-store access") + if attempt + 1 < attempts: + sleep(5) + raise ValueError("candidate run has no controller binding; refusing to guess a train") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--bucket", default="rapids-release-candidates") + parser.add_argument("--prefix", default="train-state") + parser.add_argument("--source-sha", required=True) + parser.add_argument("--script", default="") + args = parser.parse_args() + repository = os.environ["GITHUB_REPOSITORY"] + run_id = os.environ["GITHUB_RUN_ID"] + if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", repository) or not run_id.isdigit(): + raise ValueError("invalid repository or run identity") + key = f"{args.prefix}/run-bindings/{repository}/{run_id}.json" + binding = read_binding(args.bucket, key) + outputs = validate_binding(binding, os.environ, args.source_sha, args.script) + with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output: + for name, value in outputs.items(): + output.write(f"{name}={value}\n") + + +if __name__ == "__main__": + main() diff --git a/release-candidate-dependencies/action.yml b/release-candidate-dependencies/action.yml new file mode 100644 index 00000000..93cd5bbd --- /dev/null +++ b/release-candidate-dependencies/action.yml @@ -0,0 +1,70 @@ +name: Prepare release-candidate dependencies +description: Materialize only this release train's private S3 outputs for a candidate build. + +inputs: + candidate-train-sha256: + description: Canonical SHA-256 of the immutable release train. + required: true + candidate-bucket: + description: Private S3 bucket holding candidate build bundles. + required: false + default: rapids-release-candidates + candidate-train-prefix: + description: Root prefix for immutable candidate train membership records. + required: false + default: train-state + prepare-conda-channel: + description: Whether this job needs a local Conda channel for build dependencies. + required: false + default: "false" + artifact-family: + description: Release-unit family that this job is building. + required: true + arch: + description: Canonical RAPIDS architecture for this job matrix. + required: true + cuda-version: + description: CUDA version for this job matrix. + required: true + python-version: + description: Python version for this job matrix. + required: true + +runs: + using: composite + steps: + - id: implementation + name: Resolve candidate dependency implementation + shell: bash + env: + ACTION_PATH: ${{ github.action_path }} + HOST_WORKSPACE: ${{ github.workspace }} + run: | + host_root="$(dirname "$(dirname "${HOST_WORKSPACE}")")" + container_root="$(dirname "$(dirname "${PWD}")")" + action_path="${ACTION_PATH}" + if [[ ! -d "${action_path}" && "${action_path}" == "${host_root}/"* ]]; then + action_path="${container_root}/${action_path#"${host_root}/"}" + fi + if [[ ! -f "${action_path}/setup.sh" ]]; then + echo "release-candidate dependency implementation not found at ${action_path}" >&2 + exit 1 + fi + echo "path=${action_path}" >>"${GITHUB_OUTPUT}" + - name: Prepare private candidate dependency views + shell: bash + env: + RELEASE_CANDIDATE_BUCKET: ${{ inputs.candidate-bucket }} + RELEASE_CANDIDATE_TRAIN_PREFIX: ${{ inputs.candidate-train-prefix }} + RELEASE_CANDIDATE_TRAIN_SHA256: ${{ inputs.candidate-train-sha256 }} + RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL: ${{ inputs.prepare-conda-channel }} + RELEASE_CANDIDATE_ARTIFACT_FAMILY: ${{ inputs.artifact-family }} + RELEASE_CANDIDATE_ARCH: ${{ inputs.arch }} + RELEASE_CANDIDATE_CUDA_VERSION: ${{ inputs.cuda-version }} + RELEASE_CANDIDATE_PYTHON_VERSION: ${{ inputs.python-version }} + RELEASE_CANDIDATE_SHARED_ACTIONS_REPOSITORY: ${{ github.action_repository }} + RELEASE_CANDIDATE_SHARED_ACTIONS_REVISION: ${{ github.action_ref }} + RELEASE_CANDIDATE_SHARED_WORKFLOWS_REPOSITORY: ${{ job.workflow_repository }} + RELEASE_CANDIDATE_SHARED_WORKFLOWS_REVISION: ${{ job.workflow_sha }} + IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} + run: ${IMPLEMENTATION_PATH}/setup.sh diff --git a/release-candidate-dependencies/cmake.sh b/release-candidate-dependencies/cmake.sh new file mode 100644 index 00000000..f08587b7 --- /dev/null +++ b/release-candidate-dependencies/cmake.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +set -euo pipefail + +original_cmake="${RAPIDS_CANDIDATE_ORIGINAL_CMAKE:?candidate CMake path is required}" +rapids_cmake_sha="${RAPIDS_CANDIDATE_RAPIDS_CMAKE_SHA:?candidate RAPIDS CMake SHA is required}" + +# CMake's build/install/script modes do not accept configure definitions. Only +# add the immutable RAPIDS CMake override when the invocation configures a tree. +case "${1:-}" in + --build|--install|--open|-E|-P|--help|--version) + exec "${original_cmake}" "$@" + ;; +esac + +exec "${original_cmake}" "-Drapids-cmake-sha=${rapids_cmake_sha}" "$@" diff --git a/release-candidate-dependencies/patch_recipe.py b/release-candidate-dependencies/patch_recipe.py new file mode 100644 index 00000000..c8aefdb5 --- /dev/null +++ b/release-candidate-dependencies/patch_recipe.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +"""Add release-candidate lock metapackages to a disposable Rattler recipe.""" + +from __future__ import annotations + +import argparse +import hashlib +import re +import shutil +from pathlib import Path + +import yaml + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--recipe", type=Path, required=True) + parser.add_argument("--build-lock", required=True) + parser.add_argument("--host-lock", required=True) + parser.add_argument("--rapids-cmake-sha", required=True) + arguments = parser.parse_args() + + source_is_directory = arguments.recipe.is_dir() + recipe = _resolve_recipe(arguments.recipe, parser) + destination_directory = _copy_recipe_directory(recipe, arguments.build_lock, arguments.host_lock) + destination_recipe = destination_directory / "recipe.yaml" + document = yaml.safe_load(destination_recipe.read_text()) + if not isinstance(document, dict): + parser.error(f"recipe must be a YAML mapping: {recipe}") + exact_variant_keys = _local_exact_variant_keys(recipe) + outputs = document.get("outputs", []) + if outputs is not None: + if not isinstance(outputs, list): + parser.error(f"recipe outputs must be a list: {recipe}") + for output in outputs: + if not isinstance(output, dict): + parser.error(f"recipe output must be a mapping: {recipe}") + _prepare_recipe_documents( + document, + arguments.build_lock, + arguments.host_lock, + arguments.rapids_cmake_sha, + exact_variant_keys, + ) + + destination_recipe.write_text(yaml.safe_dump(document, sort_keys=False)) + print(destination_directory if source_is_directory else destination_recipe) + return 0 + + +def _add_requirements(document: dict, build_lock: str, host_lock: str) -> None: + requirements = document.setdefault("requirements", {}) + if not isinstance(requirements, dict): + raise ValueError("recipe requirements must be a mapping") + for section, lock in (("build", build_lock), ("host", host_lock)): + values = requirements.setdefault(section, []) + if not isinstance(values, list): + raise ValueError(f"recipe requirements.{section} must be a list") + if lock not in values: + values.append(lock) + + +def _candidate_cmake_preamble(rapids_cmake_sha: str) -> str: + """Return Bash setup that pins configure-mode CMake and scikit-build.""" + return f"""# Release candidates must never fetch RAPIDS CMake from a moving branch. +cmake() {{ + case "${{1:-}}" in + --build|--install|--open|-E|-P|--help|--version) command cmake "$@" ;; + *) command cmake "-Drapids-cmake-sha={rapids_cmake_sha}" "$@" ;; + esac +}} +export -f cmake +export SKBUILD_CMAKE_DEFINE="${{SKBUILD_CMAKE_DEFINE:+${{SKBUILD_CMAKE_DEFINE}};}}rapids-cmake-sha={rapids_cmake_sha}" +""" + + +def _prepend_cmake_preamble(script: object, rapids_cmake_sha: str) -> object: + preamble = _candidate_cmake_preamble(rapids_cmake_sha) + if isinstance(script, str): + return f"{preamble}\n{script}" + if isinstance(script, list): + return [preamble, *script] + if isinstance(script, dict): + content = script.get("content") + if content is None: + raise ValueError("release-candidate recipe script object must define content") + patched = dict(script) + patched["content"] = _prepend_cmake_preamble(content, rapids_cmake_sha) + return patched + raise ValueError("release-candidate recipe script must be a string, list, or content object") + + +def _pin_cmake_in_build_script(document: dict, rapids_cmake_sha: str) -> None: + """Place the exact CMake definition inside Rattler's isolated shells. + + Rattler does not inherit the workflow PATH into its build shell, so a + workflow-level CMake wrapper cannot protect scripts that invoke ``cmake`` + directly. Package tests intentionally remain unmodified: they validate + the produced package rather than the candidate build environment. + """ + build = document.get("build") + if isinstance(build, dict) and "script" in build: + build["script"] = _prepend_cmake_preamble(build["script"], rapids_cmake_sha) + + +def _resolve_recipe(candidate: Path, parser: argparse.ArgumentParser) -> Path: + recipe = candidate.resolve() + if recipe.is_dir(): + recipe = recipe / "recipe.yaml" + if not recipe.is_file(): + parser.error(f"recipe must be a file or a directory containing recipe.yaml: {candidate}") + return recipe + + +def _copy_recipe_directory(recipe: Path, build_lock: str, host_lock: str) -> Path: + """Create a same-parent disposable recipe directory named for Rattler.""" + digest = hashlib.sha256(f"{build_lock}\0{host_lock}".encode()).hexdigest()[:12] + destination = recipe.parent.parent / f".{recipe.parent.name}.release-candidate-{digest}" + if destination.exists(): + shutil.rmtree(destination) + shutil.copytree(recipe.parent, destination) + return destination + + +def _prepare_recipe_documents( + document: dict, build_lock: str, host_lock: str, rapids_cmake_sha: str, exact_variant_keys: set[str] +) -> None: + """Inject locks into the actual build target of a Rattler recipe. + + A recipe without outputs produces its root package, while every member of + a multi-output recipe is an independent build target. Rattler-Build does + not permit top-level requirements for the latter, so first distribute the + shared requirements into every output and remove the root field. + """ + _pin_cmake_in_build_script(document, rapids_cmake_sha) + cache = document.get("cache") + if cache is not None: + if not isinstance(cache, dict): + raise ValueError("recipe cache must be a mapping") + # Cache recipes have a shared build phase before output packages are + # assembled. Its solve and shell are just as much part of the + # candidate build as an output's own requirements and script. + _add_requirements(cache, build_lock, host_lock) + _pin_cmake_in_build_script(cache, rapids_cmake_sha) + outputs = document.get("outputs") + if not outputs: + _prepare_document(document, build_lock, host_lock, exact_variant_keys, add_locks=True) + return + + _move_root_requirements_to_outputs(document, outputs) + for output in outputs: + _pin_cmake_in_build_script(output, rapids_cmake_sha) + _prepare_document(output, build_lock, host_lock, exact_variant_keys, add_locks=True) + + +def _prepare_document( + document: dict, build_lock: str, host_lock: str, exact_variant_keys: set[str], *, add_locks: bool +) -> None: + if add_locks: + _add_requirements(document, build_lock, host_lock) + requirements = document["requirements"] + for section, values in requirements.items(): + if not isinstance(values, list): + continue + requirements[section] = [_add_exact_operator(value, exact_variant_keys) for value in values] + + +def _move_root_requirements_to_outputs(document: dict, outputs: list[dict]) -> None: + """Copy shared requirements into every output before removing the root field.""" + root_requirements = document.pop("requirements", None) + if root_requirements is None: + return + if not isinstance(root_requirements, dict): + raise ValueError("recipe requirements must be a mapping") + for section, shared_values in root_requirements.items(): + if not isinstance(shared_values, list): + raise ValueError(f"recipe requirements.{section} must be a list") + for output in outputs: + requirements = output.setdefault("requirements", {}) + if not isinstance(requirements, dict): + raise ValueError("recipe output requirements must be a mapping") + values = requirements.setdefault(section, []) + if not isinstance(values, list): + raise ValueError(f"recipe output requirements.{section} must be a list") + requirements[section] = [*shared_values, *(value for value in values if value not in shared_values)] + + +def _local_exact_variant_keys(recipe: Path) -> set[str]: + keys = set() + for filename in ("variants.yaml", "conda_build_config.yaml"): + path = recipe.parent / filename + if not path.is_file(): + continue + document = yaml.safe_load(path.read_text()) + if not isinstance(document, dict): + continue + for key, values in document.items(): + if ( + isinstance(key, str) + and isinstance(values, list) + and any(isinstance(value, str) and value.strip().startswith("==") for value in values) + ): + keys.add(key) + return keys + + +def _add_exact_operator(value: object, exact_variant_keys: set[str]) -> object: + if not isinstance(value, str) or not exact_variant_keys: + return value + names = "|".join(re.escape(key) for key in sorted(exact_variant_keys)) + expression = rf"^(\S+)\s+(\$\{{\{{\s*(?:{names})\s*\}}\}})$" + return re.sub(expression, r"\1 ==\2", value) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/release-candidate-dependencies/platform.sh b/release-candidate-dependencies/platform.sh new file mode 100644 index 00000000..c91f829f --- /dev/null +++ b/release-candidate-dependencies/platform.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +# Normalize the architecture names used by RAPIDS CI and Conda. +conda_platform_for_arch() { + case "$1" in + amd64|x86_64) printf 'linux-64\n' ;; + arm64|aarch64) printf 'linux-aarch64\n' ;; + *) + echo "unsupported candidate architecture: $1" >&2 + return 1 + ;; + esac +} + +wheel_platform_for_arch() { + case "$1" in + amd64|x86_64) printf 'x86_64-manylinux_2_28\n' ;; + arm64|aarch64) printf 'aarch64-manylinux_2_28\n' ;; + *) + echo "unsupported candidate architecture: $1" >&2 + return 1 + ;; + esac +} diff --git a/release-candidate-dependencies/select_lock.py b/release-candidate-dependencies/select_lock.py new file mode 100644 index 00000000..9fb956ec --- /dev/null +++ b/release-candidate-dependencies/select_lock.py @@ -0,0 +1,54 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +"""Select exactly one checksum-addressed release-train build lock.""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import PurePosixPath +from typing import Any + +SHA256 = re.compile(r"[0-9a-f]{64}") + + +def select_wheel_lock(document: dict[str, Any], platform: str, python_version: str) -> dict[str, Any]: + """Return the unique wheel lock matching one CI matrix scope.""" + records = document.get("wheel", document.get("input_locks", {}).get("wheel", [])) + if not isinstance(records, list): + raise ValueError("release train wheel locks must be a list") + matches = [ + record + for record in records + if isinstance(record, dict) and record.get("scope") == {"platform": platform, "python": python_version} + ] + if len(matches) != 1: + raise ValueError( + f"release train must contain exactly one wheel lock for {platform}, Python {python_version}; " + f"found {len(matches)}" + ) + record = matches[0] + path = record.get("path") + sha256 = record.get("sha256") + if not isinstance(path, str) or not path or PurePosixPath(path).is_absolute() or ".." in PurePosixPath(path).parts: + raise ValueError("release train contains an unsafe wheel lock path") + if not isinstance(sha256, str) or SHA256.fullmatch(sha256) is None: + raise ValueError("release train contains an invalid wheel lock checksum") + return {"path": path, "sha256": sha256} + + +def main() -> None: + """Select a wheel lock from a train or compact lock receipt.""" + parser = argparse.ArgumentParser() + parser.add_argument("--locks", required=True) + parser.add_argument("--platform", required=True) + parser.add_argument("--python-version", required=True) + arguments = parser.parse_args() + with open(arguments.locks, encoding="utf-8") as stream: + document = json.load(stream) + print(json.dumps(select_wheel_lock(document, arguments.platform, arguments.python_version), sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/release-candidate-dependencies/select_wheel.py b/release-candidate-dependencies/select_wheel.py new file mode 100644 index 00000000..abb7037b --- /dev/null +++ b/release-candidate-dependencies/select_wheel.py @@ -0,0 +1,65 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +"""Select wheel files compatible with the target Python interpreter. + +The producing job's Python matrix is useful for ranking otherwise equivalent +outputs, but it is not the wheel compatibility contract. PEP 425 wheel tags +are that contract: pure Python and ``abi3`` wheels often serve more than the +one interpreter that built them. +""" + +from __future__ import annotations + +import argparse +import re + +from packaging.utils import InvalidWheelFilename, parse_wheel_filename + + +def supports_python(wheel_name: str, python_version: str) -> bool: + """Return whether one wheel's interpreter and ABI tags allow this Python.""" + target = _python_version(python_version) + try: + _name, _version, _build, tags = parse_wheel_filename(wheel_name) + except InvalidWheelFilename: + return False + return any(_tag_supports_python(tag.interpreter, tag.abi, target) for tag in tags) + + +def _python_version(value: str) -> tuple[int, int]: + match = re.fullmatch(r"(\d+)\.(\d+)", value) + if match is None: + raise ValueError("python version must use MAJOR.MINOR form") + return int(match.group(1)), int(match.group(2)) + + +def _tag_supports_python(interpreter: str, abi: str, target: tuple[int, int]) -> bool: + if interpreter == f"py{target[0]}" or interpreter == f"py{target[0]}{target[1]}": + return True + match = re.fullmatch(r"cp(\d)(\d+)", interpreter) + if match is None: + return False + producer = int(match.group(1)), int(match.group(2)) + if producer[0] != target[0]: + return False + if abi == "abi3": + return producer <= target + return producer == target + + +def main() -> None: + """Exit successfully only when the supplied wheel supports the target.""" + parser = argparse.ArgumentParser() + parser.add_argument("--python-version", required=True) + parser.add_argument("wheel") + args = parser.parse_args() + try: + compatible = supports_python(args.wheel, args.python_version) + except ValueError as error: + parser.error(str(error)) + if not compatible: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/release-candidate-dependencies/setup.sh b/release-candidate-dependencies/setup.sh new file mode 100755 index 00000000..a41d4ba3 --- /dev/null +++ b/release-candidate-dependencies/setup.sh @@ -0,0 +1,728 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +# Candidate dependencies remain train-scoped and private. We first retrieve +# the small release train and catalog envelopes, then download only files for +# the current unit's declared *build* prerequisites. In particular, this must +# never turn all completed candidate outputs into a shared package channel. +set -euo pipefail + +script_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# The helper is adjacent to this script but its absolute action path is dynamic. +# shellcheck disable=SC1091 +source "${script_directory}/platform.sh" + +require_value() { + local name="$1" + local value="$2" + if [[ -z "${value}" || "${value}" == *$'\n'* || "${value}" == *$'\r'* ]]; then + echo "${name} must be a non-empty single-line string" >&2 + exit 1 + fi +} + +for value in GITHUB_ENV GITHUB_PATH RUNNER_TEMP GITHUB_REPOSITORY RELEASE_CANDIDATE_BUCKET RELEASE_CANDIDATE_TRAIN_PREFIX RELEASE_CANDIDATE_TRAIN_SHA256 RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL RELEASE_CANDIDATE_ARTIFACT_FAMILY RELEASE_CANDIDATE_ARCH RELEASE_CANDIDATE_CUDA_VERSION RELEASE_CANDIDATE_PYTHON_VERSION; do + require_value "${value}" "${!value:-}" +done +if [[ ! "${RELEASE_CANDIDATE_TRAIN_SHA256}" =~ ^[[:xdigit:]]{64}$ ]]; then + echo "RELEASE_CANDIDATE_TRAIN_SHA256 must be a SHA-256 digest" >&2 + exit 1 +fi +if [[ "${RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL}" != "true" && "${RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL}" != "false" ]]; then + echo "RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL must be true or false" >&2 + exit 1 +fi +if [[ "${RELEASE_CANDIDATE_ARTIFACT_FAMILY}" != "conda" && "${RELEASE_CANDIDATE_ARTIFACT_FAMILY}" != "wheel" ]]; then + echo "RELEASE_CANDIDATE_ARTIFACT_FAMILY must be conda or wheel" >&2 + exit 1 +fi + +root="s3://${RELEASE_CANDIDATE_BUCKET}/${RELEASE_CANDIDATE_TRAIN_PREFIX}/${RELEASE_CANDIDATE_TRAIN_SHA256}" +train_key_prefix="${RELEASE_CANDIDATE_TRAIN_PREFIX}/${RELEASE_CANDIDATE_TRAIN_SHA256}" +repository="${GITHUB_REPOSITORY#*/}" +cuda_major="${RELEASE_CANDIDATE_CUDA_VERSION%%.*}" +conda_platform="$(conda_platform_for_arch "${RELEASE_CANDIDATE_ARCH}")" +wheel_platform="$(wheel_platform_for_arch "${RELEASE_CANDIDATE_ARCH}")" +workspace="${RUNNER_TEMP}/release-candidate-dependencies" +channel="${workspace}/conda-channel" +wheelhouse="${workspace}/wheelhouse" +tools="${workspace}/tools" +manifests="${workspace}/catalog-manifests" +mkdir -p "${wheelhouse}" "${tools}" "${manifests}" +if [[ "${RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL}" == "true" ]]; then + mkdir -p "${channel}"/{linux-64,linux-aarch64,noarch} +fi + +# The train is the authorization boundary. A missing record means the +# coordinator did not stage this candidate correctly; guessing from S3 paths +# would make unrelated outputs available to the job. +aws s3 cp "${root}/release-train.json" "${workspace}/release-train.json" +actual_train_sha256="$(jq -cSj 'del(.train_sha256)' "${workspace}/release-train.json" | sha256sum | awk '{print $1}')" +if [[ "${actual_train_sha256}" != "${RELEASE_CANDIDATE_TRAIN_SHA256}" ]]; then + echo "stored release train does not match RELEASE_CANDIDATE_TRAIN_SHA256" >&2 + exit 1 +fi +rapids_cmake_sha="$(jq -r '.build_tool_revisions["rapids-cmake"] // empty' "${workspace}/release-train.json")" +if [[ ! "${rapids_cmake_sha}" =~ ^[[:xdigit:]]{40}$ ]]; then + echo "release train must lock rapids-cmake with build_tool_revisions.rapids-cmake; regenerate the train" >&2 + exit 1 +fi +build_implementation_revisions="$(jq -ceS ' + .build_implementation_revisions as $revisions + | if ($revisions | type) != "object" then error("build implementation revisions must be an object") + elif (["gha-tools", "shared-actions", "shared-workflows"] | all(.[]; . as $name | $revisions | has($name))) | not + then error("missing required build implementation revision") + elif ($revisions | all(to_entries[]; (.key | type) == "string" and (.value | type) == "string" and (.value | test("^[0-9a-f]{40}$")))) + then $revisions + else error("invalid build implementation revision") + end +' "${workspace}/release-train.json")" || { + echo "release train must lock shared-workflows, shared-actions, and gha-tools; regenerate the train" >&2 + exit 1 +} +gha_tools_revision="$(jq -r '."gha-tools"' <<<"${build_implementation_revisions}")" +shared_actions_revision="$(jq -r '."shared-actions"' <<<"${build_implementation_revisions}")" +shared_workflows_revision="$(jq -r '."shared-workflows"' <<<"${build_implementation_revisions}")" +candidate_sccache="$(jq -r ' + if ((.candidate_build_policy | type) == "object") + and ((.candidate_build_policy.sccache | type) == "boolean") + then .candidate_build_policy.sccache + else empty + end +' "${workspace}/release-train.json")" +if [[ "${candidate_sccache}" != "true" && "${candidate_sccache}" != "false" ]]; then + echo "release train must declare candidate_build_policy.sccache as true or false; regenerate the train" >&2 + exit 1 +fi + +require_expected_revision() { + local label="$1" + local expected_repository="$2" + local expected_revision="$3" + local actual_repository="$4" + local actual_revision="$5" + local usage_repository="$6" + if [[ "${actual_repository}" != "${expected_repository}" || "${actual_revision}" != "${expected_revision}" ]]; then + echo "${label} usage in ${usage_repository} does not match the frozen release train: train expects ${expected_repository}@${expected_revision:0:8}, got ${actual_repository}@${actual_revision:0:8}" >&2 + exit 1 + fi +} + +require_expected_revision \ + "shared-actions" \ + "rapidsai/shared-actions" \ + "${shared_actions_revision}" \ + "${RELEASE_CANDIDATE_SHARED_ACTIONS_REPOSITORY:-}" \ + "${RELEASE_CANDIDATE_SHARED_ACTIONS_REVISION:-}" \ + "${RELEASE_CANDIDATE_SHARED_WORKFLOWS_REPOSITORY:-${GITHUB_REPOSITORY}}" +require_expected_revision \ + "shared-workflows" \ + "rapidsai/shared-workflows" \ + "${shared_workflows_revision}" \ + "${RELEASE_CANDIDATE_SHARED_WORKFLOWS_REPOSITORY:-}" \ + "${RELEASE_CANDIDATE_SHARED_WORKFLOWS_REVISION:-}" \ + "${GITHUB_REPOSITORY}" + +if ! command -v git >/dev/null; then + echo "git is required to install the train-locked gha-tools revision" >&2 + exit 1 +fi +gha_tools_directory="${workspace}/gha-tools" +git init --quiet "${gha_tools_directory}" +git -C "${gha_tools_directory}" remote add origin https://github.com/rapidsai/gha-tools.git +git -C "${gha_tools_directory}" fetch --quiet --depth 1 origin "${gha_tools_revision}" +git -C "${gha_tools_directory}" checkout --quiet --detach FETCH_HEAD +actual_gha_tools_revision="$(git -C "${gha_tools_directory}" rev-parse HEAD)" +if [[ "${actual_gha_tools_revision}" != "${gha_tools_revision}" ]]; then + echo "gha-tools checkout does not match the frozen release train: expected ${gha_tools_revision}, got ${actual_gha_tools_revision}" >&2 + exit 1 +fi +PATH="${gha_tools_directory}/tools:${PATH}" +# scikit-build-core may use the CMake wheel rather than resolving `cmake` from +# PATH, so its configure call can bypass the candidate CMake wrapper below. +# Its supported environment configuration is additive and reaches both the +# bundled and system CMake implementations. Put the immutable train value +# last so it wins over an inherited generic definition of the same cache key. +skbuild_cmake_define="rapids-cmake-sha=${rapids_cmake_sha}" +if [[ -n "${SKBUILD_CMAKE_DEFINE:-}" ]]; then + require_value "SKBUILD_CMAKE_DEFINE" "${SKBUILD_CMAKE_DEFINE}" + skbuild_cmake_define="${SKBUILD_CMAKE_DEFINE};${skbuild_cmake_define}" +fi + +# Schema-2 trains keep generated input evidence in compact, checksum-addressed +# receipts. Download those before resolving this job's lock packages. Older +# trains retain the inline fields during the migration period. +if ! jq -e '.input_receipts | type == "object"' "${workspace}/release-train.json" >/dev/null; then + echo "release train must contain checksum-addressed input receipts; regenerate the train" >&2 + exit 1 +fi +lock_receipt_sha256="" +variant_receipt_sha256="" +for receipt_name in locks variants; do + receipt_path="$(jq -r --arg name "${receipt_name}" '.input_receipts[$name].path // empty' "${workspace}/release-train.json")" + receipt_sha256="$(jq -r --arg name "${receipt_name}" '.input_receipts[$name].sha256 // empty' "${workspace}/release-train.json")" + if [[ -z "${receipt_path}" || "${receipt_path}" == /* || "${receipt_path}" == *".."* || ! "${receipt_sha256}" =~ ^[[:xdigit:]]{64}$ ]]; then + echo "release train contains an unsafe ${receipt_name} receipt reference" >&2 + exit 1 + fi + receipt_file="${workspace}/${receipt_name}-receipt.json" + aws s3 cp "${root}/inputs/${receipt_path}" "${receipt_file}" + if [[ "$(sha256sum "${receipt_file}" | awk '{print $1}')" != "${receipt_sha256}" ]]; then + echo "stored ${receipt_name} receipt does not match the release train" >&2 + exit 1 + fi + if [[ "${receipt_name}" == "locks" ]]; then + lock_receipt_sha256="${receipt_sha256}" + else + variant_receipt_sha256="${receipt_sha256}" + fi +done +lock_source="${workspace}/locks-receipt.json" +variant_source="${workspace}/variants-receipt.json" +candidate_build_inputs="${workspace}/candidate-build-inputs.json" +jq -cnS \ + --arg lock_receipt_sha256 "${lock_receipt_sha256}" \ + --arg variant_receipt_sha256 "${variant_receipt_sha256}" \ + --argjson sccache "${candidate_sccache}" \ + '{schema_version: 1, + train_inputs: { + lock_receipt_sha256: $lock_receipt_sha256, + variant_receipt_sha256: $variant_receipt_sha256 + }, + build_policy: {sccache: $sccache}}' >"${candidate_build_inputs}" + +if [[ "${RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL}" == "true" ]]; then + mapfile -t bootstrap_records < <(jq -cer '.bootstrap_conda[]?' "${lock_source}") + bootstrap_packages="$(printf '%s\n' "${bootstrap_records[@]}" | jq -Rsc 'split("\n") | map(select(length > 0) | fromjson | .package) | sort')" + if [[ "${bootstrap_packages}" != '["rapids-build-backend","rapids-dependency-file-generator"]' ]]; then + echo "release train must contain the complete Conda bootstrap tool bundle" >&2 + exit 1 + fi + for record in "${bootstrap_records[@]}"; do + bootstrap_path="$(jq -r '.path // empty' <<<"${record}")" + bootstrap_sha256="$(jq -r '.sha256 // empty' <<<"${record}")" + bootstrap_filename="${bootstrap_path##*/}" + if [[ "${bootstrap_path}" != bootstrap-channel/noarch/* \ + || "${bootstrap_filename}" == *".."* \ + || ! "${bootstrap_filename}" =~ \.(conda|tar\.bz2)$ \ + || ! "${bootstrap_sha256}" =~ ^[[:xdigit:]]{64}$ ]]; then + echo "release train contains an unsafe Conda bootstrap package reference" >&2 + exit 1 + fi + bootstrap_destination="${channel}/noarch/${bootstrap_filename}" + aws s3 cp "${root}/inputs/${bootstrap_path}" "${bootstrap_destination}" + if [[ "$(sha256sum "${bootstrap_destination}" | awk '{print $1}')" != "${bootstrap_sha256}" ]]; then + echo "stored Conda bootstrap package does not match the release train: ${bootstrap_filename}" >&2 + exit 1 + fi + done +fi + +wheel_constraint="" +if [[ "${RELEASE_CANDIDATE_ARTIFACT_FAMILY}" == "wheel" ]]; then + wheel_lock_record="$(python3 "${script_directory}/select_lock.py" \ + --locks "${lock_source}" \ + --platform "${wheel_platform}" \ + --python-version "${RELEASE_CANDIDATE_PYTHON_VERSION}")" + wheel_lock_path="$(jq -r '.path' <<<"${wheel_lock_record}")" + wheel_lock_sha256="$(jq -r '.sha256' <<<"${wheel_lock_record}")" + wheel_constraint="${workspace}/wheel-constraints.txt" + aws s3 cp "${root}/inputs/${wheel_lock_path}" "${wheel_constraint}" + if [[ "$(sha256sum "${wheel_constraint}" | awk '{print $1}')" != "${wheel_lock_sha256}" ]]; then + echo "stored wheel lock does not match the release train" >&2 + exit 1 + fi +fi +# The scheduler binds each repository to one exact GitHub run. Candidate +# references are append-only across retries, so consuming that binding avoids +# accidentally mixing a prior failed attempt into this build's dependencies. +scheduler_state="${workspace}/scheduler-state.json" +if ! aws s3 cp "${root}/scheduler-state.json" "${scheduler_state}" 2>/dev/null; then + printf '{"workflow_runs":{}}\n' >"${scheduler_state}" +fi + +# Train records are metadata, not package payloads. Each one names canonical +# reusable release bytes plus an attempt-specific catalog/provenance envelope. +# Resolve its bundle descriptors before downloading only the catalog envelopes +# needed to choose build dependencies. +aws s3 cp "${root}/" "${manifests}" --recursive --exclude '*' --include '*/bundle-reference.*.json' +shopt -s globstar nullglob +reference_paths=("${manifests}"/**/bundle-reference.*.json) +declare -A selected_attempts=() +declare -A selected_reference_paths=() +for reference_path in "${reference_paths[@]}"; do + if ! jq -e '.schema_version == 4 and .storage_layout == 2' "${reference_path}" >/dev/null; then + echo "candidate train contains a reference that does not use storage layout 2" >&2 + exit 1 + fi + source_repository="$(jq -r '.repository // empty' "${reference_path}")" + source_artifact="$(jq -r '.source_artifact // empty' "${reference_path}")" + if [[ "${reference_path##*/}" != "bundle-reference.${source_artifact}.json" ]]; then + echo "candidate train contains a reference whose filename does not match its source artifact" >&2 + exit 1 + fi + source_run_id="$(jq -r '.source_run_id // empty' "${reference_path}")" + source_run_attempt="$(jq -r '.source_run_attempt // empty' "${reference_path}")" + repository_name="${source_repository##*/}" + selected_run_id="$(jq -r --arg repository "${repository_name}" '.workflow_runs[$repository] // empty' "${scheduler_state}")" + if [[ -n "${selected_run_id}" && "${source_run_id}" != "${selected_run_id}" ]]; then + rm -f "${reference_path}" + continue + fi + reference_identity="${source_repository}/${source_artifact}" + if [[ -n "${selected_run_id}" ]]; then + if [[ ! "${source_run_attempt}" =~ ^[0-9]+$ ]]; then + echo "candidate train has an invalid selected run attempt" >&2 + exit 1 + fi + if [[ -n "${selected_attempts[${reference_identity}]:-}" ]] \ + && (( source_run_attempt < selected_attempts[${reference_identity}] )); then + rm -f "${reference_path}" + continue + fi + if [[ -n "${selected_reference_paths[${reference_identity}]:-}" ]]; then + rm -f "${selected_reference_paths[${reference_identity}]}" + fi + selected_attempts[${reference_identity}]="${source_run_attempt}" + selected_reference_paths[${reference_identity}]="${reference_path}" + fi + artifact_key="$(jq -r '.artifact_key // empty' "${reference_path}")" + if [[ -z "${artifact_key}" || ! "${artifact_key}" =~ ^artifacts/[^/]+/[0-9a-f]{64}$ ]]; then + echo "candidate train contains an unsafe artifact reference" >&2 + exit 1 + fi + build_input_digest="$(jq -r '.build_input_digest // empty' "${reference_path}")" + if [[ -z "${build_input_digest}" || ! "${build_input_digest}" =~ ^[0-9a-f]{64}$ ]]; then + echo "candidate train contains a missing or unsafe producer build-input digest" >&2 + exit 1 + fi + bundle_key="$(jq -r '.bundle_key // empty' "${reference_path}")" + bundle_name="${bundle_key##*/}" + if [[ ! "${bundle_name}" =~ ^release-catalog-entries\.[A-Za-z0-9_-]+\.json$ ]] \ + || [[ "${bundle_key}" != "${train_key_prefix}"/* ]] \ + || [[ "${bundle_key}" == *".."* ]]; then + echo "candidate train contains an unsafe catalog bundle reference" >&2 + exit 1 + fi + manifest_path="${reference_path%/*}/${bundle_name}" + aws s3 cp "s3://${RELEASE_CANDIDATE_BUCKET}/${bundle_key}" "${manifest_path}" + jq \ + --arg artifact_key "${artifact_key}" \ + --arg build_input_digest "${build_input_digest}" \ + '. + {artifact_key: $artifact_key, build_input_digest: $build_input_digest}' \ + "${manifest_path}" >"${manifest_path}.tmp" + mv "${manifest_path}.tmp" "${manifest_path}" +done + +repository="${GITHUB_REPOSITORY##*/}" +target_unit="${RELEASE_CANDIDATE_ARTIFACT_FAMILY}:${repository}" +selected="${workspace}/selected-artifacts.json" +resolved_inputs="${workspace}/resolved-upstream-inputs.jsonl" +upstream_inputs="${workspace}/upstream-inputs.json" +: >"${resolved_inputs}" +if ! jq -e --arg target "${target_unit}" '.release_units[] | select(.id == $target and .artifact_family != "source")' \ + "${workspace}/release-train.json" >/dev/null; then + echo "release train has no candidate build unit for ${target_unit}" >&2 + exit 1 +fi + +# Shell globstar has a clear, portable meaning here and keeps the jq program +# focused on schema fields rather than filesystem traversal. +manifest_paths=("${manifests}"/**/release-catalog-entries.*.json) +dependency_count="$(jq -r --arg target "${target_unit}" ' + (.release_units | map({key: .id, value: .}) | from_entries) as $units + | def closure($id): + ($units[$id].dependencies // []) + | map(select($units[.] != null and $units[.].artifact_family != "source") | ., closure(.)) + | flatten | unique; + closure($target) | length +' "${workspace}/release-train.json")" +if [[ "${#manifest_paths[@]}" -eq 0 ]]; then + if [[ "${dependency_count}" -ne 0 ]]; then + echo "no completed candidate catalog envelopes found for ${target_unit}'s declared dependencies" >&2 + exit 1 + fi + printf '[]\n' >"${workspace}/catalog-records.json" +else + jq -s '.' "${manifest_paths[@]}" >"${workspace}/catalog-records.json" +fi + +# Package metadata determines compatibility. A producing job's matrix only +# describes how the package was made: rapids-logger, for example, can be built +# in a CUDA 13 job but has no CUDA dependency and is valid for CUDA 12. Python +# compatibility is intentionally left to the published wheel tag and pip. +jq -n \ + --slurpfile train "${workspace}/release-train.json" \ + --arg target "${target_unit}" \ + --arg family "${RELEASE_CANDIDATE_ARTIFACT_FAMILY}" \ + --arg arch "${RELEASE_CANDIDATE_ARCH}" \ + --arg cuda_major "${cuda_major}" \ + --arg python_version "${RELEASE_CANDIDATE_PYTHON_VERSION}" \ + --arg conda_platform "${conda_platform}" \ + --slurpfile records "${workspace}/catalog-records.json" \ + ' + ($train[0].release_units | map({key: .id, value: .}) | from_entries) as $units + | def closure($id): + ($units[$id].dependencies // []) + | map(select($units[.] != null and $units[.].artifact_family != "source") | ., closure(.)) + | flatten | unique; + closure($target) as $dependencies + | [ $dependencies[] | select(startswith($family + ":")) ] as $family_dependencies + | [ $records[0][] + | . as $record + | .entries[] + | select(.release_catalog_key as $key | $family_dependencies | index($key)) + # Older envelopes have no cuda_major. Treating that as compatible is a + # deliberate transition rule; new Conda envelopes set it only when the + # package declares a cuda-version dependency in info/index.json. + | select((.package.cuda_major // $cuda_major) == $cuda_major) + | select( + $family != "conda" + or (.package.platform == "noarch" or .package.platform == $conda_platform) + ) + | { + release_catalog_key: .release_catalog_key, + source: $record.source, + artifact_key: $record.artifact_key, + producer_build_input_digest: $record.build_input_digest, + path: .path, + package: .package + } + ] + | sort_by([.package.name, .package.version, if .source.matrix.python_version == $python_version then 0 else 1 end]) + | group_by([.package.name, .package.version, .package.build, .package.platform, .path]) + | map(.[0]) + ' >"${selected}" + +if [[ "$(jq 'length' "${selected}")" -eq 0 ]]; then + # A root node has no upstream candidate packages. Its own C++/Python + # intermediate artifacts are still served by the exact-name wrapper below. + echo "no upstream candidate package dependencies declared for ${target_unit}" +fi + +record_resolved_input() { + local selection="$1" + local downloaded_path="$2" + local artifact_key artifact_path release_catalog_key package sha256 + local producer_repository producer_sha producer_build_input_digest + artifact_key="$(jq -r '.artifact_key' <<<"${selection}")" + artifact_path="$(jq -r '.path' <<<"${selection}")" + release_catalog_key="$(jq -r '.release_catalog_key // empty' <<<"${selection}")" + producer_repository="$(jq -r '.source.repository // empty' <<<"${selection}")" + producer_sha="$(jq -r '.source.sha // empty' <<<"${selection}")" + producer_build_input_digest="$(jq -r '.producer_build_input_digest // empty' <<<"${selection}")" + package="$(jq -c '.package' <<<"${selection}")" + sha256="$(sha256sum "${downloaded_path}" | awk '{print $1}')" + jq -cn \ + --arg artifact_key "${artifact_key}" \ + --arg path "${artifact_path}" \ + --arg release_catalog_key "${release_catalog_key}" \ + --arg producer_repository "${producer_repository}" \ + --arg producer_sha "${producer_sha}" \ + --arg producer_build_input_digest "${producer_build_input_digest}" \ + --arg sha256 "${sha256}" \ + --argjson package "${package}" \ + '{artifact_key: $artifact_key, path: $path, release_catalog_key: $release_catalog_key, + producer: {repository: $producer_repository, sha: $producer_sha, + build_input_digest: $producer_build_input_digest}, + sha256: $sha256, package: $package}' >>"${resolved_inputs}" +} + +while IFS= read -r selection; do + artifact_key="$(jq -r '.artifact_key // empty' <<<"${selection}")" + artifact_path="$(jq -r '.path' <<<"${selection}")" + if [[ -z "${artifact_key}" || ! "${artifact_key}" =~ ^artifacts/[^/]+/[0-9a-f]{64}$ || "${artifact_path}" == /* || "${artifact_path}" == *".."* ]]; then + echo "candidate catalog contains an unsafe dependency location" >&2 + exit 1 + fi + # Canonical candidate storage preserves the producer's bundle-relative + # layout without tying dependency retrieval to a GitHub run attempt. + source="s3://${RELEASE_CANDIDATE_BUCKET}/${artifact_key}/${artifact_path}" + case "${artifact_path}" in + *.conda|*.tar.bz2) + [[ "${RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL}" == "true" ]] || continue + platform="$(jq -r '.package.platform // ""' <<<"${selection}")" + case "${platform}" in + linux-64|linux-aarch64|noarch) destination="${channel}/${platform}" ;; + *) echo "candidate Conda package has unsupported platform: ${platform}" >&2; exit 1 ;; + esac + aws s3 cp "${source}" "${destination}/$(basename "${artifact_path}")" + record_resolved_input "${selection}" "${destination}/$(basename "${artifact_path}")" + ;; + *.whl) + if ! python3 "$(dirname "$0")/select_wheel.py" \ + --python-version "${RELEASE_CANDIDATE_PYTHON_VERSION}" "$(basename "${artifact_path}")"; then + continue + fi + aws s3 cp "${source}" "${wheelhouse}/$(basename "${artifact_path}")" + record_resolved_input "${selection}" "${wheelhouse}/$(basename "${artifact_path}")" + ;; + *) echo "candidate catalog has unsupported package type: ${artifact_path}" >&2; exit 1 ;; + esac +done < <(jq -c '.[]' "${selected}") + +# The uploader uses this lock as part of the repository build-input digest. It +# records only the exact upstream package bytes materialized for this matrix, +# never their GitHub execution IDs. Sorting makes the lock independent of S3 +# listing order and avoids rebuilding when an equivalent dependency view is +# prepared again. +jq -sS \ + '{schema_version: 1, + dependencies: (unique_by([.release_catalog_key, .artifact_key, .path, .sha256]) + | sort_by(.release_catalog_key, .artifact_key, .path, .sha256))}' \ + "${resolved_inputs}" >"${upstream_inputs}" + +if [[ "${RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL}" == "true" ]]; then + if ! command -v conda >/dev/null; then + echo "conda is required to index the private candidate channel" >&2 + exit 1 + fi + conda index "${channel}" +fi + +original_download="$(command -v rapids-download-from-github || true)" +original_rattler="$(command -v rapids-rattler-channel-string || true)" +original_rattler_build="$(command -v rattler-build || true)" +original_cmake="$(command -v cmake || true)" +if [[ -z "${original_download}" ]]; then + echo "RAPIDS gha-tools must be installed before candidate dependency setup" >&2 + exit 1 +fi +if [[ "${RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL}" == "true" && -z "${original_rattler}" ]]; then + echo "rapids-rattler-channel-string is required for a candidate Conda build" >&2 + exit 1 +fi +if [[ "${RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL}" == "true" && -z "${original_rattler_build}" ]]; then + echo "rattler-build is required for a candidate Conda build" >&2 + exit 1 +fi +# A lock metapackage has no payload. Its run constraints are nevertheless part +# of the build/host solve once this package is injected into a recipe. Keeping +# it in the job-local candidate channel avoids publishing CI-only lock packages. +if [[ "${RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL}" == "true" ]]; then + lock_workspace="${workspace}/train-locks" + mkdir -p "${lock_workspace}" + + prepare_lock_metapackage() { + local section="$1" + local record relative_path expected_sha256 lock_path actual_sha256 constraints package_name recipe activation_script package_script + record="$(jq -cer \ + --arg repository "${repository}" \ + --arg platform "${conda_platform}" \ + --arg cuda "${cuda_major}" \ + --arg environment "${section}" \ + '[ (.conda // .input_locks.conda)[] + | select(.scope.repository == $repository + and .scope.platform == $platform + and .scope.cuda == $cuda + and .scope.environment == $environment) ] + | if length == 1 then .[0] + else error("release train must contain exactly one repository-scoped Conda lock") + end' \ + "${lock_source}")" + relative_path="$(jq -r '.path' <<<"${record}")" + expected_sha256="$(jq -r '.sha256' <<<"${record}")" + if [[ "${relative_path}" == /* || "${relative_path}" == *".."* || ! "${expected_sha256}" =~ ^[[:xdigit:]]{64}$ ]]; then + echo "release train contains an unsafe ${section} lock reference" >&2 + exit 1 + fi + lock_path="${lock_workspace}/${section}-constraints.txt" + # The enclosing function is called through command substitution. AWS also + # writes transfer progress to stdout, so keep it out of the captured + # package name for the later one-line GITHUB_ENV assignment. + aws s3 cp "${root}/inputs/${relative_path}" "${lock_path}" >&2 + actual_sha256="$(sha256sum "${lock_path}" | awk '{print $1}')" + if [[ "${actual_sha256}" != "${expected_sha256}" ]]; then + echo "stored ${section} lock does not match the release train" >&2 + exit 1 + fi + constraints="$(jq -Rsc 'split("\n") | map(select(length > 0))' "${lock_path}")" + package_name="rapids-release-lock-${RELEASE_CANDIDATE_TRAIN_SHA256:0:12}-${conda_platform}-cuda${cuda_major}-${section}" + recipe="${lock_workspace}/${section}-recipe.yaml" + # Rattler-Build activates build and host prefixes in an isolated shell. + # Compiler activation creates CMAKE_ARGS there, after the outer candidate + # action has finished. Put the frozen RAPIDS CMake definition in each lock + # package's late-named activation script so ordinary Conda CMake builds + # receive it as well as scikit-build-core and PATH-based invocations. + activation_script="if [[ \" \${CMAKE_ARGS:-} \" != *\" -Drapids-cmake-sha=${rapids_cmake_sha} \"* ]]; then export CMAKE_ARGS=\"\${CMAKE_ARGS:+\${CMAKE_ARGS} }-Drapids-cmake-sha=${rapids_cmake_sha}\"; fi" + package_script=$'mkdir -p "${PREFIX}/etc/conda/activate.d"\n' + package_script+="printf '%s\\n' '${activation_script}' > \"\${PREFIX}/etc/conda/activate.d/zz-rapids-release-cmake.sh\"" + jq -n \ + --arg name "${package_name}" \ + --argjson constraints "${constraints}" \ + --arg script "${package_script}" \ + '{schema_version: 1, package: {name: $name, version: "0"}, build: {number: 0, noarch: "generic", script: $script}, requirements: {run_constraints: $constraints}}' \ + >"${recipe}" + # This function is called through command substitution. Keep Rattler's + # human build log on stderr so the captured value is exactly the package + # name, which can safely be written as one GITHUB_ENV assignment. + "${original_rattler_build}" build --recipe "${recipe}" --output-dir "${channel}" --color never >&2 + printf '%s' "${package_name}" + } + + lock_build_package="$(prepare_lock_metapackage build)" + lock_host_package="$(prepare_lock_metapackage host)" + variant_record="$(jq -cer 'if has("configuration_path") then {path: .configuration_path, sha256: .configuration_sha256} else .input_variants | {path: .configuration_path, sha256: .configuration_sha256} end' "${variant_source}")" + variant_path="$(jq -r '.path' <<<"${variant_record}")" + variant_sha256="$(jq -r '.sha256' <<<"${variant_record}")" + if [[ "${variant_path}" == /* || "${variant_path}" == *".."* || ! "${variant_sha256}" =~ ^[[:xdigit:]]{64}$ ]]; then + echo "release train contains an unsafe variant configuration reference" >&2 + exit 1 + fi + variant_config="${lock_workspace}/train-variants.yaml" + aws s3 cp "${root}/inputs/${variant_path}" "${variant_config}" + if [[ "$(sha256sum "${variant_config}" | awk '{print $1}')" != "${variant_sha256}" ]]; then + echo "stored variant configuration does not match the release train" >&2 + exit 1 + fi + conda index "${channel}" +fi + +# Repository build scripts already name their same-run intermediate artifact. +# Preserve that interface, but resolve only that exact bundle from candidate S3. +# Capture the family decision now: the generated helper runs later and should +# not need a Conda-only setup variable in a wheel job. +{ + printf '%s\n' '#!/usr/bin/env bash' 'set -euo pipefail' + printf 'prepare_conda_channel=%q\n' "${RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL}" + cat <<'EOF' +artifact_name="${1:?artifact name is required}" +destination="${RAPIDS_UNZIP_DIR:-$(mktemp -d)}" +reference_root="${RELEASE_CANDIDATE_DEPENDENCY_MANIFESTS:?candidate dependency manifests are required}/${GITHUB_REPOSITORY}" +mapfile -t references < <( + find "${reference_root}" -type f -name 'bundle-reference.*.json' -print \ + | while IFS= read -r reference; do + if [[ "$(jq -r '.source_artifact // empty' "${reference}")" == "${artifact_name}" ]]; then + printf '%s\n' "${reference}" + fi + done \ + | sort +) +if [[ "${#references[@]}" -ne 1 ]]; then + echo "candidate artifact must have exactly one selected train reference: ${artifact_name}" >&2 + exit 1 +fi +reference="${references[0]}" +artifact_key="$(jq -r '.artifact_key // empty' "${reference}")" +if [[ -z "${artifact_key}" || ! "${artifact_key}" =~ ^artifacts/[^/]+/[0-9a-f]{64}$ ]]; then + echo "candidate artifact has an unsafe artifact reference: ${artifact_name}" >&2 + exit 1 +fi +source_prefix="s3://${RELEASE_CANDIDATE_BUCKET}/${artifact_key}/" +# Callers capture this helper's stdout as the downloaded directory. Keep AWS +# transfer diagnostics on stderr so they can never become part of a Conda URL. +if ! aws s3 cp "${source_prefix}" "${destination}" --recursive >&2; then + echo "candidate artifact is not available in this train: ${artifact_name}" >&2 + exit 1 +fi +if [[ "${prepare_conda_channel}" == "true" ]] \ + && find "${destination}" -type f \( -name '*.conda' -o -name '*.tar.bz2' \) -print -quit | grep -q .; then + # C++ outputs are also an input to the repository's Python build. That + # temporary same-job directory is a Conda channel too, so it needs empty + # noarch and platform indexes in addition to the downloaded package files. + mkdir -p "${destination}"/{linux-64,linux-aarch64,noarch} + conda index "${destination}" >&2 +fi +printf '%s' "${destination}" +EOF +} >"${tools}/rapids-download-from-github" +chmod +x "${tools}/rapids-download-from-github" + +# Not every release unit uses CMake. When it does, shadow `cmake` so the +# repository's existing RAPIDS.cmake bootstrap receives the train-locked +# revision. Pure-Python jobs remain independent of a CMake installation. +if [[ -n "${original_cmake}" ]]; then + cp "${script_directory}/cmake.sh" "${tools}/cmake" + chmod +x "${tools}/cmake" +fi + +printf 'RELEASE_CANDIDATE_DEPENDENCY_MANIFESTS=%s\n' "${manifests}" >>"${GITHUB_ENV}" + +if [[ "${RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL}" == "true" ]]; then +cat >"${tools}/rapids-rattler-channel-string" <<'EOF' +#!/usr/bin/env bash +RAPIDS_PREPENDED_CONDA_CHANNELS=("${RAPIDS_CANDIDATE_CONDA_CHANNEL}" "${RAPIDS_PREPENDED_CONDA_CHANNELS[@]:-}") +source "${RAPIDS_CANDIDATE_ORIGINAL_RATTLER_CHANNEL_STRING}" + +# Candidate builds must resolve RAPIDS packages only from exact local outputs +# and the train-scoped candidate channel. The standard helper adds the public +# RAPIDS channel, so remove those known defaults and reject any other external +# source that is not conda-forge. +candidate_channels=() +for ((index = 0; index < ${#RATTLER_CHANNELS[@]}; index += 2)); do + if [[ "${RATTLER_CHANNELS[${index}]}" != "--channel" || -z "${RATTLER_CHANNELS[$((index + 1))]:-}" ]]; then + echo "candidate build received an invalid Conda channel argument" >&2 + return 1 + fi + channel="${RATTLER_CHANNELS[$((index + 1))]%/}" + case "${channel}" in + rapidsai|rapidsai-nightly|nvidia|https://conda.anaconda.org/rapidsai|https://conda.anaconda.org/rapidsai-nightly|https://conda.anaconda.org/nvidia) + ;; + conda-forge|https://conda.anaconda.org/conda-forge|file://*|/*) + candidate_channels+=(--channel "${channel}") + ;; + *) + echo "candidate build rejected external Conda channel: ${channel}" >&2 + return 1 + ;; + esac +done +RATTLER_CHANNELS=("${candidate_channels[@]}") +export RATTLER_CHANNELS +EOF +chmod +x "${tools}/rapids-rattler-channel-string" +cat >"${tools}/rattler-build" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +arguments=("$@") +if [[ "${arguments[0]:-}" == "build" ]]; then + for index in "${!arguments[@]}"; do + if [[ "${arguments[${index}]}" == "--recipe" && -n "${arguments[$((index + 1))]:-}" ]]; then + arguments[$((index + 1))]="$(python3 "${RAPIDS_CANDIDATE_RECIPE_PATCHER}" \ + --recipe "${arguments[$((index + 1))]}" \ + --build-lock "${RAPIDS_CANDIDATE_BUILD_LOCK_PACKAGE}" \ + --host-lock "${RAPIDS_CANDIDATE_HOST_LOCK_PACKAGE}" \ + --rapids-cmake-sha "${RAPIDS_CANDIDATE_RAPIDS_CMAKE_SHA}")" + arguments+=(--ignore-recipe-variants --variant-config "${RAPIDS_CANDIDATE_RATTLER_VARIANT_CONFIG}") + break + fi + done +fi +exec "${RAPIDS_CANDIDATE_ORIGINAL_RATTLER_BUILD}" "${arguments[@]}" +EOF +chmod +x "${tools}/rattler-build" +fi + +{ + printf 'RELEASE_CANDIDATE_BUCKET=%s\n' "${RELEASE_CANDIDATE_BUCKET}" + printf 'RELEASE_CANDIDATE_TRAIN_PREFIX=%s\n' "${RELEASE_CANDIDATE_TRAIN_PREFIX}" + printf 'RELEASE_CANDIDATE_TRAIN_SHA256=%s\n' "${RELEASE_CANDIDATE_TRAIN_SHA256}" + printf 'RELEASE_CANDIDATE_BUILD_IMPLEMENTATION_REVISIONS=%s\n' "${build_implementation_revisions}" + printf 'RELEASE_CANDIDATE_GHA_TOOLS_REVISION=%s\n' "${actual_gha_tools_revision}" + printf 'RELEASE_CANDIDATE_UPSTREAM_INPUTS=%s\n' "${upstream_inputs}" + printf 'RELEASE_CANDIDATE_BUILD_INPUTS=%s\n' "${candidate_build_inputs}" + printf 'RAPIDS_RELEASE_CANDIDATE_SCCACHE=%s\n' "${candidate_sccache}" + # Rattler installs CMake in its isolated prefixes, so this train value must + # be available to the recipe patcher even when CMake is absent from the + # outer workflow container at setup time. + printf 'RAPIDS_CANDIDATE_RAPIDS_CMAKE_SHA=%s\n' "${rapids_cmake_sha}" + printf 'SKBUILD_CMAKE_DEFINE=%s\n' "${skbuild_cmake_define}" + if [[ -n "${original_cmake}" ]]; then + printf 'RAPIDS_CANDIDATE_ORIGINAL_CMAKE=%s\n' "${original_cmake}" + fi + if [[ "${RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL}" == "true" ]]; then + printf 'RAPIDS_CANDIDATE_CONDA_CHANNEL=%s\n' "${channel}" + printf 'RAPIDS_CANDIDATE_ORIGINAL_RATTLER_CHANNEL_STRING=%s\n' "${original_rattler}" + printf 'RAPIDS_CANDIDATE_ORIGINAL_RATTLER_BUILD=%s\n' "${original_rattler_build}" + printf 'RAPIDS_CANDIDATE_RECIPE_PATCHER=%s\n' "${IMPLEMENTATION_PATH}/patch_recipe.py" + printf 'RAPIDS_CANDIDATE_BUILD_LOCK_PACKAGE=%s\n' "${lock_build_package}" + printf 'RAPIDS_CANDIDATE_HOST_LOCK_PACKAGE=%s\n' "${lock_host_package}" + printf 'RAPIDS_CANDIDATE_RATTLER_VARIANT_CONFIG=%s\n' "${variant_config}" + fi + if [[ -n "${wheel_constraint}" ]]; then + printf 'PIP_CONSTRAINT=%s\n' "${wheel_constraint}" + fi + printf 'PIP_FIND_LINKS=%s\n' "${wheelhouse}${PIP_FIND_LINKS:+ ${PIP_FIND_LINKS}}" +} >>"${GITHUB_ENV}" +printf '%s\n' "${gha_tools_directory}/tools" >>"${GITHUB_PATH}" +printf '%s\n' "${tools}" >>"${GITHUB_PATH}" diff --git a/release-candidate-docs/action.yml b/release-candidate-docs/action.yml new file mode 100644 index 00000000..b3fdf290 --- /dev/null +++ b/release-candidate-docs/action.yml @@ -0,0 +1,45 @@ +name: Stage release-candidate documentation +description: Replace public documentation publication with train-scoped private S3 staging. + +inputs: + candidate-train-sha256: + description: Canonical SHA-256 of the immutable release train. + required: true + candidate-bucket: + description: Private S3 bucket holding candidate outputs. + required: false + default: rapids-release-candidates + candidate-train-prefix: + description: Root prefix for immutable candidate train records and documentation. + required: false + default: train-state + +runs: + using: composite + steps: + - id: implementation + name: Resolve candidate documentation implementation + shell: bash + env: + ACTION_PATH: ${{ github.action_path }} + HOST_WORKSPACE: ${{ github.workspace }} + run: | + host_root="$(dirname "$(dirname "${HOST_WORKSPACE}")")" + container_root="$(dirname "$(dirname "${PWD}")")" + action_path="${ACTION_PATH}" + if [[ ! -d "${action_path}" && "${action_path}" == "${host_root}/"* ]]; then + action_path="${container_root}/${action_path#"${host_root}/"}" + fi + if [[ ! -f "${action_path}/stage.sh" ]]; then + echo "release-candidate documentation implementation not found at ${action_path}" >&2 + exit 1 + fi + echo "path=${action_path}" >>"${GITHUB_OUTPUT}" + - name: Replace public docs uploader with candidate staging + shell: bash + env: + RELEASE_CANDIDATE_BUCKET: ${{ inputs.candidate-bucket }} + RELEASE_CANDIDATE_TRAIN_PREFIX: ${{ inputs.candidate-train-prefix }} + RELEASE_CANDIDATE_TRAIN_SHA256: ${{ inputs.candidate-train-sha256 }} + IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} + run: bash "${IMPLEMENTATION_PATH}/stage.sh" diff --git a/release-candidate-docs/stage.sh b/release-candidate-docs/stage.sh new file mode 100644 index 00000000..3413a19a --- /dev/null +++ b/release-candidate-docs/stage.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +# Install a private candidate replacement for rapids-upload-docs. +# +# Repository docs scripts retain their normal interface; the custom job calls +# this only after candidate dependency setup has selected the private train. +set -euo pipefail + +for value in GITHUB_PATH GITHUB_REPOSITORY GITHUB_RUN_ID RELEASE_CANDIDATE_BUCKET RELEASE_CANDIDATE_TRAIN_PREFIX RELEASE_CANDIDATE_TRAIN_SHA256 RUNNER_TEMP; do + if [[ -z "${!value:-}" ]]; then + echo "${value} must be set for release-candidate documentation staging" >&2 + exit 1 + fi +done + +tools="${RUNNER_TEMP}/release-candidate-tools" +mkdir -p "${tools}" + +{ + printf '%s\n' '#!/usr/bin/env bash' 'set -euo pipefail' + printf 'candidate_bucket=%q\n' "${RELEASE_CANDIDATE_BUCKET}" + printf 'candidate_train_prefix=%q\n' "${RELEASE_CANDIDATE_TRAIN_PREFIX}" + printf 'candidate_train_sha256=%q\n' "${RELEASE_CANDIDATE_TRAIN_SHA256}" + cat <<'EOF' +docs_directory="${RAPIDS_DOCS_DIR:?RAPIDS_DOCS_DIR must name the generated documentation}" +docs_version="${RAPIDS_VERSION_NUMBER:?RAPIDS_VERSION_NUMBER must name the staged documentation version}" +destination="s3://${candidate_bucket}/${candidate_train_prefix}/${candidate_train_sha256}/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}/docs/${docs_version}/" +aws s3 sync "${docs_directory}/" "${destination}" --no-progress +printf 'Staged release-candidate documentation: %s\n' "${destination}" +EOF +} >"${tools}/rapids-upload-docs" + +chmod +x "${tools}/rapids-upload-docs" +printf '%s\n' "${tools}" >>"${GITHUB_PATH}" diff --git a/release-catalog-dispatch/action.yml b/release-catalog-dispatch/action.yml new file mode 100644 index 00000000..aef2a9ab --- /dev/null +++ b/release-catalog-dispatch/action.yml @@ -0,0 +1,100 @@ +name: Dispatch release catalog +description: Write a release-candidate catalog bundle directly to the private S3 store. + +inputs: + config: + description: >- + JSON string with configuration for this action. Schema and field documentation: + https://github.com/rapidsai/shared-actions/blob/main/release-catalog/config.schema.json + required: true + source-artifact-name: + description: Stable name of this producing artifact bundle. + required: true + candidate-train-sha256: + description: Canonical SHA-256 of the immutable release-train JSON. + required: true + candidate-bucket: + description: Private S3 bucket that holds candidate-build bytes. + required: false + default: rapids-release-candidates + candidate-content-prefix: + description: Root prefix for reusable, immutable candidate build bundles. + required: false + default: artifacts + candidate-train-prefix: + description: Root prefix for immutable train membership records. + required: false + default: train-state + upload-to-s3: + description: Whether to upload the generated bundle to the candidate store. + required: false + default: "false" + candidate-matrix-arch: + description: Canonical RAPIDS architecture for the producing job. + required: false + default: "" + candidate-matrix-cuda-version: + description: CUDA version for the producing job. + required: false + default: "" + candidate-matrix-python-version: + description: Python version for the producing job. + required: false + default: "" +runs: + using: composite + steps: + - id: implementation + name: Resolve release catalog implementation + shell: bash + env: + ACTION_PATH: ${{ github.action_path }} + HOST_WORKSPACE: ${{ github.workspace }} + run: | + host_root="$(dirname "$(dirname "${HOST_WORKSPACE}")")" + container_root="$(dirname "$(dirname "${PWD}")")" + action_path="${ACTION_PATH}" + if [[ ! -d "${action_path}" && "${action_path}" == "${host_root}/"* ]]; then + action_path="${container_root}/${action_path#"${host_root}/"}" + fi + implementation_path="$(dirname "${action_path}")/release-catalog" + if [[ ! -d "${implementation_path}" ]]; then + echo "release-catalog implementation not found at ${implementation_path}" >&2 + exit 1 + fi + echo "path=${implementation_path}" >>"${GITHUB_OUTPUT}" + - id: configuration + name: Validate release catalog configuration + shell: bash + env: + IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} + RELEASE_CATALOG_CONFIG: ${{ inputs.config }} + run: "${IMPLEMENTATION_PATH}/validate-config.sh" + - name: Materialize release catalog entries + shell: bash + env: + IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} + RELEASE_ARTIFACTS: ${{ steps.configuration.outputs.artifacts }} + RELEASE_ENTRIES_NAME: release-catalog-entries.${{ inputs.source-artifact-name }}.json + RELEASE_ARTIFACT_DIRECTORY: ${{ steps.configuration.outputs.artifact_directory }} + RELEASE_SOURCE_SHA: ${{ env.RAPIDS_SHA }} + RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} + RELEASE_CATALOG_KEY: ${{ steps.configuration.outputs.release_catalog_key }} + RELEASE_CANDIDATE_MATRIX_ARCH: ${{ inputs.candidate-matrix-arch }} + RELEASE_CANDIDATE_MATRIX_CUDA_VERSION: ${{ inputs.candidate-matrix-cuda-version }} + RELEASE_CANDIDATE_MATRIX_PYTHON_VERSION: ${{ inputs.candidate-matrix-python-version }} + run: "${IMPLEMENTATION_PATH}/materialize.sh" + - name: Write release-candidate bundle to S3 + if: ${{ inputs.upload-to-s3 == 'true' }} + shell: bash + env: + IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} + RELEASE_ARTIFACT_DIRECTORY: ${{ steps.configuration.outputs.artifact_directory }} + RELEASE_CANDIDATE_BUCKET: ${{ inputs.candidate-bucket }} + RELEASE_CANDIDATE_CONTENT_PREFIX: ${{ inputs.candidate-content-prefix }} + RELEASE_CANDIDATE_TRAIN_PREFIX: ${{ inputs.candidate-train-prefix }} + RELEASE_CANDIDATE_TRAIN_SHA256: ${{ inputs.candidate-train-sha256 }} + RELEASE_CANDIDATE_CATALOG_SHARED_ACTIONS_REPOSITORY: ${{ github.action_repository }} + RELEASE_CANDIDATE_CATALOG_SHARED_ACTIONS_REVISION: ${{ github.action_ref }} + RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} + run: "${IMPLEMENTATION_PATH}/upload-candidate-s3.sh" diff --git a/release-catalog/README.md b/release-catalog/README.md index 16a5ffa4..5b799779 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -95,6 +95,12 @@ dependency SBOM will add. That will be a new `sbom_kind` value and a new The action is used via `rapidsai/shared-actions/release-catalog`. +Candidate builds use the separate `release-catalog-dispatch` action. It names +each job's envelope `release-catalog-entries..json` and uploads +through `upload-candidate-s3.sh`, which implements immutable content bundles and +train membership records. The standard `release-catalog` action retains its +existing envelope name and `upload-s3.sh` upload contract. + Configuration, materialization, and upload failures appear as GitHub Actions error annotations. When the scripts run outside GitHub Actions, the same errors are written to stderr with the responsible script name. @@ -217,3 +223,50 @@ provenance without an SBOM cannot be matched to a package by name and version. Although CycloneDX can carry some build metadata, embedding provenance in the SBOM would not replace the in-toto/SLSA statement and would couple evidence with different consumers and lifecycles. + +## Prerequisite state + +The action requires `RAPIDS_SHA` in the job environment. It must identify the +repository revision actually checked out and built: + +- Conda build workflows set `RAPIDS_SHA` to `git rev-parse HEAD` immediately + after checkout. +- Wheel and custom workflows use `rapids-github-info`; it uses `inputs.sha` + when supplied and otherwise sets `RAPIDS_SHA` to `git rev-parse HEAD`. + +This distinction matters when a reusable workflow checks out a repository or +revision different from the workflow event. Direct callers must likewise set +`RAPIDS_SHA` to the checked-out commit instead of assuming `${{ github.sha }}` +names the built source. Variables written to `GITHUB_ENV` are available to the +action when it runs as a subsequent job step. + +## Candidate build reuse + +For a release-candidate build, `release-candidate-dependencies` writes +`RELEASE_CANDIDATE_UPSTREAM_INPUTS` to `GITHUB_ENV`. It is a sorted lock of the +exact upstream Conda and wheel files actually materialized for the current job +matrix. Each entry contains the file SHA-256 plus the producing RAPIDS +repository SHA and build-input digest. `release-catalog` incorporates that lock +into `build-record.json`. The same setup action writes +`RELEASE_CANDIDATE_BUILD_INPUTS`, containing the checksum of the train lock +receipt, the checksum of the train variant receipt, and the train's boolean +`sccache` policy. The record also includes the frozen `shared-workflows`, +`shared-actions`, and `gha-tools` revisions and any build timestamp supplied to +the package build. The timestamp is included because Conda packages and wheels +can embed it in their bytes; workflows that do not use one record an empty, +stable value. The record's SHA-256 determines the canonical +artifact location. Therefore an unchanged build can be reused +across candidate trains and GitHub retries, while a downstream build is +necessarily distinct when an upstream package byte, selected upstream build, or +shared build implementation changes. Lock, variant, or cache-policy changes +also create a different canonical artifact identity, so an `sccache`-assisted +build cannot be reused as a clean build. The locks intentionally omit GitHub +run and attempt IDs. + +Before a candidate job can use this mechanism, its dependency setup compares +the running reusable workflow SHA and composite-action SHA with the train's +`shared-workflows` and `shared-actions` entries. It also installs `gha-tools` +at the train's exact SHA and makes that checkout the job's tool path. The +catalog uploader repeats the `shared-actions` and `gha-tools` comparison before +writing S3 content. A mismatch is a build error, not an informational warning: +one train must not combine artifacts produced by different shared build logic. diff --git a/release-catalog/entries.schema.json b/release-catalog/entries.schema.json index c51669d9..520c9d19 100644 --- a/release-catalog/entries.schema.json +++ b/release-catalog/entries.schema.json @@ -37,7 +37,16 @@ "sha": {"type": "string", "pattern": "^(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})$"}, "workflow_ref": {"type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$"}, "run_id": {"type": "string", "pattern": "^[1-9][0-9]*$"}, - "run_attempt": {"type": "string", "pattern": "^[1-9][0-9]*$"} + "run_attempt": {"type": "string", "pattern": "^[1-9][0-9]*$"}, + "matrix": { + "type": "object", "additionalProperties": false, + "required": ["arch", "cuda_version", "python_version"], + "properties": { + "arch": {"type": "string"}, + "cuda_version": {"type": "string"}, + "python_version": {"type": "string"} + } + } } }, "package": { @@ -50,7 +59,8 @@ "name": {"type": "string", "minLength": 1}, "version": {"type": "string", "minLength": 1}, "build": {"type": "string", "minLength": 1}, - "platform": {"type": "string", "minLength": 1} + "platform": {"type": "string", "minLength": 1}, + "cuda_major": {"type": "string", "pattern": "^[0-9]+$"} } }, "entry": { diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh index 7eb0baad..2c9df33c 100755 --- a/release-catalog/materialize.sh +++ b/release-catalog/materialize.sh @@ -146,8 +146,20 @@ describe_wheel_package() { exit 1 fi - jq -cn --arg name "${package_name}" --arg version "${package_version}" \ - '{ecosystem: "wheel", name: $name, version: $version}' + # RAPIDS CUDA wheels use a distribution suffix such as rmm-cu12. Keep that + # artifact-level compatibility fact in the catalog; the producing job's + # matrix is not a reliable substitute for it. + local cuda_major="" + if [[ "${package_name}" =~ -cu([0-9]+)$ ]]; then + cuda_major="${BASH_REMATCH[1]}" + fi + if [[ -n "${cuda_major}" ]]; then + jq -cn --arg name "${package_name}" --arg version "${package_version}" --arg cuda_major "${cuda_major}" \ + '{ecosystem: "wheel", name: $name, version: $version, cuda_major: $cuda_major}' + else + jq -cn --arg name "${package_name}" --arg version "${package_version}" \ + '{ecosystem: "wheel", name: $name, version: $version}' + fi } # Maven-built JARs normally expose their coordinates in exactly one @@ -227,7 +239,29 @@ describe_conda_package() { exit 1 fi - jq -c '{ecosystem: "conda", name, version, build, platform: .subdir}' <<<"${index_json}" + # The embedded package dependency is the compatibility contract. The build + # matrix is only how this package was produced; it must not prevent a + # CUDA-independent package from serving another CUDA build in the train. + local cuda_major + cuda_major="$(jq -r ' + [ + .depends[]? + | select(type == "string" and startswith("cuda-version")) + | capture("(?[0-9]+)").major + ] + | unique + | if length == 0 then "" + elif length == 1 then .[0] + else error("Conda package has conflicting cuda-version dependencies") + end + ' <<<"${index_json}")" + if [[ -n "${cuda_major}" ]]; then + jq -c --arg cuda_major "${cuda_major}" \ + '{ecosystem: "conda", name, version, build, platform: .subdir, cuda_major: $cuda_major}' \ + <<<"${index_json}" + else + jq -c '{ecosystem: "conda", name, version, build, platform: .subdir}' <<<"${index_json}" + fi } prepare_artifacts() { @@ -346,7 +380,12 @@ prepare_artifacts() { # package identity or a caller-created package identity file. artifacts="$(prepare_artifacts)" # Fixed name: upload-s3.sh, the schema, and consumers all depend on it. -entries_path="${artifact_directory}/release-catalog-entries.json" +entries_name="${RELEASE_ENTRIES_NAME:-release-catalog-entries.json}" +if [[ "$entries_name" == */* || "$entries_name" == *..* ]]; then + release_catalog_error "release entry document must be a plain filename" + exit 1 +fi +entries_path="${artifact_directory}/${entries_name}" temporary_manifest="$(mktemp "${artifact_directory}/.release-catalog.XXXXXX")" trap 'rm -f "${temporary_manifest}"' EXIT @@ -355,12 +394,13 @@ printf '%s\n' '{"entries":[]}' >"${temporary_manifest}" validate_package_identity() { jq -e ' type == "object" - and (keys - ["ecosystem", "name", "version", "build", "platform"] | length == 0) + and (keys - ["ecosystem", "name", "version", "build", "platform", "cuda_major"] | length == 0) and (.ecosystem | type == "string" and length > 0) and (.name | type == "string" and length > 0) and (.version | type == "string" and length > 0) and ((has("build") | not) or (.build | type == "string" and length > 0)) and ((has("platform") | not) or (.platform | type == "string" and length > 0)) + and ((has("cuda_major") | not) or (.cuda_major | type == "string" and test("^[0-9]+$"))) ' >/dev/null } @@ -518,6 +558,9 @@ jq -n -S \ --arg run_id "${github_run_id}" \ --arg sha "${source_sha}" \ --arg workflow_ref "${github_workflow_ref}" \ + --arg matrix_arch "${RELEASE_CANDIDATE_MATRIX_ARCH:-}" \ + --arg matrix_cuda_version "${RELEASE_CANDIDATE_MATRIX_CUDA_VERSION:-}" \ + --arg matrix_python_version "${RELEASE_CANDIDATE_MATRIX_PYTHON_VERSION:-}" \ --argjson entries "$(jq -c '.entries' "${temporary_manifest}")" \ '{ schema_version: 1, @@ -528,7 +571,12 @@ jq -n -S \ sha: $sha, workflow_ref: $workflow_ref, run_id: $run_id, - run_attempt: $run_attempt + run_attempt: $run_attempt, + matrix: { + arch: $matrix_arch, + cuda_version: $matrix_cuda_version, + python_version: $matrix_python_version + } }, entries: $entries - }' >"${entries_path}" + } | if $matrix_arch == "" and $matrix_cuda_version == "" and $matrix_python_version == "" then del(.source.matrix) else . end' >"${entries_path}" diff --git a/release-catalog/upload-candidate-s3.sh b/release-catalog/upload-candidate-s3.sh new file mode 100755 index 00000000..b689bb49 --- /dev/null +++ b/release-catalog/upload-candidate-s3.sh @@ -0,0 +1,340 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +# Upload only the files declared by the generated catalog document. GitHub +# Actions retains logs and this receipt, but never a second release-artifact +# copy. Conditional S3 writes make reruns safe: an existing object is accepted +# only when its checksum exactly matches the byte about to be uploaded. +set -euo pipefail + +require_value() { + local name="$1" + local value="$2" + if [[ -z "${value}" || "${value}" == *$'\n'* || "${value}" == *$'\r'* ]]; then + echo "${name} must be a non-empty single-line string" >&2 + exit 1 + fi +} + +safe_prefix() { + local prefix="$1" + [[ "${prefix}" != /* && "${prefix}" != */ && "${prefix}" != *'//' && "${prefix}" != *'..'* ]] +} + +for value in RELEASE_ARTIFACT_DIRECTORY RELEASE_CANDIDATE_BUCKET RELEASE_CANDIDATE_CONTENT_PREFIX RELEASE_CANDIDATE_TRAIN_PREFIX RELEASE_CANDIDATE_TRAIN_SHA256 RELEASE_CANDIDATE_BUILD_IMPLEMENTATION_REVISIONS RELEASE_CANDIDATE_GHA_TOOLS_REVISION RELEASE_CANDIDATE_CATALOG_SHARED_ACTIONS_REPOSITORY RELEASE_CANDIDATE_CATALOG_SHARED_ACTIONS_REVISION RELEASE_CANDIDATE_BUILD_INPUTS RELEASE_SOURCE_ARTIFACT_NAME GITHUB_REPOSITORY; do + require_value "${value}" "${!value:-}" +done +if [[ ! "${GITHUB_REPOSITORY}" =~ ^[^/]+/[^/]+$ ]]; then + echo "GITHUB_REPOSITORY must use owner/repository format" >&2 + exit 1 +fi +repository_name="${GITHUB_REPOSITORY##*/}" +if [[ -n "${RAPIDS_DATETIME_STRING:-}" && ! "${RAPIDS_DATETIME_STRING}" =~ ^[0-9]{12}$ ]]; then + echo "RAPIDS_DATETIME_STRING must use YYMMDDhhmmss format" >&2 + exit 1 +fi +if [[ ! "${RELEASE_CANDIDATE_TRAIN_SHA256}" =~ ^[[:xdigit:]]{64}$ ]]; then + echo "RELEASE_CANDIDATE_TRAIN_SHA256 must be a SHA-256 hex digest" >&2 + exit 1 +fi +if [[ "${RELEASE_CANDIDATE_CATALOG_SHARED_ACTIONS_REPOSITORY}" != "rapidsai/shared-actions" ]]; then + echo "release catalog action must execute from rapidsai/shared-actions" >&2 + exit 1 +fi +build_implementation_revisions="$(jq -ceS ' + . as $revisions + | if ($revisions | type) != "object" then error("build implementation revisions must be an object") + elif (["gha-tools", "shared-actions", "shared-workflows"] | all(.[]; . as $name | $revisions | has($name))) | not + then error("missing required build implementation revision") + elif ($revisions | all(to_entries[]; (.key | type) == "string" and (.value | type) == "string" and (.value | test("^[0-9a-f]{40}$")))) + then $revisions + else error("invalid build implementation revision") + end +' <<<"${RELEASE_CANDIDATE_BUILD_IMPLEMENTATION_REVISIONS}")" || { + echo "RELEASE_CANDIDATE_BUILD_IMPLEMENTATION_REVISIONS is invalid" >&2 + exit 1 +} +if [[ "${RELEASE_CANDIDATE_CATALOG_SHARED_ACTIONS_REVISION}" != "$(jq -r '."shared-actions"' <<<"${build_implementation_revisions}")" ]]; then + echo "release catalog shared-actions revision does not match the frozen release train" >&2 + exit 1 +fi +if [[ "${RELEASE_CANDIDATE_GHA_TOOLS_REVISION}" != "$(jq -r '."gha-tools"' <<<"${build_implementation_revisions}")" ]]; then + echo "gha-tools revision does not match the frozen release train" >&2 + exit 1 +fi +if [[ ! "${RELEASE_SOURCE_ARTIFACT_NAME}" =~ ^[A-Za-z0-9_-]+$ ]]; then + echo "RELEASE_SOURCE_ARTIFACT_NAME must contain only letters, numbers, underscores, and hyphens" >&2 + exit 1 +fi +if ! safe_prefix "${RELEASE_CANDIDATE_CONTENT_PREFIX}" || ! safe_prefix "${RELEASE_CANDIDATE_TRAIN_PREFIX}"; then + echo "candidate content and train prefixes must be safe, relative S3 prefixes" >&2 + exit 1 +fi +if [[ ! -f "${RELEASE_CANDIDATE_BUILD_INPUTS}" ]]; then + echo "RELEASE_CANDIDATE_BUILD_INPUTS does not name a readable candidate input record" >&2 + exit 1 +fi +candidate_build_inputs="$(jq -ceS ' + if .schema_version != 1 + or ((.train_inputs | type) != "object") + or ((.train_inputs.lock_receipt_sha256 | type) != "string") + or ((.train_inputs.lock_receipt_sha256 | test("^[0-9a-f]{64}$")) | not) + or ((.train_inputs.variant_receipt_sha256 | type) != "string") + or ((.train_inputs.variant_receipt_sha256 | test("^[0-9a-f]{64}$")) | not) + or ((.build_policy | type) != "object") + or ((.build_policy.sccache | type) != "boolean") + then error("candidate build inputs must contain lock, variant, and sccache identities") + else { + schema_version: 1, + train_inputs: { + lock_receipt_sha256: .train_inputs.lock_receipt_sha256, + variant_receipt_sha256: .train_inputs.variant_receipt_sha256 + }, + build_policy: {sccache: .build_policy.sccache} + } + end +' "${RELEASE_CANDIDATE_BUILD_INPUTS}")" || { + echo "RELEASE_CANDIDATE_BUILD_INPUTS is invalid" >&2 + exit 1 +} + +bundle_root="$(realpath "${RELEASE_ARTIFACT_DIRECTORY}")" +entries_name="release-catalog-entries.${RELEASE_SOURCE_ARTIFACT_NAME}.json" +entries_path="${bundle_root}/${entries_name}" +if [[ ! -f "${entries_path}" ]]; then + echo "release catalog entries are missing: ${entries_path}" >&2 + exit 1 +fi + +# Catalog paths are relative to the build bundle. Reject traversal before an +# S3 key is constructed, even though materialize.sh already validates them. +safe_relative_path() { + local path="$1" + [[ -n "${path}" && "${path}" != /* && "${path}" != ../* && "${path}" != */../* && "${path}" != *'/..' ]] +} + +# A reusable repository build is keyed only by its declared inputs. Package +# bytes can embed the CI timestamp, so that timestamp is an explicit input. +# The full generated catalog remains an output record and must not determine +# this key. +# A digest names canonical, reusable release bytes. Execution-specific catalog +# context and provenance stay with the train attempt instead of duplicating +# payloads below the canonical artifact namespace. +# The dependency action writes this lock after resolving only the upstream +# package bytes used by the current matrix. Keep run/attempt metadata out of +# it: a downstream rebuild is required when an actual input changes, not when +# an equivalent upstream job is retried. +upstream_inputs_path="${RELEASE_CANDIDATE_UPSTREAM_INPUTS:-}" +if [[ -z "${upstream_inputs_path}" ]]; then + upstream_dependencies='[]' +elif [[ ! -f "${upstream_inputs_path}" ]]; then + echo "RELEASE_CANDIDATE_UPSTREAM_INPUTS does not name a readable input lock" >&2 + exit 1 +else + upstream_dependencies="$(jq -ceS ' + if .schema_version != 1 or (.dependencies | type) != "array" then + error("upstream input lock must contain schema_version 1 and dependencies") + else + [.dependencies[] + | if (((.artifact_key | type) == "string") + and ((.path | type) == "string") + and ((.release_catalog_key | type) == "string") + and ((.sha256 | type) == "string") + and ((.sha256 | test("^[0-9a-f]{64}$"))) + and ((.producer | type) == "object") + and ((.producer.repository | type) == "string") + and ((.producer.sha | type) == "string") + and ((.producer.sha | test("^[0-9a-f]{40}$"))) + and ((.producer.build_input_digest | type) == "string") + and ((.producer.build_input_digest | test("^[0-9a-f]{64}$"))) + and ((.package | type) == "object")) + then {artifact_key, path, release_catalog_key, producer, sha256, package} + else error("upstream dependency has invalid identity") + end] + | unique_by([.release_catalog_key, .artifact_key, .path, .sha256]) + | sort_by(.release_catalog_key, .artifact_key, .path, .sha256) + end + ' "${upstream_inputs_path}")" || { + echo "RELEASE_CANDIDATE_UPSTREAM_INPUTS is invalid" >&2 + exit 1 + } +fi + +build_record_path="$(mktemp)" +jq -cS --argjson upstream_dependencies "${upstream_dependencies}" \ + --argjson build_implementation_revisions "${build_implementation_revisions}" \ + --argjson candidate_build_inputs "${candidate_build_inputs}" \ + --arg build_datetime "${RAPIDS_DATETIME_STRING:-}" ' + { + schema_version: 5, + storage_layout: 2, + producer, + source: (.source | {artifact, repository, sha, workflow_ref, matrix}), + build_datetime: $build_datetime, + build_implementation_revisions: $build_implementation_revisions, + candidate_build_inputs: $candidate_build_inputs, + upstream_dependencies: $upstream_dependencies, + packages: [ + .entries[] + | { + release_catalog_key, + package: (.package | {ecosystem, name, version, platform}) + } + ] | sort_by(.release_catalog_key, .package.ecosystem, .package.name, .package.version, .package.platform) + } +' "${entries_path}" >"${build_record_path}" +build_input_digest="$(sha256sum "${build_record_path}" | awk '{print $1}')" +attempt_id="${GITHUB_RUN_ID:-}.${GITHUB_RUN_ATTEMPT:-}" +if [[ ! "${attempt_id}" =~ ^[0-9]+\.[0-9]+$ ]]; then + echo "GITHUB_RUN_ID and GITHUB_RUN_ATTEMPT must be numeric for a candidate upload" >&2 + exit 1 +fi +artifact_key="${RELEASE_CANDIDATE_CONTENT_PREFIX}/${repository_name}/${build_input_digest}" +build_record_key="${artifact_key}/build-record.json" +artifact_index_key="${artifact_key}/artifact-index.json" +train_bundle_key="${RELEASE_CANDIDATE_TRAIN_PREFIX}/${RELEASE_CANDIDATE_TRAIN_SHA256}/${repository_name}/${attempt_id}/${build_input_digest}" +bundle_key="${train_bundle_key}/${entries_name}" +train_key="${train_bundle_key}/bundle-reference.${RELEASE_SOURCE_ARTIFACT_NAME}.json" +# Primary package bytes and deterministic SBOMs are canonical release content. +# Generated provenance identifies a specific GitHub execution, so it remains +# under the train attempt alongside the execution catalog. A signature is kept +# there as well until its determinism contract is defined. +mapfile -t artifact_paths < <(jq -r '([.entries[] | .path, .sbom | strings] | unique) | .[]' "${entries_path}") +mapfile -t execution_paths < <(jq -r '([.entries[] | .provenance, .signature? | strings] | unique) | .[]' "${entries_path}") +head_metadata="$(mktemp)" +artifact_index_entries="$(mktemp)" +artifact_index_path="$(mktemp)" +trap 'rm -f "${head_metadata}" "${build_record_path}" "${artifact_index_entries}" "${artifact_index_path}"' EXIT + +upload_immutable() { + local local_path="$1" + local object_key="$2" + local checksum + checksum="$(openssl dgst -sha256 -binary "${local_path}" | base64)" + + if aws s3api head-object --bucket "${RELEASE_CANDIDATE_BUCKET}" --key "${object_key}" --checksum-mode ENABLED >"${head_metadata}" 2>/dev/null; then + if [[ "$(jq -r '.ChecksumSHA256 // empty' "${head_metadata}")" != "${checksum}" ]]; then + echo "existing candidate object has different bytes: ${object_key}" >&2 + exit 1 + fi + return + fi + + if ! aws s3api put-object --bucket "${RELEASE_CANDIDATE_BUCKET}" --key "${object_key}" --body "${local_path}" --checksum-algorithm SHA256 --checksum-sha256 "${checksum}" --if-none-match '*' --tagging 'release-candidate-status=candidate' >/dev/null 2>&1; then + aws s3api head-object --bucket "${RELEASE_CANDIDATE_BUCKET}" --key "${object_key}" --checksum-mode ENABLED >"${head_metadata}" + if [[ "$(jq -r '.ChecksumSHA256 // empty' "${head_metadata}")" != "${checksum}" ]]; then + echo "candidate object could not be written immutably: ${object_key}" >&2 + exit 1 + fi + fi +} + +upload_artifact() { + local relative_path="$1" + local local_path="${bundle_root}/${relative_path}" + upload_immutable "${local_path}" "${artifact_key}/${relative_path}" +} + +upload_execution() { + local relative_path="$1" + local local_path="${bundle_root}/${relative_path}" + upload_immutable "${local_path}" "${train_bundle_key}/${relative_path}" +} + +for relative_path in "${artifact_paths[@]}"; do + if ! safe_relative_path "${relative_path}" || [[ ! -f "${bundle_root}/${relative_path}" ]]; then + echo "catalog declares a missing or unsafe candidate file: ${relative_path}" >&2 + exit 1 + fi +done +for relative_path in "${execution_paths[@]}"; do + if ! safe_relative_path "${relative_path}" || [[ ! -f "${bundle_root}/${relative_path}" ]]; then + echo "catalog declares a missing or unsafe candidate file: ${relative_path}" >&2 + exit 1 + fi +done + +# This is the stable artifact receipt: it identifies the canonical package +# bytes without carrying GitHub run/attempt data. A retry with different bytes +# for these declared inputs must fail the immutable write rather than create a +# second implicit release candidate. +while IFS= read -r entry; do + primary_path="$(jq -r '.path' <<<"${entry}")" + package="$(jq -c '.package' <<<"${entry}")" + primary_sha256="$(sha256sum "${bundle_root}/${primary_path}" | awk '{print $1}')" + sbom_path="$(jq -r '.sbom // empty' <<<"${entry}")" + if [[ -n "${sbom_path}" ]]; then + sbom_sha256="$(sha256sum "${bundle_root}/${sbom_path}" | awk '{print $1}')" + else + sbom_sha256="" + fi + jq -cn \ + --arg release_catalog_key "$(jq -r '.release_catalog_key' <<<"${entry}")" \ + --arg path "${primary_path}" \ + --arg sha256 "${primary_sha256}" \ + --arg sbom_path "${sbom_path}" \ + --arg sbom_sha256 "${sbom_sha256}" \ + --argjson package "${package}" \ + '{release_catalog_key: $release_catalog_key, path: $path, sha256: $sha256, package: $package} + + if $sbom_path == "" then {} else {sbom: {path: $sbom_path, sha256: $sbom_sha256}} end' +done < <(jq -c '.entries[]' "${entries_path}") >"${artifact_index_entries}" +jq -csS \ + --arg build_input_digest "${build_input_digest}" \ + --arg source_artifact "${RELEASE_SOURCE_ARTIFACT_NAME}" \ + '{schema_version: 3, producer: "shared-workflows", build_input_digest: $build_input_digest, + source_artifact: $source_artifact, + entries: sort_by(.release_catalog_key, .path)}' \ + "${artifact_index_entries}" >"${artifact_index_path}" + +upload_immutable "${build_record_path}" "${build_record_key}" +for relative_path in "${artifact_paths[@]}"; do + upload_artifact "${relative_path}" +done +# The index is the canonical artifact commit marker. Write it only after every +# byte and deterministic SBOM that it certifies is durably present. A failed +# or racing upload may leave harmless unreferenced content, but never a +# complete-looking index for a partial candidate. +upload_immutable "${artifact_index_path}" "${artifact_index_key}" +for relative_path in "${execution_paths[@]}"; do + upload_execution "${relative_path}" +done +upload_immutable "${entries_path}" "${bundle_key}" + +reference_path="$(mktemp)" +trap 'rm -f "${head_metadata}" "${build_record_path}" "${reference_path}"' EXIT +jq -n \ + --arg train_sha256 "${RELEASE_CANDIDATE_TRAIN_SHA256}" \ + --arg repository "${GITHUB_REPOSITORY}" \ + --arg source_artifact "${RELEASE_SOURCE_ARTIFACT_NAME}" \ + --arg source_run_id "${GITHUB_RUN_ID}" \ + --arg source_run_attempt "${GITHUB_RUN_ATTEMPT}" \ + --arg build_input_digest "${build_input_digest}" \ + --arg artifact_key "${artifact_key}" \ + --arg build_record_key "${build_record_key}" \ + --arg artifact_index_key "${artifact_index_key}" \ + --arg bundle_key "${bundle_key}" \ + '{schema_version: 4, producer: "shared-workflows", storage_layout: 2, train_sha256: $train_sha256, repository: $repository, source_artifact: $source_artifact, source_run_id: $source_run_id, source_run_attempt: $source_run_attempt, build_input_digest: $build_input_digest, artifact_key: $artifact_key, build_record_key: $build_record_key, artifact_index_key: $artifact_index_key, bundle_key: $bundle_key}' \ + >"${reference_path}" + +upload_reference() { + local checksum + checksum="$(openssl dgst -sha256 -binary "${reference_path}" | base64)" + if aws s3api head-object --bucket "${RELEASE_CANDIDATE_BUCKET}" --key "${train_key}" --checksum-mode ENABLED >"${head_metadata}" 2>/dev/null; then + if [[ "$(jq -r '.ChecksumSHA256 // empty' "${head_metadata}")" != "${checksum}" ]]; then + echo "existing train reference has different bytes: ${train_key}" >&2 + exit 1 + fi + return + fi + if ! aws s3api put-object --bucket "${RELEASE_CANDIDATE_BUCKET}" --key "${train_key}" --body "${reference_path}" --checksum-algorithm SHA256 --checksum-sha256 "${checksum}" --if-none-match '*' --tagging 'release-candidate-status=candidate' >/dev/null 2>&1; then + aws s3api head-object --bucket "${RELEASE_CANDIDATE_BUCKET}" --key "${train_key}" --checksum-mode ENABLED >"${head_metadata}" + if [[ "$(jq -r '.ChecksumSHA256 // empty' "${head_metadata}")" != "${checksum}" ]]; then + echo "candidate train reference could not be written immutably: ${train_key}" >&2 + exit 1 + fi + fi +} + +upload_reference +printf 'Reusable candidate artifacts: s3://%s/%s\n' "${RELEASE_CANDIDATE_BUCKET}" "${artifact_key}" >>"${GITHUB_STEP_SUMMARY}" +printf 'Candidate train reference: s3://%s/%s\n' "${RELEASE_CANDIDATE_BUCKET}" "${train_key}" >>"${GITHUB_STEP_SUMMARY}" diff --git a/tests/release_catalog_discovery_test.sh b/tests/release_catalog_discovery_test.sh index 72728211..1e6d063c 100755 --- a/tests/release_catalog_discovery_test.sh +++ b/tests/release_catalog_discovery_test.sh @@ -41,7 +41,7 @@ export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY jq -e ' [.entries[] | {path, package}] == [{ path: "libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl", - package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0"} + package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0", cuda_major: "12"} }] ' "${temporary_directory}/release-catalog-entries.json" >/dev/null @@ -95,7 +95,7 @@ export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/materialize.sh" jq -e ' .entries[0].path == "libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl" - and .entries[0].package == {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0"} + and .entries[0].package == {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0", cuda_major: "12"} and .entries[0].sbom_kind == "generated-identity" ' "${temporary_directory}/release-catalog-entries.json" >/dev/null diff --git a/tests/release_catalog_package_parsing_test.sh b/tests/release_catalog_package_parsing_test.sh index 726f55f1..2217f32a 100755 --- a/tests/release_catalog_package_parsing_test.sh +++ b/tests/release_catalog_package_parsing_test.sh @@ -37,7 +37,7 @@ export RELEASE_ARTIFACT_DIRECTORY jq -e ' [.entries[] | {path, package}] == [{ path: "libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl", - package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"} + package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32", cuda_major: "12"} }] ' "${wheel_output_directory}/release-catalog-entries.json" >/dev/null @@ -105,7 +105,7 @@ tar -cjf \ info/index.json jq -n \ - '{name: "librmm", version: "26.08.00a32", build: "cuda12_260714_2f567060", subdir: "linux-64"}' \ + '{name: "librmm", version: "26.08.00a32", build: "cuda12_260714_2f567060", subdir: "linux-64", depends: ["cuda-version >=12,<13"]}' \ >"${conda_staging_directory}/info/index.json" tar -cf "${temporary_directory}/info-librmm.tar" -C "${conda_staging_directory}" info/index.json zstd -q -f "${temporary_directory}/info-librmm.tar" -o "${temporary_directory}/info-librmm.tar.zst" @@ -128,7 +128,8 @@ jq -e ' name: "librmm", version: "26.08.00a32", build: "cuda12_260714_2f567060", - platform: "linux-64" + platform: "linux-64", + cuda_major: "12" } ) and any(.entries[]; diff --git a/tests/release_catalog_upload_layout_test.sh b/tests/release_catalog_upload_layout_test.sh new file mode 100755 index 00000000..50b48211 --- /dev/null +++ b/tests/release_catalog_upload_layout_test.sh @@ -0,0 +1,250 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +# Exercise the immutable candidate-store split without AWS credentials. The +# fake CLI deliberately implements only the S3 calls made by upload-s3.sh. +set -euo pipefail + +repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +temporary_directory="$(mktemp -d)" +trap 'rm -rf "${temporary_directory}"' EXIT +mkdir -p "${temporary_directory}/bin" "${temporary_directory}/bundle/release-evidence" + +cat >"${temporary_directory}/bin/aws" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +[[ "$1" == "s3api" ]] +operation="$2" +shift 2 +bucket="" +key="" +body="" +while [[ "$#" -gt 0 ]]; do + case "$1" in + --bucket) bucket="$2"; shift 2 ;; + --key) key="$2"; shift 2 ;; + --body) body="$2"; shift 2 ;; + *) shift ;; + esac +done +path="${FAKE_S3}/${bucket}/${key}" +case "${operation}" in + head-object) + [[ -f "${path}" ]] + checksum="$(openssl dgst -sha256 -binary "${path}" | base64)" + printf '{"ChecksumSHA256":"%s"}\n' "${checksum}" + ;; + put-object) + [[ ! -e "${path}" ]] + mkdir -p "$(dirname "${path}")" + cp "${body}" "${path}" + printf '%s\n' "${key}" >>"${FAKE_LOG}" + ;; + *) + echo "unsupported fake AWS operation: ${operation}" >&2 + exit 2 + ;; +esac +EOF +chmod +x "${temporary_directory}/bin/aws" + +cat >"${temporary_directory}/bundle/release-catalog-entries.conda.json" <<'EOF' +{ + "schema_version": 1, + "producer": "shared-workflows", + "source": { + "artifact": "conda", + "repository": "rapidsai/example", + "sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "workflow_ref": "rapidsai/example/.github/workflows/build.yaml@refs/heads/main", + "run_id": "101", + "run_attempt": "1", + "matrix": {"arch": "x86_64", "cuda_version": "12.0", "python_version": "3.12"} + }, + "entries": [{ + "release_catalog_key": "conda:example", + "path": "linux-64/example-26.10.00.conda", + "sbom": "release-evidence/example.spdx.json", + "provenance": "release-evidence/example.intoto.jsonl", + "package": {"ecosystem": "conda", "name": "example", "version": "26.10.00", "platform": "linux-64"} + }] +} +EOF +mkdir -p "${temporary_directory}/bundle/linux-64" +printf 'package bytes\n' >"${temporary_directory}/bundle/linux-64/example-26.10.00.conda" +printf '{"spdxVersion":"SPDX-2.3"}\n' >"${temporary_directory}/bundle/release-evidence/example.spdx.json" +printf '{"run":"101"}\n' >"${temporary_directory}/bundle/release-evidence/example.intoto.jsonl" +printf '{"schema_version":1,"dependencies":[]}\n' >"${temporary_directory}/upstream-inputs.json" +printf '%s\n' '{"schema_version":1,"train_inputs":{"lock_receipt_sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","variant_receipt_sha256":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"},"build_policy":{"sccache":true}}' >"${temporary_directory}/candidate-build-inputs.json" + +run_upload() { + local implementation_revisions="$2" + local gha_tools_revision + local shared_actions_revision + local build_datetime + gha_tools_revision="$(jq -r '."gha-tools"' <<<"${implementation_revisions}")" + shared_actions_revision="$(jq -r '."shared-actions"' <<<"${implementation_revisions}")" + gha_tools_revision="${3:-${gha_tools_revision}}" + shared_actions_revision="${4:-${shared_actions_revision}}" + build_datetime="${5-260901120000}" + GITHUB_RUN_ATTEMPT="$1" \ + PATH="${temporary_directory}/bin:${PATH}" \ + FAKE_S3="${temporary_directory}/s3" \ + FAKE_LOG="${temporary_directory}/put-objects.log" \ + RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}/bundle" \ + RELEASE_CANDIDATE_BUCKET="candidate-store" \ + RELEASE_CANDIDATE_CONTENT_PREFIX="artifacts" \ + RELEASE_CANDIDATE_TRAIN_PREFIX="train-state" \ + RELEASE_CANDIDATE_TRAIN_SHA256="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" \ + RELEASE_SOURCE_ARTIFACT_NAME="conda" \ + GITHUB_REPOSITORY="rapidsai/example" \ + GITHUB_RUN_ID="101" \ + GITHUB_STEP_SUMMARY="${temporary_directory}/summary" \ + RELEASE_CANDIDATE_BUILD_IMPLEMENTATION_REVISIONS="${implementation_revisions}" \ + RELEASE_CANDIDATE_GHA_TOOLS_REVISION="${gha_tools_revision}" \ + RELEASE_CANDIDATE_CATALOG_SHARED_ACTIONS_REPOSITORY="rapidsai/shared-actions" \ + RELEASE_CANDIDATE_CATALOG_SHARED_ACTIONS_REVISION="${shared_actions_revision}" \ + RELEASE_CANDIDATE_UPSTREAM_INPUTS="${temporary_directory}/upstream-inputs.json" \ + RELEASE_CANDIDATE_BUILD_INPUTS="${temporary_directory}/candidate-build-inputs.json" \ + RAPIDS_DATETIME_STRING="${build_datetime}" \ + "${repository_root}/release-catalog/upload-candidate-s3.sh" +} + +base_implementation_revisions='{"gha-tools":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","shared-actions":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","shared-workflows":"cccccccccccccccccccccccccccccccccccccccc"}' +changed_implementation_revisions='{"gha-tools":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","shared-actions":"dddddddddddddddddddddddddddddddddddddddd","shared-workflows":"cccccccccccccccccccccccccccccccccccccccc"}' + +run_upload 1 "${base_implementation_revisions}" +artifact_root="${temporary_directory}/s3/candidate-store/artifacts/example" +artifact_digest="$(find "${artifact_root}" -mindepth 1 -maxdepth 1 -type d -exec basename {} \;)" +canonical="${artifact_root}/${artifact_digest}" +train="${temporary_directory}/s3/candidate-store/train-state/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/example" +test -f "${canonical}/artifact-index.json" +test -f "${canonical}/linux-64/example-26.10.00.conda" +test -f "${train}/101.1/${artifact_digest}/release-catalog-entries.conda.json" +test -f "${train}/101.1/${artifact_digest}/release-evidence/example.intoto.jsonl" +test ! -d "${artifact_root}/${artifact_digest}/attempts" +jq -e '.upstream_dependencies == []' "${canonical}/build-record.json" >/dev/null +jq -e '.schema_version == 5 and .storage_layout == 2 and .build_datetime == "260901120000"' "${canonical}/build-record.json" >/dev/null +jq -e '.candidate_build_inputs == {"schema_version":1,"train_inputs":{"lock_receipt_sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","variant_receipt_sha256":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"},"build_policy":{"sccache":true}}' "${canonical}/build-record.json" >/dev/null +jq -e '.schema_version == 3 and .source_artifact == "conda"' "${canonical}/artifact-index.json" >/dev/null +artifact_line="$(grep -nF "${artifact_digest}/linux-64/example-26.10.00.conda" "${temporary_directory}/put-objects.log" | cut -d: -f1)" +index_line="$(grep -nF "${artifact_digest}/artifact-index.json" "${temporary_directory}/put-objects.log" | cut -d: -f1)" +if [[ -z "${artifact_line}" || -z "${index_line}" || "${index_line}" -le "${artifact_line}" ]]; then + echo "canonical artifact index was not committed after its artifact bytes" >&2 + exit 1 +fi +jq -e '.build_implementation_revisions == {"gha-tools":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","shared-actions":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","shared-workflows":"cccccccccccccccccccccccccccccccccccccccc"}' "${canonical}/build-record.json" >/dev/null +if jq -e 'tostring | test("run_id|run_attempt")' "${canonical}/artifact-index.json" >/dev/null; then + echo "canonical artifact index contains GitHub execution metadata" >&2 + exit 1 +fi +jq -e '.source_run_id == "101" and .source_run_attempt == "1"' \ + "${train}/101.1/${artifact_digest}/bundle-reference.conda.json" >/dev/null +if run_upload 9 "${base_implementation_revisions}" "dddddddddddddddddddddddddddddddddddddddd" >/dev/null 2>&1; then + echo "upload accepted a gha-tools revision that does not match the release train" >&2 + exit 1 +fi +if run_upload 9 "${base_implementation_revisions}" "" "dddddddddddddddddddddddddddddddddddddddd" >/dev/null 2>&1; then + echo "upload accepted a shared-actions revision that does not match the release train" >&2 + exit 1 +fi + +run_upload 2 "${changed_implementation_revisions}" +test -n "$(find "${train}/101.2" -name 'bundle-reference.conda.json' -print -quit)" +mapfile -t artifact_digests < <(find "${artifact_root}" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sort) +if [[ "${#artifact_digests[@]}" -ne 2 || "${artifact_digests[0]}" == "${artifact_digests[1]}" ]]; then + echo "shared build implementation change did not create a distinct build-input digest" >&2 + exit 1 +fi +cat >"${temporary_directory}/upstream-inputs.json" <<'EOF' +{ + "schema_version": 1, + "dependencies": [{ + "artifact_key": "artifacts/rapids-logger/input", + "path": "noarch/rapids-logger-0.3.0.conda", + "release_catalog_key": "conda:rapids-logger", + "producer": { + "repository": "rapidsai/rapids-logger", + "sha": "cccccccccccccccccccccccccccccccccccccccc", + "build_input_digest": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" + }, + "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "package": {"ecosystem": "conda", "name": "rapids-logger", "version": "0.3.0", "platform": "noarch"} + }] +} +EOF +run_upload 3 "${base_implementation_revisions}" +mapfile -t artifact_digests < <(find "${artifact_root}" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sort) +if [[ "${#artifact_digests[@]}" -ne 3 ]]; then + echo "upstream input change did not create a distinct build-input digest" >&2 + exit 1 +fi +test -n "$(find "${train}/101.3" -name 'bundle-reference.conda.json' -print -quit)" +upstream_digest="" +for candidate_digest in "${artifact_digests[@]}"; do + candidate_record="${artifact_root}/${candidate_digest}/build-record.json" + if jq -e '.upstream_dependencies | length == 1' "${candidate_record}" >/dev/null; then + upstream_digest="${candidate_digest}" + break + fi +done +if [[ -z "${upstream_digest}" ]]; then + echo "unable to find the canonical artifact record with the upstream dependency" >&2 + exit 1 +fi +jq -e ' + .upstream_dependencies == [{ + artifact_key: "artifacts/rapids-logger/input", + path: "noarch/rapids-logger-0.3.0.conda", + release_catalog_key: "conda:rapids-logger", + producer: { + repository: "rapidsai/rapids-logger", + sha: "cccccccccccccccccccccccccccccccccccccccc", + build_input_digest: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" + }, + sha256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + package: { + ecosystem: "conda", name: "rapids-logger", version: "0.3.0", platform: "noarch" + } + }] +' "${artifact_root}/${upstream_digest}/build-record.json" >/dev/null +run_upload 4 "${base_implementation_revisions}" "" "" "260901120001" +mapfile -t artifact_digests < <(find "${artifact_root}" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sort) +if [[ "${#artifact_digests[@]}" -ne 4 ]]; then + echo "build timestamp change did not create a distinct build-input digest" >&2 + exit 1 +fi +run_upload 5 "${base_implementation_revisions}" "" "" "" +mapfile -t artifact_digests < <(find "${artifact_root}" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sort) +if [[ "${#artifact_digests[@]}" -ne 5 ]]; then + echo "empty build timestamp was not accepted as a stable build input" >&2 + exit 1 +fi +printf '%s\n' '{"schema_version":1,"train_inputs":{"lock_receipt_sha256":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","variant_receipt_sha256":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"},"build_policy":{"sccache":true}}' >"${temporary_directory}/candidate-build-inputs.json" +run_upload 7 "${base_implementation_revisions}" +mapfile -t artifact_digests < <(find "${artifact_root}" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sort) +if [[ "${#artifact_digests[@]}" -ne 6 ]]; then + echo "train lock receipt change did not create a distinct build-input digest" >&2 + exit 1 +fi +printf '%s\n' '{"schema_version":1,"train_inputs":{"lock_receipt_sha256":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","variant_receipt_sha256":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"},"build_policy":{"sccache":true}}' >"${temporary_directory}/candidate-build-inputs.json" +run_upload 8 "${base_implementation_revisions}" +mapfile -t artifact_digests < <(find "${artifact_root}" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sort) +if [[ "${#artifact_digests[@]}" -ne 7 ]]; then + echo "train variant receipt change did not create a distinct build-input digest" >&2 + exit 1 +fi +printf '%s\n' '{"schema_version":1,"train_inputs":{"lock_receipt_sha256":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","variant_receipt_sha256":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"},"build_policy":{"sccache":false}}' >"${temporary_directory}/candidate-build-inputs.json" +run_upload 9 "${base_implementation_revisions}" +mapfile -t artifact_digests < <(find "${artifact_root}" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sort) +if [[ "${#artifact_digests[@]}" -ne 8 ]]; then + echo "sccache policy change did not create a distinct build-input digest" >&2 + exit 1 +fi +printf '%s\n' '{"schema_version":1,"train_inputs":{"lock_receipt_sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","variant_receipt_sha256":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"},"build_policy":{"sccache":true}}' >"${temporary_directory}/candidate-build-inputs.json" +printf 'different bytes\n' >"${temporary_directory}/bundle/linux-64/example-26.10.00.conda" +if run_upload 6 "${base_implementation_revisions}" >/dev/null 2>&1; then + echo "upload accepted different canonical package bytes" >&2 + exit 1 +fi diff --git a/tests/test_candidate_context.py b/tests/test_candidate_context.py new file mode 100644 index 00000000..23ad6ed6 --- /dev/null +++ b/tests/test_candidate_context.py @@ -0,0 +1,107 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Candidate identity must survive unchanged project caller workflows.""" + +import importlib.util +import json +import subprocess +from pathlib import Path + +import pytest + +SPEC = importlib.util.spec_from_file_location( + "candidate_context", Path(__file__).parents[1] / "release-candidate-context/resolver.py" +) +assert SPEC is not None and SPEC.loader is not None +resolver = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(resolver) + + +@pytest.fixture +def environment(): + return { + "GITHUB_REPOSITORY": "rapidsai/rapids-logger", + "GITHUB_RUN_ID": "42", + "GITHUB_WORKFLOW_REF": "rapidsai/rapids-logger/.github/workflows/build.yaml@refs/heads/main", + "GITHUB_EVENT_NAME": "workflow_dispatch", + } + + +@pytest.fixture +def binding(): + return { + "schema_version": 1, + "repository": "rapidsai/rapids-logger", + "run_id": "42", + "source_sha": "a" * 40, + "workflow": "build.yaml", + "workflow_ref": "main", + "train_sha256": "f" * 64, + "custom_jobs": {"ci/build_static.sh": "test", "ci/build_dynamic.sh": "test"}, + } + + +def test_standard_build_recovers_digest_without_caller_input(binding, environment): + assert resolver.validate_binding(binding, environment, "a" * 40, "") == { + "train-sha256": "f" * 64, + "custom-mode": "build", + } + + +@pytest.mark.parametrize("script", ["ci/build_static.sh", "ci/build_dynamic.sh"]) +def test_logger_custom_tests_need_no_project_changes(binding, environment, script): + assert resolver.validate_binding(binding, environment, "a" * 40, script)["custom-mode"] == "test" + + +@pytest.mark.parametrize( + "key,value", + [ + ("run_id", "43"), + ("repository", "rapidsai/kvikio"), + ("source_sha", "b" * 40), + ("workflow_ref", "canary"), + ("train_sha256", "bad\noutput=wrong"), + ("schema_version", 2), + ], +) +def test_mismatched_binding_fails_closed(binding, environment, key, value): + binding[key] = value + with pytest.raises(ValueError): + resolver.validate_binding(binding, environment, "a" * 40, "") + + +def test_unknown_custom_release_job_fails_closed(binding, environment): + with pytest.raises(ValueError, match="reviewed policy"): + resolver.validate_binding(binding, environment, "a" * 40, "ci/build_java.sh") + + +def test_missing_binding_retries_only_not_found(binding): + responses = iter( + [ + subprocess.CompletedProcess([], 1, stderr="404 Not Found"), + subprocess.CompletedProcess([], 0, stdout=json.dumps(binding)), + ] + ) + sleeps = [] + assert ( + resolver.read_binding("bucket", "key", run=lambda *_args, **_kwargs: next(responses), sleep=sleeps.append) + == binding + ) + assert sleeps == [5] + + +def test_access_denied_does_not_retry_or_guess(): + def run(*_args, **_kwargs): + return subprocess.CompletedProcess([], 1, stderr="AccessDenied") + + with pytest.raises(ValueError, match="access"): + resolver.read_binding("bucket", "key", run=run, sleep=lambda _: pytest.fail("unexpected sleep")) + + +def test_missing_binding_is_bounded(): + def run(*_args, **_kwargs): + return subprocess.CompletedProcess([], 1, stderr="NoSuchKey") + + sleeps = [] + with pytest.raises(ValueError, match="refusing to guess"): + resolver.read_binding("bucket", "key", run=run, sleep=sleeps.append) + assert sum(sleeps) == 55 diff --git a/tests/test_release_candidate_dependencies.py b/tests/test_release_candidate_dependencies.py new file mode 100644 index 00000000..31c6f8b6 --- /dev/null +++ b/tests/test_release_candidate_dependencies.py @@ -0,0 +1,387 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +import argparse +import importlib.util +import subprocess +from pathlib import Path + +import pytest + +_SCRIPT = Path(__file__).parents[1] / "release-candidate-dependencies" / "select_wheel.py" +_SPEC = importlib.util.spec_from_file_location("select_wheel", _SCRIPT) +if _SPEC is None or _SPEC.loader is None: + raise RuntimeError(f"unable to load wheel selector: {_SCRIPT}") +_MODULE = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(_MODULE) +supports_python = _MODULE.supports_python + +_PATCHER = Path(__file__).parents[1] / "release-candidate-dependencies" / "patch_recipe.py" +_PATCHER_SPEC = importlib.util.spec_from_file_location("patch_recipe", _PATCHER) +if _PATCHER_SPEC is None or _PATCHER_SPEC.loader is None: + raise RuntimeError(f"unable to load recipe patcher: {_PATCHER}") +_PATCHER_MODULE = importlib.util.module_from_spec(_PATCHER_SPEC) +_PATCHER_SPEC.loader.exec_module(_PATCHER_MODULE) + +_LOCK_SELECTOR = Path(__file__).parents[1] / "release-candidate-dependencies" / "select_lock.py" +_SETUP = Path(__file__).parents[1] / "release-candidate-dependencies" / "setup.sh" +_LOCK_SELECTOR_SPEC = importlib.util.spec_from_file_location("select_lock", _LOCK_SELECTOR) +if _LOCK_SELECTOR_SPEC is None or _LOCK_SELECTOR_SPEC.loader is None: + raise RuntimeError(f"unable to load lock selector: {_LOCK_SELECTOR}") +_LOCK_SELECTOR_MODULE = importlib.util.module_from_spec(_LOCK_SELECTOR_SPEC) +_LOCK_SELECTOR_SPEC.loader.exec_module(_LOCK_SELECTOR_MODULE) + + +def test_candidate_conda_channels_exclude_public_rapids_and_nvidia_channels(): + setup = _SETUP.read_text() + + expected_contracts = ( + "candidate build rejected external Conda channel", + "release train must contain the complete Conda bootstrap tool bundle", + "bootstrap-channel/noarch/*", + "rapidsai|rapidsai-nightly|nvidia|", + "conda-forge|https://conda.anaconda.org/conda-forge|file://*|/*", + ) + missing = [contract for contract in expected_contracts if contract not in setup] + if missing: + pytest.fail(f"candidate channel policy is incomplete: {missing}") + + +def test_candidate_conda_lock_selection_is_repository_scoped(): + setup = _SETUP.read_text() + + expected_contracts = ( + 'repository="${GITHUB_REPOSITORY#*/}"', + ".scope.repository == $repository", + "exactly one repository-scoped Conda lock", + ) + missing = [contract for contract in expected_contracts if contract not in setup] + if missing: + pytest.fail(f"repository-scoped Conda lock selection is incomplete: {missing}") + + +def test_candidate_setup_exports_cache_policy_and_train_input_identity(): + setup = _SETUP.read_text() + + expected_contracts = ( + "candidate_build_policy.sccache", + "lock_receipt_sha256", + "variant_receipt_sha256", + "RELEASE_CANDIDATE_BUILD_INPUTS", + "RAPIDS_RELEASE_CANDIDATE_SCCACHE", + ) + missing = [contract for contract in expected_contracts if contract not in setup] + if missing: + pytest.fail(f"candidate build-input identity is incomplete: {missing}") + + +def test_wheel_lock_selector_requires_one_exact_matrix_scope(): + expected = { + "path": "locks/wheel/x86_64-manylinux_2_28/python-3.14/requirements.txt", + "sha256": "a" * 64, + } + receipt = { + "wheel": [ + {**expected, "scope": {"platform": "x86_64-manylinux_2_28", "python": "3.14"}}, + { + "path": "locks/wheel/aarch64-manylinux_2_28/python-3.14/requirements.txt", + "sha256": "b" * 64, + "scope": {"platform": "aarch64-manylinux_2_28", "python": "3.14"}, + }, + ] + } + + selected = _LOCK_SELECTOR_MODULE.select_wheel_lock(receipt, "x86_64-manylinux_2_28", "3.14") + + if selected != expected: + pytest.fail(f"wheel lock selector returned the wrong matrix lock: {selected}") + + +def test_wheel_lock_selector_rejects_missing_and_unsafe_records(): + with pytest.raises(ValueError, match="exactly one wheel lock"): + _LOCK_SELECTOR_MODULE.select_wheel_lock({"wheel": []}, "x86_64-manylinux_2_28", "3.14") + with pytest.raises(ValueError, match="unsafe wheel lock path"): + _LOCK_SELECTOR_MODULE.select_wheel_lock( + { + "wheel": [ + { + "path": "../requirements.txt", + "sha256": "a" * 64, + "scope": {"platform": "x86_64-manylinux_2_28", "python": "3.14"}, + } + ] + }, + "x86_64-manylinux_2_28", + "3.14", + ) + + +@pytest.mark.parametrize( + ("wheel", "python_version"), + [ + ("rapids_logger-0.3.0-py3-none-any.whl", "3.10"), + ("package-1.0.0-cp310-abi3-manylinux_2_17_x86_64.whl", "3.12"), + ("package-1.0.0-cp312-cp312-manylinux_2_17_x86_64.whl", "3.12"), + ], +) +def test_wheel_tags_allow_compatible_python_versions(wheel, python_version): + if not supports_python(wheel, python_version): + pytest.fail(f"expected {wheel} to support Python {python_version}") + + +@pytest.mark.parametrize( + ("wheel", "python_version"), + [ + ("package-1.0.0-cp310-cp310-manylinux_2_17_x86_64.whl", "3.12"), + ("package-1.0.0-cp312-abi3-manylinux_2_17_x86_64.whl", "3.10"), + ("not-a-wheel", "3.12"), + ], +) +def test_wheel_tags_reject_incompatible_python_versions(wheel, python_version): + if supports_python(wheel, python_version): + pytest.fail(f"expected {wheel} not to support Python {python_version}") + + +def test_recipe_patcher_adds_lock_metapackages_to_root_and_outputs(): + recipe = { + "requirements": {"build": ["cmake"], "host": ["zlib"]}, + "outputs": [{"package": {"name": "example"}, "requirements": {"host": ["python"]}}], + } + + _PATCHER_MODULE._add_requirements(recipe, "candidate-build-lock", "candidate-host-lock") + _PATCHER_MODULE._add_requirements(recipe["outputs"][0], "candidate-build-lock", "candidate-host-lock") + + expected_root = { + "build": ["cmake", "candidate-build-lock"], + "host": ["zlib", "candidate-host-lock"], + } + expected_output = { + "build": ["candidate-build-lock"], + "host": ["python", "candidate-host-lock"], + } + if recipe["requirements"] != expected_root: + pytest.fail("root recipe requirements did not receive both lock metapackages") + if recipe["outputs"][0]["requirements"] != expected_output: + pytest.fail("output requirements did not receive both lock metapackages") + + +def test_recipe_patcher_injects_locks_only_into_multi_output_build_targets(): + recipe = { + "requirements": {"build": ["cmake"], "host": ["zlib"]}, + "outputs": [ + { + "package": {"name": "library"}, + "build": {"script": {"content": "cmake --install build", "env": {"KEEP": "true"}}}, + "requirements": {"host": ["python"]}, + }, + {"package": {"name": "tests"}, "requirements": {"build": ["ninja"]}}, + ], + } + + _PATCHER_MODULE._prepare_recipe_documents(recipe, "candidate-build-lock", "candidate-host-lock", "deadbeef", set()) + + if "requirements" in recipe: + pytest.fail("multi-output recipe retained a forbidden top-level requirements field") + expected_library_requirements = { + "build": ["cmake", "candidate-build-lock"], + "host": ["zlib", "python", "candidate-host-lock"], + } + if recipe["outputs"][0]["requirements"] != expected_library_requirements: + pytest.fail("library output did not inherit shared and candidate requirements") + output_script = recipe["outputs"][0]["build"]["script"] + if output_script["env"] != {"KEEP": "true"} or "-Drapids-cmake-sha=deadbeef" not in output_script["content"]: + pytest.fail("script-object content was not pinned without losing its environment") + expected_test_requirements = { + "build": ["cmake", "ninja", "candidate-build-lock"], + "host": ["zlib", "candidate-host-lock"], + } + if recipe["outputs"][1]["requirements"] != expected_test_requirements: + pytest.fail("test output did not inherit shared and candidate requirements") + + +def test_recipe_patcher_injects_locks_into_single_output_root(): + recipe = { + "build": {"script": "cmake -S . -B build"}, + "requirements": {"build": ["cmake"], "host": ["zlib"]}, + "tests": [{"script": "cmake -S tests -B build"}], + } + + _PATCHER_MODULE._prepare_recipe_documents(recipe, "candidate-build-lock", "candidate-host-lock", "deadbeef", set()) + + expected_requirements = { + "build": ["cmake", "candidate-build-lock"], + "host": ["zlib", "candidate-host-lock"], + } + if recipe["requirements"] != expected_requirements: + pytest.fail("single-output root did not receive candidate requirements") + if "-Drapids-cmake-sha=deadbeef" not in recipe["build"]["script"]: + pytest.fail("package build script did not pin RAPIDS CMake") + if "-Drapids-cmake-sha=deadbeef" in recipe["tests"][0]["script"]: + pytest.fail("package test script unexpectedly pinned RAPIDS CMake") + + +def test_recipe_patcher_leaves_multi_output_tests_unmodified(): + recipe = { + "outputs": [ + {"package": {"name": "library"}, "tests": [{"script": "ctest"}]}, + {"package": {"name": "bindings"}, "tests": [{"python": {"imports": ["bindings"]}}]}, + ] + } + + _PATCHER_MODULE._prepare_recipe_documents(recipe, "candidate-build-lock", "candidate-host-lock", "deadbeef", set()) + + for output in recipe["outputs"]: + if "requirements" in output["tests"][0]: + pytest.fail("candidate recipe patcher modified a package test") + + +def test_recipe_patcher_pins_and_locks_shared_cache_build(): + recipe = { + "cache": { + "build": {"script": {"content": "./build.sh", "secrets": ["AWS_TOKEN"]}}, + "requirements": {"build": ["cmake"], "host": ["zlib"]}, + }, + "outputs": [{"package": {"name": "library"}}], + } + + _PATCHER_MODULE._prepare_recipe_documents(recipe, "candidate-build-lock", "candidate-host-lock", "deadbeef", set()) + + if recipe["cache"]["requirements"] != { + "build": ["cmake", "candidate-build-lock"], + "host": ["zlib", "candidate-host-lock"], + }: + pytest.fail("shared cache build did not receive candidate lock packages") + cache_script = recipe["cache"]["build"]["script"] + if cache_script["secrets"] != ["AWS_TOKEN"] or "-Drapids-cmake-sha=deadbeef" not in cache_script["content"]: + pytest.fail("shared cache build script was not pinned without losing its configuration") + + +def test_recipe_patcher_adapts_exact_variant_spelling_from_local_config(tmp_path): + recipe = tmp_path / "recipe.yaml" + recipe.write_text("package:\n name: example\n version: 1\n") + (tmp_path / "conda_build_config.yaml").write_text("libcurl_version:\n - ==8.13.0\n") + + keys = _PATCHER_MODULE._local_exact_variant_keys(recipe) + + expected = "libcurl ==${{ libcurl_version }}" + if _PATCHER_MODULE._add_exact_operator("libcurl ${{ libcurl_version }}", keys) != expected: + pytest.fail("recipe patcher did not add the local exact MatchSpec operator") + if _PATCHER_MODULE._add_exact_operator(expected, keys) != expected: + pytest.fail("recipe patcher duplicated the local exact MatchSpec operator") + + +def test_recipe_patcher_accepts_recipe_directory(tmp_path): + recipe = tmp_path / "recipe" + recipe.mkdir() + expected = recipe / "recipe.yaml" + expected.write_text("package:\n name: example\n version: 1\n") + + resolved = _PATCHER_MODULE._resolve_recipe(recipe, argparse.ArgumentParser()) + + if resolved != expected: + pytest.fail("recipe patcher did not resolve a recipe directory to recipe.yaml") + + +def test_recipe_patcher_copies_recipe_directory_with_recipe_yaml(tmp_path): + recipe = tmp_path / "recipes" / "example" + recipe.mkdir(parents=True) + source_recipe = recipe / "recipe.yaml" + source_recipe.write_text("package:\n name: example\n version: 1\n") + (recipe / "build.sh").write_text("#!/usr/bin/env bash\n") + + destination = _PATCHER_MODULE._copy_recipe_directory( + source_recipe, + "candidate-build-lock", + "candidate-host-lock", + ) + + if not destination.name.startswith(".example.release-candidate-"): + pytest.fail("recipe patcher did not use a deterministic disposable directory") + if not (destination / "recipe.yaml").is_file(): + pytest.fail("disposable recipe directory did not retain recipe.yaml") + if not (destination / "build.sh").is_file(): + pytest.fail("disposable recipe directory did not retain recipe-local files") + + +@pytest.mark.parametrize( + ("architecture", "platform"), + [("amd64", "linux-64"), ("x86_64", "linux-64"), ("arm64", "linux-aarch64"), ("aarch64", "linux-aarch64")], +) +def test_candidate_conda_platform_normalizes_ci_architecture_names(architecture, platform): + script = Path(__file__).parents[1] / "release-candidate-dependencies" / "platform.sh" + # The command runs the repository-local helper with a fixed shell program. + result = subprocess.run( # noqa: S603 + ["/bin/bash", "-c", 'source "$1"; conda_platform_for_arch "$2"', "bash", str(script), architecture], + check=True, + capture_output=True, + text=True, + ) + + if result.stdout.strip() != platform: + pytest.fail(f"expected {architecture} to normalize to {platform}") + + +def test_candidate_conda_platform_rejects_unknown_architecture(): + script = Path(__file__).parents[1] / "release-candidate-dependencies" / "platform.sh" + # The command runs the repository-local helper with a fixed shell program. + result = subprocess.run( # noqa: S603 + ["/bin/bash", "-c", 'source "$1"; conda_platform_for_arch "$2"', "bash", str(script), "not-an-arch"], + check=False, + capture_output=True, + text=True, + ) + + if result.returncode == 0 or "unsupported candidate architecture" not in result.stderr: + pytest.fail("unknown candidate architecture was accepted") + + +def test_candidate_cmake_wrapper_pins_rapids_cmake_for_configure(tmp_path): + output = tmp_path / "arguments.txt" + original = tmp_path / "cmake" + original.write_text('#!/bin/bash\nprintf \'%s\\n\' "$@" >"${TEST_OUTPUT}"\n') + original.chmod(0o755) + wrapper = Path(__file__).parents[1] / "release-candidate-dependencies" / "cmake.sh" + result = subprocess.run( # noqa: S603 + ["/bin/bash", str(wrapper), "-S", ".", "-B", "build"], + check=True, + capture_output=True, + text=True, + env={ + "RAPIDS_CANDIDATE_ORIGINAL_CMAKE": str(original), + "RAPIDS_CANDIDATE_RAPIDS_CMAKE_SHA": "a" * 40, + "TEST_OUTPUT": str(output), + }, + ) + + if result.stdout: + pytest.fail("candidate CMake wrapper wrote unexpected standard output") + expected_arguments = [ + "-Drapids-cmake-sha=" + "a" * 40, + "-S", + ".", + "-B", + "build", + ] + if output.read_text().splitlines() != expected_arguments: + pytest.fail("candidate CMake wrapper did not pin RAPIDS CMake during configuration") + + +def test_candidate_cmake_wrapper_does_not_modify_build_mode(tmp_path): + output = tmp_path / "arguments.txt" + original = tmp_path / "cmake" + original.write_text('#!/bin/bash\nprintf \'%s\\n\' "$@" >"${TEST_OUTPUT}"\n') + original.chmod(0o755) + wrapper = Path(__file__).parents[1] / "release-candidate-dependencies" / "cmake.sh" + subprocess.run( # noqa: S603 + ["/bin/bash", str(wrapper), "--build", "build"], + check=True, + capture_output=True, + text=True, + env={ + "RAPIDS_CANDIDATE_ORIGINAL_CMAKE": str(original), + "RAPIDS_CANDIDATE_RAPIDS_CMAKE_SHA": "a" * 40, + "TEST_OUTPUT": str(output), + }, + ) + + if output.read_text().splitlines() != ["--build", "build"]: + pytest.fail("candidate CMake wrapper modified a build-mode invocation")