diff --git a/.gitignore b/.gitignore index 9c92e4ec..850a2c2e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ *.swp .venv +__pycache__/ +*.py[cod] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 641716fc..6ea3a749 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,6 +35,19 @@ repos: args: ["--fix", "--config=pyproject.toml"] - id: ruff-format args: ["--config=pyproject.toml"] + - repo: https://github.com/PyCQA/bandit + rev: 1.8.6 + hooks: + - id: bandit + files: ^(release-candidate-context|release-candidate-dependencies|release-catalog)/.*\.py$ + args: ["--quiet"] + - repo: https://github.com/PyCQA/flake8 + rev: 7.3.0 + hooks: + - id: flake8 + files: ^(release-candidate-context|release-candidate-dependencies|release-catalog)/.*\.py$ + additional_dependencies: [flake8-bugbear==25.10.21] + args: ["--select=B"] - repo: https://github.com/rapidsai/pre-commit-hooks rev: v1.7.0 hooks: @@ -81,3 +94,12 @@ repos: additional_dependencies: [jsonschema==4.25.1] pass_filenames: false files: ^(release-catalog/(config|entries)\.schema\.json|release-catalog/examples/cuvs-java/release-catalog-entries\.json|tests/release-catalog-config/.*\.json|ci/test_release_catalog_schemas\.py)$ + - repo: local + hooks: + - id: candidate-context-tests + name: test candidate run bindings + entry: python -m pytest -q tests/test_candidate_context.py + language: python + additional_dependencies: [pytest] + pass_filenames: false + files: ^(release-candidate-context/|tests/test_candidate_context.py) diff --git a/pyproject.toml b/pyproject.toml index 23c81e9f..cc18e9d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,12 @@ -# Copyright (c) 2024-2026, NVIDIA CORPORATION. +# Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. [tool.ruff] line-length = 120 target-version = "py310" +[tool.ruff.lint.per-file-ignores] +"tests/test_candidate_context.py" = ["S101"] + [tool.ruff.lint] select = [ # flake8-bugbear diff --git a/release-candidate-context/action.yml b/release-candidate-context/action.yml new file mode 100644 index 00000000..de4dba2a --- /dev/null +++ b/release-candidate-context/action.yml @@ -0,0 +1,42 @@ +name: Resolve release-candidate run context +description: Resolve an immutable controller binding without adding project workflow inputs. +inputs: + source-sha: + description: Full source SHA checked out for this build. + required: true + script: + description: Custom job script requiring a centrally reviewed policy; empty for standard builds. + default: '' +outputs: + train-sha256: + description: Exact release train bound to this workflow run. + value: ${{ steps.resolve.outputs.train-sha256 }} + custom-mode: + description: Build, test, or intermediate artifact policy. + value: ${{ steps.resolve.outputs.custom-mode }} +runs: + using: composite + steps: + - name: Locate context resolver in the job container + id: implementation + shell: bash + env: + ACTION_PATH: ${{ github.action_path }} + HOST_WORKSPACE: ${{ github.workspace }} + run: | + host_root="$(dirname "$(dirname "${HOST_WORKSPACE}")")" + container_root="$(dirname "$(dirname "${PWD}")")" + action_path="${ACTION_PATH}" + if [[ ! -d "${action_path}" && "${action_path}" == "${host_root}/"* ]]; then + action_path="${container_root}/${action_path#"${host_root}/"}" + fi + test -f "${action_path}/resolver.py" + echo "path=${action_path}" >> "${GITHUB_OUTPUT}" + - name: Resolve exact run binding + id: resolve + shell: bash + env: + IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} + SOURCE_SHA: ${{ inputs.source-sha }} + CUSTOM_SCRIPT: ${{ inputs.script }} + run: python "${IMPLEMENTATION_PATH}/resolver.py" --source-sha "${SOURCE_SHA}" --script "${CUSTOM_SCRIPT}" diff --git a/release-candidate-context/resolver.py b/release-candidate-context/resolver.py new file mode 100644 index 00000000..bf756b7a --- /dev/null +++ b/release-candidate-context/resolver.py @@ -0,0 +1,83 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Resolve candidate metadata for unchanged caller workflows from an exact run binding.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess # nosec B404 - invokes AWS CLI with an argument list, never a shell. +import time +from collections.abc import Mapping +from pathlib import Path + + +def validate_binding(binding: Mapping, environment: Mapping[str, str], source_sha: str, script: str) -> dict[str, str]: + """Reject unrelated runs/sources and custom jobs not enrolled by the controller.""" + expected = { + "schema_version": 1, + "repository": environment["GITHUB_REPOSITORY"], + "run_id": environment["GITHUB_RUN_ID"], + "source_sha": source_sha, + } + if any(binding.get(key) != value for key, value in expected.items()): + raise ValueError("candidate binding does not match this repository, run, or source SHA") + if not re.fullmatch(r"[0-9a-f]{40}|[0-9a-f]{64}", source_sha): + raise ValueError("candidate source SHA must be a full object ID") + if binding.get("workflow") != "build.yaml": + raise ValueError("candidate binding has an unsupported workflow") + workflow_ref = environment["GITHUB_WORKFLOW_REF"] + expected_prefix = f"{binding['repository']}/.github/workflows/build.yaml@refs/heads/" + if workflow_ref != expected_prefix + binding.get("workflow_ref", ""): + raise ValueError("candidate binding does not match the caller workflow ref") + if environment["GITHUB_EVENT_NAME"] != "workflow_dispatch": + raise ValueError("candidate binding requires a workflow dispatch") + digest = binding.get("train_sha256", "") + if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest): + raise ValueError("candidate binding has no valid train digest") + mode = "build" + if script: + mode = binding.get("custom_jobs", {}).get(script) + if mode not in {"test", "artifact"}: + raise ValueError(f"custom candidate job has no reviewed policy: {script}") + return {"train-sha256": digest, "custom-mode": mode} + + +def read_binding(bucket: str, key: str, run=subprocess.run, sleep=time.sleep, attempts: int = 12) -> Mapping: + """Wait at most 55 seconds for the controller's post-dispatch binding write.""" + for attempt in range(attempts): + response = run(("aws", "s3", "cp", f"s3://{bucket}/{key}", "-"), text=True, capture_output=True, check=False) + if response.returncode == 0: + binding = json.loads(response.stdout) + if not isinstance(binding, dict): + raise ValueError("candidate binding must be a JSON object") + return binding + if "404" not in response.stderr and "NoSuchKey" not in response.stderr: + raise ValueError("unable to read candidate binding; check candidate-store access") + if attempt + 1 < attempts: + sleep(5) + raise ValueError("candidate run has no controller binding; refusing to guess a train") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--bucket", default="rapids-release-candidates") + parser.add_argument("--prefix", default="train-state") + parser.add_argument("--source-sha", required=True) + parser.add_argument("--script", default="") + args = parser.parse_args() + repository = os.environ["GITHUB_REPOSITORY"] + run_id = os.environ["GITHUB_RUN_ID"] + if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", repository) or not run_id.isdigit(): + raise ValueError("invalid repository or run identity") + key = f"{args.prefix}/run-bindings/{repository}/{run_id}.json" + binding = read_binding(args.bucket, key) + outputs = validate_binding(binding, os.environ, args.source_sha, args.script) + with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output: + for name, value in outputs.items(): + output.write(f"{name}={value}\n") + + +if __name__ == "__main__": + main() diff --git a/release-candidate-dependencies/action.yml b/release-candidate-dependencies/action.yml new file mode 100644 index 00000000..93cd5bbd --- /dev/null +++ b/release-candidate-dependencies/action.yml @@ -0,0 +1,70 @@ +name: Prepare release-candidate dependencies +description: Materialize only this release train's private S3 outputs for a candidate build. + +inputs: + candidate-train-sha256: + description: Canonical SHA-256 of the immutable release train. + required: true + candidate-bucket: + description: Private S3 bucket holding candidate build bundles. + required: false + default: rapids-release-candidates + candidate-train-prefix: + description: Root prefix for immutable candidate train membership records. + required: false + default: train-state + prepare-conda-channel: + description: Whether this job needs a local Conda channel for build dependencies. + required: false + default: "false" + artifact-family: + description: Release-unit family that this job is building. + required: true + arch: + description: Canonical RAPIDS architecture for this job matrix. + required: true + cuda-version: + description: CUDA version for this job matrix. + required: true + python-version: + description: Python version for this job matrix. + required: true + +runs: + using: composite + steps: + - id: implementation + name: Resolve candidate dependency implementation + shell: bash + env: + ACTION_PATH: ${{ github.action_path }} + HOST_WORKSPACE: ${{ github.workspace }} + run: | + host_root="$(dirname "$(dirname "${HOST_WORKSPACE}")")" + container_root="$(dirname "$(dirname "${PWD}")")" + action_path="${ACTION_PATH}" + if [[ ! -d "${action_path}" && "${action_path}" == "${host_root}/"* ]]; then + action_path="${container_root}/${action_path#"${host_root}/"}" + fi + if [[ ! -f "${action_path}/setup.sh" ]]; then + echo "release-candidate dependency implementation not found at ${action_path}" >&2 + exit 1 + fi + echo "path=${action_path}" >>"${GITHUB_OUTPUT}" + - name: Prepare private candidate dependency views + shell: bash + env: + RELEASE_CANDIDATE_BUCKET: ${{ inputs.candidate-bucket }} + RELEASE_CANDIDATE_TRAIN_PREFIX: ${{ inputs.candidate-train-prefix }} + RELEASE_CANDIDATE_TRAIN_SHA256: ${{ inputs.candidate-train-sha256 }} + RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL: ${{ inputs.prepare-conda-channel }} + RELEASE_CANDIDATE_ARTIFACT_FAMILY: ${{ inputs.artifact-family }} + RELEASE_CANDIDATE_ARCH: ${{ inputs.arch }} + RELEASE_CANDIDATE_CUDA_VERSION: ${{ inputs.cuda-version }} + RELEASE_CANDIDATE_PYTHON_VERSION: ${{ inputs.python-version }} + RELEASE_CANDIDATE_SHARED_ACTIONS_REPOSITORY: ${{ github.action_repository }} + RELEASE_CANDIDATE_SHARED_ACTIONS_REVISION: ${{ github.action_ref }} + RELEASE_CANDIDATE_SHARED_WORKFLOWS_REPOSITORY: ${{ job.workflow_repository }} + RELEASE_CANDIDATE_SHARED_WORKFLOWS_REVISION: ${{ job.workflow_sha }} + IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} + run: ${IMPLEMENTATION_PATH}/setup.sh diff --git a/release-candidate-dependencies/cmake.sh b/release-candidate-dependencies/cmake.sh new file mode 100644 index 00000000..f08587b7 --- /dev/null +++ b/release-candidate-dependencies/cmake.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +set -euo pipefail + +original_cmake="${RAPIDS_CANDIDATE_ORIGINAL_CMAKE:?candidate CMake path is required}" +rapids_cmake_sha="${RAPIDS_CANDIDATE_RAPIDS_CMAKE_SHA:?candidate RAPIDS CMake SHA is required}" + +# CMake's build/install/script modes do not accept configure definitions. Only +# add the immutable RAPIDS CMake override when the invocation configures a tree. +case "${1:-}" in + --build|--install|--open|-E|-P|--help|--version) + exec "${original_cmake}" "$@" + ;; +esac + +exec "${original_cmake}" "-Drapids-cmake-sha=${rapids_cmake_sha}" "$@" diff --git a/release-candidate-dependencies/patch_recipe.py b/release-candidate-dependencies/patch_recipe.py new file mode 100644 index 00000000..c8aefdb5 --- /dev/null +++ b/release-candidate-dependencies/patch_recipe.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +"""Add release-candidate lock metapackages to a disposable Rattler recipe.""" + +from __future__ import annotations + +import argparse +import hashlib +import re +import shutil +from pathlib import Path + +import yaml + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--recipe", type=Path, required=True) + parser.add_argument("--build-lock", required=True) + parser.add_argument("--host-lock", required=True) + parser.add_argument("--rapids-cmake-sha", required=True) + arguments = parser.parse_args() + + source_is_directory = arguments.recipe.is_dir() + recipe = _resolve_recipe(arguments.recipe, parser) + destination_directory = _copy_recipe_directory(recipe, arguments.build_lock, arguments.host_lock) + destination_recipe = destination_directory / "recipe.yaml" + document = yaml.safe_load(destination_recipe.read_text()) + if not isinstance(document, dict): + parser.error(f"recipe must be a YAML mapping: {recipe}") + exact_variant_keys = _local_exact_variant_keys(recipe) + outputs = document.get("outputs", []) + if outputs is not None: + if not isinstance(outputs, list): + parser.error(f"recipe outputs must be a list: {recipe}") + for output in outputs: + if not isinstance(output, dict): + parser.error(f"recipe output must be a mapping: {recipe}") + _prepare_recipe_documents( + document, + arguments.build_lock, + arguments.host_lock, + arguments.rapids_cmake_sha, + exact_variant_keys, + ) + + destination_recipe.write_text(yaml.safe_dump(document, sort_keys=False)) + print(destination_directory if source_is_directory else destination_recipe) + return 0 + + +def _add_requirements(document: dict, build_lock: str, host_lock: str) -> None: + requirements = document.setdefault("requirements", {}) + if not isinstance(requirements, dict): + raise ValueError("recipe requirements must be a mapping") + for section, lock in (("build", build_lock), ("host", host_lock)): + values = requirements.setdefault(section, []) + if not isinstance(values, list): + raise ValueError(f"recipe requirements.{section} must be a list") + if lock not in values: + values.append(lock) + + +def _candidate_cmake_preamble(rapids_cmake_sha: str) -> str: + """Return Bash setup that pins configure-mode CMake and scikit-build.""" + return f"""# Release candidates must never fetch RAPIDS CMake from a moving branch. +cmake() {{ + case "${{1:-}}" in + --build|--install|--open|-E|-P|--help|--version) command cmake "$@" ;; + *) command cmake "-Drapids-cmake-sha={rapids_cmake_sha}" "$@" ;; + esac +}} +export -f cmake +export SKBUILD_CMAKE_DEFINE="${{SKBUILD_CMAKE_DEFINE:+${{SKBUILD_CMAKE_DEFINE}};}}rapids-cmake-sha={rapids_cmake_sha}" +""" + + +def _prepend_cmake_preamble(script: object, rapids_cmake_sha: str) -> object: + preamble = _candidate_cmake_preamble(rapids_cmake_sha) + if isinstance(script, str): + return f"{preamble}\n{script}" + if isinstance(script, list): + return [preamble, *script] + if isinstance(script, dict): + content = script.get("content") + if content is None: + raise ValueError("release-candidate recipe script object must define content") + patched = dict(script) + patched["content"] = _prepend_cmake_preamble(content, rapids_cmake_sha) + return patched + raise ValueError("release-candidate recipe script must be a string, list, or content object") + + +def _pin_cmake_in_build_script(document: dict, rapids_cmake_sha: str) -> None: + """Place the exact CMake definition inside Rattler's isolated shells. + + Rattler does not inherit the workflow PATH into its build shell, so a + workflow-level CMake wrapper cannot protect scripts that invoke ``cmake`` + directly. Package tests intentionally remain unmodified: they validate + the produced package rather than the candidate build environment. + """ + build = document.get("build") + if isinstance(build, dict) and "script" in build: + build["script"] = _prepend_cmake_preamble(build["script"], rapids_cmake_sha) + + +def _resolve_recipe(candidate: Path, parser: argparse.ArgumentParser) -> Path: + recipe = candidate.resolve() + if recipe.is_dir(): + recipe = recipe / "recipe.yaml" + if not recipe.is_file(): + parser.error(f"recipe must be a file or a directory containing recipe.yaml: {candidate}") + return recipe + + +def _copy_recipe_directory(recipe: Path, build_lock: str, host_lock: str) -> Path: + """Create a same-parent disposable recipe directory named for Rattler.""" + digest = hashlib.sha256(f"{build_lock}\0{host_lock}".encode()).hexdigest()[:12] + destination = recipe.parent.parent / f".{recipe.parent.name}.release-candidate-{digest}" + if destination.exists(): + shutil.rmtree(destination) + shutil.copytree(recipe.parent, destination) + return destination + + +def _prepare_recipe_documents( + document: dict, build_lock: str, host_lock: str, rapids_cmake_sha: str, exact_variant_keys: set[str] +) -> None: + """Inject locks into the actual build target of a Rattler recipe. + + A recipe without outputs produces its root package, while every member of + a multi-output recipe is an independent build target. Rattler-Build does + not permit top-level requirements for the latter, so first distribute the + shared requirements into every output and remove the root field. + """ + _pin_cmake_in_build_script(document, rapids_cmake_sha) + cache = document.get("cache") + if cache is not None: + if not isinstance(cache, dict): + raise ValueError("recipe cache must be a mapping") + # Cache recipes have a shared build phase before output packages are + # assembled. Its solve and shell are just as much part of the + # candidate build as an output's own requirements and script. + _add_requirements(cache, build_lock, host_lock) + _pin_cmake_in_build_script(cache, rapids_cmake_sha) + outputs = document.get("outputs") + if not outputs: + _prepare_document(document, build_lock, host_lock, exact_variant_keys, add_locks=True) + return + + _move_root_requirements_to_outputs(document, outputs) + for output in outputs: + _pin_cmake_in_build_script(output, rapids_cmake_sha) + _prepare_document(output, build_lock, host_lock, exact_variant_keys, add_locks=True) + + +def _prepare_document( + document: dict, build_lock: str, host_lock: str, exact_variant_keys: set[str], *, add_locks: bool +) -> None: + if add_locks: + _add_requirements(document, build_lock, host_lock) + requirements = document["requirements"] + for section, values in requirements.items(): + if not isinstance(values, list): + continue + requirements[section] = [_add_exact_operator(value, exact_variant_keys) for value in values] + + +def _move_root_requirements_to_outputs(document: dict, outputs: list[dict]) -> None: + """Copy shared requirements into every output before removing the root field.""" + root_requirements = document.pop("requirements", None) + if root_requirements is None: + return + if not isinstance(root_requirements, dict): + raise ValueError("recipe requirements must be a mapping") + for section, shared_values in root_requirements.items(): + if not isinstance(shared_values, list): + raise ValueError(f"recipe requirements.{section} must be a list") + for output in outputs: + requirements = output.setdefault("requirements", {}) + if not isinstance(requirements, dict): + raise ValueError("recipe output requirements must be a mapping") + values = requirements.setdefault(section, []) + if not isinstance(values, list): + raise ValueError(f"recipe output requirements.{section} must be a list") + requirements[section] = [*shared_values, *(value for value in values if value not in shared_values)] + + +def _local_exact_variant_keys(recipe: Path) -> set[str]: + keys = set() + for filename in ("variants.yaml", "conda_build_config.yaml"): + path = recipe.parent / filename + if not path.is_file(): + continue + document = yaml.safe_load(path.read_text()) + if not isinstance(document, dict): + continue + for key, values in document.items(): + if ( + isinstance(key, str) + and isinstance(values, list) + and any(isinstance(value, str) and value.strip().startswith("==") for value in values) + ): + keys.add(key) + return keys + + +def _add_exact_operator(value: object, exact_variant_keys: set[str]) -> object: + if not isinstance(value, str) or not exact_variant_keys: + return value + names = "|".join(re.escape(key) for key in sorted(exact_variant_keys)) + expression = rf"^(\S+)\s+(\$\{{\{{\s*(?:{names})\s*\}}\}})$" + return re.sub(expression, r"\1 ==\2", value) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/release-candidate-dependencies/platform.sh b/release-candidate-dependencies/platform.sh new file mode 100644 index 00000000..c91f829f --- /dev/null +++ b/release-candidate-dependencies/platform.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +# Normalize the architecture names used by RAPIDS CI and Conda. +conda_platform_for_arch() { + case "$1" in + amd64|x86_64) printf 'linux-64\n' ;; + arm64|aarch64) printf 'linux-aarch64\n' ;; + *) + echo "unsupported candidate architecture: $1" >&2 + return 1 + ;; + esac +} + +wheel_platform_for_arch() { + case "$1" in + amd64|x86_64) printf 'x86_64-manylinux_2_28\n' ;; + arm64|aarch64) printf 'aarch64-manylinux_2_28\n' ;; + *) + echo "unsupported candidate architecture: $1" >&2 + return 1 + ;; + esac +} diff --git a/release-candidate-dependencies/select_lock.py b/release-candidate-dependencies/select_lock.py new file mode 100644 index 00000000..9fb956ec --- /dev/null +++ b/release-candidate-dependencies/select_lock.py @@ -0,0 +1,54 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +"""Select exactly one checksum-addressed release-train build lock.""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import PurePosixPath +from typing import Any + +SHA256 = re.compile(r"[0-9a-f]{64}") + + +def select_wheel_lock(document: dict[str, Any], platform: str, python_version: str) -> dict[str, Any]: + """Return the unique wheel lock matching one CI matrix scope.""" + records = document.get("wheel", document.get("input_locks", {}).get("wheel", [])) + if not isinstance(records, list): + raise ValueError("release train wheel locks must be a list") + matches = [ + record + for record in records + if isinstance(record, dict) and record.get("scope") == {"platform": platform, "python": python_version} + ] + if len(matches) != 1: + raise ValueError( + f"release train must contain exactly one wheel lock for {platform}, Python {python_version}; " + f"found {len(matches)}" + ) + record = matches[0] + path = record.get("path") + sha256 = record.get("sha256") + if not isinstance(path, str) or not path or PurePosixPath(path).is_absolute() or ".." in PurePosixPath(path).parts: + raise ValueError("release train contains an unsafe wheel lock path") + if not isinstance(sha256, str) or SHA256.fullmatch(sha256) is None: + raise ValueError("release train contains an invalid wheel lock checksum") + return {"path": path, "sha256": sha256} + + +def main() -> None: + """Select a wheel lock from a train or compact lock receipt.""" + parser = argparse.ArgumentParser() + parser.add_argument("--locks", required=True) + parser.add_argument("--platform", required=True) + parser.add_argument("--python-version", required=True) + arguments = parser.parse_args() + with open(arguments.locks, encoding="utf-8") as stream: + document = json.load(stream) + print(json.dumps(select_wheel_lock(document, arguments.platform, arguments.python_version), sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/release-candidate-dependencies/select_wheel.py b/release-candidate-dependencies/select_wheel.py new file mode 100644 index 00000000..abb7037b --- /dev/null +++ b/release-candidate-dependencies/select_wheel.py @@ -0,0 +1,65 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +"""Select wheel files compatible with the target Python interpreter. + +The producing job's Python matrix is useful for ranking otherwise equivalent +outputs, but it is not the wheel compatibility contract. PEP 425 wheel tags +are that contract: pure Python and ``abi3`` wheels often serve more than the +one interpreter that built them. +""" + +from __future__ import annotations + +import argparse +import re + +from packaging.utils import InvalidWheelFilename, parse_wheel_filename + + +def supports_python(wheel_name: str, python_version: str) -> bool: + """Return whether one wheel's interpreter and ABI tags allow this Python.""" + target = _python_version(python_version) + try: + _name, _version, _build, tags = parse_wheel_filename(wheel_name) + except InvalidWheelFilename: + return False + return any(_tag_supports_python(tag.interpreter, tag.abi, target) for tag in tags) + + +def _python_version(value: str) -> tuple[int, int]: + match = re.fullmatch(r"(\d+)\.(\d+)", value) + if match is None: + raise ValueError("python version must use MAJOR.MINOR form") + return int(match.group(1)), int(match.group(2)) + + +def _tag_supports_python(interpreter: str, abi: str, target: tuple[int, int]) -> bool: + if interpreter == f"py{target[0]}" or interpreter == f"py{target[0]}{target[1]}": + return True + match = re.fullmatch(r"cp(\d)(\d+)", interpreter) + if match is None: + return False + producer = int(match.group(1)), int(match.group(2)) + if producer[0] != target[0]: + return False + if abi == "abi3": + return producer <= target + return producer == target + + +def main() -> None: + """Exit successfully only when the supplied wheel supports the target.""" + parser = argparse.ArgumentParser() + parser.add_argument("--python-version", required=True) + parser.add_argument("wheel") + args = parser.parse_args() + try: + compatible = supports_python(args.wheel, args.python_version) + except ValueError as error: + parser.error(str(error)) + if not compatible: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/release-candidate-dependencies/setup.sh b/release-candidate-dependencies/setup.sh new file mode 100755 index 00000000..a41d4ba3 --- /dev/null +++ b/release-candidate-dependencies/setup.sh @@ -0,0 +1,728 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +# Candidate dependencies remain train-scoped and private. We first retrieve +# the small release train and catalog envelopes, then download only files for +# the current unit's declared *build* prerequisites. In particular, this must +# never turn all completed candidate outputs into a shared package channel. +set -euo pipefail + +script_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# The helper is adjacent to this script but its absolute action path is dynamic. +# shellcheck disable=SC1091 +source "${script_directory}/platform.sh" + +require_value() { + local name="$1" + local value="$2" + if [[ -z "${value}" || "${value}" == *$'\n'* || "${value}" == *$'\r'* ]]; then + echo "${name} must be a non-empty single-line string" >&2 + exit 1 + fi +} + +for value in GITHUB_ENV GITHUB_PATH RUNNER_TEMP GITHUB_REPOSITORY RELEASE_CANDIDATE_BUCKET RELEASE_CANDIDATE_TRAIN_PREFIX RELEASE_CANDIDATE_TRAIN_SHA256 RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL RELEASE_CANDIDATE_ARTIFACT_FAMILY RELEASE_CANDIDATE_ARCH RELEASE_CANDIDATE_CUDA_VERSION RELEASE_CANDIDATE_PYTHON_VERSION; do + require_value "${value}" "${!value:-}" +done +if [[ ! "${RELEASE_CANDIDATE_TRAIN_SHA256}" =~ ^[[:xdigit:]]{64}$ ]]; then + echo "RELEASE_CANDIDATE_TRAIN_SHA256 must be a SHA-256 digest" >&2 + exit 1 +fi +if [[ "${RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL}" != "true" && "${RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL}" != "false" ]]; then + echo "RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL must be true or false" >&2 + exit 1 +fi +if [[ "${RELEASE_CANDIDATE_ARTIFACT_FAMILY}" != "conda" && "${RELEASE_CANDIDATE_ARTIFACT_FAMILY}" != "wheel" ]]; then + echo "RELEASE_CANDIDATE_ARTIFACT_FAMILY must be conda or wheel" >&2 + exit 1 +fi + +root="s3://${RELEASE_CANDIDATE_BUCKET}/${RELEASE_CANDIDATE_TRAIN_PREFIX}/${RELEASE_CANDIDATE_TRAIN_SHA256}" +train_key_prefix="${RELEASE_CANDIDATE_TRAIN_PREFIX}/${RELEASE_CANDIDATE_TRAIN_SHA256}" +repository="${GITHUB_REPOSITORY#*/}" +cuda_major="${RELEASE_CANDIDATE_CUDA_VERSION%%.*}" +conda_platform="$(conda_platform_for_arch "${RELEASE_CANDIDATE_ARCH}")" +wheel_platform="$(wheel_platform_for_arch "${RELEASE_CANDIDATE_ARCH}")" +workspace="${RUNNER_TEMP}/release-candidate-dependencies" +channel="${workspace}/conda-channel" +wheelhouse="${workspace}/wheelhouse" +tools="${workspace}/tools" +manifests="${workspace}/catalog-manifests" +mkdir -p "${wheelhouse}" "${tools}" "${manifests}" +if [[ "${RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL}" == "true" ]]; then + mkdir -p "${channel}"/{linux-64,linux-aarch64,noarch} +fi + +# The train is the authorization boundary. A missing record means the +# coordinator did not stage this candidate correctly; guessing from S3 paths +# would make unrelated outputs available to the job. +aws s3 cp "${root}/release-train.json" "${workspace}/release-train.json" +actual_train_sha256="$(jq -cSj 'del(.train_sha256)' "${workspace}/release-train.json" | sha256sum | awk '{print $1}')" +if [[ "${actual_train_sha256}" != "${RELEASE_CANDIDATE_TRAIN_SHA256}" ]]; then + echo "stored release train does not match RELEASE_CANDIDATE_TRAIN_SHA256" >&2 + exit 1 +fi +rapids_cmake_sha="$(jq -r '.build_tool_revisions["rapids-cmake"] // empty' "${workspace}/release-train.json")" +if [[ ! "${rapids_cmake_sha}" =~ ^[[:xdigit:]]{40}$ ]]; then + echo "release train must lock rapids-cmake with build_tool_revisions.rapids-cmake; regenerate the train" >&2 + exit 1 +fi +build_implementation_revisions="$(jq -ceS ' + .build_implementation_revisions as $revisions + | if ($revisions | type) != "object" then error("build implementation revisions must be an object") + elif (["gha-tools", "shared-actions", "shared-workflows"] | all(.[]; . as $name | $revisions | has($name))) | not + then error("missing required build implementation revision") + elif ($revisions | all(to_entries[]; (.key | type) == "string" and (.value | type) == "string" and (.value | test("^[0-9a-f]{40}$")))) + then $revisions + else error("invalid build implementation revision") + end +' "${workspace}/release-train.json")" || { + echo "release train must lock shared-workflows, shared-actions, and gha-tools; regenerate the train" >&2 + exit 1 +} +gha_tools_revision="$(jq -r '."gha-tools"' <<<"${build_implementation_revisions}")" +shared_actions_revision="$(jq -r '."shared-actions"' <<<"${build_implementation_revisions}")" +shared_workflows_revision="$(jq -r '."shared-workflows"' <<<"${build_implementation_revisions}")" +candidate_sccache="$(jq -r ' + if ((.candidate_build_policy | type) == "object") + and ((.candidate_build_policy.sccache | type) == "boolean") + then .candidate_build_policy.sccache + else empty + end +' "${workspace}/release-train.json")" +if [[ "${candidate_sccache}" != "true" && "${candidate_sccache}" != "false" ]]; then + echo "release train must declare candidate_build_policy.sccache as true or false; regenerate the train" >&2 + exit 1 +fi + +require_expected_revision() { + local label="$1" + local expected_repository="$2" + local expected_revision="$3" + local actual_repository="$4" + local actual_revision="$5" + local usage_repository="$6" + if [[ "${actual_repository}" != "${expected_repository}" || "${actual_revision}" != "${expected_revision}" ]]; then + echo "${label} usage in ${usage_repository} does not match the frozen release train: train expects ${expected_repository}@${expected_revision:0:8}, got ${actual_repository}@${actual_revision:0:8}" >&2 + exit 1 + fi +} + +require_expected_revision \ + "shared-actions" \ + "rapidsai/shared-actions" \ + "${shared_actions_revision}" \ + "${RELEASE_CANDIDATE_SHARED_ACTIONS_REPOSITORY:-}" \ + "${RELEASE_CANDIDATE_SHARED_ACTIONS_REVISION:-}" \ + "${RELEASE_CANDIDATE_SHARED_WORKFLOWS_REPOSITORY:-${GITHUB_REPOSITORY}}" +require_expected_revision \ + "shared-workflows" \ + "rapidsai/shared-workflows" \ + "${shared_workflows_revision}" \ + "${RELEASE_CANDIDATE_SHARED_WORKFLOWS_REPOSITORY:-}" \ + "${RELEASE_CANDIDATE_SHARED_WORKFLOWS_REVISION:-}" \ + "${GITHUB_REPOSITORY}" + +if ! command -v git >/dev/null; then + echo "git is required to install the train-locked gha-tools revision" >&2 + exit 1 +fi +gha_tools_directory="${workspace}/gha-tools" +git init --quiet "${gha_tools_directory}" +git -C "${gha_tools_directory}" remote add origin https://github.com/rapidsai/gha-tools.git +git -C "${gha_tools_directory}" fetch --quiet --depth 1 origin "${gha_tools_revision}" +git -C "${gha_tools_directory}" checkout --quiet --detach FETCH_HEAD +actual_gha_tools_revision="$(git -C "${gha_tools_directory}" rev-parse HEAD)" +if [[ "${actual_gha_tools_revision}" != "${gha_tools_revision}" ]]; then + echo "gha-tools checkout does not match the frozen release train: expected ${gha_tools_revision}, got ${actual_gha_tools_revision}" >&2 + exit 1 +fi +PATH="${gha_tools_directory}/tools:${PATH}" +# scikit-build-core may use the CMake wheel rather than resolving `cmake` from +# PATH, so its configure call can bypass the candidate CMake wrapper below. +# Its supported environment configuration is additive and reaches both the +# bundled and system CMake implementations. Put the immutable train value +# last so it wins over an inherited generic definition of the same cache key. +skbuild_cmake_define="rapids-cmake-sha=${rapids_cmake_sha}" +if [[ -n "${SKBUILD_CMAKE_DEFINE:-}" ]]; then + require_value "SKBUILD_CMAKE_DEFINE" "${SKBUILD_CMAKE_DEFINE}" + skbuild_cmake_define="${SKBUILD_CMAKE_DEFINE};${skbuild_cmake_define}" +fi + +# Schema-2 trains keep generated input evidence in compact, checksum-addressed +# receipts. Download those before resolving this job's lock packages. Older +# trains retain the inline fields during the migration period. +if ! jq -e '.input_receipts | type == "object"' "${workspace}/release-train.json" >/dev/null; then + echo "release train must contain checksum-addressed input receipts; regenerate the train" >&2 + exit 1 +fi +lock_receipt_sha256="" +variant_receipt_sha256="" +for receipt_name in locks variants; do + receipt_path="$(jq -r --arg name "${receipt_name}" '.input_receipts[$name].path // empty' "${workspace}/release-train.json")" + receipt_sha256="$(jq -r --arg name "${receipt_name}" '.input_receipts[$name].sha256 // empty' "${workspace}/release-train.json")" + if [[ -z "${receipt_path}" || "${receipt_path}" == /* || "${receipt_path}" == *".."* || ! "${receipt_sha256}" =~ ^[[:xdigit:]]{64}$ ]]; then + echo "release train contains an unsafe ${receipt_name} receipt reference" >&2 + exit 1 + fi + receipt_file="${workspace}/${receipt_name}-receipt.json" + aws s3 cp "${root}/inputs/${receipt_path}" "${receipt_file}" + if [[ "$(sha256sum "${receipt_file}" | awk '{print $1}')" != "${receipt_sha256}" ]]; then + echo "stored ${receipt_name} receipt does not match the release train" >&2 + exit 1 + fi + if [[ "${receipt_name}" == "locks" ]]; then + lock_receipt_sha256="${receipt_sha256}" + else + variant_receipt_sha256="${receipt_sha256}" + fi +done +lock_source="${workspace}/locks-receipt.json" +variant_source="${workspace}/variants-receipt.json" +candidate_build_inputs="${workspace}/candidate-build-inputs.json" +jq -cnS \ + --arg lock_receipt_sha256 "${lock_receipt_sha256}" \ + --arg variant_receipt_sha256 "${variant_receipt_sha256}" \ + --argjson sccache "${candidate_sccache}" \ + '{schema_version: 1, + train_inputs: { + lock_receipt_sha256: $lock_receipt_sha256, + variant_receipt_sha256: $variant_receipt_sha256 + }, + build_policy: {sccache: $sccache}}' >"${candidate_build_inputs}" + +if [[ "${RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL}" == "true" ]]; then + mapfile -t bootstrap_records < <(jq -cer '.bootstrap_conda[]?' "${lock_source}") + bootstrap_packages="$(printf '%s\n' "${bootstrap_records[@]}" | jq -Rsc 'split("\n") | map(select(length > 0) | fromjson | .package) | sort')" + if [[ "${bootstrap_packages}" != '["rapids-build-backend","rapids-dependency-file-generator"]' ]]; then + echo "release train must contain the complete Conda bootstrap tool bundle" >&2 + exit 1 + fi + for record in "${bootstrap_records[@]}"; do + bootstrap_path="$(jq -r '.path // empty' <<<"${record}")" + bootstrap_sha256="$(jq -r '.sha256 // empty' <<<"${record}")" + bootstrap_filename="${bootstrap_path##*/}" + if [[ "${bootstrap_path}" != bootstrap-channel/noarch/* \ + || "${bootstrap_filename}" == *".."* \ + || ! "${bootstrap_filename}" =~ \.(conda|tar\.bz2)$ \ + || ! "${bootstrap_sha256}" =~ ^[[:xdigit:]]{64}$ ]]; then + echo "release train contains an unsafe Conda bootstrap package reference" >&2 + exit 1 + fi + bootstrap_destination="${channel}/noarch/${bootstrap_filename}" + aws s3 cp "${root}/inputs/${bootstrap_path}" "${bootstrap_destination}" + if [[ "$(sha256sum "${bootstrap_destination}" | awk '{print $1}')" != "${bootstrap_sha256}" ]]; then + echo "stored Conda bootstrap package does not match the release train: ${bootstrap_filename}" >&2 + exit 1 + fi + done +fi + +wheel_constraint="" +if [[ "${RELEASE_CANDIDATE_ARTIFACT_FAMILY}" == "wheel" ]]; then + wheel_lock_record="$(python3 "${script_directory}/select_lock.py" \ + --locks "${lock_source}" \ + --platform "${wheel_platform}" \ + --python-version "${RELEASE_CANDIDATE_PYTHON_VERSION}")" + wheel_lock_path="$(jq -r '.path' <<<"${wheel_lock_record}")" + wheel_lock_sha256="$(jq -r '.sha256' <<<"${wheel_lock_record}")" + wheel_constraint="${workspace}/wheel-constraints.txt" + aws s3 cp "${root}/inputs/${wheel_lock_path}" "${wheel_constraint}" + if [[ "$(sha256sum "${wheel_constraint}" | awk '{print $1}')" != "${wheel_lock_sha256}" ]]; then + echo "stored wheel lock does not match the release train" >&2 + exit 1 + fi +fi +# The scheduler binds each repository to one exact GitHub run. Candidate +# references are append-only across retries, so consuming that binding avoids +# accidentally mixing a prior failed attempt into this build's dependencies. +scheduler_state="${workspace}/scheduler-state.json" +if ! aws s3 cp "${root}/scheduler-state.json" "${scheduler_state}" 2>/dev/null; then + printf '{"workflow_runs":{}}\n' >"${scheduler_state}" +fi + +# Train records are metadata, not package payloads. Each one names canonical +# reusable release bytes plus an attempt-specific catalog/provenance envelope. +# Resolve its bundle descriptors before downloading only the catalog envelopes +# needed to choose build dependencies. +aws s3 cp "${root}/" "${manifests}" --recursive --exclude '*' --include '*/bundle-reference.*.json' +shopt -s globstar nullglob +reference_paths=("${manifests}"/**/bundle-reference.*.json) +declare -A selected_attempts=() +declare -A selected_reference_paths=() +for reference_path in "${reference_paths[@]}"; do + if ! jq -e '.schema_version == 4 and .storage_layout == 2' "${reference_path}" >/dev/null; then + echo "candidate train contains a reference that does not use storage layout 2" >&2 + exit 1 + fi + source_repository="$(jq -r '.repository // empty' "${reference_path}")" + source_artifact="$(jq -r '.source_artifact // empty' "${reference_path}")" + if [[ "${reference_path##*/}" != "bundle-reference.${source_artifact}.json" ]]; then + echo "candidate train contains a reference whose filename does not match its source artifact" >&2 + exit 1 + fi + source_run_id="$(jq -r '.source_run_id // empty' "${reference_path}")" + source_run_attempt="$(jq -r '.source_run_attempt // empty' "${reference_path}")" + repository_name="${source_repository##*/}" + selected_run_id="$(jq -r --arg repository "${repository_name}" '.workflow_runs[$repository] // empty' "${scheduler_state}")" + if [[ -n "${selected_run_id}" && "${source_run_id}" != "${selected_run_id}" ]]; then + rm -f "${reference_path}" + continue + fi + reference_identity="${source_repository}/${source_artifact}" + if [[ -n "${selected_run_id}" ]]; then + if [[ ! "${source_run_attempt}" =~ ^[0-9]+$ ]]; then + echo "candidate train has an invalid selected run attempt" >&2 + exit 1 + fi + if [[ -n "${selected_attempts[${reference_identity}]:-}" ]] \ + && (( source_run_attempt < selected_attempts[${reference_identity}] )); then + rm -f "${reference_path}" + continue + fi + if [[ -n "${selected_reference_paths[${reference_identity}]:-}" ]]; then + rm -f "${selected_reference_paths[${reference_identity}]}" + fi + selected_attempts[${reference_identity}]="${source_run_attempt}" + selected_reference_paths[${reference_identity}]="${reference_path}" + fi + artifact_key="$(jq -r '.artifact_key // empty' "${reference_path}")" + if [[ -z "${artifact_key}" || ! "${artifact_key}" =~ ^artifacts/[^/]+/[0-9a-f]{64}$ ]]; then + echo "candidate train contains an unsafe artifact reference" >&2 + exit 1 + fi + build_input_digest="$(jq -r '.build_input_digest // empty' "${reference_path}")" + if [[ -z "${build_input_digest}" || ! "${build_input_digest}" =~ ^[0-9a-f]{64}$ ]]; then + echo "candidate train contains a missing or unsafe producer build-input digest" >&2 + exit 1 + fi + bundle_key="$(jq -r '.bundle_key // empty' "${reference_path}")" + bundle_name="${bundle_key##*/}" + if [[ ! "${bundle_name}" =~ ^release-catalog-entries\.[A-Za-z0-9_-]+\.json$ ]] \ + || [[ "${bundle_key}" != "${train_key_prefix}"/* ]] \ + || [[ "${bundle_key}" == *".."* ]]; then + echo "candidate train contains an unsafe catalog bundle reference" >&2 + exit 1 + fi + manifest_path="${reference_path%/*}/${bundle_name}" + aws s3 cp "s3://${RELEASE_CANDIDATE_BUCKET}/${bundle_key}" "${manifest_path}" + jq \ + --arg artifact_key "${artifact_key}" \ + --arg build_input_digest "${build_input_digest}" \ + '. + {artifact_key: $artifact_key, build_input_digest: $build_input_digest}' \ + "${manifest_path}" >"${manifest_path}.tmp" + mv "${manifest_path}.tmp" "${manifest_path}" +done + +repository="${GITHUB_REPOSITORY##*/}" +target_unit="${RELEASE_CANDIDATE_ARTIFACT_FAMILY}:${repository}" +selected="${workspace}/selected-artifacts.json" +resolved_inputs="${workspace}/resolved-upstream-inputs.jsonl" +upstream_inputs="${workspace}/upstream-inputs.json" +: >"${resolved_inputs}" +if ! jq -e --arg target "${target_unit}" '.release_units[] | select(.id == $target and .artifact_family != "source")' \ + "${workspace}/release-train.json" >/dev/null; then + echo "release train has no candidate build unit for ${target_unit}" >&2 + exit 1 +fi + +# Shell globstar has a clear, portable meaning here and keeps the jq program +# focused on schema fields rather than filesystem traversal. +manifest_paths=("${manifests}"/**/release-catalog-entries.*.json) +dependency_count="$(jq -r --arg target "${target_unit}" ' + (.release_units | map({key: .id, value: .}) | from_entries) as $units + | def closure($id): + ($units[$id].dependencies // []) + | map(select($units[.] != null and $units[.].artifact_family != "source") | ., closure(.)) + | flatten | unique; + closure($target) | length +' "${workspace}/release-train.json")" +if [[ "${#manifest_paths[@]}" -eq 0 ]]; then + if [[ "${dependency_count}" -ne 0 ]]; then + echo "no completed candidate catalog envelopes found for ${target_unit}'s declared dependencies" >&2 + exit 1 + fi + printf '[]\n' >"${workspace}/catalog-records.json" +else + jq -s '.' "${manifest_paths[@]}" >"${workspace}/catalog-records.json" +fi + +# Package metadata determines compatibility. A producing job's matrix only +# describes how the package was made: rapids-logger, for example, can be built +# in a CUDA 13 job but has no CUDA dependency and is valid for CUDA 12. Python +# compatibility is intentionally left to the published wheel tag and pip. +jq -n \ + --slurpfile train "${workspace}/release-train.json" \ + --arg target "${target_unit}" \ + --arg family "${RELEASE_CANDIDATE_ARTIFACT_FAMILY}" \ + --arg arch "${RELEASE_CANDIDATE_ARCH}" \ + --arg cuda_major "${cuda_major}" \ + --arg python_version "${RELEASE_CANDIDATE_PYTHON_VERSION}" \ + --arg conda_platform "${conda_platform}" \ + --slurpfile records "${workspace}/catalog-records.json" \ + ' + ($train[0].release_units | map({key: .id, value: .}) | from_entries) as $units + | def closure($id): + ($units[$id].dependencies // []) + | map(select($units[.] != null and $units[.].artifact_family != "source") | ., closure(.)) + | flatten | unique; + closure($target) as $dependencies + | [ $dependencies[] | select(startswith($family + ":")) ] as $family_dependencies + | [ $records[0][] + | . as $record + | .entries[] + | select(.release_catalog_key as $key | $family_dependencies | index($key)) + # Older envelopes have no cuda_major. Treating that as compatible is a + # deliberate transition rule; new Conda envelopes set it only when the + # package declares a cuda-version dependency in info/index.json. + | select((.package.cuda_major // $cuda_major) == $cuda_major) + | select( + $family != "conda" + or (.package.platform == "noarch" or .package.platform == $conda_platform) + ) + | { + release_catalog_key: .release_catalog_key, + source: $record.source, + artifact_key: $record.artifact_key, + producer_build_input_digest: $record.build_input_digest, + path: .path, + package: .package + } + ] + | sort_by([.package.name, .package.version, if .source.matrix.python_version == $python_version then 0 else 1 end]) + | group_by([.package.name, .package.version, .package.build, .package.platform, .path]) + | map(.[0]) + ' >"${selected}" + +if [[ "$(jq 'length' "${selected}")" -eq 0 ]]; then + # A root node has no upstream candidate packages. Its own C++/Python + # intermediate artifacts are still served by the exact-name wrapper below. + echo "no upstream candidate package dependencies declared for ${target_unit}" +fi + +record_resolved_input() { + local selection="$1" + local downloaded_path="$2" + local artifact_key artifact_path release_catalog_key package sha256 + local producer_repository producer_sha producer_build_input_digest + artifact_key="$(jq -r '.artifact_key' <<<"${selection}")" + artifact_path="$(jq -r '.path' <<<"${selection}")" + release_catalog_key="$(jq -r '.release_catalog_key // empty' <<<"${selection}")" + producer_repository="$(jq -r '.source.repository // empty' <<<"${selection}")" + producer_sha="$(jq -r '.source.sha // empty' <<<"${selection}")" + producer_build_input_digest="$(jq -r '.producer_build_input_digest // empty' <<<"${selection}")" + package="$(jq -c '.package' <<<"${selection}")" + sha256="$(sha256sum "${downloaded_path}" | awk '{print $1}')" + jq -cn \ + --arg artifact_key "${artifact_key}" \ + --arg path "${artifact_path}" \ + --arg release_catalog_key "${release_catalog_key}" \ + --arg producer_repository "${producer_repository}" \ + --arg producer_sha "${producer_sha}" \ + --arg producer_build_input_digest "${producer_build_input_digest}" \ + --arg sha256 "${sha256}" \ + --argjson package "${package}" \ + '{artifact_key: $artifact_key, path: $path, release_catalog_key: $release_catalog_key, + producer: {repository: $producer_repository, sha: $producer_sha, + build_input_digest: $producer_build_input_digest}, + sha256: $sha256, package: $package}' >>"${resolved_inputs}" +} + +while IFS= read -r selection; do + artifact_key="$(jq -r '.artifact_key // empty' <<<"${selection}")" + artifact_path="$(jq -r '.path' <<<"${selection}")" + if [[ -z "${artifact_key}" || ! "${artifact_key}" =~ ^artifacts/[^/]+/[0-9a-f]{64}$ || "${artifact_path}" == /* || "${artifact_path}" == *".."* ]]; then + echo "candidate catalog contains an unsafe dependency location" >&2 + exit 1 + fi + # Canonical candidate storage preserves the producer's bundle-relative + # layout without tying dependency retrieval to a GitHub run attempt. + source="s3://${RELEASE_CANDIDATE_BUCKET}/${artifact_key}/${artifact_path}" + case "${artifact_path}" in + *.conda|*.tar.bz2) + [[ "${RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL}" == "true" ]] || continue + platform="$(jq -r '.package.platform // ""' <<<"${selection}")" + case "${platform}" in + linux-64|linux-aarch64|noarch) destination="${channel}/${platform}" ;; + *) echo "candidate Conda package has unsupported platform: ${platform}" >&2; exit 1 ;; + esac + aws s3 cp "${source}" "${destination}/$(basename "${artifact_path}")" + record_resolved_input "${selection}" "${destination}/$(basename "${artifact_path}")" + ;; + *.whl) + if ! python3 "$(dirname "$0")/select_wheel.py" \ + --python-version "${RELEASE_CANDIDATE_PYTHON_VERSION}" "$(basename "${artifact_path}")"; then + continue + fi + aws s3 cp "${source}" "${wheelhouse}/$(basename "${artifact_path}")" + record_resolved_input "${selection}" "${wheelhouse}/$(basename "${artifact_path}")" + ;; + *) echo "candidate catalog has unsupported package type: ${artifact_path}" >&2; exit 1 ;; + esac +done < <(jq -c '.[]' "${selected}") + +# The uploader uses this lock as part of the repository build-input digest. It +# records only the exact upstream package bytes materialized for this matrix, +# never their GitHub execution IDs. Sorting makes the lock independent of S3 +# listing order and avoids rebuilding when an equivalent dependency view is +# prepared again. +jq -sS \ + '{schema_version: 1, + dependencies: (unique_by([.release_catalog_key, .artifact_key, .path, .sha256]) + | sort_by(.release_catalog_key, .artifact_key, .path, .sha256))}' \ + "${resolved_inputs}" >"${upstream_inputs}" + +if [[ "${RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL}" == "true" ]]; then + if ! command -v conda >/dev/null; then + echo "conda is required to index the private candidate channel" >&2 + exit 1 + fi + conda index "${channel}" +fi + +original_download="$(command -v rapids-download-from-github || true)" +original_rattler="$(command -v rapids-rattler-channel-string || true)" +original_rattler_build="$(command -v rattler-build || true)" +original_cmake="$(command -v cmake || true)" +if [[ -z "${original_download}" ]]; then + echo "RAPIDS gha-tools must be installed before candidate dependency setup" >&2 + exit 1 +fi +if [[ "${RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL}" == "true" && -z "${original_rattler}" ]]; then + echo "rapids-rattler-channel-string is required for a candidate Conda build" >&2 + exit 1 +fi +if [[ "${RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL}" == "true" && -z "${original_rattler_build}" ]]; then + echo "rattler-build is required for a candidate Conda build" >&2 + exit 1 +fi +# A lock metapackage has no payload. Its run constraints are nevertheless part +# of the build/host solve once this package is injected into a recipe. Keeping +# it in the job-local candidate channel avoids publishing CI-only lock packages. +if [[ "${RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL}" == "true" ]]; then + lock_workspace="${workspace}/train-locks" + mkdir -p "${lock_workspace}" + + prepare_lock_metapackage() { + local section="$1" + local record relative_path expected_sha256 lock_path actual_sha256 constraints package_name recipe activation_script package_script + record="$(jq -cer \ + --arg repository "${repository}" \ + --arg platform "${conda_platform}" \ + --arg cuda "${cuda_major}" \ + --arg environment "${section}" \ + '[ (.conda // .input_locks.conda)[] + | select(.scope.repository == $repository + and .scope.platform == $platform + and .scope.cuda == $cuda + and .scope.environment == $environment) ] + | if length == 1 then .[0] + else error("release train must contain exactly one repository-scoped Conda lock") + end' \ + "${lock_source}")" + relative_path="$(jq -r '.path' <<<"${record}")" + expected_sha256="$(jq -r '.sha256' <<<"${record}")" + if [[ "${relative_path}" == /* || "${relative_path}" == *".."* || ! "${expected_sha256}" =~ ^[[:xdigit:]]{64}$ ]]; then + echo "release train contains an unsafe ${section} lock reference" >&2 + exit 1 + fi + lock_path="${lock_workspace}/${section}-constraints.txt" + # The enclosing function is called through command substitution. AWS also + # writes transfer progress to stdout, so keep it out of the captured + # package name for the later one-line GITHUB_ENV assignment. + aws s3 cp "${root}/inputs/${relative_path}" "${lock_path}" >&2 + actual_sha256="$(sha256sum "${lock_path}" | awk '{print $1}')" + if [[ "${actual_sha256}" != "${expected_sha256}" ]]; then + echo "stored ${section} lock does not match the release train" >&2 + exit 1 + fi + constraints="$(jq -Rsc 'split("\n") | map(select(length > 0))' "${lock_path}")" + package_name="rapids-release-lock-${RELEASE_CANDIDATE_TRAIN_SHA256:0:12}-${conda_platform}-cuda${cuda_major}-${section}" + recipe="${lock_workspace}/${section}-recipe.yaml" + # Rattler-Build activates build and host prefixes in an isolated shell. + # Compiler activation creates CMAKE_ARGS there, after the outer candidate + # action has finished. Put the frozen RAPIDS CMake definition in each lock + # package's late-named activation script so ordinary Conda CMake builds + # receive it as well as scikit-build-core and PATH-based invocations. + activation_script="if [[ \" \${CMAKE_ARGS:-} \" != *\" -Drapids-cmake-sha=${rapids_cmake_sha} \"* ]]; then export CMAKE_ARGS=\"\${CMAKE_ARGS:+\${CMAKE_ARGS} }-Drapids-cmake-sha=${rapids_cmake_sha}\"; fi" + package_script=$'mkdir -p "${PREFIX}/etc/conda/activate.d"\n' + package_script+="printf '%s\\n' '${activation_script}' > \"\${PREFIX}/etc/conda/activate.d/zz-rapids-release-cmake.sh\"" + jq -n \ + --arg name "${package_name}" \ + --argjson constraints "${constraints}" \ + --arg script "${package_script}" \ + '{schema_version: 1, package: {name: $name, version: "0"}, build: {number: 0, noarch: "generic", script: $script}, requirements: {run_constraints: $constraints}}' \ + >"${recipe}" + # This function is called through command substitution. Keep Rattler's + # human build log on stderr so the captured value is exactly the package + # name, which can safely be written as one GITHUB_ENV assignment. + "${original_rattler_build}" build --recipe "${recipe}" --output-dir "${channel}" --color never >&2 + printf '%s' "${package_name}" + } + + lock_build_package="$(prepare_lock_metapackage build)" + lock_host_package="$(prepare_lock_metapackage host)" + variant_record="$(jq -cer 'if has("configuration_path") then {path: .configuration_path, sha256: .configuration_sha256} else .input_variants | {path: .configuration_path, sha256: .configuration_sha256} end' "${variant_source}")" + variant_path="$(jq -r '.path' <<<"${variant_record}")" + variant_sha256="$(jq -r '.sha256' <<<"${variant_record}")" + if [[ "${variant_path}" == /* || "${variant_path}" == *".."* || ! "${variant_sha256}" =~ ^[[:xdigit:]]{64}$ ]]; then + echo "release train contains an unsafe variant configuration reference" >&2 + exit 1 + fi + variant_config="${lock_workspace}/train-variants.yaml" + aws s3 cp "${root}/inputs/${variant_path}" "${variant_config}" + if [[ "$(sha256sum "${variant_config}" | awk '{print $1}')" != "${variant_sha256}" ]]; then + echo "stored variant configuration does not match the release train" >&2 + exit 1 + fi + conda index "${channel}" +fi + +# Repository build scripts already name their same-run intermediate artifact. +# Preserve that interface, but resolve only that exact bundle from candidate S3. +# Capture the family decision now: the generated helper runs later and should +# not need a Conda-only setup variable in a wheel job. +{ + printf '%s\n' '#!/usr/bin/env bash' 'set -euo pipefail' + printf 'prepare_conda_channel=%q\n' "${RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL}" + cat <<'EOF' +artifact_name="${1:?artifact name is required}" +destination="${RAPIDS_UNZIP_DIR:-$(mktemp -d)}" +reference_root="${RELEASE_CANDIDATE_DEPENDENCY_MANIFESTS:?candidate dependency manifests are required}/${GITHUB_REPOSITORY}" +mapfile -t references < <( + find "${reference_root}" -type f -name 'bundle-reference.*.json' -print \ + | while IFS= read -r reference; do + if [[ "$(jq -r '.source_artifact // empty' "${reference}")" == "${artifact_name}" ]]; then + printf '%s\n' "${reference}" + fi + done \ + | sort +) +if [[ "${#references[@]}" -ne 1 ]]; then + echo "candidate artifact must have exactly one selected train reference: ${artifact_name}" >&2 + exit 1 +fi +reference="${references[0]}" +artifact_key="$(jq -r '.artifact_key // empty' "${reference}")" +if [[ -z "${artifact_key}" || ! "${artifact_key}" =~ ^artifacts/[^/]+/[0-9a-f]{64}$ ]]; then + echo "candidate artifact has an unsafe artifact reference: ${artifact_name}" >&2 + exit 1 +fi +source_prefix="s3://${RELEASE_CANDIDATE_BUCKET}/${artifact_key}/" +# Callers capture this helper's stdout as the downloaded directory. Keep AWS +# transfer diagnostics on stderr so they can never become part of a Conda URL. +if ! aws s3 cp "${source_prefix}" "${destination}" --recursive >&2; then + echo "candidate artifact is not available in this train: ${artifact_name}" >&2 + exit 1 +fi +if [[ "${prepare_conda_channel}" == "true" ]] \ + && find "${destination}" -type f \( -name '*.conda' -o -name '*.tar.bz2' \) -print -quit | grep -q .; then + # C++ outputs are also an input to the repository's Python build. That + # temporary same-job directory is a Conda channel too, so it needs empty + # noarch and platform indexes in addition to the downloaded package files. + mkdir -p "${destination}"/{linux-64,linux-aarch64,noarch} + conda index "${destination}" >&2 +fi +printf '%s' "${destination}" +EOF +} >"${tools}/rapids-download-from-github" +chmod +x "${tools}/rapids-download-from-github" + +# Not every release unit uses CMake. When it does, shadow `cmake` so the +# repository's existing RAPIDS.cmake bootstrap receives the train-locked +# revision. Pure-Python jobs remain independent of a CMake installation. +if [[ -n "${original_cmake}" ]]; then + cp "${script_directory}/cmake.sh" "${tools}/cmake" + chmod +x "${tools}/cmake" +fi + +printf 'RELEASE_CANDIDATE_DEPENDENCY_MANIFESTS=%s\n' "${manifests}" >>"${GITHUB_ENV}" + +if [[ "${RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL}" == "true" ]]; then +cat >"${tools}/rapids-rattler-channel-string" <<'EOF' +#!/usr/bin/env bash +RAPIDS_PREPENDED_CONDA_CHANNELS=("${RAPIDS_CANDIDATE_CONDA_CHANNEL}" "${RAPIDS_PREPENDED_CONDA_CHANNELS[@]:-}") +source "${RAPIDS_CANDIDATE_ORIGINAL_RATTLER_CHANNEL_STRING}" + +# Candidate builds must resolve RAPIDS packages only from exact local outputs +# and the train-scoped candidate channel. The standard helper adds the public +# RAPIDS channel, so remove those known defaults and reject any other external +# source that is not conda-forge. +candidate_channels=() +for ((index = 0; index < ${#RATTLER_CHANNELS[@]}; index += 2)); do + if [[ "${RATTLER_CHANNELS[${index}]}" != "--channel" || -z "${RATTLER_CHANNELS[$((index + 1))]:-}" ]]; then + echo "candidate build received an invalid Conda channel argument" >&2 + return 1 + fi + channel="${RATTLER_CHANNELS[$((index + 1))]%/}" + case "${channel}" in + rapidsai|rapidsai-nightly|nvidia|https://conda.anaconda.org/rapidsai|https://conda.anaconda.org/rapidsai-nightly|https://conda.anaconda.org/nvidia) + ;; + conda-forge|https://conda.anaconda.org/conda-forge|file://*|/*) + candidate_channels+=(--channel "${channel}") + ;; + *) + echo "candidate build rejected external Conda channel: ${channel}" >&2 + return 1 + ;; + esac +done +RATTLER_CHANNELS=("${candidate_channels[@]}") +export RATTLER_CHANNELS +EOF +chmod +x "${tools}/rapids-rattler-channel-string" +cat >"${tools}/rattler-build" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +arguments=("$@") +if [[ "${arguments[0]:-}" == "build" ]]; then + for index in "${!arguments[@]}"; do + if [[ "${arguments[${index}]}" == "--recipe" && -n "${arguments[$((index + 1))]:-}" ]]; then + arguments[$((index + 1))]="$(python3 "${RAPIDS_CANDIDATE_RECIPE_PATCHER}" \ + --recipe "${arguments[$((index + 1))]}" \ + --build-lock "${RAPIDS_CANDIDATE_BUILD_LOCK_PACKAGE}" \ + --host-lock "${RAPIDS_CANDIDATE_HOST_LOCK_PACKAGE}" \ + --rapids-cmake-sha "${RAPIDS_CANDIDATE_RAPIDS_CMAKE_SHA}")" + arguments+=(--ignore-recipe-variants --variant-config "${RAPIDS_CANDIDATE_RATTLER_VARIANT_CONFIG}") + break + fi + done +fi +exec "${RAPIDS_CANDIDATE_ORIGINAL_RATTLER_BUILD}" "${arguments[@]}" +EOF +chmod +x "${tools}/rattler-build" +fi + +{ + printf 'RELEASE_CANDIDATE_BUCKET=%s\n' "${RELEASE_CANDIDATE_BUCKET}" + printf 'RELEASE_CANDIDATE_TRAIN_PREFIX=%s\n' "${RELEASE_CANDIDATE_TRAIN_PREFIX}" + printf 'RELEASE_CANDIDATE_TRAIN_SHA256=%s\n' "${RELEASE_CANDIDATE_TRAIN_SHA256}" + printf 'RELEASE_CANDIDATE_BUILD_IMPLEMENTATION_REVISIONS=%s\n' "${build_implementation_revisions}" + printf 'RELEASE_CANDIDATE_GHA_TOOLS_REVISION=%s\n' "${actual_gha_tools_revision}" + printf 'RELEASE_CANDIDATE_UPSTREAM_INPUTS=%s\n' "${upstream_inputs}" + printf 'RELEASE_CANDIDATE_BUILD_INPUTS=%s\n' "${candidate_build_inputs}" + printf 'RAPIDS_RELEASE_CANDIDATE_SCCACHE=%s\n' "${candidate_sccache}" + # Rattler installs CMake in its isolated prefixes, so this train value must + # be available to the recipe patcher even when CMake is absent from the + # outer workflow container at setup time. + printf 'RAPIDS_CANDIDATE_RAPIDS_CMAKE_SHA=%s\n' "${rapids_cmake_sha}" + printf 'SKBUILD_CMAKE_DEFINE=%s\n' "${skbuild_cmake_define}" + if [[ -n "${original_cmake}" ]]; then + printf 'RAPIDS_CANDIDATE_ORIGINAL_CMAKE=%s\n' "${original_cmake}" + fi + if [[ "${RELEASE_CANDIDATE_PREPARE_CONDA_CHANNEL}" == "true" ]]; then + printf 'RAPIDS_CANDIDATE_CONDA_CHANNEL=%s\n' "${channel}" + printf 'RAPIDS_CANDIDATE_ORIGINAL_RATTLER_CHANNEL_STRING=%s\n' "${original_rattler}" + printf 'RAPIDS_CANDIDATE_ORIGINAL_RATTLER_BUILD=%s\n' "${original_rattler_build}" + printf 'RAPIDS_CANDIDATE_RECIPE_PATCHER=%s\n' "${IMPLEMENTATION_PATH}/patch_recipe.py" + printf 'RAPIDS_CANDIDATE_BUILD_LOCK_PACKAGE=%s\n' "${lock_build_package}" + printf 'RAPIDS_CANDIDATE_HOST_LOCK_PACKAGE=%s\n' "${lock_host_package}" + printf 'RAPIDS_CANDIDATE_RATTLER_VARIANT_CONFIG=%s\n' "${variant_config}" + fi + if [[ -n "${wheel_constraint}" ]]; then + printf 'PIP_CONSTRAINT=%s\n' "${wheel_constraint}" + fi + printf 'PIP_FIND_LINKS=%s\n' "${wheelhouse}${PIP_FIND_LINKS:+ ${PIP_FIND_LINKS}}" +} >>"${GITHUB_ENV}" +printf '%s\n' "${gha_tools_directory}/tools" >>"${GITHUB_PATH}" +printf '%s\n' "${tools}" >>"${GITHUB_PATH}" diff --git a/release-candidate-docs/action.yml b/release-candidate-docs/action.yml new file mode 100644 index 00000000..b3fdf290 --- /dev/null +++ b/release-candidate-docs/action.yml @@ -0,0 +1,45 @@ +name: Stage release-candidate documentation +description: Replace public documentation publication with train-scoped private S3 staging. + +inputs: + candidate-train-sha256: + description: Canonical SHA-256 of the immutable release train. + required: true + candidate-bucket: + description: Private S3 bucket holding candidate outputs. + required: false + default: rapids-release-candidates + candidate-train-prefix: + description: Root prefix for immutable candidate train records and documentation. + required: false + default: train-state + +runs: + using: composite + steps: + - id: implementation + name: Resolve candidate documentation implementation + shell: bash + env: + ACTION_PATH: ${{ github.action_path }} + HOST_WORKSPACE: ${{ github.workspace }} + run: | + host_root="$(dirname "$(dirname "${HOST_WORKSPACE}")")" + container_root="$(dirname "$(dirname "${PWD}")")" + action_path="${ACTION_PATH}" + if [[ ! -d "${action_path}" && "${action_path}" == "${host_root}/"* ]]; then + action_path="${container_root}/${action_path#"${host_root}/"}" + fi + if [[ ! -f "${action_path}/stage.sh" ]]; then + echo "release-candidate documentation implementation not found at ${action_path}" >&2 + exit 1 + fi + echo "path=${action_path}" >>"${GITHUB_OUTPUT}" + - name: Replace public docs uploader with candidate staging + shell: bash + env: + RELEASE_CANDIDATE_BUCKET: ${{ inputs.candidate-bucket }} + RELEASE_CANDIDATE_TRAIN_PREFIX: ${{ inputs.candidate-train-prefix }} + RELEASE_CANDIDATE_TRAIN_SHA256: ${{ inputs.candidate-train-sha256 }} + IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} + run: bash "${IMPLEMENTATION_PATH}/stage.sh" diff --git a/release-candidate-docs/stage.sh b/release-candidate-docs/stage.sh new file mode 100644 index 00000000..3413a19a --- /dev/null +++ b/release-candidate-docs/stage.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +# Install a private candidate replacement for rapids-upload-docs. +# +# Repository docs scripts retain their normal interface; the custom job calls +# this only after candidate dependency setup has selected the private train. +set -euo pipefail + +for value in GITHUB_PATH GITHUB_REPOSITORY GITHUB_RUN_ID RELEASE_CANDIDATE_BUCKET RELEASE_CANDIDATE_TRAIN_PREFIX RELEASE_CANDIDATE_TRAIN_SHA256 RUNNER_TEMP; do + if [[ -z "${!value:-}" ]]; then + echo "${value} must be set for release-candidate documentation staging" >&2 + exit 1 + fi +done + +tools="${RUNNER_TEMP}/release-candidate-tools" +mkdir -p "${tools}" + +{ + printf '%s\n' '#!/usr/bin/env bash' 'set -euo pipefail' + printf 'candidate_bucket=%q\n' "${RELEASE_CANDIDATE_BUCKET}" + printf 'candidate_train_prefix=%q\n' "${RELEASE_CANDIDATE_TRAIN_PREFIX}" + printf 'candidate_train_sha256=%q\n' "${RELEASE_CANDIDATE_TRAIN_SHA256}" + cat <<'EOF' +docs_directory="${RAPIDS_DOCS_DIR:?RAPIDS_DOCS_DIR must name the generated documentation}" +docs_version="${RAPIDS_VERSION_NUMBER:?RAPIDS_VERSION_NUMBER must name the staged documentation version}" +destination="s3://${candidate_bucket}/${candidate_train_prefix}/${candidate_train_sha256}/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}/docs/${docs_version}/" +aws s3 sync "${docs_directory}/" "${destination}" --no-progress +printf 'Staged release-candidate documentation: %s\n' "${destination}" +EOF +} >"${tools}/rapids-upload-docs" + +chmod +x "${tools}/rapids-upload-docs" +printf '%s\n' "${tools}" >>"${GITHUB_PATH}" diff --git a/release-catalog-dispatch/action.yml b/release-catalog-dispatch/action.yml new file mode 100644 index 00000000..aef2a9ab --- /dev/null +++ b/release-catalog-dispatch/action.yml @@ -0,0 +1,100 @@ +name: Dispatch release catalog +description: Write a release-candidate catalog bundle directly to the private S3 store. + +inputs: + config: + description: >- + JSON string with configuration for this action. Schema and field documentation: + https://github.com/rapidsai/shared-actions/blob/main/release-catalog/config.schema.json + required: true + source-artifact-name: + description: Stable name of this producing artifact bundle. + required: true + candidate-train-sha256: + description: Canonical SHA-256 of the immutable release-train JSON. + required: true + candidate-bucket: + description: Private S3 bucket that holds candidate-build bytes. + required: false + default: rapids-release-candidates + candidate-content-prefix: + description: Root prefix for reusable, immutable candidate build bundles. + required: false + default: artifacts + candidate-train-prefix: + description: Root prefix for immutable train membership records. + required: false + default: train-state + upload-to-s3: + description: Whether to upload the generated bundle to the candidate store. + required: false + default: "false" + candidate-matrix-arch: + description: Canonical RAPIDS architecture for the producing job. + required: false + default: "" + candidate-matrix-cuda-version: + description: CUDA version for the producing job. + required: false + default: "" + candidate-matrix-python-version: + description: Python version for the producing job. + required: false + default: "" +runs: + using: composite + steps: + - id: implementation + name: Resolve release catalog implementation + shell: bash + env: + ACTION_PATH: ${{ github.action_path }} + HOST_WORKSPACE: ${{ github.workspace }} + run: | + host_root="$(dirname "$(dirname "${HOST_WORKSPACE}")")" + container_root="$(dirname "$(dirname "${PWD}")")" + action_path="${ACTION_PATH}" + if [[ ! -d "${action_path}" && "${action_path}" == "${host_root}/"* ]]; then + action_path="${container_root}/${action_path#"${host_root}/"}" + fi + implementation_path="$(dirname "${action_path}")/release-catalog" + if [[ ! -d "${implementation_path}" ]]; then + echo "release-catalog implementation not found at ${implementation_path}" >&2 + exit 1 + fi + echo "path=${implementation_path}" >>"${GITHUB_OUTPUT}" + - id: configuration + name: Validate release catalog configuration + shell: bash + env: + IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} + RELEASE_CATALOG_CONFIG: ${{ inputs.config }} + run: "${IMPLEMENTATION_PATH}/validate-config.sh" + - name: Materialize release catalog entries + shell: bash + env: + IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} + RELEASE_ARTIFACTS: ${{ steps.configuration.outputs.artifacts }} + RELEASE_ENTRIES_NAME: release-catalog-entries.${{ inputs.source-artifact-name }}.json + RELEASE_ARTIFACT_DIRECTORY: ${{ steps.configuration.outputs.artifact_directory }} + RELEASE_SOURCE_SHA: ${{ env.RAPIDS_SHA }} + RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} + RELEASE_CATALOG_KEY: ${{ steps.configuration.outputs.release_catalog_key }} + RELEASE_CANDIDATE_MATRIX_ARCH: ${{ inputs.candidate-matrix-arch }} + RELEASE_CANDIDATE_MATRIX_CUDA_VERSION: ${{ inputs.candidate-matrix-cuda-version }} + RELEASE_CANDIDATE_MATRIX_PYTHON_VERSION: ${{ inputs.candidate-matrix-python-version }} + run: "${IMPLEMENTATION_PATH}/materialize.sh" + - name: Write release-candidate bundle to S3 + if: ${{ inputs.upload-to-s3 == 'true' }} + shell: bash + env: + IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} + RELEASE_ARTIFACT_DIRECTORY: ${{ steps.configuration.outputs.artifact_directory }} + RELEASE_CANDIDATE_BUCKET: ${{ inputs.candidate-bucket }} + RELEASE_CANDIDATE_CONTENT_PREFIX: ${{ inputs.candidate-content-prefix }} + RELEASE_CANDIDATE_TRAIN_PREFIX: ${{ inputs.candidate-train-prefix }} + RELEASE_CANDIDATE_TRAIN_SHA256: ${{ inputs.candidate-train-sha256 }} + RELEASE_CANDIDATE_CATALOG_SHARED_ACTIONS_REPOSITORY: ${{ github.action_repository }} + RELEASE_CANDIDATE_CATALOG_SHARED_ACTIONS_REVISION: ${{ github.action_ref }} + RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} + run: "${IMPLEMENTATION_PATH}/upload-candidate-s3.sh" diff --git a/release-catalog/README.md b/release-catalog/README.md index 16a5ffa4..5b799779 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -95,6 +95,12 @@ dependency SBOM will add. That will be a new `sbom_kind` value and a new The action is used via `rapidsai/shared-actions/release-catalog`. +Candidate builds use the separate `release-catalog-dispatch` action. It names +each job's envelope `release-catalog-entries..json` and uploads +through `upload-candidate-s3.sh`, which implements immutable content bundles and +train membership records. The standard `release-catalog` action retains its +existing envelope name and `upload-s3.sh` upload contract. + Configuration, materialization, and upload failures appear as GitHub Actions error annotations. When the scripts run outside GitHub Actions, the same errors are written to stderr with the responsible script name. @@ -217,3 +223,50 @@ provenance without an SBOM cannot be matched to a package by name and version. Although CycloneDX can carry some build metadata, embedding provenance in the SBOM would not replace the in-toto/SLSA statement and would couple evidence with different consumers and lifecycles. + +## Prerequisite state + +The action requires `RAPIDS_SHA` in the job environment. It must identify the +repository revision actually checked out and built: + +- Conda build workflows set `RAPIDS_SHA` to `git rev-parse HEAD` immediately + after checkout. +- Wheel and custom workflows use `rapids-github-info`; it uses `inputs.sha` + when supplied and otherwise sets `RAPIDS_SHA` to `git rev-parse HEAD`. + +This distinction matters when a reusable workflow checks out a repository or +revision different from the workflow event. Direct callers must likewise set +`RAPIDS_SHA` to the checked-out commit instead of assuming `${{ github.sha }}` +names the built source. Variables written to `GITHUB_ENV` are available to the +action when it runs as a subsequent job step. + +## Candidate build reuse + +For a release-candidate build, `release-candidate-dependencies` writes +`RELEASE_CANDIDATE_UPSTREAM_INPUTS` to `GITHUB_ENV`. It is a sorted lock of the +exact upstream Conda and wheel files actually materialized for the current job +matrix. Each entry contains the file SHA-256 plus the producing RAPIDS +repository SHA and build-input digest. `release-catalog` incorporates that lock +into `build-record.json`. The same setup action writes +`RELEASE_CANDIDATE_BUILD_INPUTS`, containing the checksum of the train lock +receipt, the checksum of the train variant receipt, and the train's boolean +`sccache` policy. The record also includes the frozen `shared-workflows`, +`shared-actions`, and `gha-tools` revisions and any build timestamp supplied to +the package build. The timestamp is included because Conda packages and wheels +can embed it in their bytes; workflows that do not use one record an empty, +stable value. The record's SHA-256 determines the canonical +artifact location. Therefore an unchanged build can be reused +across candidate trains and GitHub retries, while a downstream build is +necessarily distinct when an upstream package byte, selected upstream build, or +shared build implementation changes. Lock, variant, or cache-policy changes +also create a different canonical artifact identity, so an `sccache`-assisted +build cannot be reused as a clean build. The locks intentionally omit GitHub +run and attempt IDs. + +Before a candidate job can use this mechanism, its dependency setup compares +the running reusable workflow SHA and composite-action SHA with the train's +`shared-workflows` and `shared-actions` entries. It also installs `gha-tools` +at the train's exact SHA and makes that checkout the job's tool path. The +catalog uploader repeats the `shared-actions` and `gha-tools` comparison before +writing S3 content. A mismatch is a build error, not an informational warning: +one train must not combine artifacts produced by different shared build logic. diff --git a/release-catalog/entries.schema.json b/release-catalog/entries.schema.json index c51669d9..520c9d19 100644 --- a/release-catalog/entries.schema.json +++ b/release-catalog/entries.schema.json @@ -37,7 +37,16 @@ "sha": {"type": "string", "pattern": "^(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})$"}, "workflow_ref": {"type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$"}, "run_id": {"type": "string", "pattern": "^[1-9][0-9]*$"}, - "run_attempt": {"type": "string", "pattern": "^[1-9][0-9]*$"} + "run_attempt": {"type": "string", "pattern": "^[1-9][0-9]*$"}, + "matrix": { + "type": "object", "additionalProperties": false, + "required": ["arch", "cuda_version", "python_version"], + "properties": { + "arch": {"type": "string"}, + "cuda_version": {"type": "string"}, + "python_version": {"type": "string"} + } + } } }, "package": { @@ -50,7 +59,8 @@ "name": {"type": "string", "minLength": 1}, "version": {"type": "string", "minLength": 1}, "build": {"type": "string", "minLength": 1}, - "platform": {"type": "string", "minLength": 1} + "platform": {"type": "string", "minLength": 1}, + "cuda_major": {"type": "string", "pattern": "^[0-9]+$"} } }, "entry": { diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh index 7eb0baad..2c9df33c 100755 --- a/release-catalog/materialize.sh +++ b/release-catalog/materialize.sh @@ -146,8 +146,20 @@ describe_wheel_package() { exit 1 fi - jq -cn --arg name "${package_name}" --arg version "${package_version}" \ - '{ecosystem: "wheel", name: $name, version: $version}' + # RAPIDS CUDA wheels use a distribution suffix such as rmm-cu12. Keep that + # artifact-level compatibility fact in the catalog; the producing job's + # matrix is not a reliable substitute for it. + local cuda_major="" + if [[ "${package_name}" =~ -cu([0-9]+)$ ]]; then + cuda_major="${BASH_REMATCH[1]}" + fi + if [[ -n "${cuda_major}" ]]; then + jq -cn --arg name "${package_name}" --arg version "${package_version}" --arg cuda_major "${cuda_major}" \ + '{ecosystem: "wheel", name: $name, version: $version, cuda_major: $cuda_major}' + else + jq -cn --arg name "${package_name}" --arg version "${package_version}" \ + '{ecosystem: "wheel", name: $name, version: $version}' + fi } # Maven-built JARs normally expose their coordinates in exactly one @@ -227,7 +239,29 @@ describe_conda_package() { exit 1 fi - jq -c '{ecosystem: "conda", name, version, build, platform: .subdir}' <<<"${index_json}" + # The embedded package dependency is the compatibility contract. The build + # matrix is only how this package was produced; it must not prevent a + # CUDA-independent package from serving another CUDA build in the train. + local cuda_major + cuda_major="$(jq -r ' + [ + .depends[]? + | select(type == "string" and startswith("cuda-version")) + | capture("(?[0-9]+)").major + ] + | unique + | if length == 0 then "" + elif length == 1 then .[0] + else error("Conda package has conflicting cuda-version dependencies") + end + ' <<<"${index_json}")" + if [[ -n "${cuda_major}" ]]; then + jq -c --arg cuda_major "${cuda_major}" \ + '{ecosystem: "conda", name, version, build, platform: .subdir, cuda_major: $cuda_major}' \ + <<<"${index_json}" + else + jq -c '{ecosystem: "conda", name, version, build, platform: .subdir}' <<<"${index_json}" + fi } prepare_artifacts() { @@ -346,7 +380,12 @@ prepare_artifacts() { # package identity or a caller-created package identity file. artifacts="$(prepare_artifacts)" # Fixed name: upload-s3.sh, the schema, and consumers all depend on it. -entries_path="${artifact_directory}/release-catalog-entries.json" +entries_name="${RELEASE_ENTRIES_NAME:-release-catalog-entries.json}" +if [[ "$entries_name" == */* || "$entries_name" == *..* ]]; then + release_catalog_error "release entry document must be a plain filename" + exit 1 +fi +entries_path="${artifact_directory}/${entries_name}" temporary_manifest="$(mktemp "${artifact_directory}/.release-catalog.XXXXXX")" trap 'rm -f "${temporary_manifest}"' EXIT @@ -355,12 +394,13 @@ printf '%s\n' '{"entries":[]}' >"${temporary_manifest}" validate_package_identity() { jq -e ' type == "object" - and (keys - ["ecosystem", "name", "version", "build", "platform"] | length == 0) + and (keys - ["ecosystem", "name", "version", "build", "platform", "cuda_major"] | length == 0) and (.ecosystem | type == "string" and length > 0) and (.name | type == "string" and length > 0) and (.version | type == "string" and length > 0) and ((has("build") | not) or (.build | type == "string" and length > 0)) and ((has("platform") | not) or (.platform | type == "string" and length > 0)) + and ((has("cuda_major") | not) or (.cuda_major | type == "string" and test("^[0-9]+$"))) ' >/dev/null } @@ -518,6 +558,9 @@ jq -n -S \ --arg run_id "${github_run_id}" \ --arg sha "${source_sha}" \ --arg workflow_ref "${github_workflow_ref}" \ + --arg matrix_arch "${RELEASE_CANDIDATE_MATRIX_ARCH:-}" \ + --arg matrix_cuda_version "${RELEASE_CANDIDATE_MATRIX_CUDA_VERSION:-}" \ + --arg matrix_python_version "${RELEASE_CANDIDATE_MATRIX_PYTHON_VERSION:-}" \ --argjson entries "$(jq -c '.entries' "${temporary_manifest}")" \ '{ schema_version: 1, @@ -528,7 +571,12 @@ jq -n -S \ sha: $sha, workflow_ref: $workflow_ref, run_id: $run_id, - run_attempt: $run_attempt + run_attempt: $run_attempt, + matrix: { + arch: $matrix_arch, + cuda_version: $matrix_cuda_version, + python_version: $matrix_python_version + } }, entries: $entries - }' >"${entries_path}" + } | if $matrix_arch == "" and $matrix_cuda_version == "" and $matrix_python_version == "" then del(.source.matrix) else . end' >"${entries_path}" diff --git a/release-catalog/upload-candidate-s3.sh b/release-catalog/upload-candidate-s3.sh new file mode 100755 index 00000000..b689bb49 --- /dev/null +++ b/release-catalog/upload-candidate-s3.sh @@ -0,0 +1,340 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +# Upload only the files declared by the generated catalog document. GitHub +# Actions retains logs and this receipt, but never a second release-artifact +# copy. Conditional S3 writes make reruns safe: an existing object is accepted +# only when its checksum exactly matches the byte about to be uploaded. +set -euo pipefail + +require_value() { + local name="$1" + local value="$2" + if [[ -z "${value}" || "${value}" == *$'\n'* || "${value}" == *$'\r'* ]]; then + echo "${name} must be a non-empty single-line string" >&2 + exit 1 + fi +} + +safe_prefix() { + local prefix="$1" + [[ "${prefix}" != /* && "${prefix}" != */ && "${prefix}" != *'//' && "${prefix}" != *'..'* ]] +} + +for value in RELEASE_ARTIFACT_DIRECTORY RELEASE_CANDIDATE_BUCKET RELEASE_CANDIDATE_CONTENT_PREFIX RELEASE_CANDIDATE_TRAIN_PREFIX RELEASE_CANDIDATE_TRAIN_SHA256 RELEASE_CANDIDATE_BUILD_IMPLEMENTATION_REVISIONS RELEASE_CANDIDATE_GHA_TOOLS_REVISION RELEASE_CANDIDATE_CATALOG_SHARED_ACTIONS_REPOSITORY RELEASE_CANDIDATE_CATALOG_SHARED_ACTIONS_REVISION RELEASE_CANDIDATE_BUILD_INPUTS RELEASE_SOURCE_ARTIFACT_NAME GITHUB_REPOSITORY; do + require_value "${value}" "${!value:-}" +done +if [[ ! "${GITHUB_REPOSITORY}" =~ ^[^/]+/[^/]+$ ]]; then + echo "GITHUB_REPOSITORY must use owner/repository format" >&2 + exit 1 +fi +repository_name="${GITHUB_REPOSITORY##*/}" +if [[ -n "${RAPIDS_DATETIME_STRING:-}" && ! "${RAPIDS_DATETIME_STRING}" =~ ^[0-9]{12}$ ]]; then + echo "RAPIDS_DATETIME_STRING must use YYMMDDhhmmss format" >&2 + exit 1 +fi +if [[ ! "${RELEASE_CANDIDATE_TRAIN_SHA256}" =~ ^[[:xdigit:]]{64}$ ]]; then + echo "RELEASE_CANDIDATE_TRAIN_SHA256 must be a SHA-256 hex digest" >&2 + exit 1 +fi +if [[ "${RELEASE_CANDIDATE_CATALOG_SHARED_ACTIONS_REPOSITORY}" != "rapidsai/shared-actions" ]]; then + echo "release catalog action must execute from rapidsai/shared-actions" >&2 + exit 1 +fi +build_implementation_revisions="$(jq -ceS ' + . as $revisions + | if ($revisions | type) != "object" then error("build implementation revisions must be an object") + elif (["gha-tools", "shared-actions", "shared-workflows"] | all(.[]; . as $name | $revisions | has($name))) | not + then error("missing required build implementation revision") + elif ($revisions | all(to_entries[]; (.key | type) == "string" and (.value | type) == "string" and (.value | test("^[0-9a-f]{40}$")))) + then $revisions + else error("invalid build implementation revision") + end +' <<<"${RELEASE_CANDIDATE_BUILD_IMPLEMENTATION_REVISIONS}")" || { + echo "RELEASE_CANDIDATE_BUILD_IMPLEMENTATION_REVISIONS is invalid" >&2 + exit 1 +} +if [[ "${RELEASE_CANDIDATE_CATALOG_SHARED_ACTIONS_REVISION}" != "$(jq -r '."shared-actions"' <<<"${build_implementation_revisions}")" ]]; then + echo "release catalog shared-actions revision does not match the frozen release train" >&2 + exit 1 +fi +if [[ "${RELEASE_CANDIDATE_GHA_TOOLS_REVISION}" != "$(jq -r '."gha-tools"' <<<"${build_implementation_revisions}")" ]]; then + echo "gha-tools revision does not match the frozen release train" >&2 + exit 1 +fi +if [[ ! "${RELEASE_SOURCE_ARTIFACT_NAME}" =~ ^[A-Za-z0-9_-]+$ ]]; then + echo "RELEASE_SOURCE_ARTIFACT_NAME must contain only letters, numbers, underscores, and hyphens" >&2 + exit 1 +fi +if ! safe_prefix "${RELEASE_CANDIDATE_CONTENT_PREFIX}" || ! safe_prefix "${RELEASE_CANDIDATE_TRAIN_PREFIX}"; then + echo "candidate content and train prefixes must be safe, relative S3 prefixes" >&2 + exit 1 +fi +if [[ ! -f "${RELEASE_CANDIDATE_BUILD_INPUTS}" ]]; then + echo "RELEASE_CANDIDATE_BUILD_INPUTS does not name a readable candidate input record" >&2 + exit 1 +fi +candidate_build_inputs="$(jq -ceS ' + if .schema_version != 1 + or ((.train_inputs | type) != "object") + or ((.train_inputs.lock_receipt_sha256 | type) != "string") + or ((.train_inputs.lock_receipt_sha256 | test("^[0-9a-f]{64}$")) | not) + or ((.train_inputs.variant_receipt_sha256 | type) != "string") + or ((.train_inputs.variant_receipt_sha256 | test("^[0-9a-f]{64}$")) | not) + or ((.build_policy | type) != "object") + or ((.build_policy.sccache | type) != "boolean") + then error("candidate build inputs must contain lock, variant, and sccache identities") + else { + schema_version: 1, + train_inputs: { + lock_receipt_sha256: .train_inputs.lock_receipt_sha256, + variant_receipt_sha256: .train_inputs.variant_receipt_sha256 + }, + build_policy: {sccache: .build_policy.sccache} + } + end +' "${RELEASE_CANDIDATE_BUILD_INPUTS}")" || { + echo "RELEASE_CANDIDATE_BUILD_INPUTS is invalid" >&2 + exit 1 +} + +bundle_root="$(realpath "${RELEASE_ARTIFACT_DIRECTORY}")" +entries_name="release-catalog-entries.${RELEASE_SOURCE_ARTIFACT_NAME}.json" +entries_path="${bundle_root}/${entries_name}" +if [[ ! -f "${entries_path}" ]]; then + echo "release catalog entries are missing: ${entries_path}" >&2 + exit 1 +fi + +# Catalog paths are relative to the build bundle. Reject traversal before an +# S3 key is constructed, even though materialize.sh already validates them. +safe_relative_path() { + local path="$1" + [[ -n "${path}" && "${path}" != /* && "${path}" != ../* && "${path}" != */../* && "${path}" != *'/..' ]] +} + +# A reusable repository build is keyed only by its declared inputs. Package +# bytes can embed the CI timestamp, so that timestamp is an explicit input. +# The full generated catalog remains an output record and must not determine +# this key. +# A digest names canonical, reusable release bytes. Execution-specific catalog +# context and provenance stay with the train attempt instead of duplicating +# payloads below the canonical artifact namespace. +# The dependency action writes this lock after resolving only the upstream +# package bytes used by the current matrix. Keep run/attempt metadata out of +# it: a downstream rebuild is required when an actual input changes, not when +# an equivalent upstream job is retried. +upstream_inputs_path="${RELEASE_CANDIDATE_UPSTREAM_INPUTS:-}" +if [[ -z "${upstream_inputs_path}" ]]; then + upstream_dependencies='[]' +elif [[ ! -f "${upstream_inputs_path}" ]]; then + echo "RELEASE_CANDIDATE_UPSTREAM_INPUTS does not name a readable input lock" >&2 + exit 1 +else + upstream_dependencies="$(jq -ceS ' + if .schema_version != 1 or (.dependencies | type) != "array" then + error("upstream input lock must contain schema_version 1 and dependencies") + else + [.dependencies[] + | if (((.artifact_key | type) == "string") + and ((.path | type) == "string") + and ((.release_catalog_key | type) == "string") + and ((.sha256 | type) == "string") + and ((.sha256 | test("^[0-9a-f]{64}$"))) + and ((.producer | type) == "object") + and ((.producer.repository | type) == "string") + and ((.producer.sha | type) == "string") + and ((.producer.sha | test("^[0-9a-f]{40}$"))) + and ((.producer.build_input_digest | type) == "string") + and ((.producer.build_input_digest | test("^[0-9a-f]{64}$"))) + and ((.package | type) == "object")) + then {artifact_key, path, release_catalog_key, producer, sha256, package} + else error("upstream dependency has invalid identity") + end] + | unique_by([.release_catalog_key, .artifact_key, .path, .sha256]) + | sort_by(.release_catalog_key, .artifact_key, .path, .sha256) + end + ' "${upstream_inputs_path}")" || { + echo "RELEASE_CANDIDATE_UPSTREAM_INPUTS is invalid" >&2 + exit 1 + } +fi + +build_record_path="$(mktemp)" +jq -cS --argjson upstream_dependencies "${upstream_dependencies}" \ + --argjson build_implementation_revisions "${build_implementation_revisions}" \ + --argjson candidate_build_inputs "${candidate_build_inputs}" \ + --arg build_datetime "${RAPIDS_DATETIME_STRING:-}" ' + { + schema_version: 5, + storage_layout: 2, + producer, + source: (.source | {artifact, repository, sha, workflow_ref, matrix}), + build_datetime: $build_datetime, + build_implementation_revisions: $build_implementation_revisions, + candidate_build_inputs: $candidate_build_inputs, + upstream_dependencies: $upstream_dependencies, + packages: [ + .entries[] + | { + release_catalog_key, + package: (.package | {ecosystem, name, version, platform}) + } + ] | sort_by(.release_catalog_key, .package.ecosystem, .package.name, .package.version, .package.platform) + } +' "${entries_path}" >"${build_record_path}" +build_input_digest="$(sha256sum "${build_record_path}" | awk '{print $1}')" +attempt_id="${GITHUB_RUN_ID:-}.${GITHUB_RUN_ATTEMPT:-}" +if [[ ! "${attempt_id}" =~ ^[0-9]+\.[0-9]+$ ]]; then + echo "GITHUB_RUN_ID and GITHUB_RUN_ATTEMPT must be numeric for a candidate upload" >&2 + exit 1 +fi +artifact_key="${RELEASE_CANDIDATE_CONTENT_PREFIX}/${repository_name}/${build_input_digest}" +build_record_key="${artifact_key}/build-record.json" +artifact_index_key="${artifact_key}/artifact-index.json" +train_bundle_key="${RELEASE_CANDIDATE_TRAIN_PREFIX}/${RELEASE_CANDIDATE_TRAIN_SHA256}/${repository_name}/${attempt_id}/${build_input_digest}" +bundle_key="${train_bundle_key}/${entries_name}" +train_key="${train_bundle_key}/bundle-reference.${RELEASE_SOURCE_ARTIFACT_NAME}.json" +# Primary package bytes and deterministic SBOMs are canonical release content. +# Generated provenance identifies a specific GitHub execution, so it remains +# under the train attempt alongside the execution catalog. A signature is kept +# there as well until its determinism contract is defined. +mapfile -t artifact_paths < <(jq -r '([.entries[] | .path, .sbom | strings] | unique) | .[]' "${entries_path}") +mapfile -t execution_paths < <(jq -r '([.entries[] | .provenance, .signature? | strings] | unique) | .[]' "${entries_path}") +head_metadata="$(mktemp)" +artifact_index_entries="$(mktemp)" +artifact_index_path="$(mktemp)" +trap 'rm -f "${head_metadata}" "${build_record_path}" "${artifact_index_entries}" "${artifact_index_path}"' EXIT + +upload_immutable() { + local local_path="$1" + local object_key="$2" + local checksum + checksum="$(openssl dgst -sha256 -binary "${local_path}" | base64)" + + if aws s3api head-object --bucket "${RELEASE_CANDIDATE_BUCKET}" --key "${object_key}" --checksum-mode ENABLED >"${head_metadata}" 2>/dev/null; then + if [[ "$(jq -r '.ChecksumSHA256 // empty' "${head_metadata}")" != "${checksum}" ]]; then + echo "existing candidate object has different bytes: ${object_key}" >&2 + exit 1 + fi + return + fi + + if ! aws s3api put-object --bucket "${RELEASE_CANDIDATE_BUCKET}" --key "${object_key}" --body "${local_path}" --checksum-algorithm SHA256 --checksum-sha256 "${checksum}" --if-none-match '*' --tagging 'release-candidate-status=candidate' >/dev/null 2>&1; then + aws s3api head-object --bucket "${RELEASE_CANDIDATE_BUCKET}" --key "${object_key}" --checksum-mode ENABLED >"${head_metadata}" + if [[ "$(jq -r '.ChecksumSHA256 // empty' "${head_metadata}")" != "${checksum}" ]]; then + echo "candidate object could not be written immutably: ${object_key}" >&2 + exit 1 + fi + fi +} + +upload_artifact() { + local relative_path="$1" + local local_path="${bundle_root}/${relative_path}" + upload_immutable "${local_path}" "${artifact_key}/${relative_path}" +} + +upload_execution() { + local relative_path="$1" + local local_path="${bundle_root}/${relative_path}" + upload_immutable "${local_path}" "${train_bundle_key}/${relative_path}" +} + +for relative_path in "${artifact_paths[@]}"; do + if ! safe_relative_path "${relative_path}" || [[ ! -f "${bundle_root}/${relative_path}" ]]; then + echo "catalog declares a missing or unsafe candidate file: ${relative_path}" >&2 + exit 1 + fi +done +for relative_path in "${execution_paths[@]}"; do + if ! safe_relative_path "${relative_path}" || [[ ! -f "${bundle_root}/${relative_path}" ]]; then + echo "catalog declares a missing or unsafe candidate file: ${relative_path}" >&2 + exit 1 + fi +done + +# This is the stable artifact receipt: it identifies the canonical package +# bytes without carrying GitHub run/attempt data. A retry with different bytes +# for these declared inputs must fail the immutable write rather than create a +# second implicit release candidate. +while IFS= read -r entry; do + primary_path="$(jq -r '.path' <<<"${entry}")" + package="$(jq -c '.package' <<<"${entry}")" + primary_sha256="$(sha256sum "${bundle_root}/${primary_path}" | awk '{print $1}')" + sbom_path="$(jq -r '.sbom // empty' <<<"${entry}")" + if [[ -n "${sbom_path}" ]]; then + sbom_sha256="$(sha256sum "${bundle_root}/${sbom_path}" | awk '{print $1}')" + else + sbom_sha256="" + fi + jq -cn \ + --arg release_catalog_key "$(jq -r '.release_catalog_key' <<<"${entry}")" \ + --arg path "${primary_path}" \ + --arg sha256 "${primary_sha256}" \ + --arg sbom_path "${sbom_path}" \ + --arg sbom_sha256 "${sbom_sha256}" \ + --argjson package "${package}" \ + '{release_catalog_key: $release_catalog_key, path: $path, sha256: $sha256, package: $package} + + if $sbom_path == "" then {} else {sbom: {path: $sbom_path, sha256: $sbom_sha256}} end' +done < <(jq -c '.entries[]' "${entries_path}") >"${artifact_index_entries}" +jq -csS \ + --arg build_input_digest "${build_input_digest}" \ + --arg source_artifact "${RELEASE_SOURCE_ARTIFACT_NAME}" \ + '{schema_version: 3, producer: "shared-workflows", build_input_digest: $build_input_digest, + source_artifact: $source_artifact, + entries: sort_by(.release_catalog_key, .path)}' \ + "${artifact_index_entries}" >"${artifact_index_path}" + +upload_immutable "${build_record_path}" "${build_record_key}" +for relative_path in "${artifact_paths[@]}"; do + upload_artifact "${relative_path}" +done +# The index is the canonical artifact commit marker. Write it only after every +# byte and deterministic SBOM that it certifies is durably present. A failed +# or racing upload may leave harmless unreferenced content, but never a +# complete-looking index for a partial candidate. +upload_immutable "${artifact_index_path}" "${artifact_index_key}" +for relative_path in "${execution_paths[@]}"; do + upload_execution "${relative_path}" +done +upload_immutable "${entries_path}" "${bundle_key}" + +reference_path="$(mktemp)" +trap 'rm -f "${head_metadata}" "${build_record_path}" "${reference_path}"' EXIT +jq -n \ + --arg train_sha256 "${RELEASE_CANDIDATE_TRAIN_SHA256}" \ + --arg repository "${GITHUB_REPOSITORY}" \ + --arg source_artifact "${RELEASE_SOURCE_ARTIFACT_NAME}" \ + --arg source_run_id "${GITHUB_RUN_ID}" \ + --arg source_run_attempt "${GITHUB_RUN_ATTEMPT}" \ + --arg build_input_digest "${build_input_digest}" \ + --arg artifact_key "${artifact_key}" \ + --arg build_record_key "${build_record_key}" \ + --arg artifact_index_key "${artifact_index_key}" \ + --arg bundle_key "${bundle_key}" \ + '{schema_version: 4, producer: "shared-workflows", storage_layout: 2, train_sha256: $train_sha256, repository: $repository, source_artifact: $source_artifact, source_run_id: $source_run_id, source_run_attempt: $source_run_attempt, build_input_digest: $build_input_digest, artifact_key: $artifact_key, build_record_key: $build_record_key, artifact_index_key: $artifact_index_key, bundle_key: $bundle_key}' \ + >"${reference_path}" + +upload_reference() { + local checksum + checksum="$(openssl dgst -sha256 -binary "${reference_path}" | base64)" + if aws s3api head-object --bucket "${RELEASE_CANDIDATE_BUCKET}" --key "${train_key}" --checksum-mode ENABLED >"${head_metadata}" 2>/dev/null; then + if [[ "$(jq -r '.ChecksumSHA256 // empty' "${head_metadata}")" != "${checksum}" ]]; then + echo "existing train reference has different bytes: ${train_key}" >&2 + exit 1 + fi + return + fi + if ! aws s3api put-object --bucket "${RELEASE_CANDIDATE_BUCKET}" --key "${train_key}" --body "${reference_path}" --checksum-algorithm SHA256 --checksum-sha256 "${checksum}" --if-none-match '*' --tagging 'release-candidate-status=candidate' >/dev/null 2>&1; then + aws s3api head-object --bucket "${RELEASE_CANDIDATE_BUCKET}" --key "${train_key}" --checksum-mode ENABLED >"${head_metadata}" + if [[ "$(jq -r '.ChecksumSHA256 // empty' "${head_metadata}")" != "${checksum}" ]]; then + echo "candidate train reference could not be written immutably: ${train_key}" >&2 + exit 1 + fi + fi +} + +upload_reference +printf 'Reusable candidate artifacts: s3://%s/%s\n' "${RELEASE_CANDIDATE_BUCKET}" "${artifact_key}" >>"${GITHUB_STEP_SUMMARY}" +printf 'Candidate train reference: s3://%s/%s\n' "${RELEASE_CANDIDATE_BUCKET}" "${train_key}" >>"${GITHUB_STEP_SUMMARY}" diff --git a/tests/release_catalog_discovery_test.sh b/tests/release_catalog_discovery_test.sh index 72728211..1e6d063c 100755 --- a/tests/release_catalog_discovery_test.sh +++ b/tests/release_catalog_discovery_test.sh @@ -41,7 +41,7 @@ export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY jq -e ' [.entries[] | {path, package}] == [{ path: "libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl", - package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0"} + package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0", cuda_major: "12"} }] ' "${temporary_directory}/release-catalog-entries.json" >/dev/null @@ -95,7 +95,7 @@ export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/materialize.sh" jq -e ' .entries[0].path == "libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl" - and .entries[0].package == {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0"} + and .entries[0].package == {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0", cuda_major: "12"} and .entries[0].sbom_kind == "generated-identity" ' "${temporary_directory}/release-catalog-entries.json" >/dev/null diff --git a/tests/release_catalog_package_parsing_test.sh b/tests/release_catalog_package_parsing_test.sh index 726f55f1..2217f32a 100755 --- a/tests/release_catalog_package_parsing_test.sh +++ b/tests/release_catalog_package_parsing_test.sh @@ -37,7 +37,7 @@ export RELEASE_ARTIFACT_DIRECTORY jq -e ' [.entries[] | {path, package}] == [{ path: "libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl", - package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"} + package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32", cuda_major: "12"} }] ' "${wheel_output_directory}/release-catalog-entries.json" >/dev/null @@ -105,7 +105,7 @@ tar -cjf \ info/index.json jq -n \ - '{name: "librmm", version: "26.08.00a32", build: "cuda12_260714_2f567060", subdir: "linux-64"}' \ + '{name: "librmm", version: "26.08.00a32", build: "cuda12_260714_2f567060", subdir: "linux-64", depends: ["cuda-version >=12,<13"]}' \ >"${conda_staging_directory}/info/index.json" tar -cf "${temporary_directory}/info-librmm.tar" -C "${conda_staging_directory}" info/index.json zstd -q -f "${temporary_directory}/info-librmm.tar" -o "${temporary_directory}/info-librmm.tar.zst" @@ -128,7 +128,8 @@ jq -e ' name: "librmm", version: "26.08.00a32", build: "cuda12_260714_2f567060", - platform: "linux-64" + platform: "linux-64", + cuda_major: "12" } ) and any(.entries[]; diff --git a/tests/release_catalog_upload_layout_test.sh b/tests/release_catalog_upload_layout_test.sh new file mode 100755 index 00000000..50b48211 --- /dev/null +++ b/tests/release_catalog_upload_layout_test.sh @@ -0,0 +1,250 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +# Exercise the immutable candidate-store split without AWS credentials. The +# fake CLI deliberately implements only the S3 calls made by upload-s3.sh. +set -euo pipefail + +repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +temporary_directory="$(mktemp -d)" +trap 'rm -rf "${temporary_directory}"' EXIT +mkdir -p "${temporary_directory}/bin" "${temporary_directory}/bundle/release-evidence" + +cat >"${temporary_directory}/bin/aws" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +[[ "$1" == "s3api" ]] +operation="$2" +shift 2 +bucket="" +key="" +body="" +while [[ "$#" -gt 0 ]]; do + case "$1" in + --bucket) bucket="$2"; shift 2 ;; + --key) key="$2"; shift 2 ;; + --body) body="$2"; shift 2 ;; + *) shift ;; + esac +done +path="${FAKE_S3}/${bucket}/${key}" +case "${operation}" in + head-object) + [[ -f "${path}" ]] + checksum="$(openssl dgst -sha256 -binary "${path}" | base64)" + printf '{"ChecksumSHA256":"%s"}\n' "${checksum}" + ;; + put-object) + [[ ! -e "${path}" ]] + mkdir -p "$(dirname "${path}")" + cp "${body}" "${path}" + printf '%s\n' "${key}" >>"${FAKE_LOG}" + ;; + *) + echo "unsupported fake AWS operation: ${operation}" >&2 + exit 2 + ;; +esac +EOF +chmod +x "${temporary_directory}/bin/aws" + +cat >"${temporary_directory}/bundle/release-catalog-entries.conda.json" <<'EOF' +{ + "schema_version": 1, + "producer": "shared-workflows", + "source": { + "artifact": "conda", + "repository": "rapidsai/example", + "sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "workflow_ref": "rapidsai/example/.github/workflows/build.yaml@refs/heads/main", + "run_id": "101", + "run_attempt": "1", + "matrix": {"arch": "x86_64", "cuda_version": "12.0", "python_version": "3.12"} + }, + "entries": [{ + "release_catalog_key": "conda:example", + "path": "linux-64/example-26.10.00.conda", + "sbom": "release-evidence/example.spdx.json", + "provenance": "release-evidence/example.intoto.jsonl", + "package": {"ecosystem": "conda", "name": "example", "version": "26.10.00", "platform": "linux-64"} + }] +} +EOF +mkdir -p "${temporary_directory}/bundle/linux-64" +printf 'package bytes\n' >"${temporary_directory}/bundle/linux-64/example-26.10.00.conda" +printf '{"spdxVersion":"SPDX-2.3"}\n' >"${temporary_directory}/bundle/release-evidence/example.spdx.json" +printf '{"run":"101"}\n' >"${temporary_directory}/bundle/release-evidence/example.intoto.jsonl" +printf '{"schema_version":1,"dependencies":[]}\n' >"${temporary_directory}/upstream-inputs.json" +printf '%s\n' '{"schema_version":1,"train_inputs":{"lock_receipt_sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","variant_receipt_sha256":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"},"build_policy":{"sccache":true}}' >"${temporary_directory}/candidate-build-inputs.json" + +run_upload() { + local implementation_revisions="$2" + local gha_tools_revision + local shared_actions_revision + local build_datetime + gha_tools_revision="$(jq -r '."gha-tools"' <<<"${implementation_revisions}")" + shared_actions_revision="$(jq -r '."shared-actions"' <<<"${implementation_revisions}")" + gha_tools_revision="${3:-${gha_tools_revision}}" + shared_actions_revision="${4:-${shared_actions_revision}}" + build_datetime="${5-260901120000}" + GITHUB_RUN_ATTEMPT="$1" \ + PATH="${temporary_directory}/bin:${PATH}" \ + FAKE_S3="${temporary_directory}/s3" \ + FAKE_LOG="${temporary_directory}/put-objects.log" \ + RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}/bundle" \ + RELEASE_CANDIDATE_BUCKET="candidate-store" \ + RELEASE_CANDIDATE_CONTENT_PREFIX="artifacts" \ + RELEASE_CANDIDATE_TRAIN_PREFIX="train-state" \ + RELEASE_CANDIDATE_TRAIN_SHA256="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" \ + RELEASE_SOURCE_ARTIFACT_NAME="conda" \ + GITHUB_REPOSITORY="rapidsai/example" \ + GITHUB_RUN_ID="101" \ + GITHUB_STEP_SUMMARY="${temporary_directory}/summary" \ + RELEASE_CANDIDATE_BUILD_IMPLEMENTATION_REVISIONS="${implementation_revisions}" \ + RELEASE_CANDIDATE_GHA_TOOLS_REVISION="${gha_tools_revision}" \ + RELEASE_CANDIDATE_CATALOG_SHARED_ACTIONS_REPOSITORY="rapidsai/shared-actions" \ + RELEASE_CANDIDATE_CATALOG_SHARED_ACTIONS_REVISION="${shared_actions_revision}" \ + RELEASE_CANDIDATE_UPSTREAM_INPUTS="${temporary_directory}/upstream-inputs.json" \ + RELEASE_CANDIDATE_BUILD_INPUTS="${temporary_directory}/candidate-build-inputs.json" \ + RAPIDS_DATETIME_STRING="${build_datetime}" \ + "${repository_root}/release-catalog/upload-candidate-s3.sh" +} + +base_implementation_revisions='{"gha-tools":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","shared-actions":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","shared-workflows":"cccccccccccccccccccccccccccccccccccccccc"}' +changed_implementation_revisions='{"gha-tools":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","shared-actions":"dddddddddddddddddddddddddddddddddddddddd","shared-workflows":"cccccccccccccccccccccccccccccccccccccccc"}' + +run_upload 1 "${base_implementation_revisions}" +artifact_root="${temporary_directory}/s3/candidate-store/artifacts/example" +artifact_digest="$(find "${artifact_root}" -mindepth 1 -maxdepth 1 -type d -exec basename {} \;)" +canonical="${artifact_root}/${artifact_digest}" +train="${temporary_directory}/s3/candidate-store/train-state/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/example" +test -f "${canonical}/artifact-index.json" +test -f "${canonical}/linux-64/example-26.10.00.conda" +test -f "${train}/101.1/${artifact_digest}/release-catalog-entries.conda.json" +test -f "${train}/101.1/${artifact_digest}/release-evidence/example.intoto.jsonl" +test ! -d "${artifact_root}/${artifact_digest}/attempts" +jq -e '.upstream_dependencies == []' "${canonical}/build-record.json" >/dev/null +jq -e '.schema_version == 5 and .storage_layout == 2 and .build_datetime == "260901120000"' "${canonical}/build-record.json" >/dev/null +jq -e '.candidate_build_inputs == {"schema_version":1,"train_inputs":{"lock_receipt_sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","variant_receipt_sha256":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"},"build_policy":{"sccache":true}}' "${canonical}/build-record.json" >/dev/null +jq -e '.schema_version == 3 and .source_artifact == "conda"' "${canonical}/artifact-index.json" >/dev/null +artifact_line="$(grep -nF "${artifact_digest}/linux-64/example-26.10.00.conda" "${temporary_directory}/put-objects.log" | cut -d: -f1)" +index_line="$(grep -nF "${artifact_digest}/artifact-index.json" "${temporary_directory}/put-objects.log" | cut -d: -f1)" +if [[ -z "${artifact_line}" || -z "${index_line}" || "${index_line}" -le "${artifact_line}" ]]; then + echo "canonical artifact index was not committed after its artifact bytes" >&2 + exit 1 +fi +jq -e '.build_implementation_revisions == {"gha-tools":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","shared-actions":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","shared-workflows":"cccccccccccccccccccccccccccccccccccccccc"}' "${canonical}/build-record.json" >/dev/null +if jq -e 'tostring | test("run_id|run_attempt")' "${canonical}/artifact-index.json" >/dev/null; then + echo "canonical artifact index contains GitHub execution metadata" >&2 + exit 1 +fi +jq -e '.source_run_id == "101" and .source_run_attempt == "1"' \ + "${train}/101.1/${artifact_digest}/bundle-reference.conda.json" >/dev/null +if run_upload 9 "${base_implementation_revisions}" "dddddddddddddddddddddddddddddddddddddddd" >/dev/null 2>&1; then + echo "upload accepted a gha-tools revision that does not match the release train" >&2 + exit 1 +fi +if run_upload 9 "${base_implementation_revisions}" "" "dddddddddddddddddddddddddddddddddddddddd" >/dev/null 2>&1; then + echo "upload accepted a shared-actions revision that does not match the release train" >&2 + exit 1 +fi + +run_upload 2 "${changed_implementation_revisions}" +test -n "$(find "${train}/101.2" -name 'bundle-reference.conda.json' -print -quit)" +mapfile -t artifact_digests < <(find "${artifact_root}" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sort) +if [[ "${#artifact_digests[@]}" -ne 2 || "${artifact_digests[0]}" == "${artifact_digests[1]}" ]]; then + echo "shared build implementation change did not create a distinct build-input digest" >&2 + exit 1 +fi +cat >"${temporary_directory}/upstream-inputs.json" <<'EOF' +{ + "schema_version": 1, + "dependencies": [{ + "artifact_key": "artifacts/rapids-logger/input", + "path": "noarch/rapids-logger-0.3.0.conda", + "release_catalog_key": "conda:rapids-logger", + "producer": { + "repository": "rapidsai/rapids-logger", + "sha": "cccccccccccccccccccccccccccccccccccccccc", + "build_input_digest": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" + }, + "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "package": {"ecosystem": "conda", "name": "rapids-logger", "version": "0.3.0", "platform": "noarch"} + }] +} +EOF +run_upload 3 "${base_implementation_revisions}" +mapfile -t artifact_digests < <(find "${artifact_root}" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sort) +if [[ "${#artifact_digests[@]}" -ne 3 ]]; then + echo "upstream input change did not create a distinct build-input digest" >&2 + exit 1 +fi +test -n "$(find "${train}/101.3" -name 'bundle-reference.conda.json' -print -quit)" +upstream_digest="" +for candidate_digest in "${artifact_digests[@]}"; do + candidate_record="${artifact_root}/${candidate_digest}/build-record.json" + if jq -e '.upstream_dependencies | length == 1' "${candidate_record}" >/dev/null; then + upstream_digest="${candidate_digest}" + break + fi +done +if [[ -z "${upstream_digest}" ]]; then + echo "unable to find the canonical artifact record with the upstream dependency" >&2 + exit 1 +fi +jq -e ' + .upstream_dependencies == [{ + artifact_key: "artifacts/rapids-logger/input", + path: "noarch/rapids-logger-0.3.0.conda", + release_catalog_key: "conda:rapids-logger", + producer: { + repository: "rapidsai/rapids-logger", + sha: "cccccccccccccccccccccccccccccccccccccccc", + build_input_digest: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" + }, + sha256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + package: { + ecosystem: "conda", name: "rapids-logger", version: "0.3.0", platform: "noarch" + } + }] +' "${artifact_root}/${upstream_digest}/build-record.json" >/dev/null +run_upload 4 "${base_implementation_revisions}" "" "" "260901120001" +mapfile -t artifact_digests < <(find "${artifact_root}" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sort) +if [[ "${#artifact_digests[@]}" -ne 4 ]]; then + echo "build timestamp change did not create a distinct build-input digest" >&2 + exit 1 +fi +run_upload 5 "${base_implementation_revisions}" "" "" "" +mapfile -t artifact_digests < <(find "${artifact_root}" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sort) +if [[ "${#artifact_digests[@]}" -ne 5 ]]; then + echo "empty build timestamp was not accepted as a stable build input" >&2 + exit 1 +fi +printf '%s\n' '{"schema_version":1,"train_inputs":{"lock_receipt_sha256":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","variant_receipt_sha256":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"},"build_policy":{"sccache":true}}' >"${temporary_directory}/candidate-build-inputs.json" +run_upload 7 "${base_implementation_revisions}" +mapfile -t artifact_digests < <(find "${artifact_root}" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sort) +if [[ "${#artifact_digests[@]}" -ne 6 ]]; then + echo "train lock receipt change did not create a distinct build-input digest" >&2 + exit 1 +fi +printf '%s\n' '{"schema_version":1,"train_inputs":{"lock_receipt_sha256":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","variant_receipt_sha256":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"},"build_policy":{"sccache":true}}' >"${temporary_directory}/candidate-build-inputs.json" +run_upload 8 "${base_implementation_revisions}" +mapfile -t artifact_digests < <(find "${artifact_root}" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sort) +if [[ "${#artifact_digests[@]}" -ne 7 ]]; then + echo "train variant receipt change did not create a distinct build-input digest" >&2 + exit 1 +fi +printf '%s\n' '{"schema_version":1,"train_inputs":{"lock_receipt_sha256":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","variant_receipt_sha256":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"},"build_policy":{"sccache":false}}' >"${temporary_directory}/candidate-build-inputs.json" +run_upload 9 "${base_implementation_revisions}" +mapfile -t artifact_digests < <(find "${artifact_root}" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sort) +if [[ "${#artifact_digests[@]}" -ne 8 ]]; then + echo "sccache policy change did not create a distinct build-input digest" >&2 + exit 1 +fi +printf '%s\n' '{"schema_version":1,"train_inputs":{"lock_receipt_sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","variant_receipt_sha256":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"},"build_policy":{"sccache":true}}' >"${temporary_directory}/candidate-build-inputs.json" +printf 'different bytes\n' >"${temporary_directory}/bundle/linux-64/example-26.10.00.conda" +if run_upload 6 "${base_implementation_revisions}" >/dev/null 2>&1; then + echo "upload accepted different canonical package bytes" >&2 + exit 1 +fi diff --git a/tests/test_candidate_context.py b/tests/test_candidate_context.py new file mode 100644 index 00000000..23ad6ed6 --- /dev/null +++ b/tests/test_candidate_context.py @@ -0,0 +1,107 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Candidate identity must survive unchanged project caller workflows.""" + +import importlib.util +import json +import subprocess +from pathlib import Path + +import pytest + +SPEC = importlib.util.spec_from_file_location( + "candidate_context", Path(__file__).parents[1] / "release-candidate-context/resolver.py" +) +assert SPEC is not None and SPEC.loader is not None +resolver = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(resolver) + + +@pytest.fixture +def environment(): + return { + "GITHUB_REPOSITORY": "rapidsai/rapids-logger", + "GITHUB_RUN_ID": "42", + "GITHUB_WORKFLOW_REF": "rapidsai/rapids-logger/.github/workflows/build.yaml@refs/heads/main", + "GITHUB_EVENT_NAME": "workflow_dispatch", + } + + +@pytest.fixture +def binding(): + return { + "schema_version": 1, + "repository": "rapidsai/rapids-logger", + "run_id": "42", + "source_sha": "a" * 40, + "workflow": "build.yaml", + "workflow_ref": "main", + "train_sha256": "f" * 64, + "custom_jobs": {"ci/build_static.sh": "test", "ci/build_dynamic.sh": "test"}, + } + + +def test_standard_build_recovers_digest_without_caller_input(binding, environment): + assert resolver.validate_binding(binding, environment, "a" * 40, "") == { + "train-sha256": "f" * 64, + "custom-mode": "build", + } + + +@pytest.mark.parametrize("script", ["ci/build_static.sh", "ci/build_dynamic.sh"]) +def test_logger_custom_tests_need_no_project_changes(binding, environment, script): + assert resolver.validate_binding(binding, environment, "a" * 40, script)["custom-mode"] == "test" + + +@pytest.mark.parametrize( + "key,value", + [ + ("run_id", "43"), + ("repository", "rapidsai/kvikio"), + ("source_sha", "b" * 40), + ("workflow_ref", "canary"), + ("train_sha256", "bad\noutput=wrong"), + ("schema_version", 2), + ], +) +def test_mismatched_binding_fails_closed(binding, environment, key, value): + binding[key] = value + with pytest.raises(ValueError): + resolver.validate_binding(binding, environment, "a" * 40, "") + + +def test_unknown_custom_release_job_fails_closed(binding, environment): + with pytest.raises(ValueError, match="reviewed policy"): + resolver.validate_binding(binding, environment, "a" * 40, "ci/build_java.sh") + + +def test_missing_binding_retries_only_not_found(binding): + responses = iter( + [ + subprocess.CompletedProcess([], 1, stderr="404 Not Found"), + subprocess.CompletedProcess([], 0, stdout=json.dumps(binding)), + ] + ) + sleeps = [] + assert ( + resolver.read_binding("bucket", "key", run=lambda *_args, **_kwargs: next(responses), sleep=sleeps.append) + == binding + ) + assert sleeps == [5] + + +def test_access_denied_does_not_retry_or_guess(): + def run(*_args, **_kwargs): + return subprocess.CompletedProcess([], 1, stderr="AccessDenied") + + with pytest.raises(ValueError, match="access"): + resolver.read_binding("bucket", "key", run=run, sleep=lambda _: pytest.fail("unexpected sleep")) + + +def test_missing_binding_is_bounded(): + def run(*_args, **_kwargs): + return subprocess.CompletedProcess([], 1, stderr="NoSuchKey") + + sleeps = [] + with pytest.raises(ValueError, match="refusing to guess"): + resolver.read_binding("bucket", "key", run=run, sleep=sleeps.append) + assert sum(sleeps) == 55 diff --git a/tests/test_release_candidate_dependencies.py b/tests/test_release_candidate_dependencies.py new file mode 100644 index 00000000..31c6f8b6 --- /dev/null +++ b/tests/test_release_candidate_dependencies.py @@ -0,0 +1,387 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +import argparse +import importlib.util +import subprocess +from pathlib import Path + +import pytest + +_SCRIPT = Path(__file__).parents[1] / "release-candidate-dependencies" / "select_wheel.py" +_SPEC = importlib.util.spec_from_file_location("select_wheel", _SCRIPT) +if _SPEC is None or _SPEC.loader is None: + raise RuntimeError(f"unable to load wheel selector: {_SCRIPT}") +_MODULE = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(_MODULE) +supports_python = _MODULE.supports_python + +_PATCHER = Path(__file__).parents[1] / "release-candidate-dependencies" / "patch_recipe.py" +_PATCHER_SPEC = importlib.util.spec_from_file_location("patch_recipe", _PATCHER) +if _PATCHER_SPEC is None or _PATCHER_SPEC.loader is None: + raise RuntimeError(f"unable to load recipe patcher: {_PATCHER}") +_PATCHER_MODULE = importlib.util.module_from_spec(_PATCHER_SPEC) +_PATCHER_SPEC.loader.exec_module(_PATCHER_MODULE) + +_LOCK_SELECTOR = Path(__file__).parents[1] / "release-candidate-dependencies" / "select_lock.py" +_SETUP = Path(__file__).parents[1] / "release-candidate-dependencies" / "setup.sh" +_LOCK_SELECTOR_SPEC = importlib.util.spec_from_file_location("select_lock", _LOCK_SELECTOR) +if _LOCK_SELECTOR_SPEC is None or _LOCK_SELECTOR_SPEC.loader is None: + raise RuntimeError(f"unable to load lock selector: {_LOCK_SELECTOR}") +_LOCK_SELECTOR_MODULE = importlib.util.module_from_spec(_LOCK_SELECTOR_SPEC) +_LOCK_SELECTOR_SPEC.loader.exec_module(_LOCK_SELECTOR_MODULE) + + +def test_candidate_conda_channels_exclude_public_rapids_and_nvidia_channels(): + setup = _SETUP.read_text() + + expected_contracts = ( + "candidate build rejected external Conda channel", + "release train must contain the complete Conda bootstrap tool bundle", + "bootstrap-channel/noarch/*", + "rapidsai|rapidsai-nightly|nvidia|", + "conda-forge|https://conda.anaconda.org/conda-forge|file://*|/*", + ) + missing = [contract for contract in expected_contracts if contract not in setup] + if missing: + pytest.fail(f"candidate channel policy is incomplete: {missing}") + + +def test_candidate_conda_lock_selection_is_repository_scoped(): + setup = _SETUP.read_text() + + expected_contracts = ( + 'repository="${GITHUB_REPOSITORY#*/}"', + ".scope.repository == $repository", + "exactly one repository-scoped Conda lock", + ) + missing = [contract for contract in expected_contracts if contract not in setup] + if missing: + pytest.fail(f"repository-scoped Conda lock selection is incomplete: {missing}") + + +def test_candidate_setup_exports_cache_policy_and_train_input_identity(): + setup = _SETUP.read_text() + + expected_contracts = ( + "candidate_build_policy.sccache", + "lock_receipt_sha256", + "variant_receipt_sha256", + "RELEASE_CANDIDATE_BUILD_INPUTS", + "RAPIDS_RELEASE_CANDIDATE_SCCACHE", + ) + missing = [contract for contract in expected_contracts if contract not in setup] + if missing: + pytest.fail(f"candidate build-input identity is incomplete: {missing}") + + +def test_wheel_lock_selector_requires_one_exact_matrix_scope(): + expected = { + "path": "locks/wheel/x86_64-manylinux_2_28/python-3.14/requirements.txt", + "sha256": "a" * 64, + } + receipt = { + "wheel": [ + {**expected, "scope": {"platform": "x86_64-manylinux_2_28", "python": "3.14"}}, + { + "path": "locks/wheel/aarch64-manylinux_2_28/python-3.14/requirements.txt", + "sha256": "b" * 64, + "scope": {"platform": "aarch64-manylinux_2_28", "python": "3.14"}, + }, + ] + } + + selected = _LOCK_SELECTOR_MODULE.select_wheel_lock(receipt, "x86_64-manylinux_2_28", "3.14") + + if selected != expected: + pytest.fail(f"wheel lock selector returned the wrong matrix lock: {selected}") + + +def test_wheel_lock_selector_rejects_missing_and_unsafe_records(): + with pytest.raises(ValueError, match="exactly one wheel lock"): + _LOCK_SELECTOR_MODULE.select_wheel_lock({"wheel": []}, "x86_64-manylinux_2_28", "3.14") + with pytest.raises(ValueError, match="unsafe wheel lock path"): + _LOCK_SELECTOR_MODULE.select_wheel_lock( + { + "wheel": [ + { + "path": "../requirements.txt", + "sha256": "a" * 64, + "scope": {"platform": "x86_64-manylinux_2_28", "python": "3.14"}, + } + ] + }, + "x86_64-manylinux_2_28", + "3.14", + ) + + +@pytest.mark.parametrize( + ("wheel", "python_version"), + [ + ("rapids_logger-0.3.0-py3-none-any.whl", "3.10"), + ("package-1.0.0-cp310-abi3-manylinux_2_17_x86_64.whl", "3.12"), + ("package-1.0.0-cp312-cp312-manylinux_2_17_x86_64.whl", "3.12"), + ], +) +def test_wheel_tags_allow_compatible_python_versions(wheel, python_version): + if not supports_python(wheel, python_version): + pytest.fail(f"expected {wheel} to support Python {python_version}") + + +@pytest.mark.parametrize( + ("wheel", "python_version"), + [ + ("package-1.0.0-cp310-cp310-manylinux_2_17_x86_64.whl", "3.12"), + ("package-1.0.0-cp312-abi3-manylinux_2_17_x86_64.whl", "3.10"), + ("not-a-wheel", "3.12"), + ], +) +def test_wheel_tags_reject_incompatible_python_versions(wheel, python_version): + if supports_python(wheel, python_version): + pytest.fail(f"expected {wheel} not to support Python {python_version}") + + +def test_recipe_patcher_adds_lock_metapackages_to_root_and_outputs(): + recipe = { + "requirements": {"build": ["cmake"], "host": ["zlib"]}, + "outputs": [{"package": {"name": "example"}, "requirements": {"host": ["python"]}}], + } + + _PATCHER_MODULE._add_requirements(recipe, "candidate-build-lock", "candidate-host-lock") + _PATCHER_MODULE._add_requirements(recipe["outputs"][0], "candidate-build-lock", "candidate-host-lock") + + expected_root = { + "build": ["cmake", "candidate-build-lock"], + "host": ["zlib", "candidate-host-lock"], + } + expected_output = { + "build": ["candidate-build-lock"], + "host": ["python", "candidate-host-lock"], + } + if recipe["requirements"] != expected_root: + pytest.fail("root recipe requirements did not receive both lock metapackages") + if recipe["outputs"][0]["requirements"] != expected_output: + pytest.fail("output requirements did not receive both lock metapackages") + + +def test_recipe_patcher_injects_locks_only_into_multi_output_build_targets(): + recipe = { + "requirements": {"build": ["cmake"], "host": ["zlib"]}, + "outputs": [ + { + "package": {"name": "library"}, + "build": {"script": {"content": "cmake --install build", "env": {"KEEP": "true"}}}, + "requirements": {"host": ["python"]}, + }, + {"package": {"name": "tests"}, "requirements": {"build": ["ninja"]}}, + ], + } + + _PATCHER_MODULE._prepare_recipe_documents(recipe, "candidate-build-lock", "candidate-host-lock", "deadbeef", set()) + + if "requirements" in recipe: + pytest.fail("multi-output recipe retained a forbidden top-level requirements field") + expected_library_requirements = { + "build": ["cmake", "candidate-build-lock"], + "host": ["zlib", "python", "candidate-host-lock"], + } + if recipe["outputs"][0]["requirements"] != expected_library_requirements: + pytest.fail("library output did not inherit shared and candidate requirements") + output_script = recipe["outputs"][0]["build"]["script"] + if output_script["env"] != {"KEEP": "true"} or "-Drapids-cmake-sha=deadbeef" not in output_script["content"]: + pytest.fail("script-object content was not pinned without losing its environment") + expected_test_requirements = { + "build": ["cmake", "ninja", "candidate-build-lock"], + "host": ["zlib", "candidate-host-lock"], + } + if recipe["outputs"][1]["requirements"] != expected_test_requirements: + pytest.fail("test output did not inherit shared and candidate requirements") + + +def test_recipe_patcher_injects_locks_into_single_output_root(): + recipe = { + "build": {"script": "cmake -S . -B build"}, + "requirements": {"build": ["cmake"], "host": ["zlib"]}, + "tests": [{"script": "cmake -S tests -B build"}], + } + + _PATCHER_MODULE._prepare_recipe_documents(recipe, "candidate-build-lock", "candidate-host-lock", "deadbeef", set()) + + expected_requirements = { + "build": ["cmake", "candidate-build-lock"], + "host": ["zlib", "candidate-host-lock"], + } + if recipe["requirements"] != expected_requirements: + pytest.fail("single-output root did not receive candidate requirements") + if "-Drapids-cmake-sha=deadbeef" not in recipe["build"]["script"]: + pytest.fail("package build script did not pin RAPIDS CMake") + if "-Drapids-cmake-sha=deadbeef" in recipe["tests"][0]["script"]: + pytest.fail("package test script unexpectedly pinned RAPIDS CMake") + + +def test_recipe_patcher_leaves_multi_output_tests_unmodified(): + recipe = { + "outputs": [ + {"package": {"name": "library"}, "tests": [{"script": "ctest"}]}, + {"package": {"name": "bindings"}, "tests": [{"python": {"imports": ["bindings"]}}]}, + ] + } + + _PATCHER_MODULE._prepare_recipe_documents(recipe, "candidate-build-lock", "candidate-host-lock", "deadbeef", set()) + + for output in recipe["outputs"]: + if "requirements" in output["tests"][0]: + pytest.fail("candidate recipe patcher modified a package test") + + +def test_recipe_patcher_pins_and_locks_shared_cache_build(): + recipe = { + "cache": { + "build": {"script": {"content": "./build.sh", "secrets": ["AWS_TOKEN"]}}, + "requirements": {"build": ["cmake"], "host": ["zlib"]}, + }, + "outputs": [{"package": {"name": "library"}}], + } + + _PATCHER_MODULE._prepare_recipe_documents(recipe, "candidate-build-lock", "candidate-host-lock", "deadbeef", set()) + + if recipe["cache"]["requirements"] != { + "build": ["cmake", "candidate-build-lock"], + "host": ["zlib", "candidate-host-lock"], + }: + pytest.fail("shared cache build did not receive candidate lock packages") + cache_script = recipe["cache"]["build"]["script"] + if cache_script["secrets"] != ["AWS_TOKEN"] or "-Drapids-cmake-sha=deadbeef" not in cache_script["content"]: + pytest.fail("shared cache build script was not pinned without losing its configuration") + + +def test_recipe_patcher_adapts_exact_variant_spelling_from_local_config(tmp_path): + recipe = tmp_path / "recipe.yaml" + recipe.write_text("package:\n name: example\n version: 1\n") + (tmp_path / "conda_build_config.yaml").write_text("libcurl_version:\n - ==8.13.0\n") + + keys = _PATCHER_MODULE._local_exact_variant_keys(recipe) + + expected = "libcurl ==${{ libcurl_version }}" + if _PATCHER_MODULE._add_exact_operator("libcurl ${{ libcurl_version }}", keys) != expected: + pytest.fail("recipe patcher did not add the local exact MatchSpec operator") + if _PATCHER_MODULE._add_exact_operator(expected, keys) != expected: + pytest.fail("recipe patcher duplicated the local exact MatchSpec operator") + + +def test_recipe_patcher_accepts_recipe_directory(tmp_path): + recipe = tmp_path / "recipe" + recipe.mkdir() + expected = recipe / "recipe.yaml" + expected.write_text("package:\n name: example\n version: 1\n") + + resolved = _PATCHER_MODULE._resolve_recipe(recipe, argparse.ArgumentParser()) + + if resolved != expected: + pytest.fail("recipe patcher did not resolve a recipe directory to recipe.yaml") + + +def test_recipe_patcher_copies_recipe_directory_with_recipe_yaml(tmp_path): + recipe = tmp_path / "recipes" / "example" + recipe.mkdir(parents=True) + source_recipe = recipe / "recipe.yaml" + source_recipe.write_text("package:\n name: example\n version: 1\n") + (recipe / "build.sh").write_text("#!/usr/bin/env bash\n") + + destination = _PATCHER_MODULE._copy_recipe_directory( + source_recipe, + "candidate-build-lock", + "candidate-host-lock", + ) + + if not destination.name.startswith(".example.release-candidate-"): + pytest.fail("recipe patcher did not use a deterministic disposable directory") + if not (destination / "recipe.yaml").is_file(): + pytest.fail("disposable recipe directory did not retain recipe.yaml") + if not (destination / "build.sh").is_file(): + pytest.fail("disposable recipe directory did not retain recipe-local files") + + +@pytest.mark.parametrize( + ("architecture", "platform"), + [("amd64", "linux-64"), ("x86_64", "linux-64"), ("arm64", "linux-aarch64"), ("aarch64", "linux-aarch64")], +) +def test_candidate_conda_platform_normalizes_ci_architecture_names(architecture, platform): + script = Path(__file__).parents[1] / "release-candidate-dependencies" / "platform.sh" + # The command runs the repository-local helper with a fixed shell program. + result = subprocess.run( # noqa: S603 + ["/bin/bash", "-c", 'source "$1"; conda_platform_for_arch "$2"', "bash", str(script), architecture], + check=True, + capture_output=True, + text=True, + ) + + if result.stdout.strip() != platform: + pytest.fail(f"expected {architecture} to normalize to {platform}") + + +def test_candidate_conda_platform_rejects_unknown_architecture(): + script = Path(__file__).parents[1] / "release-candidate-dependencies" / "platform.sh" + # The command runs the repository-local helper with a fixed shell program. + result = subprocess.run( # noqa: S603 + ["/bin/bash", "-c", 'source "$1"; conda_platform_for_arch "$2"', "bash", str(script), "not-an-arch"], + check=False, + capture_output=True, + text=True, + ) + + if result.returncode == 0 or "unsupported candidate architecture" not in result.stderr: + pytest.fail("unknown candidate architecture was accepted") + + +def test_candidate_cmake_wrapper_pins_rapids_cmake_for_configure(tmp_path): + output = tmp_path / "arguments.txt" + original = tmp_path / "cmake" + original.write_text('#!/bin/bash\nprintf \'%s\\n\' "$@" >"${TEST_OUTPUT}"\n') + original.chmod(0o755) + wrapper = Path(__file__).parents[1] / "release-candidate-dependencies" / "cmake.sh" + result = subprocess.run( # noqa: S603 + ["/bin/bash", str(wrapper), "-S", ".", "-B", "build"], + check=True, + capture_output=True, + text=True, + env={ + "RAPIDS_CANDIDATE_ORIGINAL_CMAKE": str(original), + "RAPIDS_CANDIDATE_RAPIDS_CMAKE_SHA": "a" * 40, + "TEST_OUTPUT": str(output), + }, + ) + + if result.stdout: + pytest.fail("candidate CMake wrapper wrote unexpected standard output") + expected_arguments = [ + "-Drapids-cmake-sha=" + "a" * 40, + "-S", + ".", + "-B", + "build", + ] + if output.read_text().splitlines() != expected_arguments: + pytest.fail("candidate CMake wrapper did not pin RAPIDS CMake during configuration") + + +def test_candidate_cmake_wrapper_does_not_modify_build_mode(tmp_path): + output = tmp_path / "arguments.txt" + original = tmp_path / "cmake" + original.write_text('#!/bin/bash\nprintf \'%s\\n\' "$@" >"${TEST_OUTPUT}"\n') + original.chmod(0o755) + wrapper = Path(__file__).parents[1] / "release-candidate-dependencies" / "cmake.sh" + subprocess.run( # noqa: S603 + ["/bin/bash", str(wrapper), "--build", "build"], + check=True, + capture_output=True, + text=True, + env={ + "RAPIDS_CANDIDATE_ORIGINAL_CMAKE": str(original), + "RAPIDS_CANDIDATE_RAPIDS_CMAKE_SHA": "a" * 40, + "TEST_OUTPUT": str(output), + }, + ) + + if output.read_text().splitlines() != ["--build", "build"]: + pytest.fail("candidate CMake wrapper modified a build-mode invocation")