diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc4f0999200..0d3b3561892 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -476,6 +476,25 @@ jobs: single-cuda-major: ${{ needs.ci-vars.outputs.WINDOWS_ARM64_SINGLE_CUDA_MAJOR == 'true' }} python-versions: ${{ needs.ci-vars.outputs.PYTHON_SUPPORTED }} + woa-main-dispatch: + name: Dispatch WoA integration + needs: + - build-linux-64 + - build-windows-arm64 + if: ${{ github.repository_owner == 'nvidia' && + github.event_name == 'push' && + github.ref == 'refs/heads/main' && + needs.build-linux-64.result == 'success' && + needs.build-windows-arm64.result == 'success' }} + permissions: + actions: read + checks: read + contents: read + uses: ./.github/workflows/woa-main-ci.yml + with: + public_run_id: ${{ github.run_id }} + secrets: inherit + # NOTE: test-sdist jobs are split by platform (mirroring build-* and test-wheel-*) # so platform-specific sources (e.g. cuda_bindings/*_windows.pyx selected by # build_hooks.py) are exercised on their target OS. Keep these job definitions diff --git a/.github/workflows/woa-validation-dispatch.yml b/.github/workflows/woa-main-ci.yml similarity index 76% rename from .github/workflows/woa-validation-dispatch.yml rename to .github/workflows/woa-main-ci.yml index cead1752f24..2c044f072d1 100644 --- a/.github/workflows/woa-validation-dispatch.yml +++ b/.github/workflows/woa-main-ci.yml @@ -2,19 +2,33 @@ # # SPDX-License-Identifier: Apache-2.0 -name: "WoA validation dispatch" -run-name: "WoA exact dispatch for ${{ inputs.public_run_id || 'latest successful public main CI run' }}" +name: "WoA main CI" +run-name: "WoA main CI for ${{ inputs.public_run_id || 'latest successful public main CI run' }}" on: + workflow_call: + inputs: + public_run_id: + required: true + type: string + secrets: + CUDA_PYTHON_WOA_XREPO_CI_PRIVATE_APP_PRIVATE_KEY: + required: true + schedule: + - cron: "17 * * * *" workflow_dispatch: inputs: public_run_id: - description: Successful public main CI run; leave empty to find the latest successful main CI run + description: Eligible public main CI run; leave empty to find the latest successful main CI run required: false type: string permissions: {} +concurrency: + group: cuda-python-woa-public-selector + cancel-in-progress: false + jobs: resolve-public-build: if: ${{ github.repository == 'NVIDIA/cuda-python' }} @@ -25,20 +39,22 @@ jobs: contents: read checks: read outputs: + dispatch: ${{ steps.select.outputs.dispatch }} correlation-id: ${{ steps.resolve.outputs.correlation_id }} public-run-id: ${{ steps.resolve.outputs.public_run_id }} public-sha: ${{ steps.resolve.outputs.public_sha }} steps: - - name: Checkout main run lookup helper - if: ${{ inputs.public_run_id == '' }} + - name: Checkout CI selection helpers uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: main + ref: ${{ github.sha }} persist-credentials: false - sparse-checkout: ci/tools/lookup-run-id + sparse-checkout: | + ci/tools/lookup-run-id + ci/tools/select-woa-batch sparse-checkout-cone-mode: false - - name: Resolve exact successful public build + - name: Resolve exact public build id: resolve env: GH_TOKEN: ${{ github.token }} @@ -47,7 +63,7 @@ jobs: set -euo pipefail public_sha='' if [[ -z "$PUBLIC_RUN_ID" ]]; then - lookup_output=$(./ci/tools/lookup-run-id --branch main --head-sha NVIDIA/cuda-python CI) + lookup_output=$(./ci/tools/lookup-run-id --branch main --event push --head-sha NVIDIA/cuda-python CI) mapfile -t lookup_result <<< "$lookup_output" [[ ${#lookup_result[@]} == 2 ]] PUBLIC_RUN_ID=${lookup_result[0]} @@ -64,8 +80,8 @@ jobs: .path == ".github/workflows/ci.yml" and .event == "push" and .head_branch == "main" and - .status == "completed" and - .conclusion == "success" and + ((.status == "in_progress" and .conclusion == null) or + (.status == "completed" and .conclusion == "success")) and ($sha == "" or .head_sha == $sha) and (.head_sha | test("^[0-9a-f]{40}$")) ' <<< "$run" >/dev/null @@ -83,11 +99,20 @@ jobs: echo "public_run_id=$PUBLIC_RUN_ID" >> "$GITHUB_OUTPUT" echo "public_sha=$public_sha" >> "$GITHUB_OUTPUT" { - echo "Resolved successful public run \`$PUBLIC_RUN_ID\` at commit \`$public_sha\`." + echo "Resolved public run \`$PUBLIC_RUN_ID\` at commit \`$public_sha\`." } >> "$GITHUB_STEP_SUMMARY" + - name: Select public batch + id: select + env: + GH_TOKEN: ${{ github.token }} + PUBLIC_RUN_ID: ${{ steps.resolve.outputs.public_run_id }} + PUBLIC_SHA: ${{ steps.resolve.outputs.public_sha }} + run: bash ./ci/tools/select-woa-batch + dispatch-private-validation: needs: resolve-public-build + if: ${{ needs.resolve-public-build.outputs.dispatch == 'true' }} runs-on: ubuntu-latest timeout-minutes: 10 permissions: {} @@ -123,15 +148,13 @@ jobs: public_workflow_id: "155304118", public_run_id: $run_id, public_sha: $sha, - baseline_sha: "", - commit_count: "1", correlation_id: $correlation_id } }') response=$(gh api \ --method POST \ -H 'X-GitHub-Api-Version: 2026-03-10' \ - repos/NVIDIA-dev/cuda-python-private/actions/workflows/ctk-next-woa-validation.yml/dispatches \ + repos/NVIDIA-dev/cuda-python-private/actions/workflows/ctk-next-woa-main-ci.yml/dispatches \ --input - <<< "$payload") private_run_id=$(jq -er '.workflow_run_id | tostring' <<< "$response") run=$(gh api \ @@ -139,7 +162,7 @@ jobs: "repos/NVIDIA-dev/cuda-python-private/actions/runs/$private_run_id") jq -e ' .repository.full_name == "NVIDIA-dev/cuda-python-private" and - .path == ".github/workflows/ctk-next-woa-validation.yml" and + .path == ".github/workflows/ctk-next-woa-main-ci.yml" and .event == "workflow_dispatch" and .head_branch == "ctk-next" ' <<< "$run" >/dev/null diff --git a/.github/workflows/woa-transport-check.yml b/.github/workflows/woa-transport-check.yml deleted file mode 100644 index 34bfadd386e..00000000000 --- a/.github/workflows/woa-transport-check.yml +++ /dev/null @@ -1,108 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -name: "WoA transport check" -run-name: "WoA transport smoke for current public main" - -on: - workflow_dispatch: - -permissions: {} - -jobs: - resolve-public-main: - if: ${{ github.repository == 'NVIDIA/cuda-python' }} - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - outputs: - correlation-id: ${{ steps.resolve.outputs.correlation_id }} - public-sha: ${{ steps.resolve.outputs.public_sha }} - steps: - - name: Resolve exact public main commit - id: resolve - env: - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - - public_sha=$(gh api \ - -H 'X-GitHub-Api-Version: 2026-03-10' \ - repos/NVIDIA/cuda-python/git/ref/heads/main \ - --jq .object.sha) - [[ "$public_sha" =~ ^[0-9a-f]{40}$ ]] - - commit=$(gh api \ - -H 'X-GitHub-Api-Version: 2026-03-10' \ - "repos/NVIDIA/cuda-python/commits/$public_sha") - jq -e --arg sha "$public_sha" '.sha == $sha' <<< "$commit" >/dev/null - - comparison=$(gh api \ - -H 'X-GitHub-Api-Version: 2026-03-10' \ - "repos/NVIDIA/cuda-python/compare/$public_sha...main") - jq -e \ - --arg sha "$public_sha" \ - '.base_commit.sha == $sha and - .merge_base_commit.sha == $sha and - (.status == "ahead" or .status == "identical")' \ - <<< "$comparison" >/dev/null - - correlation_id="smoke:NVIDIA/cuda-python:main:$public_sha:${GITHUB_RUN_ID}:${GITHUB_RUN_ATTEMPT}" - echo "public_sha=$public_sha" >> "$GITHUB_OUTPUT" - echo "correlation_id=$correlation_id" >> "$GITHUB_OUTPUT" - echo "Transport smoke will report on public main commit \`$public_sha\`." >> "$GITHUB_STEP_SUMMARY" - - dispatch-private-smoke: - needs: resolve-public-main - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: {} - steps: - - name: Create private dispatch token - id: private-app-token - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 - with: - client-id: ${{ vars.CUDA_PYTHON_WOA_XREPO_CI_PRIVATE_APP_CLIENT_ID }} - private-key: ${{ secrets.CUDA_PYTHON_WOA_XREPO_CI_PRIVATE_APP_PRIVATE_KEY }} - owner: NVIDIA-dev - repositories: cuda-python-private - permission-actions: write - - - name: Dispatch and verify private smoke workflow - env: - CORRELATION_ID: ${{ needs.resolve-public-main.outputs.correlation-id }} - GH_TOKEN: ${{ steps.private-app-token.outputs.token }} - PUBLIC_SHA: ${{ needs.resolve-public-main.outputs.public-sha }} - run: | - set -euo pipefail - payload=$(jq -n \ - --arg correlation_id "$CORRELATION_ID" \ - --arg public_sha "$PUBLIC_SHA" \ - '{ - ref: "ctk-next", - return_run_details: true, - inputs: { - schema_version: "2", - public_repository: "NVIDIA/cuda-python", - public_sha: $public_sha, - correlation_id: $correlation_id - } - }') - response=$(gh api \ - --method POST \ - -H 'X-GitHub-Api-Version: 2026-03-10' \ - repos/NVIDIA-dev/cuda-python-private/actions/workflows/ctk-next-woa-transport-check.yml/dispatches \ - --input - <<< "$payload") - private_run_id=$(jq -er '.workflow_run_id | tostring' <<< "$response") - run=$(gh api \ - -H 'X-GitHub-Api-Version: 2026-03-10' \ - "repos/NVIDIA-dev/cuda-python-private/actions/runs/$private_run_id") - jq -e ' - .repository.full_name == "NVIDIA-dev/cuda-python-private" and - .path == ".github/workflows/ctk-next-woa-transport-check.yml" and - .event == "workflow_dispatch" and - .head_branch == "ctk-next" - ' <<< "$run" >/dev/null - echo "Private transport-smoke workflow accepted the exact dispatch." >> "$GITHUB_STEP_SUMMARY" diff --git a/ci/tools/lookup-run-id b/ci/tools/lookup-run-id index 89833cfe1bd..4f7eb2c043f 100755 --- a/ci/tools/lookup-run-id +++ b/ci/tools/lookup-run-id @@ -24,6 +24,7 @@ Usage: Options: --tag Find run by git tag (requires local git repo with the tag) --branch Find latest successful run on the given branch + --event Restrict branch-mode runs to a trigger event (for example push) --artifact Require an unexpired artifact matching this shell glob (repeatable; branch mode only) --head-sha Also print the run's head commit SHA (second line) @@ -33,6 +34,7 @@ Examples: $0 --branch main NVIDIA/cuda-python $0 --branch 12.9.x --artifact 'cuda-bindings-python313-*' NVIDIA/cuda-python $0 --branch main --head-sha NVIDIA/cuda-python "CI" + $0 --branch main --event push --head-sha NVIDIA/cuda-python "CI" EOF exit 1 } @@ -41,6 +43,7 @@ EOF MODE="" REF="" HEAD_SHA_FLAG=0 +EVENT="" ARTIFACT_PATTERNS=() while [[ $# -gt 0 ]]; do @@ -56,6 +59,9 @@ while [[ $# -gt 0 ]]; do --artifact) [[ $# -ge 2 ]] || usage ARTIFACT_PATTERNS+=("${2}"); shift 2 ;; + --event) + [[ $# -ge 2 && -n "${2}" ]] || usage + EVENT="${2}"; shift 2 ;; --head-sha) HEAD_SHA_FLAG=1; shift ;; -h|--help) @@ -83,6 +89,11 @@ if [[ "${MODE}" != "branch" && ${#ARTIFACT_PATTERNS[@]} -gt 0 ]]; then exit 1 fi +if [[ "${MODE}" != "branch" && -n "${EVENT}" ]]; then + echo "Error: --event is only supported with --branch" >&2 + exit 1 +fi + # ── Prerequisite checks ── if [[ -z "${GH_TOKEN:-}" ]]; then echo "Error: GH_TOKEN environment variable is required" >&2 @@ -176,7 +187,12 @@ if [[ "${MODE}" == "branch" ]]; then "repos/${REPOSITORY}/actions/workflows/${WORKFLOW_ID}/runs" \ -f "head_sha=${commit_sha}" \ -F "per_page=100" \ - --jq '[.workflow_runs[] | select(.conclusion == "success") | {id, head_sha, created_at}] | sort_by(.created_at) | reverse'); then + | jq -c --arg event "${EVENT}" ' + [.workflow_runs[] + | select(.conclusion == "success" and ($event == "" or .event == $event)) + | {id, head_sha, created_at}] + | sort_by(.created_at) | reverse + '); then echo "Error: Failed to list runs for commit ${commit_sha}" >&2 exit 1 fi diff --git a/ci/tools/select-woa-batch b/ci/tools/select-woa-batch new file mode 100644 index 00000000000..91be1836d81 --- /dev/null +++ b/ci/tools/select-woa-batch @@ -0,0 +1,78 @@ +#!/usr/bin/env bash + +set -euo pipefail +# The resolver has already validated this public main CI run and commit. +run_id=$PUBLIC_RUN_ID +sha=$PUBLIC_SHA +external_id="woa:v1:NVIDIA/cuda-python:$run_id:$sha" + +# Any matching check from the reporting App suppresses a duplicate dispatch, +# including checks that are still running, failed, or cancelled. +checks=$(gh api \ + -H 'X-GitHub-Api-Version: 2026-03-10' \ + "repos/NVIDIA/cuda-python/commits/$sha/check-runs?check_name=cuda-python%20WoA%20integration&per_page=100") +if jq -e --arg id "$external_id" ' + any(.check_runs[]; .app.id == 4954254 and .external_id == $id) +' <<< "$checks" >/dev/null; then + echo "dispatch=false" >> "$GITHUB_OUTPUT" + echo "This exact public run already has an App-owned attempt." >> "$GITHUB_STEP_SUMMARY" + exit 0 +fi + +# Scan at most 100 main commits, newest first. The candidate must be in this window. +commits=$(gh api --method GET \ + -H 'X-GitHub-Api-Version: 2026-03-10' \ + repos/NVIDIA/cuda-python/commits -f sha=main -F per_page=100) +candidate_index=$(jq -er --arg sha "$sha" ' + [to_entries[] | select(.value.sha == $sha) | .key][0] +' <<< "$commits") +commit_total=$(jq length <<< "$commits") +baseline_index="" +baseline_sha="" +# The nearest older successful App-owned WoA check is the batching baseline. +# Failed or cancelled validations do not advance it. +for ((index = candidate_index + 1; index < commit_total; index++)); do + older_sha=$(jq -r --argjson index "$index" '.[$index].sha' <<< "$commits") + older_checks=$(gh api \ + -H 'X-GitHub-Api-Version: 2026-03-10' \ + "repos/NVIDIA/cuda-python/commits/$older_sha/check-runs?check_name=cuda-python%20WoA%20integration&per_page=100") + if jq -e ' + any(.check_runs[]; + .app.id == 4954254 and + .status == "completed" and + .conclusion == "success") + ' <<< "$older_checks" >/dev/null; then + baseline_index=$index + baseline_sha=$older_sha + break + fi +done + +# Without a baseline in the scan window, validate immediately to bootstrap. +dispatch=true +reason="Bootstrap: no successful App-owned baseline exists in the latest 100 commits." +if [[ -n "$baseline_index" ]]; then + # Count commits after the baseline through the candidate; test only the candidate. + commit_count=$((baseline_index - candidate_index)) + oldest_pending_index=$((baseline_index - 1)) + oldest_pending_at=$(jq -r \ + --argjson index "$oldest_pending_index" \ + '.[$index].commit.committer.date' <<< "$commits") + # Age starts at the oldest pending commit's timestamp, not workflow start time. + age_seconds=$(($(date +%s) - $(date -d "$oldest_pending_at" +%s))) + # Dispatch after three commits or two hours; hourly cron revisits deferred batches. + if (( commit_count >= 3 )); then + reason="Batch threshold reached: $commit_count commits." + elif (( age_seconds >= 7200 )); then + reason="Two-hour fallback reached: oldest pending commit is $age_seconds seconds old." + else + dispatch=false + reason="Deferred: $commit_count/3 commits and oldest pending age $age_seconds/7200 seconds." + fi +fi +# Only dispatch=true enables the separate credential-bearing dispatch job. +echo "dispatch=$dispatch" >> "$GITHUB_OUTPUT" +if [[ -n "$baseline_sha" ]]; then + echo "Baseline: $baseline_sha; pending commits through $sha: $commit_count." >> "$GITHUB_STEP_SUMMARY" +fi +echo "$reason" >> "$GITHUB_STEP_SUMMARY" diff --git a/ci/tools/tests/test_lookup_run_id.py b/ci/tools/tests/test_lookup_run_id.py new file mode 100644 index 00000000000..e2c08c9df10 --- /dev/null +++ b/ci/tools/tests/test_lookup_run_id.py @@ -0,0 +1,121 @@ +# ruff: noqa: S603 + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +LOOKUP_RUN_ID = Path(__file__).resolve().parents[1] / "lookup-run-id" +HEAD_SHA = "a" * 40 +OLDER_SHA = "b" * 40 + + +@pytest.fixture +def lookup(tmp_path): + fixture_path = tmp_path / "fixture.json" + mock = tmp_path / "gh" + mock.write_text( + f"#!{sys.executable}\n" + "import json, os, sys\n" + "from pathlib import Path\n" + "fixture = json.loads(Path(os.environ['LOOKUP_FIXTURE']).read_text())\n" + "arguments = sys.argv[1:]\n" + "endpoint = next(value for value in arguments if value.startswith('repos/'))\n" + "if endpoint.endswith('/actions/workflows/ci.yml'):\n" + " print(155304118)\n" + "elif '/branches/' in endpoint:\n" + " print(fixture['commits'][0])\n" + "elif endpoint.endswith('/commits'):\n" + " print('\\n'.join(fixture['commits']))\n" + "elif endpoint.endswith('/runs'):\n" + " sha = next(value.split('=', 1)[1] for value in arguments if value.startswith('head_sha='))\n" + " print(json.dumps({'workflow_runs': fixture['runs'].get(sha, [])}))\n" + "else:\n" + " raise RuntimeError(endpoint)\n" + ) + mock.chmod(0o755) + + def invoke(runs, *, event=None, commits=None, mode=None): + fixture_path.write_text(json.dumps({"commits": commits or [HEAD_SHA], "runs": runs})) + arguments = ["bash", str(LOOKUP_RUN_ID), *(mode or ["--branch", "main"]), "--head-sha"] + if event is not None: + arguments.extend(["--event", event]) + arguments.extend(["NVIDIA/cuda-python", "ci.yml"]) + return subprocess.run( + arguments, + env=dict( + os.environ, + PATH=str(tmp_path) + os.pathsep + os.environ["PATH"], + GH_TOKEN=tmp_path.name, + LOOKUP_FIXTURE=str(fixture_path), + LOOKUP_RUN_ID_COMMIT_SCAN_LIMIT="2", + ), + capture_output=True, + text=True, + timeout=15, + ) + + return invoke + + +def make_run(run_id, event, *, sha=HEAD_SHA, conclusion="success"): + return { + "id": run_id, + "head_sha": sha, + "created_at": f"2026-10-14T00:00:{run_id:02d}Z", + "event": event, + "conclusion": conclusion, + } + + +@pytest.mark.agent_authored(model="gpt-6") +@pytest.mark.parametrize("event,expected", [(None, 30), ("push", 20), ("workflow_dispatch", 10)]) +def test_optional_event_filter_selects_latest_matching_successful_run(lookup, event, expected): + result = lookup( + {HEAD_SHA: [make_run(30, "schedule"), make_run(20, "push"), make_run(10, "workflow_dispatch")]}, + event=event, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.splitlines() == [str(expected), HEAD_SHA] + + +@pytest.mark.agent_authored(model="gpt-6") +def test_push_lookup_continues_to_older_commit_after_scheduled_head_run(lookup): + result = lookup( + { + HEAD_SHA: [make_run(30, "schedule")], + OLDER_SHA: [make_run(20, "push", sha=OLDER_SHA)], + }, + event="push", + commits=[HEAD_SHA, OLDER_SHA], + ) + assert result.returncode == 0, result.stderr + assert result.stdout.splitlines() == ["20", OLDER_SHA] + + +@pytest.mark.agent_authored(model="gpt-6") +def test_push_lookup_rejects_failed_push_even_when_scheduled_run_succeeds(lookup): + result = lookup( + {HEAD_SHA: [make_run(30, "schedule"), make_run(20, "push", conclusion="failure")]}, + event="push", + ) + assert result.returncode != 0 + assert result.stdout == "" + assert "No successful" in result.stderr + + +@pytest.mark.agent_authored(model="gpt-6") +def test_event_filter_is_rejected_in_tag_mode(lookup): + result = lookup({}, event="push", mode=["--tag", "v1.0.0"]) + assert result.returncode != 0 + assert "--event is only supported with --branch" in result.stderr + + +@pytest.mark.agent_authored(model="gpt-6") +def test_empty_event_is_rejected(lookup): + result = lookup({}, event="") + assert result.returncode != 0 + assert "Usage:" in result.stderr