diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 313be5ff77e..26d17f65b1e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -658,6 +658,10 @@ jobs: name: Check job status if: ${{ always() && github.repository_owner == 'nvidia' }} runs-on: ubuntu-latest + permissions: + actions: read # download the test-results artifacts + contents: write # publish the report to the gh-pages branch + pull-requests: write # sticky comment with the report link needs: - ci-vars - should-skip @@ -676,6 +680,102 @@ jobs: - doc - precommit-windows steps: + # DEMO: render every test job's JUnit XML as an Allure report and publish + # it to GitHub Pages next to the docs previews. Every step is + # informational: continue-on-error keeps it away from the gate below, and + # none of them fails on test failures. + - name: Checkout repository + if: ${{ needs.should-skip.outputs.skip != 'true' }} + continue-on-error: true + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + + - name: Download test results + if: ${{ needs.should-skip.outputs.skip != 'true' }} + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: test-results-* + path: test-results + + - name: Prepare the Allure results + # One flat directory of JUnit files, each suite tagged with its CI + # configuration, plus an allurerc.mjs that maps the tags to Allure + # environments so the configurations stay apart in the report. + if: ${{ needs.should-skip.outputs.skip != 'true' }} + continue-on-error: true + run: | + python3 ci/tools/allure_prepare.py \ + --results test-results \ + --output allure-results \ + --config allurerc.mjs \ + --report-name "cuda-python CI test results (${{ github.sha }})" + + - name: Generate the Allure report + # Allure Report 3 is a CLI from npm, run here as a plain command. + if: ${{ needs.should-skip.outputs.skip != 'true' }} + continue-on-error: true + run: | + npx --yes allure@3.20.0 generate allure-results -o allure-report + size=$(du -sh allure-report | cut -f1) + files=$(find allure-report -type f | wc -l) + compressed=$(tar czf - allure-report | wc -c | awk '{ printf "%.0f MB", $1 / 1048576 }') + echo "ALLURE_REPORT_SIZE=${size}" >> "$GITHUB_ENV" + echo "ALLURE_REPORT_FILES=${files}" >> "$GITHUB_ENV" + echo "ALLURE_REPORT_COMPRESSED=${compressed}" >> "$GITHUB_ENV" + { + echo "## Allure report" + echo + echo "Size on disk: ${size} in ${files} files (${compressed} compressed)." + echo + echo '```json' + cat allure-report/widgets/statistic.json + echo + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload the Allure report + if: ${{ needs.should-skip.outputs.skip != 'true' }} + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: allure-report + path: allure-report/ + if-no-files-found: ignore + retention-days: 14 + + # Only mirror branches have a PR to publish for and to comment on. + - name: Get PR number + if: ${{ needs.should-skip.outputs.skip != 'true' && startsWith(github.ref_name, 'pull-request/') }} + continue-on-error: true + uses: ./.github/actions/get_pr_number + + - name: Publish the Allure report to GitHub Pages + # Same mechanism as the docs preview: a folder in the gh-pages branch, + # served at https://nvidia.github.io/cuda-python/test-report/pr-N/. + if: ${{ needs.should-skip.outputs.skip != 'true' && startsWith(github.ref_name, 'pull-request/') && env.PR_NUMBER != '' }} + continue-on-error: true + uses: JamesIves/github-pages-deploy-action@fa24774553152dd7873cd16ebd8d959b010c5445 # v4.9.0 + with: + git-config-name: cuda-python-bot + git-config-email: cuda-python-bot@users.noreply.github.com + folder: allure-report + target-folder: docs/test-report/pr-${{ env.PR_NUMBER }}/ + commit-message: "Deploy test report for PR ${{ env.PR_NUMBER }} (${{ github.sha }})" + + - name: Post the test-report comment + if: ${{ needs.should-skip.outputs.skip != 'true' && startsWith(github.ref_name, 'pull-request/') && env.PR_NUMBER != '' }} + continue-on-error: true + uses: marocchino/sticky-pull-request-comment@d4d6b0936434b21bc8345ad45a440c5f7d2c40ff # v3.0.3 + with: + header: test-report + number: ${{ env.PR_NUMBER }} + message: | + Allure test report for ${{ github.sha }}: https://nvidia.github.io/cuda-python/test-report/pr-${{ env.PR_NUMBER }}/ + + The report is ${{ env.ALLURE_REPORT_SIZE }} on disk in ${{ env.ALLURE_REPORT_FILES }} files (${{ env.ALLURE_REPORT_COMPRESSED }} compressed), all committed to the gh-pages branch. + - name: Exit env: NEEDS_JSON: ${{ toJSON(needs) }} diff --git a/ci/test-matrix.yml b/ci/test-matrix.yml index 9897df54702..b215a4e7c59 100644 --- a/ci/test-matrix.yml +++ b/ci/test-matrix.yml @@ -40,55 +40,11 @@ platforms: linux: pull-request: - # linux-64 - - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '12.9.1', LOCAL_CTK: '1', GPU: 'v100', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.11', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.11', CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.11', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } - # disabled: compute-sanitizer install broken on CUDA 12.9.1 local CTK (TODO: re-enable once fixed) - # - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.9.1', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', ENV: { CUDA_PYTHON_CUDA_PER_THREAD_DEFAULT_STREAM: '1' } } - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } + # TEMPORARY: trimmed to one fast row per platform for the Allure report demo + # (PR stacked on #2963). The aarch64 row uses the L4 pool, which had runners + # when the A100 pool did not. Revert this commit before any merge. - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'v100', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: '610.43.02' } - - { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 't4', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: '610.43.02' } - - { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '12.9.1', LOCAL_CTK: '1', GPU: 't4', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.15', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.15t', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } - # linux-aarch64 - - { ARCH: 'arm64', PY_VER: '3.10', CUDA_VER: '12.9.1', LOCAL_CTK: '1', GPU: 'a100', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'arm64', PY_VER: '3.10', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'arm64', PY_VER: '3.10', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'a100', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'arm64', PY_VER: '3.11', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'arm64', PY_VER: '3.11', CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'a100', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'arm64', PY_VER: '3.11', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } - # disabled: compute-sanitizer install broken on CUDA 12.9.1 local CTK (TODO: re-enable once fixed) - # - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '12.9.1', LOCAL_CTK: '1', GPU: 'a100', GPU_COUNT: '1', DRIVER: 'latest' } - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'a100', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'arm64', PY_VER: '3.13', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'arm64', PY_VER: '3.13', CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'a100', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'arm64', PY_VER: '3.13', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'arm64', PY_VER: '3.14', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'a100', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'arm64', PY_VER: '3.14', CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'arm64', PY_VER: '3.14', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'a100', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'arm64', PY_VER: '3.14t', CUDA_VER: '12.9.1', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'arm64', PY_VER: '3.14t', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'a100', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'arm64', PY_VER: '3.14t', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } - # special runners - - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'h100', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'h100', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 't4', GPU_COUNT: '2', DRIVER: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'h100', GPU_COUNT: '2', DRIVER: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.11', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 't4', GPU_COUNT: '1', DRIVER: 'latest', FLAVOR: 'wsl' } - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'rtx4090', GPU_COUNT: '1', DRIVER: 'latest', FLAVOR: 'wsl' } nightly: # nightly-pytorch - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.6.3', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', ENV: { MODE: 'nightly-pytorch', TORCH_VER: '2.12.1', TORCH_CUDA: 'cu126' } } @@ -118,29 +74,8 @@ linux: windows: pull-request: - # win-64 - - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'rtx2080', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'WDDM' } - - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC' } - - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC' } - - { ARCH: 'amd64', PY_VER: '3.11', CUDA_VER: '12.9.1', LOCAL_CTK: '1', GPU: 'v100', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM' } - - { ARCH: 'amd64', PY_VER: '3.11', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'rtx4090', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'WDDM' } - - { ARCH: 'amd64', PY_VER: '3.11', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'rtx4090', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'WDDM' } - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM' } - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'a100', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC' } + # TEMPORARY: see the linux section above. - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'a100', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC' } - - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '12.9.1', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC' } - - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM' } - - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM', ENV: { CUDA_PYTHON_CUDA_PER_THREAD_DEFAULT_STREAM: '1' } } - - { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'v100', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC' } - - { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM' } - - { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM' } - - { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '12.9.1', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC' } - - { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'a100', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM' } - - { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'a100', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM' } - - { ARCH: 'amd64', PY_VER: '3.15', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM' } - # special runners - - { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 't4', GPU_COUNT: '2', DRIVER: 'latest', DRIVER_MODE: 'TCC' } - - { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'h100', GPU_COUNT: '2', DRIVER: 'latest', DRIVER_MODE: 'MCDM' } nightly: # nightly-pytorch - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.6.3', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC', ENV: { MODE: 'nightly-pytorch', TORCH_VER: '2.12.1', TORCH_CUDA: 'cu126' } } diff --git a/ci/tools/allure_prepare.py b/ci/tools/allure_prepare.py new file mode 100644 index 00000000000..7133f51facc --- /dev/null +++ b/ci/tools/allure_prepare.py @@ -0,0 +1,121 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Prepare the JUnit XML files from every CI configuration for Allure Report 3. + +The test workflows upload one ``test-results-`` artifact per +job, each holding one JUnit XML file per suite. Allure reads JUnit XML +directly, but it identifies a test by class name and test name only, so the +same test from several configurations would collapse into "retries" of one +result. This script gives every ```` a ``package`` attribute that +names its configuration, which Allure turns into a ``package`` label, and +writes an ``allurerc.mjs`` whose ``environments`` map that label back to one +environment per configuration. The report then keeps the configurations +apart and shows a test's result in every configuration side by side. + +Usage:: + + python ci/tools/allure_prepare.py --results test-results \\ + --output allure-results --config allurerc.mjs +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +ARTIFACT_PREFIX = "test-results-" +MODE_PREFIX = "standard-" + + +def configuration_name(artifact_dir: Path) -> str: + """Return the configuration name encoded in an artifact directory name.""" + name = artifact_dir.name + if name.startswith(ARTIFACT_PREFIX): + name = name[len(ARTIFACT_PREFIX) :] + if name.startswith(MODE_PREFIX): + name = name[len(MODE_PREFIX) :] + return name + + +def environment_id(name: str, taken: set[str]) -> str: + """Return a unique Allure environment id: ``[A-Za-z0-9_-]``, at most 64 characters.""" + base = re.sub(r"[^A-Za-z0-9_-]", "-", name)[:60] or "configuration" + candidate = base + counter = 2 + while candidate in taken: + candidate = f"{base}-{counter}" + counter += 1 + taken.add(candidate) + return candidate + + +def prepare(results: Path, output: Path) -> dict[str, int]: + """Copy every JUnit file into ``output`` with its configuration as the suite package. + + Returns the number of files copied per configuration. + """ + output.mkdir(parents=True, exist_ok=True) + counts: dict[str, int] = {} + for xml_path in sorted(results.rglob("*.xml")): + configuration = configuration_name(xml_path.parent) + # The files come from our own test jobs, not from an untrusted source. + tree = ET.parse(xml_path) # noqa: S314 + for suite in tree.iter("testsuite"): + suite.set("package", configuration) + tree.write(output / f"{configuration}-{xml_path.name}", encoding="utf-8", xml_declaration=True) + counts[configuration] = counts.get(configuration, 0) + 1 + return counts + + +def render_config(report_name: str, configurations: list[str]) -> str: + """Return the allurerc.mjs text with one environment per configuration.""" + taken: set[str] = set() + entries = [] + for configuration in configurations: + env_id = json.dumps(environment_id(configuration, taken)) + value = json.dumps(configuration) + entries.append( + f" {env_id}: {{\n" + f" name: {value},\n" + f' matcher: ({{ labels }}) => labels.some(({{ name, value }}) => name === "package" && value === {value}),\n' + f" }}," + ) + return ( + "// Generated by ci/tools/allure_prepare.py: one environment per CI configuration.\n" + "export default {\n" + f" name: {json.dumps(report_name)},\n" + " environments: {\n" + "\n".join(entries) + "\n },\n};\n" + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Prepare CI JUnit XML files for Allure Report 3.") + parser.add_argument( + "--results", type=Path, required=True, help="directory with the downloaded test-results-* artifacts" + ) + parser.add_argument( + "--output", type=Path, required=True, help="flat directory of JUnit files for 'allure generate'" + ) + parser.add_argument("--config", type=Path, required=True, help="path of the allurerc.mjs to write") + parser.add_argument("--report-name", default="cuda-python test results", help="title of the report") + args = parser.parse_args(argv) + + counts = prepare(args.results, args.output) + if not counts: + print(f"error: no JUnit XML files under {args.results}", file=sys.stderr) + return 1 + args.config.write_text(render_config(args.report_name, sorted(counts)), encoding="utf-8") + for configuration, count in sorted(counts.items()): + print(f"{configuration}: {count} files") + print(f"{len(counts)} configurations -> {args.output}; config written to {args.config}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/tools/tests/test_allure_prepare.py b/ci/tools/tests/test_allure_prepare.py new file mode 100644 index 00000000000..3901f1c0c94 --- /dev/null +++ b/ci/tools/tests/test_allure_prepare.py @@ -0,0 +1,79 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import os +import sys +import xml.etree.ElementTree as ET + +import pytest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from allure_prepare import configuration_name, environment_id, main, prepare, render_config + +JUNIT = """ + + +boom + +""" + + +def write_artifacts(root): + for artifact in ("test-results-standard-linux-64-py3.12", "test-results-nightly-cuda-core-win-64"): + directory = root / artifact + directory.mkdir(parents=True) + (directory / "junit-core.xml").write_text(JUNIT, encoding="utf-8") + (directory / "junit-bindings.xml").write_text(JUNIT, encoding="utf-8") + + +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_configuration_name_strips_the_artifact_and_standard_prefixes(tmp_path): + assert configuration_name(tmp_path / "test-results-standard-linux-64-py3.12") == "linux-64-py3.12" + assert configuration_name(tmp_path / "test-results-nightly-cuda-core-win-64") == "nightly-cuda-core-win-64" + + +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_prepare_flattens_the_files_and_tags_every_suite_with_its_configuration(tmp_path): + write_artifacts(tmp_path / "in") + counts = prepare(tmp_path / "in", tmp_path / "out") + assert counts == {"linux-64-py3.12": 2, "nightly-cuda-core-win-64": 2} + assert sorted(p.name for p in (tmp_path / "out").iterdir()) == [ + "linux-64-py3.12-junit-bindings.xml", + "linux-64-py3.12-junit-core.xml", + "nightly-cuda-core-win-64-junit-bindings.xml", + "nightly-cuda-core-win-64-junit-core.xml", + ] + tree = ET.parse(tmp_path / "out" / "linux-64-py3.12-junit-core.xml") # noqa: S314 + assert [suite.get("package") for suite in tree.iter("testsuite")] == ["linux-64-py3.12"] + assert len(list(tree.iter("testcase"))) == 2 + + +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_render_config_has_one_environment_per_configuration(): + text = render_config("demo", ["linux-64-py3.12", "win-64 (TCC)"]) + assert 'name: "demo"' in text + assert '"linux-64-py3-12": {' in text + assert '"win-64--TCC-": {' in text + assert 'value === "win-64 (TCC)"' in text + assert text.count("matcher:") == 2 + + +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_environment_id_is_sanitized_and_unique(): + taken: set[str] = set() + assert environment_id("a.b", taken) == "a-b" + assert environment_id("a_b", taken) == "a_b" + assert environment_id("a-b", taken) == "a-b-2" + + +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_main_fails_when_there_are_no_results(tmp_path, capsys): + (tmp_path / "empty").mkdir() + rc = main( + ["--results", str(tmp_path / "empty"), "--output", str(tmp_path / "out"), "--config", str(tmp_path / "rc.mjs")] + ) + assert rc == 1 + assert "no JUnit XML" in capsys.readouterr().err diff --git a/cuda_core/tests/test_zz_injected_failures.py b/cuda_core/tests/test_zz_injected_failures.py new file mode 100644 index 00000000000..debfca3d5b1 --- /dev/null +++ b/cuda_core/tests/test_zz_injected_failures.py @@ -0,0 +1,71 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""TEMPORARY: deliberate failures that exercise the CI failure-visibility changes. + +REMOVE THIS FILE BEFORE MERGE. + +Each test fails in a different way, so one CI run shows how the folded log, the +reordered short test summary, and the GitHub annotations render: + +* a plain assertion in a parallel test (pytest-run-parallel reports it as ERROR) +* two failing cases of a parametrized test, with a passing case between them +* a thread-unsafe test that raises (the FAILED path, outside the parallel runner) +* an assertion with a multi-line diff +* a fixture error (ERROR at setup; the annotate plugin covers only the call phase) +* a failure preceded by a warning (the warning must not become an annotation) +* an exception raised inside a helper (two traceback frames) +""" + +import warnings + +import pytest + + +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_injected_plain_assert(): + assert 1 + 1 == 3 + + +@pytest.mark.agent_authored(model="claude-fable-5-1") +@pytest.mark.parametrize("value", [1, 2, 3]) +def test_injected_parametrized(value): + assert value % 2 == 0, f"value {value} is odd" + + +@pytest.mark.agent_authored(model="claude-fable-5-1") +@pytest.mark.thread_unsafe(reason="injected: exercises the non-parallel FAILED path") +def test_injected_thread_unsafe(): + raise RuntimeError("injected failure outside the parallel runner") + + +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_injected_dict_diff(): + expected = {"a": 1, "b": 2, "c": 3} + actual = {"a": 1, "b": 20, "d": 4} + assert actual == expected + + +@pytest.fixture +def broken_fixture(): + raise ValueError("injected fixture error") + + +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_injected_setup_error(broken_fixture): + pass + + +@pytest.mark.agent_authored(model="claude-fable-5-1") +@pytest.mark.thread_unsafe(reason="records process-global warnings") +def test_injected_warning_then_fail(): + warnings.warn("injected warning: must not become an annotation", UserWarning, stacklevel=1) + raise AssertionError("injected failure after a warning") + + +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_injected_exception_in_helper(): + def helper(n): + return 10 // n + + assert helper(0) == 1