github-pr.yml

# SPDX-FileCopyrightText: Copyright (c) 2023 - 2026 NVIDIA CORPORATION & AFFILIATES.
# SPDX-FileCopyrightText: All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This CI runs on pull requests and uses testmon to skip tests
# that don't have changed dependencies based on the nightly cache.
# The tests run here will only use UV.
#
# ----------------------------------------------------------------------------
# Cache design (see .github/CACHE_CONTRACT.md):
#
#   Each job restores the nightly's uv download cache fail-open, then
#   rebuilds .venv from the committed lockfile via `uv sync --frozen`.
#   Testmon and coverage baseline caches are restored by lockhash with a
#   prefix fallback (best-effort; testmon handles stale DBs gracefully).
#
#   No cross-run .venv cache: a pinned container + frozen lockfile +
#   pinned uv version already make the sync deterministic, and caching
#   the realized venv was not worth the complexity it cost.
# ----------------------------------------------------------------------------

name: Pull Request Github CI
on:
  workflow_dispatch:
  push:
    branches:
      - "pull-request/[0-9]+"

permissions:
  contents: read
  actions: write
  checks: write

defaults:
  run:
    shell: bash

env:
  # ---- Container baseline identity ---------------------------------------
  # MUST match the nightly workflow so cache keys align.  See
  # .github/CACHE_CONTRACT.md for the full design.
  PYTHON_VERSION: "3.12"
  UV_VERSION: "0.13.5"
  CONTAINER_ID: "cuda13.3.1-devel-ubuntu24.04"
  # All feature extras + cu13 backend + matching natten wheel index.  This
  # powers the @requires_module / pytest.importorskip tests that would
  # otherwise be skipped due to missing optional dependencies.  Modules
  # with no extras home (moto, scikit-image, pyg_lib, earth2grid, ...) are
  # installed by the `Install CI-only test dependencies` step inside the
  # setup-uv-env composite action.
  EXTRAS_TAG: "cu13,natten-cu13,utils-extras,mesh-extras,nn-extras,model-extras,datapipes-extras,uq-extras,gnns,sym,transformer-engine-cu13"

  # ---- Cache key prefixes (shared with nightly) --------------------------
  # The `-fullextras` suffix is bumped relative to the previous prefix so
  # the first run under the expanded EXTRAS_TAG rebuilds the wheel cache
  # from scratch instead of layering on top of a stale narrower cache.
  UV_CACHE_KEY_PREFIX:   "uv-cache-nightly-cuda13.3.1-devel-ubuntu24.04-py3.12-uv0.13.5-torch2.13-fullextras"
  TESTMON_CACHE_KEY_PREFIX:  "testmon-nightly-cu13"
  COVERAGE_CACHE_KEY_PREFIX: "coverage-nightly-cu13"
  JIT_CACHE_KEY_PREFIX: "jit-cache-cuda13.3.1-devel-ubuntu24.04-py3.12"
  JIT_CACHE_DIR: "/root/.cache/jit"

  # ---- uv read-only defaults --------------------------------------------
  UV_FROZEN: "1"
  UV_NO_SYNC: "1"

  PYVISTA_OFF_SCREEN: "true"

jobs:
  # Stage 1: Run testmon tests
  #
  # Each GPU job sets up its own environment: restore the nightly's uv
  # download cache fail-open, then `uv sync --frozen --group dev --extra
  # cu13`.  The sync is fast because the warm uv cache already has the
  # wheels.  If the PR bumped uv.lock, any new wheels are downloaded on
  # demand and the cache is simply not quite as warm.
  testmon:
    name: Testmon
    runs-on: linux-amd64-gpu-h100-latest-1
    container:
      image: nvidia/cuda:13.3.1-devel-ubuntu24.04
      # /dev/shm defaults to 64 MiB in docker, which DALI's multiprocess
      # worker pool exhausts via SemLock allocations and trips ENOSPC
      # ("No space left on device") in datapipes tests.  2 GiB is plenty
      # for the test suite and matches the PyTorch container default.
      options: --shm-size=2g

    steps:
    - uses: actions/checkout@v5

    - name: Bootstrap cuDNN CI container
      uses: ./.github/actions/bootstrap-cudnn-ci
      with:
        python-version: ${{ env.PYTHON_VERSION }}
        uv-version: ${{ env.UV_VERSION }}

    - name: Setup uv environment from cache
      uses: ./.github/actions/setup-uv-env
      with:
        uv-cache-key-prefix: ${{ env.UV_CACHE_KEY_PREFIX }}
        uv-cache-key-suffix: "latest"
        extras: ${{ env.EXTRAS_TAG }}

    # Restore the nightly testmon DB so testmon can skip unchanged tests.
    # Pulls from the mutable -latest slot the nightly publishes via
    # replace-cache.  Fail-open: a miss only costs full-suite runtime,
    # never correctness, and testmon handles stale DBs gracefully by
    # re-running tests whose dependency hashes differ.
    - name: Restore testmon database from nightly cache
      uses: actions/cache/restore@v5
      with:
        path: |
          .testmondata
          .testmondata-shm
          .testmondata-wal
        key: ${{ env.TESTMON_CACHE_KEY_PREFIX }}-latest

    - name: Restore JIT compilation cache from nightly
      uses: actions/cache/restore@v5
      with:
        path: ${{ env.JIT_CACHE_DIR }}
        key: ${{ env.JIT_CACHE_KEY_PREFIX }}-latest

    - name: Download CI test data
      uses: ./.github/actions/download-ci-data
      with:
        hf-token: ${{ secrets.HF_CI_DATA_TOKEN }}

    - name: Run core tests (with testmon)
      env:
        WARP_CACHE_PATH: ${{ env.JIT_CACHE_DIR }}/warp
        TRITON_CACHE_DIR: ${{ env.JIT_CACHE_DIR }}/triton
        TORCHINDUCTOR_CACHE_DIR: ${{ env.JIT_CACHE_DIR }}/inductor
      run: |
        uv run --no-sync python -m pytest --testmon --ignore-glob="*docs*" --ignore-glob="*examples*"

  # Stage 2: Run coverage tests and upload artifacts
  coverage:
    name: Coverage
    runs-on: linux-amd64-gpu-h100-latest-1
    env:
      # Single source of truth for the coverage gate; read by both the
      # "Coverage %" check run (display) and the enforcement step (gate).
      COVERAGE_FAIL_UNDER: "70"
    container:
      image: nvidia/cuda:13.3.1-devel-ubuntu24.04
      # /dev/shm defaults to 64 MiB in docker, which DALI's multiprocess
      # worker pool exhausts via SemLock allocations and trips ENOSPC
      # ("No space left on device") in datapipes tests.  2 GiB is plenty
      # for the test suite and matches the PyTorch container default.
      options: --shm-size=2g

    steps:
    - uses: actions/checkout@v5

    - name: Bootstrap cuDNN CI container
      uses: ./.github/actions/bootstrap-cudnn-ci
      with:
        python-version: ${{ env.PYTHON_VERSION }}
        uv-version: ${{ env.UV_VERSION }}

    - name: Setup uv environment from cache
      uses: ./.github/actions/setup-uv-env
      with:
        uv-cache-key-prefix: ${{ env.UV_CACHE_KEY_PREFIX }}
        uv-cache-key-suffix: "latest"
        extras: ${{ env.EXTRAS_TAG }}

    - name: Restore testmon database from nightly cache
      uses: actions/cache/restore@v5
      with:
        path: |
          .testmondata
          .testmondata-shm
          .testmondata-wal
        key: ${{ env.TESTMON_CACHE_KEY_PREFIX }}-latest

    - name: Restore nightly coverage baseline from cache
      id: cache-coverage-restore
      uses: actions/cache/restore@v5
      with:
        path: .coverage*
        key: ${{ env.COVERAGE_CACHE_KEY_PREFIX }}-latest

    # Nightly multi-GPU coverage shards (published by github-multigpu.yml)
    # use the canonical `.coverage.pytest.` prefix so the plain `coverage
    # combine` below folds them into the full report.  Restores fail open:
    # a missing slot just means the report lacks multi-GPU lines.
    - name: Restore nightly dynamic multi-GPU coverage from cache
      uses: actions/cache/restore@v5
      with:
        path: .coverage.pytest.multigpu-dynamic*
        key: coverage-multigpu-dynamic-latest

    - name: Restore nightly static multi-GPU coverage from cache
      uses: actions/cache/restore@v5
      with:
        path: .coverage.pytest.multigpu-static*
        key: coverage-multigpu-static-latest

    - name: Restore JIT compilation cache from nightly
      uses: actions/cache/restore@v5
      with:
        path: ${{ env.JIT_CACHE_DIR }}
        key: ${{ env.JIT_CACHE_KEY_PREFIX }}-latest

    - name: Download CI test data
      uses: ./.github/actions/download-ci-data
      with:
        hf-token: ${{ secrets.HF_CI_DATA_TOKEN }}

    # --testmon-nocollect: testmon's own tracer would otherwise displace
    # `coverage run` and drop everything executed inside tests.
    - name: Run core tests for coverage report (testmon-selected)
      env:
        WARP_CACHE_PATH: ${{ env.JIT_CACHE_DIR }}/warp
        TRITON_CACHE_DIR: ${{ env.JIT_CACHE_DIR }}/triton
        TORCHINDUCTOR_CACHE_DIR: ${{ env.JIT_CACHE_DIR }}/inductor
      run: |
        uv run --no-sync coverage run --rcfile='test/coverage.pytest.rc' -m pytest --testmon --testmon-nocollect --ignore-glob="*docs*" --ignore-glob="*examples*"

    - name: Run doc tests (testmon not supported for doctests)
      env:
        WARP_CACHE_PATH: ${{ env.JIT_CACHE_DIR }}/warp
        TRITON_CACHE_DIR: ${{ env.JIT_CACHE_DIR }}/triton
        TORCHINDUCTOR_CACHE_DIR: ${{ env.JIT_CACHE_DIR }}/inductor
      run: |
        uv run --no-sync coverage run --rcfile='test/coverage.docstring.rc' -m pytest --doctest-modules physicsnemo/ --ignore-glob="*internal*" --ignore-glob="*experimental*"

    - name: Merge coverage reports
      run: |
        echo "Coverage files to combine:"
        ls -la .coverage* 2>/dev/null || echo "No coverage files found"

        uv run --no-sync coverage combine

        # -i / --ignore-errors downgrades coverage's fatal "No source for
        # code" error to a warning.  The restored nightly baseline can
        # reference files this PR has moved or deleted (e.g. a renamed
        # module); without -i the combined report aborts on the stale path.
        #
        # Report generation is intentionally kept free of --fail-under: it
        # used to gate this step, which meant html/xml/json (and the PR's
        # coverage percentage) were silently skipped whenever a PR fell
        # under the threshold -- exactly the case where seeing the number
        # matters most. The threshold is now enforced by its own step below.
        uv run --no-sync coverage report -i --show-missing --omit="*test*" --omit="*internal*" --omit="*experimental*"
        uv run --no-sync coverage html -i
        # xml carries the same omits as report/json so the number Codecov
        # publishes agrees with the "Coverage %" check and the gate below.
        uv run --no-sync coverage xml -i -o coverage.xml --omit="*test*" --omit="*internal*" --omit="*experimental*"
        uv run --no-sync coverage json -i -o coverage.json --omit="*test*" --omit="*internal*" --omit="*experimental*"

    - name: Upload coverage HTML report
      uses: actions/upload-artifact@v4
      with:
        name: coverage-report-pr
        path: htmlcov/
        retention-days: 7

    - name: Upload coverage to Codecov
      if: ${{ !cancelled() }}
      # Best-effort (NVIDIA/warp's pattern): inert until an admin installs
      # the Codecov GitHub App and provisions the CODECOV_TOKEN secret, and
      # never fails the job on upload errors. Behavior config: codecov.yml
      # at the repo root. Dashboard: app.codecov.io/gh/NVIDIA/physicsnemo
      continue-on-error: true
      uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f  # v7.0.0
      with:
        disable_search: true
        files: ./coverage.xml
        flags: pr-core
        token: ${{ secrets.CODECOV_TOKEN }}

    - name: Report coverage percentage on the PR
      if: always()
      # Informational: a failure here (corrupt coverage.json, transient
      # Checks API error) must never fail the Coverage job itself.
      continue-on-error: true
      uses: actions/github-script@v7
      with:
        script: |
          const fs = require('fs');

          // Informational only -- the existing "Coverage" job/check
          // already gates pass/fail on the threshold; this check just
          // surfaces the actual number and never blocks the PR itself.
          const check = {
            owner: context.repo.owner,
            repo: context.repo.repo,
            name: 'Coverage %',
            head_sha: context.sha,
            status: 'completed',
            conclusion: 'neutral',
          };

          // With `if: always()` this step runs even when an earlier step
          // died before writing coverage.json; report that instead of
          // throwing on the missing file.
          if (!fs.existsSync('coverage.json')) {
            check.output = {
              title: 'coverage.json not produced',
              summary:
                'An earlier step in the Coverage job failed before the ' +
                'report was generated -- see that step\'s log for the cause.',
            };
            await github.rest.checks.create(check);
            return;
          }

          const data = JSON.parse(fs.readFileSync('coverage.json', 'utf8'));
          const pct = data.totals.percent_covered.toFixed(2);
          const threshold = Number(process.env.COVERAGE_FAIL_UNDER);
          const passed = data.totals.percent_covered >= threshold;

          check.output = {
            title: `${pct}% covered (excl. test/internal/experimental)`,
            summary:
              `**Line coverage: ${pct}%** (threshold: ${threshold}%, ` +
              `${passed ? 'passing' : 'currently failing -- see the "Coverage" check'})\n\n` +
              `- Lines covered: ${data.totals.covered_lines} / ${data.totals.num_statements}\n` +
              `- Branches covered: ${data.totals.covered_branches} / ${data.totals.num_branches}\n\n` +
              `Full annotated report: see the \`coverage-report-pr\` artifact on this workflow run.`,
          };
          await github.rest.checks.create(check);

    - name: Enforce coverage threshold
      run: |
        uv run --no-sync coverage report -i --omit="*test*" --omit="*internal*" --omit="*experimental*" --fail-under="${COVERAGE_FAIL_UNDER}"