From 63a62cde414ec8efce092af8176782ed6f009606 Mon Sep 17 00:00:00 2001
From: Andy Jost <ajost@nvidia.com>
Date: Thu, 15 Oct 2026 14:54:51 -0700
Subject: [PATCH] ci: probe fabric handle support on GPU runners (temporary)

Print the IMEX channel state, the nvidia-smi GPU and driver line, and the fabric and POSIX file descriptor handle device attributes once per bindings and core test job. This shows which runners can run fabric memory handle tests, to plan the fabric-handle follow-up to #2980.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
---
 ci/tools/run-tests | 43 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/ci/tools/run-tests b/ci/tools/run-tests
index e90517c82b9..4033b523078 100755
--- a/ci/tools/run-tests
+++ b/ci/tools/run-tests
@@ -20,6 +20,47 @@ fold_pytest_log() {
   python "${CI_TOOLS_DIR}/fold_pytest_sections.py"
 }
 
+# TEMPORARY PROBE (draft PR, never merged): report whether fabric memory handles
+# are usable on this runner. Every command is guarded so the probe cannot fail the job.
+fabric_probe() {
+  echo "=== FABRIC PROBE ==="
+  (
+    set +e
+    nvidia-smi --query-gpu=name,driver_version --format=csv,noheader || true
+    echo "--- /proc/devices (imex)"
+    grep -i imex /proc/devices || true
+    echo "--- /dev/nvidia-caps-imex-channels/"
+    ls -la /dev/nvidia-caps-imex-channels/ || true
+    echo "--- /proc/driver/nvidia/params (imex)"
+    grep -i imex /proc/driver/nvidia/params || true
+    echo "--- device attributes"
+    python - << 'PYEOF' || true
+from cuda.bindings import driver
+
+
+def check(result):
+    err = result[0]
+    if int(err) != 0:
+        raise RuntimeError(f"driver call failed: {err}")
+    return result[1]
+
+
+A = driver.CUdevice_attribute
+fabric = A.CU_DEVICE_ATTRIBUTE_HANDLE_TYPE_FABRIC_SUPPORTED
+posix_fd = A.CU_DEVICE_ATTRIBUTE_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR_SUPPORTED
+check(driver.cuInit(0))
+for i in range(check(driver.cuDeviceGetCount())):
+    dev = check(driver.cuDeviceGet(i))
+    name = check(driver.cuDeviceGetName(128, dev)).split(b"\0")[0].decode()
+    print(
+        f"device {i}: {name}: FABRIC_SUPPORTED={check(driver.cuDeviceGetAttribute(fabric, dev))}"
+        f" POSIX_FILE_DESCRIPTOR_SUPPORTED={check(driver.cuDeviceGetAttribute(posix_fd, dev))}"
+    )
+PYEOF
+  ) || true
+  echo "=== END FABRIC PROBE ==="
+}
+
 # JUnit XML, one file per suite. The test workflows upload this directory as an
 # artifact: a machine-readable record of the run for follow-up tooling.
 TEST_RESULTS_DIR="$(pwd)/test-results"
@@ -78,6 +119,7 @@ elif [[ "${test_module}" == "bindings" ]]; then
     pip install $(ls "${CUDA_BINDINGS_ARTIFACTS_DIR}"/*.whl)[all] --group test "${TEST_FT_GROUP[@]}" \
       "cuda-toolkit==${TEST_CUDA_MAJOR_MINOR}.*"
   fi
+  fabric_probe
   echo "Running bindings tests"
   ${SANITIZER_CMD} pytest -rsxXfE -v --randomly-dont-reorganize --exclude-warning-annotations "${PYTEST_PARALLEL_ARGS[@]}" --junitxml "${TEST_RESULTS_DIR}/junit-bindings.xml" tests/ | fold_pytest_log
   if [[ "${SKIP_CYTHON_TEST}" == 0 ]]; then
@@ -125,6 +167,7 @@ elif [[ "${test_module}" == "core" || "${test_module}" == nightly-* ]]; then
     pushd ./cuda_core
     echo "Installing bindings (source: ${BINDINGS_SOURCE})"
     pip install "${BINDINGS_ARGS[@]}"
+    fabric_probe
     echo "Installing core wheel"
     # Constrain cuda-toolkit to the requested CTK version to avoid
     # pip pulling in a newer nvidia-cuda-runtime that conflicts with it.