Skip to content

ci: probe fabric handle support on GPU runners (temporary, do not merge) #3125

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Andy-Jost wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

ci: probe fabric handle support on GPU runners (temporary, do not merge) #3125

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions ci/tools/run-tests
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,47 @@ fold_pytest_log() {
python "${CI_TOOLS_DIR}/fold_pytest_sections.py"
}

# TEMPORARY PROBE (draft PR, never merged): report whether fabric memory handles
# are usable on this runner. Every command is guarded so the probe cannot fail the job.
fabric_probe() {
echo "=== FABRIC PROBE ==="
(
set +e
nvidia-smi --query-gpu=name,driver_version --format=csv,noheader || true
echo "--- /proc/devices (imex)"
grep -i imex /proc/devices || true
echo "--- /dev/nvidia-caps-imex-channels/"
ls -la /dev/nvidia-caps-imex-channels/ || true
echo "--- /proc/driver/nvidia/params (imex)"
grep -i imex /proc/driver/nvidia/params || true
echo "--- device attributes"
python - << 'PYEOF' || true
from cuda.bindings import driver


def check(result):
err = result[0]
if int(err) != 0:
raise RuntimeError(f"driver call failed: {err}")
return result[1]


A = driver.CUdevice_attribute
fabric = A.CU_DEVICE_ATTRIBUTE_HANDLE_TYPE_FABRIC_SUPPORTED
posix_fd = A.CU_DEVICE_ATTRIBUTE_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR_SUPPORTED
check(driver.cuInit(0))
for i in range(check(driver.cuDeviceGetCount())):
dev = check(driver.cuDeviceGet(i))
name = check(driver.cuDeviceGetName(128, dev)).split(b"\0")[0].decode()
print(
f"device {i}: {name}: FABRIC_SUPPORTED={check(driver.cuDeviceGetAttribute(fabric, dev))}"
f" POSIX_FILE_DESCRIPTOR_SUPPORTED={check(driver.cuDeviceGetAttribute(posix_fd, dev))}"
)
PYEOF
) || true
echo "=== END FABRIC PROBE ==="
}

# JUnit XML, one file per suite. The test workflows upload this directory as an
# artifact: a machine-readable record of the run for follow-up tooling.
TEST_RESULTS_DIR="$(pwd)/test-results"
Expand Down Expand Up @@ -78,6 +119,7 @@ elif [[ "${test_module}" == "bindings" ]]; then
pip install $(ls "${CUDA_BINDINGS_ARTIFACTS_DIR}"/*.whl)[all] --group test "${TEST_FT_GROUP[@]}" \
"cuda-toolkit==${TEST_CUDA_MAJOR_MINOR}.*"
fi
fabric_probe
echo "Running bindings tests"
${SANITIZER_CMD} pytest -rsxXfE -v --randomly-dont-reorganize --exclude-warning-annotations "${PYTEST_PARALLEL_ARGS[@]}" --junitxml "${TEST_RESULTS_DIR}/junit-bindings.xml" tests/ | fold_pytest_log
if [[ "${SKIP_CYTHON_TEST}" == 0 ]]; then
Expand Down Expand Up @@ -125,6 +167,7 @@ elif [[ "${test_module}" == "core" || "${test_module}" == nightly-* ]]; then
pushd ./cuda_core
echo "Installing bindings (source: ${BINDINGS_SOURCE})"
pip install "${BINDINGS_ARGS[@]}"
fabric_probe
echo "Installing core wheel"
# Constrain cuda-toolkit to the requested CTK version to avoid
# pip pulling in a newer nvidia-cuda-runtime that conflicts with it.
Expand Down
Loading