From 0370d07561b75af50e0319738e807abd605cdc53 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 2 Oct 2026 10:54:01 -0700 Subject: [PATCH 1/3] Remove xfail_if_mempool_oom and xfail_on_graph_mempool_oom helpers Issue #2381 (mempool OOMs on Windows MCDM) was resolved by disabling ASLR on the affected machine. Drop the xfail helpers that papered over those failures so the test suite surfaces any remaining OOMs directly, and rerun to confirm the helpers are no longer needed. - Delete cuda_python_test_helpers/cuda_python_test_helpers/mempool.py (xfail_if_mempool_oom, is_windows_mcdm_device). - Remove xfail_on_graph_mempool_oom from cuda_core/tests/helpers/memory.py and drop its import across the graph test suite; de-indent the bodies of the removed 'with' blocks. - Drop direct xfail_if_mempool_oom calls in cuda_bindings tests (test_cuda.py, test_cudart.py, test_interoperability.py) and cuda_core (test_managed_memory_warning.py). - In helpers/memory.py, remove the now-dead xfail branches from skip_if_managed_memory_unsupported, create_managed_memory_resource_or_skip, and create_pinned_memory_resource_or_xfail (the latter now just returns the resource). The xfail_device parameter is retained on the public helper signatures to avoid churning call sites. --- cuda_bindings/tests/test_cuda.py | 3 - cuda_bindings/tests/test_cudart.py | 2 - cuda_bindings/tests/test_interoperability.py | 3 - .../tests/graph/test_graph_definition.py | 116 ++++++++---------- .../graph/test_graph_definition_errors.py | 41 +++---- .../test_graph_definition_integration.py | 10 +- .../graph/test_graph_definition_lifetime.py | 12 +- .../tests/graph/test_graph_memory_resource.py | 42 +++---- cuda_core/tests/helpers/memory.py | 46 +------ .../tests/test_managed_memory_warning.py | 10 +- cuda_core/tests/test_object_protocols.py | 59 ++++----- .../cuda_python_test_helpers/mempool.py | 54 -------- 12 files changed, 119 insertions(+), 279 deletions(-) delete mode 100644 cuda_python_test_helpers/cuda_python_test_helpers/mempool.py diff --git a/cuda_bindings/tests/test_cuda.py b/cuda_bindings/tests/test_cuda.py index 65212efbcb7..ef592d87b36 100644 --- a/cuda_bindings/tests/test_cuda.py +++ b/cuda_bindings/tests/test_cuda.py @@ -7,7 +7,6 @@ import numpy as np import pytest -from cuda_python_test_helpers.mempool import xfail_if_mempool_oom from cuda_python_test_helpers.subprocess_runner import run_python_snippet import cuda.bindings.driver as cuda @@ -266,7 +265,6 @@ def test_cuda_memPool_attr(): attr_list = [None] * 8 err, pool = cuda.cuMemPoolCreate(poolProps) - xfail_if_mempool_oom(err, "cuMemPoolCreate", poolProps.location.id) assert err == cuda.CUresult.CUDA_SUCCESS for idx, attr in enumerate( @@ -484,7 +482,6 @@ def test_cuda_graphMem_attr(device): assert destroy_err == cuda.CUresult.CUDA_SUCCESS (destroy_err,) = cuda.cuStreamDestroy(stream) assert destroy_err == cuda.CUresult.CUDA_SUCCESS - xfail_if_mempool_oom(err, "cuGraphAddMemAllocNode", device) assert err == cuda.CUresult.CUDA_SUCCESS err, freeNode = cuda.cuGraphAddMemFreeNode(graph, [allocNode], 1, params.dptr) assert err == cuda.CUresult.CUDA_SUCCESS diff --git a/cuda_bindings/tests/test_cudart.py b/cuda_bindings/tests/test_cudart.py index b4ac2142f35..82974586ef5 100644 --- a/cuda_bindings/tests/test_cudart.py +++ b/cuda_bindings/tests/test_cudart.py @@ -6,7 +6,6 @@ import numpy as np import pytest -from cuda_python_test_helpers.mempool import xfail_if_mempool_oom import cuda.bindings.driver as cuda import cuda.bindings.runtime as cudart @@ -524,7 +523,6 @@ def test_cudart_MemPool_attr(): attr_list = [None] * 8 err, pool = cudart.cudaMemPoolCreate(poolProps) - xfail_if_mempool_oom(err, "cudaMemPoolCreate", poolProps.location.id) assertSuccess(err) for idx, attr in enumerate( diff --git a/cuda_bindings/tests/test_interoperability.py b/cuda_bindings/tests/test_interoperability.py index 08bac311a2d..6c056f435d3 100644 --- a/cuda_bindings/tests/test_interoperability.py +++ b/cuda_bindings/tests/test_interoperability.py @@ -3,7 +3,6 @@ import numpy as np import pytest -from cuda_python_test_helpers.mempool import xfail_if_mempool_oom import cuda.bindings.driver as cuda import cuda.bindings.runtime as cudart @@ -88,14 +87,12 @@ def test_interop_graphNode(): def test_interop_memPool(): # DRV to RT err_dr, pool = cuda.cuDeviceGetDefaultMemPool(0) - xfail_if_mempool_oom(err_dr, "cuDeviceGetDefaultMemPool", 0) assert err_dr == cuda.CUresult.CUDA_SUCCESS (err_rt,) = cudart.cudaDeviceSetMemPool(0, pool) assert err_rt == cudart.cudaError_t.cudaSuccess # RT to DRV err_rt, pool = cudart.cudaDeviceGetDefaultMemPool(0) - xfail_if_mempool_oom(err_rt, "cudaDeviceGetDefaultMemPool", 0) assert err_rt == cudart.cudaError_t.cudaSuccess (err_dr,) = cuda.cuDeviceSetMemPool(0, pool) assert err_dr == cuda.CUresult.CUDA_SUCCESS diff --git a/cuda_core/tests/graph/test_graph_definition.py b/cuda_core/tests/graph/test_graph_definition.py index 551b2fa5a3e..16d44fb6034 100644 --- a/cuda_core/tests/graph/test_graph_definition.py +++ b/cuda_core/tests/graph/test_graph_definition.py @@ -12,7 +12,6 @@ import pytest from helpers.graph_kernels import compile_common_kernels -from helpers.memory import xfail_on_graph_mempool_oom from helpers.misc import try_create_condition from cuda.core import Device, LaunchConfig @@ -206,15 +205,13 @@ def _build_disconnected(): def graph_spec(request, init_cuda): if request.param is not _build_empty: _skip_if_no_mempool() - with xfail_on_graph_mempool_oom(): - return request.param() + return request.param() @pytest.fixture(params=_NONEMPTY_BUILDERS) def nonempty_graph_spec(request, init_cuda): _skip_if_no_mempool() - with xfail_on_graph_mempool_oom(): - return request.param() + return request.param() # ============================================================================= @@ -569,8 +566,7 @@ def node_spec(request, init_cuda): if spec.needs_mempool: _skip_if_no_mempool() g = GraphDefinition() - with xfail_on_graph_mempool_oom(): - node, expected_attrs = spec.builder(g) + node, expected_attrs = spec.builder(g) return spec, g, node, expected_attrs @@ -854,9 +850,8 @@ def test_free_creates_dependency(init_cuda): """Free node depends on its predecessor.""" sample_graphdef = GraphDefinition() _skip_if_no_mempool() - with xfail_on_graph_mempool_oom(): - alloc = sample_graphdef.allocate(ALLOC_SIZE) - free = alloc.deallocate(alloc.dptr) + alloc = sample_graphdef.allocate(ALLOC_SIZE) + free = alloc.deallocate(alloc.dptr) assert alloc in free.pred @@ -864,11 +859,10 @@ def test_alloc_free_chain(init_cuda): """Alloc and free can be chained.""" sample_graphdef = GraphDefinition() _skip_if_no_mempool() - with xfail_on_graph_mempool_oom(): - a1 = sample_graphdef.allocate(ALLOC_SIZE) - a2 = a1.allocate(ALLOC_SIZE) - f2 = a2.deallocate(a2.dptr) - f1 = f2.deallocate(a1.dptr) + a1 = sample_graphdef.allocate(ALLOC_SIZE) + a2 = a1.allocate(ALLOC_SIZE) + f2 = a2.deallocate(a2.dptr) + f1 = f2.deallocate(a1.dptr) assert a1 in a2.pred assert a2 in f2.pred assert f2 in f1.pred @@ -898,8 +892,7 @@ def test_alloc_device_option(init_cuda, device_spec): sample_graphdef = GraphDefinition() _skip_if_no_mempool() device = Device() - with xfail_on_graph_mempool_oom(device): - node = sample_graphdef.allocate(ALLOC_SIZE, device=device_spec(device)) + node = sample_graphdef.allocate(ALLOC_SIZE, device=device_spec(device)) assert node.dptr != 0 @@ -907,8 +900,7 @@ def test_alloc_peer_access(mempool_device_x2): """AllocNode.peer_access reflects requested peers.""" d0, d1 = mempool_device_x2 g = GraphDefinition() - with xfail_on_graph_mempool_oom(d0): - node = g.allocate(ALLOC_SIZE, device=d0.device_id, peer_access=[d1.device_id]) + node = g.allocate(ALLOC_SIZE, device=d0.device_id, peer_access=[d1.device_id]) assert d1.device_id in node.peer_access @@ -920,8 +912,7 @@ def test_alloc_memory_type_host(init_cuda): g = GraphDefinition() try: - with xfail_on_graph_mempool_oom(): - node = g.allocate(ALLOC_SIZE, memory_type=GraphMemoryType.HOST) + node = g.allocate(ALLOC_SIZE, memory_type=GraphMemoryType.HOST) except CUDAError as e: if "CUDA_ERROR_NOT_SUPPORTED" in str(e): pytest.skip("Driver does not support graph alloc memory_type='host'") @@ -950,9 +941,8 @@ def test_join_merges_branches(init_cuda, num_branches): """join() with multiple branches creates correct dependencies.""" sample_graphdef = GraphDefinition() _skip_if_no_mempool() - with xfail_on_graph_mempool_oom(): - branches = [sample_graphdef.allocate(ALLOC_SIZE) for _ in range(num_branches)] - joined = sample_graphdef.join(*branches) + branches = [sample_graphdef.allocate(ALLOC_SIZE) for _ in range(num_branches)] + joined = sample_graphdef.join(*branches) assert isinstance(joined, EmptyNode) assert set(joined.pred) == set(branches) @@ -1048,9 +1038,8 @@ def test_instantiate_with_nodes(init_cuda, inst_kwargs): """Graph with nodes can be instantiated.""" sample_graphdef = GraphDefinition() _skip_if_no_mempool() - with xfail_on_graph_mempool_oom(): - sample_graphdef.allocate(ALLOC_SIZE) - sample_graphdef.allocate(ALLOC_SIZE) + sample_graphdef.allocate(ALLOC_SIZE) + sample_graphdef.allocate(ALLOC_SIZE) graph = _instantiate(sample_graphdef, inst_kwargs) assert graph is not None @@ -1093,9 +1082,8 @@ def test_instantiate_and_execute_alloc_free(init_cuda, inst_kwargs): """Graph with alloc/free can be executed.""" sample_graphdef = GraphDefinition() _skip_if_no_mempool() - with xfail_on_graph_mempool_oom(): - alloc = sample_graphdef.allocate(ALLOC_SIZE) - alloc.deallocate(alloc.dptr) + alloc = sample_graphdef.allocate(ALLOC_SIZE) + alloc.deallocate(alloc.dptr) stream = Device().create_stream() graph = _instantiate_and_upload(sample_graphdef, inst_kwargs, stream) @@ -1108,10 +1096,9 @@ def test_instantiate_and_execute_memset(init_cuda, inst_kwargs): """Graph with alloc/memset/free can be executed.""" sample_graphdef = GraphDefinition() _skip_if_no_mempool() - with xfail_on_graph_mempool_oom(): - alloc = sample_graphdef.allocate(ALLOC_SIZE) - ms = alloc.memset(alloc.dptr, 0xAB, ALLOC_SIZE) - ms.deallocate(alloc.dptr) + alloc = sample_graphdef.allocate(ALLOC_SIZE) + ms = alloc.memset(alloc.dptr, 0xAB, ALLOC_SIZE) + ms.deallocate(alloc.dptr) stream = Device().create_stream() graph = _instantiate_and_upload(sample_graphdef, inst_kwargs, stream) @@ -1126,13 +1113,12 @@ def test_instantiate_and_execute_memcpy(init_cuda, inst_kwargs): _skip_if_no_mempool() import ctypes - with xfail_on_graph_mempool_oom(): - src_alloc = sample_graphdef.allocate(ALLOC_SIZE) - dst_alloc = sample_graphdef.allocate(ALLOC_SIZE) - dep = sample_graphdef.join(src_alloc, dst_alloc) - ms = dep.memset(src_alloc.dptr, 0xAB, ALLOC_SIZE) - cp = ms.memcpy(dst_alloc.dptr, src_alloc.dptr, ALLOC_SIZE) - cp.deallocate(src_alloc.dptr) + src_alloc = sample_graphdef.allocate(ALLOC_SIZE) + dst_alloc = sample_graphdef.allocate(ALLOC_SIZE) + dep = sample_graphdef.join(src_alloc, dst_alloc) + ms = dep.memset(src_alloc.dptr, 0xAB, ALLOC_SIZE) + cp = ms.memcpy(dst_alloc.dptr, src_alloc.dptr, ALLOC_SIZE) + cp.deallocate(src_alloc.dptr) stream = Device().create_stream() graph = _instantiate_and_upload(sample_graphdef, inst_kwargs, stream) @@ -1360,12 +1346,11 @@ def test_instantiate_and_execute_if_then(init_cuda): set_handle = mod.get_kernel("set_handle") add_one = mod.get_kernel("add_one") - with xfail_on_graph_mempool_oom(): - alloc = sample_graphdef.allocate(ctypes.sizeof(ctypes.c_int)) - ms = alloc.memset(alloc.dptr, 0, ctypes.sizeof(ctypes.c_int)) - setter = ms.launch(LaunchConfig(grid=1, block=1), set_handle, condition, 1) - if_node = setter.if_then(condition) - if_node.then.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr) + alloc = sample_graphdef.allocate(ctypes.sizeof(ctypes.c_int)) + ms = alloc.memset(alloc.dptr, 0, ctypes.sizeof(ctypes.c_int)) + setter = ms.launch(LaunchConfig(grid=1, block=1), set_handle, condition, 1) + if_node = setter.if_then(condition) + if_node.then.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr) graph = sample_graphdef.instantiate() stream = Device().create_stream() @@ -1394,14 +1379,13 @@ def test_instantiate_and_execute_if_else(init_cuda): set_handle = mod.get_kernel("set_handle") add_one = mod.get_kernel("add_one") - with xfail_on_graph_mempool_oom(): - alloc = sample_graphdef.allocate(ctypes.sizeof(ctypes.c_int)) - ms = alloc.memset(alloc.dptr, 0, ctypes.sizeof(ctypes.c_int)) - setter = ms.launch(LaunchConfig(grid=1, block=1), set_handle, condition, 0) - ie_node = setter.if_else(condition) - ie_node.then.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr) - n1 = ie_node.else_.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr) - n1.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr) + alloc = sample_graphdef.allocate(ctypes.sizeof(ctypes.c_int)) + ms = alloc.memset(alloc.dptr, 0, ctypes.sizeof(ctypes.c_int)) + setter = ms.launch(LaunchConfig(grid=1, block=1), set_handle, condition, 0) + ie_node = setter.if_else(condition) + ie_node.then.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr) + n1 = ie_node.else_.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr) + n1.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr) graph = sample_graphdef.instantiate() stream = Device().create_stream() @@ -1430,13 +1414,12 @@ def test_instantiate_and_execute_switch(init_cuda): set_handle = mod.get_kernel("set_handle") add_one = mod.get_kernel("add_one") - with xfail_on_graph_mempool_oom(): - alloc = sample_graphdef.allocate(ctypes.sizeof(ctypes.c_int)) - ms = alloc.memset(alloc.dptr, 0, ctypes.sizeof(ctypes.c_int)) - setter = ms.launch(LaunchConfig(grid=1, block=1), set_handle, condition, 2) - sw_node = setter.switch(condition, 4) - for branch in sw_node.branches: - branch.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr) + alloc = sample_graphdef.allocate(ctypes.sizeof(ctypes.c_int)) + ms = alloc.memset(alloc.dptr, 0, ctypes.sizeof(ctypes.c_int)) + setter = ms.launch(LaunchConfig(grid=1, block=1), set_handle, condition, 2) + sw_node = setter.switch(condition, 4) + for branch in sw_node.branches: + branch.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr) graph = sample_graphdef.instantiate() stream = Device().create_stream() @@ -1474,8 +1457,7 @@ def test_debug_dot_print_creates_file(init_cuda, tmp_path): sample_graphdef = GraphDefinition() dot_file = tmp_path / "graph.dot" _skip_if_no_mempool() - with xfail_on_graph_mempool_oom(): - sample_graphdef.allocate(ALLOC_SIZE) + sample_graphdef.allocate(ALLOC_SIZE) sample_graphdef.debug_dot_print(str(dot_file)) assert dot_file.exists() content = dot_file.read_text() @@ -1487,8 +1469,7 @@ def test_debug_dot_print_with_options(init_cuda, tmp_path): sample_graphdef = GraphDefinition() dot_file = tmp_path / "graph.dot" _skip_if_no_mempool() - with xfail_on_graph_mempool_oom(): - sample_graphdef.allocate(ALLOC_SIZE) + sample_graphdef.allocate(ALLOC_SIZE) options = GraphDebugPrintOptions(verbose=True, handles=True) sample_graphdef.debug_dot_print(str(dot_file), options) assert dot_file.exists() @@ -1499,7 +1480,6 @@ def test_debug_dot_print_invalid_options(init_cuda, tmp_path): sample_graphdef = GraphDefinition() dot_file = tmp_path / "graph.dot" _skip_if_no_mempool() - with xfail_on_graph_mempool_oom(): - sample_graphdef.allocate(ALLOC_SIZE) + sample_graphdef.allocate(ALLOC_SIZE) with pytest.raises(TypeError, match="options must be a GraphDebugPrintOptions"): sample_graphdef.debug_dot_print(str(dot_file), "invalid") diff --git a/cuda_core/tests/graph/test_graph_definition_errors.py b/cuda_core/tests/graph/test_graph_definition_errors.py index 923ea5fb74e..40f181e5db1 100644 --- a/cuda_core/tests/graph/test_graph_definition_errors.py +++ b/cuda_core/tests/graph/test_graph_definition_errors.py @@ -7,7 +7,6 @@ import pytest from helpers.graph_kernels import compile_common_kernels -from helpers.memory import xfail_on_graph_mempool_oom from helpers.misc import try_create_condition from cuda.core import Device, LaunchConfig @@ -70,8 +69,7 @@ def test_memset_invalid_value_size(init_cuda): """memset with 3-byte value (not 1, 2, or 4) raises ValueError.""" _skip_if_no_mempool() g = GraphDefinition() - with xfail_on_graph_mempool_oom(): - alloc = g.allocate(1024) + alloc = g.allocate(1024) with pytest.raises(ValueError): alloc.memset(alloc.dptr, b"\x01\x02\x03", 100) @@ -115,9 +113,8 @@ def test_join_single_predecessor(init_cuda): """node.join() with no extra args creates a single-dep empty node.""" _skip_if_no_mempool() g = GraphDefinition() - with xfail_on_graph_mempool_oom(): - a = g.allocate(1024) - joined = a.join() + a = g.allocate(1024) + joined = a.join() assert isinstance(joined, EmptyNode) assert set(joined.pred) == {a} @@ -139,8 +136,7 @@ def test_unmatched_alloc_succeeds(init_cuda): """Alloc without corresponding free is valid (graph-scoped lifetime).""" _skip_if_no_mempool() g = GraphDefinition() - with xfail_on_graph_mempool_oom(): - g.allocate(1024) + g.allocate(1024) graph = g.instantiate() stream = Device().create_stream() graph.launch(stream) @@ -178,11 +174,10 @@ def test_while_loop_zero_iterations(init_cuda): g = GraphDefinition() condition = g.create_condition(default_value=0) - with xfail_on_graph_mempool_oom(): - alloc = g.allocate(SIZEOF_INT) - ms = alloc.memset(alloc.dptr, 0, SIZEOF_INT) - loop = ms.while_loop(condition) - loop.body.launch(cfg, add_one, alloc.dptr) + alloc = g.allocate(SIZEOF_INT) + ms = alloc.memset(alloc.dptr, 0, SIZEOF_INT) + loop = ms.while_loop(condition) + loop.body.launch(cfg, add_one, alloc.dptr) graph = g.instantiate() stream = Device().create_stream() @@ -207,11 +202,10 @@ def test_if_then_false_skips_body(init_cuda): g = GraphDefinition() condition = g.create_condition(default_value=0) - with xfail_on_graph_mempool_oom(): - alloc = g.allocate(SIZEOF_INT) - ms = alloc.memset(alloc.dptr, 0, SIZEOF_INT) - if_node = ms.if_then(condition) - if_node.then.launch(cfg, add_one, alloc.dptr) + alloc = g.allocate(SIZEOF_INT) + ms = alloc.memset(alloc.dptr, 0, SIZEOF_INT) + if_node = ms.if_then(condition) + if_node.then.launch(cfg, add_one, alloc.dptr) graph = g.instantiate() stream = Device().create_stream() @@ -236,12 +230,11 @@ def test_switch_oob_skips_all_branches(init_cuda): g = GraphDefinition() condition = g.create_condition(default_value=99) - with xfail_on_graph_mempool_oom(): - alloc = g.allocate(SIZEOF_INT) - ms = alloc.memset(alloc.dptr, 0, SIZEOF_INT) - sw = ms.switch(condition, 3) - for branch in sw.branches: - branch.launch(cfg, add_one, alloc.dptr) + alloc = g.allocate(SIZEOF_INT) + ms = alloc.memset(alloc.dptr, 0, SIZEOF_INT) + sw = ms.switch(condition, 3) + for branch in sw.branches: + branch.launch(cfg, add_one, alloc.dptr) graph = g.instantiate() stream = Device().create_stream() diff --git a/cuda_core/tests/graph/test_graph_definition_integration.py b/cuda_core/tests/graph/test_graph_definition_integration.py index 1adce901d16..a352ffba2de 100644 --- a/cuda_core/tests/graph/test_graph_definition_integration.py +++ b/cuda_core/tests/graph/test_graph_definition_integration.py @@ -8,7 +8,6 @@ import numpy as np import pytest from helpers.graph_kernels import skip_if_nvrtc_lacks_conditional_handle -from helpers.memory import xfail_on_graph_mempool_oom from cuda.core import Device, EventOptions, LaunchConfig, Program, ProgramOptions from cuda.core._utils.cuda_utils import CUDAError, driver, handle_return @@ -208,8 +207,7 @@ def test_heat_diffusion(init_cuda): host_ptr = handle_return(driver.cuMemAllocHost(_HEAT_N * SIZEOF_FLOAT)) try: - with xfail_on_graph_mempool_oom(dev): - _run_heat_graph(dev, k_heat, k_countdown, host_ptr) + _run_heat_graph(dev, k_heat, k_countdown, host_ptr) finally: handle_return(driver.cuMemFreeHost(host_ptr)) @@ -319,8 +317,7 @@ def test_bisection_root(init_cuda): host_ptr = handle_return(driver.cuMemAllocHost(SIZEOF_FLOAT)) try: - with xfail_on_graph_mempool_oom(dev): - _run_bisection_graph(dev, k_eval, k_hi, k_lo, k_cd, k_check, k_newton, host_ptr) + _run_bisection_graph(dev, k_eval, k_hi, k_lo, k_cd, k_check, k_newton, host_ptr) finally: handle_return(driver.cuMemFreeHost(host_ptr)) @@ -422,8 +419,7 @@ def test_switch_dispatch(init_cuda, mode, expected): host_ptr = handle_return(driver.cuMemAllocHost(SIZEOF_INT)) try: - with xfail_on_graph_mempool_oom(dev): - _run_switch_graph(dev, mode, k_negate, k_double, k_square, host_ptr) + _run_switch_graph(dev, mode, k_negate, k_double, k_square, host_ptr) result = ctypes.c_int.from_address(host_ptr).value assert result == expected diff --git a/cuda_core/tests/graph/test_graph_definition_lifetime.py b/cuda_core/tests/graph/test_graph_definition_lifetime.py index adf7470d232..a33b5497ae8 100644 --- a/cuda_core/tests/graph/test_graph_definition_lifetime.py +++ b/cuda_core/tests/graph/test_graph_definition_lifetime.py @@ -13,7 +13,6 @@ import pytest from cuda_python_test_helpers.subprocess_runner import run_python_snippet from helpers.graph_kernels import compile_common_kernels -from helpers.memory import xfail_on_graph_mempool_oom from helpers.misc import try_create_condition from cuda_python_test_helpers import IS_WINDOWS, under_compute_sanitizer @@ -588,8 +587,7 @@ def test_event_record_node_keeps_event_alive(init_cuda): _skip_if_no_mempool() dev = Device() g = GraphDefinition() - with xfail_on_graph_mempool_oom(dev): - alloc = g.allocate(1024) + alloc = g.allocate(1024) event = dev.create_event(EventOptions(timing_enabled=False)) node = alloc.record(event) @@ -606,8 +604,7 @@ def test_event_wait_node_keeps_event_alive(init_cuda): _skip_if_no_mempool() dev = Device() g = GraphDefinition() - with xfail_on_graph_mempool_oom(dev): - alloc = g.allocate(1024) + alloc = g.allocate(1024) event = dev.create_event(EventOptions(timing_enabled=False)) node = alloc.wait(event) @@ -1321,9 +1318,8 @@ def check(reconstructed): def _pred_chain_free(g, bufs): _skip_if_no_mempool() - with xfail_on_graph_mempool_oom(): - alloc = g.allocate(64) - node = alloc.deallocate(alloc.dptr) + alloc = g.allocate(64) + node = alloc.deallocate(alloc.dptr) free_dptr = node.dptr succ = node.record(Device().create_event()) diff --git a/cuda_core/tests/graph/test_graph_memory_resource.py b/cuda_core/tests/graph/test_graph_memory_resource.py index f6fa792dda2..19aac877a4a 100644 --- a/cuda_core/tests/graph/test_graph_memory_resource.py +++ b/cuda_core/tests/graph/test_graph_memory_resource.py @@ -8,7 +8,6 @@ import pytest from helpers.buffers import compare_buffer_to_constant, make_scratch_buffer, set_buffer, thread_unsafe_on_windows -from helpers.memory import xfail_on_graph_mempool_oom from cuda.core import ( Device, @@ -74,9 +73,8 @@ def reset(self): def alloc(self, num, nbytes): """Allocate num buffers of size nbytes from graph memory.""" gb = self.device.create_graph_builder().begin_building(self.mode) - with xfail_on_graph_mempool_oom(self.device): - buffers = [self.gmr.allocate(nbytes, stream=gb) for _ in range(num)] - graph = gb.end_building().complete() + buffers = [self.gmr.allocate(nbytes, stream=gb) for _ in range(num)] + graph = gb.end_building().complete() graph.upload(self.stream) graph.launch(self.stream) self.stream.sync() @@ -141,9 +139,8 @@ def apply_kernels(mr, stream, out): else: # Capture work, then upload and launch. gb = stream.create_graph_builder().begin_building(mode) - with xfail_on_graph_mempool_oom(device): - apply_kernels(mr=gmr, stream=gb, out=out) - graph = gb.end_building().complete() + apply_kernels(mr=gmr, stream=gb, out=out) + graph = gb.end_building().complete() # First launch. graph.upload(stream) @@ -180,17 +177,16 @@ def test_graph_alloc_with_output(mempool_device, mode): # buffer allocated within the graph. The auto_free_on_launch option # is required to properly use the output buffer. gb = stream.create_graph_builder().begin_building(mode) - with xfail_on_graph_mempool_oom(device): - out = gmr.allocate(NBYTES, stream=gb) - out.copy_from(in_, stream=gb) - launch(gb, LaunchConfig(grid=1, block=1), add_one, out, NBYTES) - options = GraphCompleteOptions(auto_free_on_launch=True) - try: - graph = gb.end_building().complete(options) - except CUDAError as exc: - if "CUDA_ERROR_INVALID_VALUE" in str(exc): - pytest.skip("auto_free_on_launch not supported on this platform") - raise + out = gmr.allocate(NBYTES, stream=gb) + out.copy_from(in_, stream=gb) + launch(gb, LaunchConfig(grid=1, block=1), add_one, out, NBYTES) + options = GraphCompleteOptions(auto_free_on_launch=True) + try: + graph = gb.end_building().complete(options) + except CUDAError as exc: + if "CUDA_ERROR_INVALID_VALUE" in str(exc): + pytest.skip("auto_free_on_launch not supported on this platform") + raise # Launch the graph. The output buffer is allocated and set to one. graph.upload(stream) @@ -213,9 +209,8 @@ def test_graph_mem_alloc_zero(mempool_device, mode): gb = device.create_graph_builder().begin_building(mode) stream = device.create_stream() gmr = GraphMemoryResource(device) - with xfail_on_graph_mempool_oom(device): - buffer = gmr.allocate(0, stream=gb) - graph = gb.end_building().complete() + buffer = gmr.allocate(0, stream=gb) + graph = gb.end_building().complete() graph.upload(stream) graph.launch(stream) stream.sync() @@ -298,9 +293,8 @@ def test_gmr_check_capture_state(mempool_device, mode): # Capturing gb = stream.create_graph_builder().begin_building(mode=mode) - with xfail_on_graph_mempool_oom(device): - gmr.allocate(1, stream=gb) # no error - gb.end_building().complete() + gmr.allocate(1, stream=gb) # no error + gb.end_building().complete() def test_graph_memory_resource_attributes_direct_init_raises(): diff --git a/cuda_core/tests/helpers/memory.py b/cuda_core/tests/helpers/memory.py index 5dac58bd03d..da263ab81bc 100644 --- a/cuda_core/tests/helpers/memory.py +++ b/cuda_core/tests/helpers/memory.py @@ -1,12 +1,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Memory-related test helpers (skip/xfail guards and resource factories).""" - -from contextlib import contextmanager +"""Memory-related test helpers (skip guards and resource factories).""" import pytest -from cuda_python_test_helpers.mempool import xfail_if_mempool_oom from cuda.core import ManagedMemoryResource, PinnedMemoryResource from cuda.core._utils.cuda_utils import CUDAError @@ -28,41 +25,17 @@ def skip_if_managed_memory_unsupported(device): pytest.skip("ManagedMemoryResource requires CUDA 13.0 or later") try: ManagedMemoryResource() - except CUDAError as e: - xfail_if_mempool_oom(e, device) - raise except RuntimeError as e: if "requires CUDA 13.0" in str(e): pytest.skip("ManagedMemoryResource requires CUDA 13.0 or later") raise -def _device_id_from_resource_options(device, args, kwargs): - if device is not None: - return device - options = kwargs.get("options") - if options is None and args: - options = args[0] - if options is None: - return 0 - if isinstance(options, dict): - preferred_location = options.get("preferred_location") - preferred_location_type = options.get("preferred_location_type") - else: - preferred_location = getattr(options, "preferred_location", None) - preferred_location_type = getattr(options, "preferred_location_type", None) - if preferred_location_type in (None, "device") and isinstance(preferred_location, int) and preferred_location >= 0: - return preferred_location - return 0 - - def create_managed_memory_resource_or_skip(*args, xfail_device=None, **kwargs): - # Keep the established "skip" helper name for call-site readability, even though - # Windows MCDM mempool OOM setup failures are xfailed instead of skipped. + # Keep the established "skip" helper name for call-site readability. try: return ManagedMemoryResource(*args, **kwargs) except CUDAError as e: - xfail_if_mempool_oom(e, _device_id_from_resource_options(xfail_device, args, kwargs)) if "CUDA_ERROR_NOT_SUPPORTED" in str(e): pytest.skip("ManagedMemoryResource is not supported on this platform/device") raise @@ -75,17 +48,4 @@ def create_managed_memory_resource_or_skip(*args, xfail_device=None, **kwargs): def create_pinned_memory_resource_or_xfail(*args, xfail_device=None, **kwargs): - try: - return PinnedMemoryResource(*args, **kwargs) - except CUDAError as e: - xfail_if_mempool_oom(e, xfail_device) - raise - - -@contextmanager -def xfail_on_graph_mempool_oom(device=0): - try: - yield - except CUDAError as e: - xfail_if_mempool_oom(e, "cuGraphAddMemAllocNode", device) - raise + return PinnedMemoryResource(*args, **kwargs) diff --git a/cuda_core/tests/test_managed_memory_warning.py b/cuda_core/tests/test_managed_memory_warning.py index 6edf2b7c05a..68c31eb5b5e 100644 --- a/cuda_core/tests/test_managed_memory_warning.py +++ b/cuda_core/tests/test_managed_memory_warning.py @@ -11,13 +11,11 @@ import warnings import pytest -from cuda_python_test_helpers.mempool import xfail_if_mempool_oom from helpers.memory import create_managed_memory_resource_or_skip import cuda.bindings from cuda.core import Device, ManagedMemoryResource, ManagedMemoryResourceOptions from cuda.core._memory._managed_memory_resource import reset_concurrent_access_warning -from cuda.core._utils.cuda_utils import CUDAError _cuda_major = int(cuda.bindings.__version__.split(".")[0]) @@ -53,12 +51,8 @@ def device_without_concurrent_managed_access(init_cuda): @requires_cuda_13 def test_default_pool_error_without_concurrent_access(device_without_concurrent_managed_access): """ManagedMemoryResource() raises RuntimeError when the default pool doesn't support managed.""" - try: - with pytest.raises(RuntimeError, match="does not support managed allocations"): - ManagedMemoryResource() - except CUDAError as exc: - xfail_if_mempool_oom(exc, device_without_concurrent_managed_access) - raise + with pytest.raises(RuntimeError, match="does not support managed allocations"): + ManagedMemoryResource() @requires_cuda_13 diff --git a/cuda_core/tests/test_object_protocols.py b/cuda_core/tests/test_object_protocols.py index 2bfe660acba..f468b3e59c2 100644 --- a/cuda_core/tests/test_object_protocols.py +++ b/cuda_core/tests/test_object_protocols.py @@ -15,7 +15,6 @@ import pytest from helpers.constants import POOL_SIZE from helpers.graph_kernels import compile_common_kernels -from helpers.memory import xfail_on_graph_mempool_oom from helpers.misc import try_create_condition from cuda.core import ( @@ -282,36 +281,32 @@ def sample_root_node_alt(sample_graphdef_alt): def sample_empty_node(sample_graphdef): """An EmptyNode created by merging two branches.""" _skip_if_no_mempool() - with xfail_on_graph_mempool_oom(): - a = sample_graphdef.allocate(ALLOC_SIZE) - b = sample_graphdef.allocate(ALLOC_SIZE) - return sample_graphdef.join(a, b) + a = sample_graphdef.allocate(ALLOC_SIZE) + b = sample_graphdef.allocate(ALLOC_SIZE) + return sample_graphdef.join(a, b) @pytest.fixture def sample_empty_node_alt(sample_graphdef): """An alternate EmptyNode from same graph.""" _skip_if_no_mempool() - with xfail_on_graph_mempool_oom(): - c = sample_graphdef.allocate(ALLOC_SIZE) - d = sample_graphdef.allocate(ALLOC_SIZE) - return sample_graphdef.join(c, d) + c = sample_graphdef.allocate(ALLOC_SIZE) + d = sample_graphdef.allocate(ALLOC_SIZE) + return sample_graphdef.join(c, d) @pytest.fixture def sample_alloc_node(sample_graphdef): """An AllocNode.""" _skip_if_no_mempool() - with xfail_on_graph_mempool_oom(): - return sample_graphdef.allocate(ALLOC_SIZE) + return sample_graphdef.allocate(ALLOC_SIZE) @pytest.fixture def sample_alloc_node_alt(sample_graphdef): """An alternate AllocNode from same graph.""" _skip_if_no_mempool() - with xfail_on_graph_mempool_oom(): - return sample_graphdef.allocate(ALLOC_SIZE) + return sample_graphdef.allocate(ALLOC_SIZE) @pytest.fixture @@ -336,58 +331,52 @@ def sample_kernel_node_alt(sample_graphdef, init_cuda): def sample_free_node(sample_graphdef): """A FreeNode.""" _skip_if_no_mempool() - with xfail_on_graph_mempool_oom(): - alloc = sample_graphdef.allocate(ALLOC_SIZE) - return alloc.deallocate(alloc.dptr) + alloc = sample_graphdef.allocate(ALLOC_SIZE) + return alloc.deallocate(alloc.dptr) @pytest.fixture def sample_free_node_alt(sample_graphdef): """An alternate FreeNode from same graph.""" _skip_if_no_mempool() - with xfail_on_graph_mempool_oom(): - alloc = sample_graphdef.allocate(ALLOC_SIZE) - return alloc.deallocate(alloc.dptr) + alloc = sample_graphdef.allocate(ALLOC_SIZE) + return alloc.deallocate(alloc.dptr) @pytest.fixture def sample_memset_node(sample_graphdef): """A MemsetNode.""" _skip_if_no_mempool() - with xfail_on_graph_mempool_oom(): - alloc = sample_graphdef.allocate(ALLOC_SIZE) - return alloc.memset(alloc.dptr, 0, ALLOC_SIZE) + alloc = sample_graphdef.allocate(ALLOC_SIZE) + return alloc.memset(alloc.dptr, 0, ALLOC_SIZE) @pytest.fixture def sample_memset_node_alt(sample_graphdef): """An alternate MemsetNode from same graph.""" _skip_if_no_mempool() - with xfail_on_graph_mempool_oom(): - alloc = sample_graphdef.allocate(ALLOC_SIZE) - return alloc.memset(alloc.dptr, 0, ALLOC_SIZE) + alloc = sample_graphdef.allocate(ALLOC_SIZE) + return alloc.memset(alloc.dptr, 0, ALLOC_SIZE) @pytest.fixture def sample_memcpy_node(sample_graphdef): """A MemcpyNode.""" _skip_if_no_mempool() - with xfail_on_graph_mempool_oom(): - src = sample_graphdef.allocate(ALLOC_SIZE) - dst = sample_graphdef.allocate(ALLOC_SIZE) - dep = sample_graphdef.join(src, dst) - return dep.memcpy(dst.dptr, src.dptr, ALLOC_SIZE) + src = sample_graphdef.allocate(ALLOC_SIZE) + dst = sample_graphdef.allocate(ALLOC_SIZE) + dep = sample_graphdef.join(src, dst) + return dep.memcpy(dst.dptr, src.dptr, ALLOC_SIZE) @pytest.fixture def sample_memcpy_node_alt(sample_graphdef): """An alternate MemcpyNode from same graph.""" _skip_if_no_mempool() - with xfail_on_graph_mempool_oom(): - src = sample_graphdef.allocate(ALLOC_SIZE) - dst = sample_graphdef.allocate(ALLOC_SIZE) - dep = sample_graphdef.join(src, dst) - return dep.memcpy(dst.dptr, src.dptr, ALLOC_SIZE) + src = sample_graphdef.allocate(ALLOC_SIZE) + dst = sample_graphdef.allocate(ALLOC_SIZE) + dep = sample_graphdef.join(src, dst) + return dep.memcpy(dst.dptr, src.dptr, ALLOC_SIZE) @pytest.fixture diff --git a/cuda_python_test_helpers/cuda_python_test_helpers/mempool.py b/cuda_python_test_helpers/cuda_python_test_helpers/mempool.py deleted file mode 100644 index c1fad576da9..00000000000 --- a/cuda_python_test_helpers/cuda_python_test_helpers/mempool.py +++ /dev/null @@ -1,54 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import sys - -import pytest - - -def is_windows_mcdm_device(device=0): - if sys.platform != "win32": - return False - from cuda.bindings import driver, nvml - - device_id = int(getattr(device, "device_id", device)) - (err,) = driver.cuInit(0) - if err != driver.CUresult.CUDA_SUCCESS: - return False - err, pci_bus_id = driver.cuDeviceGetPCIBusId(13, device_id) - if err != driver.CUresult.CUDA_SUCCESS: - return False - pci_bus_id = pci_bus_id.split(b"\x00", 1)[0].decode("ascii") - nvml.init_v2() - try: - handle = nvml.device_get_handle_by_pci_bus_id_v2(pci_bus_id) - current, _ = nvml.device_get_driver_model_v2(handle) - return current == nvml.DriverModel.DRIVER_MCDM - finally: - nvml.shutdown() - - -def xfail_if_mempool_oom(err_or_exc, api_name=None, device=0): - from cuda.bindings import driver, runtime - - if api_name is not None and not isinstance(api_name, str): - device = api_name - api_name = None - - is_oom = err_or_exc in ( - driver.CUresult.CUDA_ERROR_OUT_OF_MEMORY, - runtime.cudaError_t.cudaErrorMemoryAllocation, - ) or "CUDA_ERROR_OUT_OF_MEMORY" in str(err_or_exc) - - if not is_oom: - return - try: - is_windows_mcdm = is_windows_mcdm_device(device) - except Exception: - # If MCDM detection fails, leave the primary test failure visible. - return - if not is_windows_mcdm: - return - - api_context = f"{api_name} " if api_name else "" - pytest.xfail(f"{api_context}could not reserve VA for mempool operations on Windows MCDM") From c140c20cb177cd21488bab093831e36dc1eac3a1 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 2 Oct 2026 13:48:31 -0700 Subject: [PATCH 2/3] test: harden test_vmm_allocate_zero_size against parallel-run flakes test_vmm_allocate_zero_size asserts that grown.close() waits on the recorded deallocation stream (done.is_done after a 200ms nanosleep). It ran as PARALLEL under the free-threaded (py3.14t) job and failed on aarch64 / A100 / CUDA 13.0.2 with assert done.is_done == False, while its sibling test_vmm_close_synchronizes_recorded_streams (which is marked thread_unsafe and wraps the close in assert_no_cuda_warning) passed. The close path never raises: if cuStreamSynchronize is skipped (capture) or fails, sync_recorded_stream reports a CUDAWarning and unmaps anyway, so the failure surfaced as a bare event-check failure with no clue why. Match the sibling test: - Mark thread_unsafe so the free-threaded job runs it serially, matching the other close-synchronizes tests that record process-global warnings. - Wrap grown.close() in assert_no_cuda_warning() so a skipped/failed sync surfaces as the warning text instead of a bare assert False. If the test still fails when run serially on that job, it is a real library bug in the VMM sync path rather than a parallel-execution artifact. --- cuda_core/tests/test_memory.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cuda_core/tests/test_memory.py b/cuda_core/tests/test_memory.py index ac67d369575..f33e17ddccf 100644 --- a/cuda_core/tests/test_memory.py +++ b/cuda_core/tests/test_memory.py @@ -1765,6 +1765,7 @@ def test_vmm_host_modify_allocation_rejects_exportable_handle_type(init_cuda): @pytest.mark.agent_authored(model="claude-fable-5-1") +@pytest.mark.thread_unsafe(reason="records process-global warnings") def test_vmm_allocate_zero_size(init_cuda): """allocate(0) returns an empty buffer without a driver call; a grow of it inherits its stream.""" device = _vmm_device_or_skip() @@ -1780,10 +1781,12 @@ def test_vmm_allocate_zero_size(init_cuda): # The grown buffer's deallocation is ordered on the stream passed to # allocate(0): its close waits for the work queued there. The sleep # kernel does not touch the buffer, so a missing wait fails the event - # check instead of faulting. + # check instead of faulting. The close is wrapped so a skipped sync + # surfaces as a CUDAWarning instead of a bare event-check failure. NanosleepKernel(device, sleep_duration_ms=200).launch(s) done = s.record() - grown.close() + with assert_no_cuda_warning(): + grown.close() assert done.is_done buf.close() finally: From dd00134c52df3c9ef0f6f1646058ebc54c9d78bd Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Wed, 7 Oct 2026 09:40:24 -0700 Subject: [PATCH 3/3] ci: nudge PR head after GitHub outage GitHub's push-event processing dropped during the 2026-10-07 partial outage, so PR #3004's head object stayed pinned at the pre-merge SHA after the merge-conflict resolution push landed. This empty commit forces a fresh push event so GitHub re-links the PR head to the branch tip (7f8ac594aed).