Skip to content

Remove xfail_if_mempool_oom and xfail_on_graph_mempool_oom helpers #3004

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
juenglin merged 4 commits into from
Oct 8, 2026
Merged

Remove xfail_if_mempool_oom and xfail_on_graph_mempool_oom helpers #3004

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0370d07
Remove xfail_if_mempool_oom and xfail_on_graph_mempool_oom helpers
juenglin Oct 2, 2026
c140c20
test: harden test_vmm_allocate_zero_size against parallel-run flakes
juenglin Oct 2, 2026
7f8ac59
Merge remote-tracking branch 'upstream/main' into rm-oom-skips
juenglin Oct 7, 2026
dd00134
ci: nudge PR head after GitHub outage
juenglin Oct 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions cuda_bindings/tests/test_cuda.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@

import numpy as np
import pytest
from cuda_python_test_helpers.mempool import xfail_if_mempool_oom
from cuda_python_test_helpers.subprocess_runner import run_python_snippet

import cuda.bindings.driver as cuda
Expand Down Expand Up @@ -269,7 +268,6 @@ def test_cuda_memPool_attr():

attr_list = [None] * 8
err, pool = cuda.cuMemPoolCreate(poolProps)
xfail_if_mempool_oom(err, "cuMemPoolCreate", poolProps.location.id)
assert err == cuda.CUresult.CUDA_SUCCESS

for idx, attr in enumerate(
Expand Down Expand Up @@ -487,7 +485,6 @@ def test_cuda_graphMem_attr(device):
assert destroy_err == cuda.CUresult.CUDA_SUCCESS
(destroy_err,) = cuda.cuStreamDestroy(stream)
assert destroy_err == cuda.CUresult.CUDA_SUCCESS
xfail_if_mempool_oom(err, "cuGraphAddMemAllocNode", device)
assert err == cuda.CUresult.CUDA_SUCCESS
err, freeNode = cuda.cuGraphAddMemFreeNode(graph, [allocNode], 1, params.dptr)
assert err == cuda.CUresult.CUDA_SUCCESS
Expand Down
2 changes: 0 additions & 2 deletions cuda_bindings/tests/test_cudart.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@

import numpy as np
import pytest
from cuda_python_test_helpers.mempool import xfail_if_mempool_oom

import cuda.bindings.driver as cuda
import cuda.bindings.runtime as cudart
Expand Down Expand Up @@ -524,7 +523,6 @@ def test_cudart_MemPool_attr():

attr_list = [None] * 8
err, pool = cudart.cudaMemPoolCreate(poolProps)
xfail_if_mempool_oom(err, "cudaMemPoolCreate", poolProps.location.id)
assertSuccess(err)

for idx, attr in enumerate(
Expand Down
116 changes: 48 additions & 68 deletions cuda_core/tests/graph/test_graph_definition.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@

import pytest
from helpers.graph_kernels import compile_common_kernels
from helpers.memory import xfail_on_graph_mempool_oom
from helpers.misc import try_create_condition

from cuda.core import Device, LaunchConfig
Expand Down Expand Up @@ -206,15 +205,13 @@ def _build_disconnected():
def graph_spec(request, init_cuda):
if request.param is not _build_empty:
_skip_if_no_mempool()
with xfail_on_graph_mempool_oom():
return request.param()
return request.param()


@pytest.fixture(params=_NONEMPTY_BUILDERS)
def nonempty_graph_spec(request, init_cuda):
_skip_if_no_mempool()
with xfail_on_graph_mempool_oom():
return request.param()
return request.param()


# =============================================================================
Expand Down Expand Up @@ -569,8 +566,7 @@ def node_spec(request, init_cuda):
if spec.needs_mempool:
_skip_if_no_mempool()
g = GraphDefinition()
with xfail_on_graph_mempool_oom():
node, expected_attrs = spec.builder(g)
node, expected_attrs = spec.builder(g)
return spec, g, node, expected_attrs


Expand Down Expand Up @@ -854,21 +850,19 @@ def test_free_creates_dependency(init_cuda):
"""Free node depends on its predecessor."""
sample_graphdef = GraphDefinition()
_skip_if_no_mempool()
with xfail_on_graph_mempool_oom():
alloc = sample_graphdef.allocate(ALLOC_SIZE)
free = alloc.deallocate(alloc.dptr)
alloc = sample_graphdef.allocate(ALLOC_SIZE)
free = alloc.deallocate(alloc.dptr)
assert alloc in free.pred


def test_alloc_free_chain(init_cuda):
"""Alloc and free can be chained."""
sample_graphdef = GraphDefinition()
_skip_if_no_mempool()
with xfail_on_graph_mempool_oom():
a1 = sample_graphdef.allocate(ALLOC_SIZE)
a2 = a1.allocate(ALLOC_SIZE)
f2 = a2.deallocate(a2.dptr)
f1 = f2.deallocate(a1.dptr)
a1 = sample_graphdef.allocate(ALLOC_SIZE)
a2 = a1.allocate(ALLOC_SIZE)
f2 = a2.deallocate(a2.dptr)
f1 = f2.deallocate(a1.dptr)
assert a1 in a2.pred
assert a2 in f2.pred
assert f2 in f1.pred
Expand Down Expand Up @@ -898,17 +892,15 @@ def test_alloc_device_option(init_cuda, device_spec):
sample_graphdef = GraphDefinition()
_skip_if_no_mempool()
device = Device()
with xfail_on_graph_mempool_oom(device):
node = sample_graphdef.allocate(ALLOC_SIZE, device=device_spec(device))
node = sample_graphdef.allocate(ALLOC_SIZE, device=device_spec(device))
assert node.dptr != 0


def test_alloc_peer_access(mempool_device_x2):
"""AllocNode.peer_access reflects requested peers."""
d0, d1 = mempool_device_x2
g = GraphDefinition()
with xfail_on_graph_mempool_oom(d0):
node = g.allocate(ALLOC_SIZE, device=d0.device_id, peer_access=[d1.device_id])
node = g.allocate(ALLOC_SIZE, device=d0.device_id, peer_access=[d1.device_id])
assert d1.device_id in node.peer_access


Expand All @@ -920,8 +912,7 @@ def test_alloc_memory_type_host(init_cuda):

g = GraphDefinition()
try:
with xfail_on_graph_mempool_oom():
node = g.allocate(ALLOC_SIZE, memory_type=GraphMemoryType.HOST)
node = g.allocate(ALLOC_SIZE, memory_type=GraphMemoryType.HOST)
except CUDAError as e:
if "CUDA_ERROR_NOT_SUPPORTED" in str(e):
pytest.skip("Driver does not support graph alloc memory_type='host'")
Expand Down Expand Up @@ -950,9 +941,8 @@ def test_join_merges_branches(init_cuda, num_branches):
"""join() with multiple branches creates correct dependencies."""
sample_graphdef = GraphDefinition()
_skip_if_no_mempool()
with xfail_on_graph_mempool_oom():
branches = [sample_graphdef.allocate(ALLOC_SIZE) for _ in range(num_branches)]
joined = sample_graphdef.join(*branches)
branches = [sample_graphdef.allocate(ALLOC_SIZE) for _ in range(num_branches)]
joined = sample_graphdef.join(*branches)
assert isinstance(joined, EmptyNode)
assert set(joined.pred) == set(branches)

Expand Down Expand Up @@ -1048,9 +1038,8 @@ def test_instantiate_with_nodes(init_cuda, inst_kwargs):
"""Graph with nodes can be instantiated."""
sample_graphdef = GraphDefinition()
_skip_if_no_mempool()
with xfail_on_graph_mempool_oom():
sample_graphdef.allocate(ALLOC_SIZE)
sample_graphdef.allocate(ALLOC_SIZE)
sample_graphdef.allocate(ALLOC_SIZE)
sample_graphdef.allocate(ALLOC_SIZE)
graph = _instantiate(sample_graphdef, inst_kwargs)
assert graph is not None

Expand Down Expand Up @@ -1093,9 +1082,8 @@ def test_instantiate_and_execute_alloc_free(init_cuda, inst_kwargs):
"""Graph with alloc/free can be executed."""
sample_graphdef = GraphDefinition()
_skip_if_no_mempool()
with xfail_on_graph_mempool_oom():
alloc = sample_graphdef.allocate(ALLOC_SIZE)
alloc.deallocate(alloc.dptr)
alloc = sample_graphdef.allocate(ALLOC_SIZE)
alloc.deallocate(alloc.dptr)

stream = Device().create_stream()
graph = _instantiate_and_upload(sample_graphdef, inst_kwargs, stream)
Expand All @@ -1108,10 +1096,9 @@ def test_instantiate_and_execute_memset(init_cuda, inst_kwargs):
"""Graph with alloc/memset/free can be executed."""
sample_graphdef = GraphDefinition()
_skip_if_no_mempool()
with xfail_on_graph_mempool_oom():
alloc = sample_graphdef.allocate(ALLOC_SIZE)
ms = alloc.memset(alloc.dptr, 0xAB, ALLOC_SIZE)
ms.deallocate(alloc.dptr)
alloc = sample_graphdef.allocate(ALLOC_SIZE)
ms = alloc.memset(alloc.dptr, 0xAB, ALLOC_SIZE)
ms.deallocate(alloc.dptr)

stream = Device().create_stream()
graph = _instantiate_and_upload(sample_graphdef, inst_kwargs, stream)
Expand All @@ -1126,13 +1113,12 @@ def test_instantiate_and_execute_memcpy(init_cuda, inst_kwargs):
_skip_if_no_mempool()
import ctypes

with xfail_on_graph_mempool_oom():
src_alloc = sample_graphdef.allocate(ALLOC_SIZE)
dst_alloc = sample_graphdef.allocate(ALLOC_SIZE)
dep = sample_graphdef.join(src_alloc, dst_alloc)
ms = dep.memset(src_alloc.dptr, 0xAB, ALLOC_SIZE)
cp = ms.memcpy(dst_alloc.dptr, src_alloc.dptr, ALLOC_SIZE)
cp.deallocate(src_alloc.dptr)
src_alloc = sample_graphdef.allocate(ALLOC_SIZE)
dst_alloc = sample_graphdef.allocate(ALLOC_SIZE)
dep = sample_graphdef.join(src_alloc, dst_alloc)
ms = dep.memset(src_alloc.dptr, 0xAB, ALLOC_SIZE)
cp = ms.memcpy(dst_alloc.dptr, src_alloc.dptr, ALLOC_SIZE)
cp.deallocate(src_alloc.dptr)

stream = Device().create_stream()
graph = _instantiate_and_upload(sample_graphdef, inst_kwargs, stream)
Expand Down Expand Up @@ -1360,12 +1346,11 @@ def test_instantiate_and_execute_if_then(init_cuda):
set_handle = mod.get_kernel("set_handle")
add_one = mod.get_kernel("add_one")

with xfail_on_graph_mempool_oom():
alloc = sample_graphdef.allocate(ctypes.sizeof(ctypes.c_int))
ms = alloc.memset(alloc.dptr, 0, ctypes.sizeof(ctypes.c_int))
setter = ms.launch(LaunchConfig(grid=1, block=1), set_handle, condition, 1)
if_node = setter.if_then(condition)
if_node.then.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr)
alloc = sample_graphdef.allocate(ctypes.sizeof(ctypes.c_int))
ms = alloc.memset(alloc.dptr, 0, ctypes.sizeof(ctypes.c_int))
setter = ms.launch(LaunchConfig(grid=1, block=1), set_handle, condition, 1)
if_node = setter.if_then(condition)
if_node.then.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr)

graph = sample_graphdef.instantiate()
stream = Device().create_stream()
Expand Down Expand Up @@ -1394,14 +1379,13 @@ def test_instantiate_and_execute_if_else(init_cuda):
set_handle = mod.get_kernel("set_handle")
add_one = mod.get_kernel("add_one")

with xfail_on_graph_mempool_oom():
alloc = sample_graphdef.allocate(ctypes.sizeof(ctypes.c_int))
ms = alloc.memset(alloc.dptr, 0, ctypes.sizeof(ctypes.c_int))
setter = ms.launch(LaunchConfig(grid=1, block=1), set_handle, condition, 0)
ie_node = setter.if_else(condition)
ie_node.then.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr)
n1 = ie_node.else_.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr)
n1.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr)
alloc = sample_graphdef.allocate(ctypes.sizeof(ctypes.c_int))
ms = alloc.memset(alloc.dptr, 0, ctypes.sizeof(ctypes.c_int))
setter = ms.launch(LaunchConfig(grid=1, block=1), set_handle, condition, 0)
ie_node = setter.if_else(condition)
ie_node.then.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr)
n1 = ie_node.else_.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr)
n1.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr)

graph = sample_graphdef.instantiate()
stream = Device().create_stream()
Expand Down Expand Up @@ -1430,13 +1414,12 @@ def test_instantiate_and_execute_switch(init_cuda):
set_handle = mod.get_kernel("set_handle")
add_one = mod.get_kernel("add_one")

with xfail_on_graph_mempool_oom():
alloc = sample_graphdef.allocate(ctypes.sizeof(ctypes.c_int))
ms = alloc.memset(alloc.dptr, 0, ctypes.sizeof(ctypes.c_int))
setter = ms.launch(LaunchConfig(grid=1, block=1), set_handle, condition, 2)
sw_node = setter.switch(condition, 4)
for branch in sw_node.branches:
branch.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr)
alloc = sample_graphdef.allocate(ctypes.sizeof(ctypes.c_int))
ms = alloc.memset(alloc.dptr, 0, ctypes.sizeof(ctypes.c_int))
setter = ms.launch(LaunchConfig(grid=1, block=1), set_handle, condition, 2)
sw_node = setter.switch(condition, 4)
for branch in sw_node.branches:
branch.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr)

graph = sample_graphdef.instantiate()
stream = Device().create_stream()
Expand Down Expand Up @@ -1474,8 +1457,7 @@ def test_debug_dot_print_creates_file(init_cuda, tmp_path):
sample_graphdef = GraphDefinition()
dot_file = tmp_path / "graph.dot"
_skip_if_no_mempool()
with xfail_on_graph_mempool_oom():
sample_graphdef.allocate(ALLOC_SIZE)
sample_graphdef.allocate(ALLOC_SIZE)
sample_graphdef.debug_dot_print(str(dot_file))
assert dot_file.exists()
content = dot_file.read_text()
Expand All @@ -1487,8 +1469,7 @@ def test_debug_dot_print_with_options(init_cuda, tmp_path):
sample_graphdef = GraphDefinition()
dot_file = tmp_path / "graph.dot"
_skip_if_no_mempool()
with xfail_on_graph_mempool_oom():
sample_graphdef.allocate(ALLOC_SIZE)
sample_graphdef.allocate(ALLOC_SIZE)
options = GraphDebugPrintOptions(verbose=True, handles=True)
sample_graphdef.debug_dot_print(str(dot_file), options)
assert dot_file.exists()
Expand All @@ -1499,7 +1480,6 @@ def test_debug_dot_print_invalid_options(init_cuda, tmp_path):
sample_graphdef = GraphDefinition()
dot_file = tmp_path / "graph.dot"
_skip_if_no_mempool()
with xfail_on_graph_mempool_oom():
sample_graphdef.allocate(ALLOC_SIZE)
sample_graphdef.allocate(ALLOC_SIZE)
with pytest.raises(TypeError, match="options must be a GraphDebugPrintOptions"):
sample_graphdef.debug_dot_print(str(dot_file), "invalid")
41 changes: 17 additions & 24 deletions cuda_core/tests/graph/test_graph_definition_errors.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@

import pytest
from helpers.graph_kernels import compile_common_kernels
from helpers.memory import xfail_on_graph_mempool_oom
from helpers.misc import try_create_condition

from cuda.core import Device, LaunchConfig
Expand Down Expand Up @@ -70,8 +69,7 @@ def test_memset_invalid_value_size(init_cuda):
"""memset with 3-byte value (not 1, 2, or 4) raises ValueError."""
_skip_if_no_mempool()
g = GraphDefinition()
with xfail_on_graph_mempool_oom():
alloc = g.allocate(1024)
alloc = g.allocate(1024)
with pytest.raises(ValueError):
alloc.memset(alloc.dptr, b"\x01\x02\x03", 100)

Expand Down Expand Up @@ -115,9 +113,8 @@ def test_join_single_predecessor(init_cuda):
"""node.join() with no extra args creates a single-dep empty node."""
_skip_if_no_mempool()
g = GraphDefinition()
with xfail_on_graph_mempool_oom():
a = g.allocate(1024)
joined = a.join()
a = g.allocate(1024)
joined = a.join()
assert isinstance(joined, EmptyNode)
assert set(joined.pred) == {a}

Expand All @@ -139,8 +136,7 @@ def test_unmatched_alloc_succeeds(init_cuda):
"""Alloc without corresponding free is valid (graph-scoped lifetime)."""
_skip_if_no_mempool()
g = GraphDefinition()
with xfail_on_graph_mempool_oom():
g.allocate(1024)
g.allocate(1024)
graph = g.instantiate()
stream = Device().create_stream()
graph.launch(stream)
Expand Down Expand Up @@ -178,11 +174,10 @@ def test_while_loop_zero_iterations(init_cuda):

g = GraphDefinition()
condition = g.create_condition(default_value=0)
with xfail_on_graph_mempool_oom():
alloc = g.allocate(SIZEOF_INT)
ms = alloc.memset(alloc.dptr, 0, SIZEOF_INT)
loop = ms.while_loop(condition)
loop.body.launch(cfg, add_one, alloc.dptr)
alloc = g.allocate(SIZEOF_INT)
ms = alloc.memset(alloc.dptr, 0, SIZEOF_INT)
loop = ms.while_loop(condition)
loop.body.launch(cfg, add_one, alloc.dptr)

graph = g.instantiate()
stream = Device().create_stream()
Expand All @@ -207,11 +202,10 @@ def test_if_then_false_skips_body(init_cuda):

g = GraphDefinition()
condition = g.create_condition(default_value=0)
with xfail_on_graph_mempool_oom():
alloc = g.allocate(SIZEOF_INT)
ms = alloc.memset(alloc.dptr, 0, SIZEOF_INT)
if_node = ms.if_then(condition)
if_node.then.launch(cfg, add_one, alloc.dptr)
alloc = g.allocate(SIZEOF_INT)
ms = alloc.memset(alloc.dptr, 0, SIZEOF_INT)
if_node = ms.if_then(condition)
if_node.then.launch(cfg, add_one, alloc.dptr)

graph = g.instantiate()
stream = Device().create_stream()
Expand All @@ -236,12 +230,11 @@ def test_switch_oob_skips_all_branches(init_cuda):

g = GraphDefinition()
condition = g.create_condition(default_value=99)
with xfail_on_graph_mempool_oom():
alloc = g.allocate(SIZEOF_INT)
ms = alloc.memset(alloc.dptr, 0, SIZEOF_INT)
sw = ms.switch(condition, 3)
for branch in sw.branches:
branch.launch(cfg, add_one, alloc.dptr)
alloc = g.allocate(SIZEOF_INT)
ms = alloc.memset(alloc.dptr, 0, SIZEOF_INT)
sw = ms.switch(condition, 3)
for branch in sw.branches:
branch.launch(cfg, add_one, alloc.dptr)

graph = g.instantiate()
stream = Device().create_stream()
Expand Down
Loading
Loading