From c6638983f689f8c14b56617cefee66258d892aac Mon Sep 17 00:00:00 2001 From: Josh Taillon Date: Fri, 9 Oct 2026 09:17:28 -0400 Subject: [PATCH 1/3] fix(py): kill a worker that stops instead of letting it freeze the host On macOS, Python 3.14's asyncio treats waitid()'s report of a stopped child as an exit and calls a blocking waitpid() on the event-loop thread, so model code running os.kill(os.getpid(), signal.SIGSTOP) froze the host until the worker really exited, which nothing could then make it do. Each local worker session now starts a thread that waits for the worker to stop or exit without reaping it, and SIGKILLs the process group if it stopped. The blocked waitpid() returns at once and the driver reports a crash. This runs on every platform with os.waitid, so a stopped worker is reported the same way everywhere rather than as a call timeout on Linux. macOS before 3.13 has no os.waitid, and keeps the timeout; its asyncio does not freeze. The write-bound test used SIGSTOP to make the worker stop reading; it now uses a stand-in worker that announces itself ready and never reads. --- pkg-py/src/commons/_execution/_backend.py | 34 ++++++++++++++++++ pkg-py/tests/test_execution_driver.py | 42 ++++++++++++++++------- 2 files changed, 64 insertions(+), 12 deletions(-) diff --git a/pkg-py/src/commons/_execution/_backend.py b/pkg-py/src/commons/_execution/_backend.py index d373a17a..9bf13d09 100644 --- a/pkg-py/src/commons/_execution/_backend.py +++ b/pkg-py/src/commons/_execution/_backend.py @@ -16,6 +16,7 @@ import shutil import signal import tempfile +import threading from collections.abc import Coroutine from pathlib import Path from typing import Any, Literal, Protocol, runtime_checkable @@ -234,6 +235,12 @@ def __init__( self._closing: asyncio.Task[None] | None = None self._exit = asyncio.ensure_future(process.wait()) self._exit.add_done_callback(self._kill_group) + threading.Thread( + target=_kill_when_stopped, + args=(process.pid,), + name=f"commons-worker-stop-{process.pid}", + daemon=True, + ).start() @property def stdin(self) -> asyncio.StreamWriter: @@ -301,6 +308,33 @@ async def _finish(self) -> None: shutil.rmtree(self.scratch, ignore_errors=True) +def _kill_when_stopped(pid: int) -> None: + """Kill the worker's process group if the worker stops; return once it exits. + + Nothing legitimate stops a worker, but model code can stop itself. On + macOS, Python 3.14's asyncio reads the stop as an exit and blocks the + event loop in ``waitpid()`` until the worker really exits, so a stopped + worker would freeze the host. Killing it ends that wait at once, and + the driver reports a crash on every platform. ``WNOWAIT`` leaves the + reaping to asyncio, which keeps the pid from being reused while this + thread can still signal it. Without ``os.waitid`` (macOS before 3.13) + a stopped worker is left to the call timeout. + """ + if not hasattr(os, "waitid"): + return + try: + info = os.waitid(os.P_PID, pid, os.WEXITED | os.WSTOPPED | os.WNOWAIT) + except ChildProcessError: + return + if info is None or info.si_code != os.CLD_STOPPED: + return + with contextlib.suppress(ProcessLookupError, PermissionError): + os.killpg(pid, signal.SIGKILL) + return + with contextlib.suppress(ProcessLookupError): + os.kill(pid, signal.SIGKILL) + + def _signal_tree(process: asyncio.subprocess.Process, sig: signal.Signals) -> None: """Signal the child's whole process group, falling back to the child alone. diff --git a/pkg-py/tests/test_execution_driver.py b/pkg-py/tests/test_execution_driver.py index 3e7795cf..1644affc 100644 --- a/pkg-py/tests/test_execution_driver.py +++ b/pkg-py/tests/test_execution_driver.py @@ -10,7 +10,6 @@ import asyncio import os import signal -import sys import time from datetime import date from typing import Annotated @@ -22,7 +21,7 @@ from commons import measure, semantic_layer from commons._execution._backend import WORKER_SCRIPT, LocalBackend, LocalSession from commons._execution._driver import Failure, Worker -from commons._execution._protocol import Error, Result, Text +from commons._execution._protocol import Error, Ready, Result, Text, encode_message from commons._execution._sandbox import protection_mode from commons._handles import HandleStore from commons._measures import Injected @@ -355,18 +354,37 @@ async def test_a_child_ignoring_sigterm_does_not_outlive_the_close(): ) -@pytest.mark.skipif( - sys.platform == "darwin" and sys.version_info >= (3, 14), - reason="asyncio on 3.14 reads macOS's waitid() report of a stopped child " - "as an exit and blocks the event loop in waitpid() until it really exits", -) -async def test_a_worker_that_stops_reading_fails_the_call_instead_of_hanging(): - async with make_worker(call_timeout=0.5) as worker: +@pytest.mark.skipif(not hasattr(os, "waitid"), reason="needs os.waitid") +async def test_a_worker_that_stops_itself_is_killed_and_reported_as_a_crash(): + async with make_worker(call_timeout=30) as worker: await worker.run("1") process = process_of(worker) - # A stopped worker cannot drain its stdin; a call bigger than the - # pipe buffer would block the write forever without a bound on it. - process.send_signal(signal.SIGSTOP) + started = time.monotonic() + reply = await worker.run( + "import os, signal\nos.kill(os.getpid(), signal.SIGSTOP)" + ) + assert isinstance(reply, Failure) + assert "crashed" in reply.message + # Killed when it stopped, rather than when the call timed out. + assert time.monotonic() - started < 10 + assert process.returncode == -signal.SIGKILL + reply = await worker.run("1 + 1") + assert isinstance(reply, Result) + assert reply.value == 2 + + +async def test_a_worker_that_stops_reading_fails_the_call_instead_of_hanging(tmp_path): + # Ready, then never read stdin: a call bigger than the pipe buffer would + # block the write forever without a bound on it. + script = tmp_path / "worker.py" + script.write_text( + "import sys, time\n" + f"sys.stdout.buffer.write({encode_message(Ready())!r})\n" + "sys.stdout.flush()\n" + "time.sleep(60)\n" + ) + backend = LocalBackend(worker_script=script, terminate_grace=0.1) + async with make_worker(backend=backend, call_timeout=0.5) as worker: reply = await worker.run("x = " + "1" * (1024 * 1024)) assert isinstance(reply, Failure) assert "stopped reading" in reply.message From 10d8f6015ea1cf64d7d4890779ebba7b7682ec7d Mon Sep 17 00:00:00 2001 From: Josh Taillon Date: Fri, 9 Oct 2026 21:41:09 -0400 Subject: [PATCH 2/3] docs(py): rewrite the stopped-worker docstring in plain language --- pkg-py/src/commons/_execution/_backend.py | 26 ++++++++++++++--------- pkg-py/tests/test_execution_driver.py | 4 ++-- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/pkg-py/src/commons/_execution/_backend.py b/pkg-py/src/commons/_execution/_backend.py index 9bf13d09..e1e49cf4 100644 --- a/pkg-py/src/commons/_execution/_backend.py +++ b/pkg-py/src/commons/_execution/_backend.py @@ -309,16 +309,22 @@ async def _finish(self) -> None: def _kill_when_stopped(pid: int) -> None: - """Kill the worker's process group if the worker stops; return once it exits. - - Nothing legitimate stops a worker, but model code can stop itself. On - macOS, Python 3.14's asyncio reads the stop as an exit and blocks the - event loop in ``waitpid()`` until the worker really exits, so a stopped - worker would freeze the host. Killing it ends that wait at once, and - the driver reports a crash on every platform. ``WNOWAIT`` leaves the - reaping to asyncio, which keeps the pid from being reused while this - thread can still signal it. Without ``os.waitid`` (macOS before 3.13) - a stopped worker is left to the call timeout. + """Kill the worker's process group if the worker is stopped. + + Waits until the worker stops or exits, and returns without acting if + it exits. Model code can stop its own process with SIGSTOP, and nothing + else has a reason to. On macOS, Python 3.14's asyncio mistakes a + stopped child for an exited one, then blocks the event loop in + ``waitpid()`` until the child really exits, which freezes the host. + Killing the worker lets that ``waitpid()`` return at once. This runs + wherever ``os.waitid`` exists, so the driver reports a stopped worker + as a crash on every such platform. + + ``WNOWAIT`` leaves the exit status for asyncio to collect. Until it is + collected, the worker's pid cannot be given to another process, so this + thread cannot signal the wrong one. Without ``os.waitid`` (macOS before + Python 3.13), a stopped worker is not killed here, and the call fails + when its timeout runs out. """ if not hasattr(os, "waitid"): return diff --git a/pkg-py/tests/test_execution_driver.py b/pkg-py/tests/test_execution_driver.py index 1644affc..ef38fcb8 100644 --- a/pkg-py/tests/test_execution_driver.py +++ b/pkg-py/tests/test_execution_driver.py @@ -374,8 +374,8 @@ async def test_a_worker_that_stops_itself_is_killed_and_reported_as_a_crash(): async def test_a_worker_that_stops_reading_fails_the_call_instead_of_hanging(tmp_path): - # Ready, then never read stdin: a call bigger than the pipe buffer would - # block the write forever without a bound on it. + # A worker that says it is ready and then never reads stdin. Writing it a + # call bigger than the pipe buffer blocks until the write's time limit. script = tmp_path / "worker.py" script.write_text( "import sys, time\n" From c705930625b61fc6fff6ede1bd06e5abc51da2ab Mon Sep 17 00:00:00 2001 From: Josh Taillon Date: Fri, 9 Oct 2026 21:50:49 -0400 Subject: [PATCH 3/3] docs(py): trim the stopped-worker docstring --- pkg-py/src/commons/_execution/_backend.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/pkg-py/src/commons/_execution/_backend.py b/pkg-py/src/commons/_execution/_backend.py index e1e49cf4..f6e1e6d9 100644 --- a/pkg-py/src/commons/_execution/_backend.py +++ b/pkg-py/src/commons/_execution/_backend.py @@ -312,19 +312,17 @@ def _kill_when_stopped(pid: int) -> None: """Kill the worker's process group if the worker is stopped. Waits until the worker stops or exits, and returns without acting if - it exits. Model code can stop its own process with SIGSTOP, and nothing - else has a reason to. On macOS, Python 3.14's asyncio mistakes a + it exits. Model code can stop its own process with SIGSTOP (nothing + else has a reason to). On macOS, Python 3.14's asyncio mistakes a stopped child for an exited one, then blocks the event loop in ``waitpid()`` until the child really exits, which freezes the host. Killing the worker lets that ``waitpid()`` return at once. This runs wherever ``os.waitid`` exists, so the driver reports a stopped worker as a crash on every such platform. - ``WNOWAIT`` leaves the exit status for asyncio to collect. Until it is - collected, the worker's pid cannot be given to another process, so this - thread cannot signal the wrong one. Without ``os.waitid`` (macOS before - Python 3.13), a stopped worker is not killed here, and the call fails - when its timeout runs out. + ``WNOWAIT`` leaves the exit status for asyncio to collect. Without + ``os.waitid`` (macOS before Python 3.13), a stopped worker is not + killed here, and the call fails when its timeout runs out. """ if not hasattr(os, "waitid"): return