From 446bf128fbbaf08ae97a20f19d832f9318db776a Mon Sep 17 00:00:00 2001 From: Bolin Chen Date: Wed, 7 Oct 2026 18:38:51 +0000 Subject: [PATCH] test: wait on observed state in five Linux-flaky tests Five backend tests failed intermittently on loaded Linux shards, or passed vacuously, because each bet on timing instead of observing the state it asserts: - design-tweak relay pump: a feeder thread paced with fixed sleeps raced the relay thread; a late relay read the handshake and the frame in one buffer and the client's handshake read took the frame. Each step now waits on bytes the other side has observed, with a named lost-run bound on every read and join. - crew container fake backend: the pidfile appeared before the SIGTERM handler, so a drain that signalled on it read -15 instead of 0. The pidfile is now written atomically after the disposition and after the forked child is recorded for --reap-on-term. - admission defer: twenty 5 ms turns, then an unconditional read of a lazily created attribute. The test now waits for the gated defer to be running, always releases the gate, and bounds the spawn await. - browser view free port: two calls must differ, which the kernel never promised. The test now proves each call binds port 0 and returns that bind's own kernel answer. - md-notebook locking: three "contender still waiting" negatives ran after a fixed sleep or yield count, so a contender that had not reached the lock passed them too. Each now waits until the contender is parked on the lock, through kiro_crew.testing.wait.until_parked. Test-only: no production code changes. Fixes #17287 Fixes #17071 Fixes #14453 Fixes #14011 Part of #16536 --- .../container_tests/test_supervisor_fakes.py | 20 +++- test/test_browser_cli_view.py | 39 +++++++- test/test_design_tweak_relay_paths.py | 95 ++++++++++++------- test/test_md_notebook.py | 75 ++++++++++----- test/test_overload_integration_glue.py | 43 ++++++--- 5 files changed, 197 insertions(+), 75 deletions(-) diff --git a/src/kiro_crew/apps/builtins/aws_control/crew/runtime/container_tests/test_supervisor_fakes.py b/src/kiro_crew/apps/builtins/aws_control/crew/runtime/container_tests/test_supervisor_fakes.py index 817b086ecf97..ffbe4c7a8701 100644 --- a/src/kiro_crew/apps/builtins/aws_control/crew/runtime/container_tests/test_supervisor_fakes.py +++ b/src/kiro_crew/apps/builtins/aws_control/crew/runtime/container_tests/test_supervisor_fakes.py @@ -38,8 +38,12 @@ p.add_argument("--leader-ttl", type=float, default=-1.0) a = p.parse_args() -if a.pidfile: - Path(a.pidfile).write_text(str(os.getpid())) +def _publish_pid(path): + # Written aside and renamed into place, so the pidfile never exists without its + # whole pid in it. + tmp = path + ".tmp" + Path(tmp).write_text(str(os.getpid())) + os.replace(tmp, path) # Holds the escaped child's pid so the SIGTERM handler can reap its group, # modelling the real backend, whose kiro-cli workers setsid into their own @@ -73,14 +77,22 @@ def _on_term(*_): # escaping the parent's process group exactly as a kiro-cli worker does. if a.child_setsid: os.setsid() - if a.child_pidfile: - Path(a.child_pidfile).write_text(str(os.getpid())) + # Disposition before pidfile, as for the leader. signal.signal(signal.SIGTERM, signal.SIG_IGN if a.ignore_sigterm else signal.SIG_DFL) + if a.child_pidfile: + _publish_pid(a.child_pidfile) time.sleep(a.ttl) os._exit(0) else: _child["pid"] = pid +# A test reads the pidfile as "this process is ready for SIGTERM", so it appears +# only after the disposition above, and after the forked child is recorded for the +# reap. Before the disposition, a SIGTERM meets the default action and the drain +# reads -15 instead of 0. +if a.pidfile: + _publish_pid(a.pidfile) + if a.port and a.run_dir: time.sleep(a.secret_delay) Path(a.run_dir).mkdir(parents=True, exist_ok=True) diff --git a/test/test_browser_cli_view.py b/test/test_browser_cli_view.py index 5122faf5d44b..698eea06cff4 100644 --- a/test/test_browser_cli_view.py +++ b/test/test_browser_cli_view.py @@ -11,6 +11,7 @@ import threading import time from collections.abc import Iterator +from types import SimpleNamespace import pytest @@ -942,9 +943,41 @@ def test_free_port_is_bindable_loopback() -> None: s.bind(("127.0.0.1", port)) -def test_free_port_is_not_hardcoded() -> None: - """A fixed port would collide with whatever else the operator runs.""" - assert mod._free_port() != mod._free_port() +def test_free_port_is_not_hardcoded(monkeypatch: pytest.MonkeyPatch) -> None: + """A fixed port would collide with whatever else the operator runs. + + So every number must be the kernel's: each call binds port 0 on loopback and + returns the port the kernel answered for that bind. That is asserted on the binds + and their answers, over two calls so a cached first answer fails too. The two + numbers are never compared: once the first socket is closed the kernel may legally + hand back the same port. + """ + binds: list[object] = [] + answers: list[int] = [] + + class _RecordingSocket(socket.socket): + def bind(self, address: object) -> None: + binds.append(address) + super().bind(address) + + def getsockname(self) -> object: + name = super().getsockname() + answers.append(name[1]) + return name + + # The module's own ``socket`` binding, so the stdlib module stays untouched. + monkeypatch.setattr( + mod, + "socket", + SimpleNamespace( + socket=_RecordingSocket, AF_INET=socket.AF_INET, SOCK_STREAM=socket.SOCK_STREAM + ), + ) + + first, second = mod._free_port(), mod._free_port() + + assert binds == [(mod.LOOPBACK_HOST, 0)] * 2, "a call did not leave the port to the kernel" + assert answers == [first, second], "a returned port is not the kernel's answer to that call" def test_ensure_running_returns_none_without_the_cli(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/test/test_design_tweak_relay_paths.py b/test/test_design_tweak_relay_paths.py index d7f33980e796..61f0309da234 100644 --- a/test/test_design_tweak_relay_paths.py +++ b/test/test_design_tweak_relay_paths.py @@ -885,6 +885,30 @@ def feed_then_break(): down_b.close() +#: Lost-run bound for each read and join the pump test unblocks itself. A passing run +#: moves a few dozen bytes across a socketpair, and its call measured under 0.01 s in +#: every repeat, so this only ends a broken run, by name, well inside the suite's 120 s +#: timeout. +_PUMP_LOST_RUN_SECS = 10.0 + + +def _recv_until(sock: socket.socket, complete, what: str) -> bytes: + """Read ``sock`` until ``complete(received)`` holds; fail by name if it never does.""" + received = b"" + sock.settimeout(_PUMP_LOST_RUN_SECS) + while not complete(received): + try: + chunk = sock.recv(4096) + except TimeoutError: + raise AssertionError( + f"{what}: nothing more arrived within {_PUMP_LOST_RUN_SECS}s; " + f"received {received!r}" + ) from None + assert chunk, f"{what}: the peer closed first; received {received!r}" + received += chunk + return received + + class TestRelayWsPump: """The bidirectional pump forwards frames and exits on hangup.""" @@ -898,52 +922,59 @@ def test_pump_forwards_and_exits_on_close(self): down_a, down_b = socket.socketpair() handshake = b"HTTP/1.1 101 OK\r\nUpgrade: websocket\r\n\r\n" + upstream_frame = b"hello from upstream" + client_frame = b"hello from client" p = _make_probe( headers={"Upgrade": "websocket", "Sec-WebSocket-Key": "k"}, connection=down_a, ) - def feed_handshake_then_frame(): - # Feed the handshake, wait for pump to start, then send a frame. - up_a.sendall(handshake) - time.sleep(0.1) - up_a.sendall(b"hello from upstream") - time.sleep(0.1) - up_a.close() - - feeder = threading.Thread(target=feed_handshake_then_frame, daemon=True) - feeder.start() - def run_relay(): with patch.object(server.socket, "create_connection", return_value=up_b): p._relay_ws() relay_thread = threading.Thread(target=run_relay, daemon=True) relay_thread.start() + try: + # Each step waits for the one before it to be observed, never for a + # duration. Upstream answers only the upgrade request it received, and + # sends its frame only once the client holds the whole relayed + # handshake, so the relay has finished reading the handshake and the + # frame can only reach the client through the pump. Sent any earlier, + # the frame can reach a late-starting relay in the same read as the + # handshake, and the client's handshake read then takes it too. + _recv_until(up_a, lambda got: b"\r\n\r\n" in got, "the relayed upgrade request") + up_a.sendall(handshake) + relayed = _recv_until(down_b, lambda got: b"\r\n\r\n" in got, "the relayed handshake") + # Bytes after the header are frame data, however the reads split them. + _head, _, early = relayed.partition(b"\r\n\r\n") + + up_a.sendall(upstream_frame) + forwarded = early + _recv_until( + down_b, + lambda got: len(early + got) >= len(upstream_frame), + "the upstream frame", + ) + assert forwarded == upstream_frame - # Read the handshake from the client side first. - down_b.settimeout(2) - handshake_received = b"" - while b"\r\n\r\n" not in handshake_received: - handshake_received += down_b.recv(4096) - - # Now read the forwarded frame. - frame_data = down_b.recv(4096) - assert frame_data == b"hello from upstream" - - # Send a frame from the "client" side back upstream. - down_b.sendall(b"hello from client") - # The relay forwards it to up_b, which comes out on up_a — but up_a - # may already be closed by the feeder. The point is the relay exits - # cleanly when up_a closes. - relay_thread.join(timeout=3) - feeder.join(timeout=1) - assert not relay_thread.is_alive() + down_b.sendall(client_frame) + assert ( + _recv_until(up_a, lambda got: len(got) >= len(client_frame), "the client frame") + == client_frame + ) - # Cleanup. - down_a.close() - down_b.close() + # Upstream hangs up: the pump must end. + up_a.close() + relay_thread.join(_PUMP_LOST_RUN_SECS) + assert ( + not relay_thread.is_alive() + ), f"the pump was still running {_PUMP_LOST_RUN_SECS}s after the upstream hangup" + finally: + # Closing both ends also ends a pump a failed assertion left running. + up_a.close() + down_a.close() + down_b.close() class TestRelayWsIdleCap: diff --git a/test/test_md_notebook.py b/test/test_md_notebook.py index 82528c02eade..71eb231454c0 100644 --- a/test/test_md_notebook.py +++ b/test/test_md_notebook.py @@ -37,6 +37,7 @@ from kiro_crew import platform_compat from kiro_crew.apps.builtins.md_notebook import git_ops from kiro_crew.loop_lock import LoopBoundLock +from kiro_crew.testing.wait import until_parked SECRET = "test-proxy-secret" @@ -3785,6 +3786,26 @@ def __getattr__(self, name): # type: ignore[no-untyped-def] assert not list(root.glob("One.md.*.tmp")), "a temp survived a successful save" +async def _parked_behind(lock: LoopBoundLock, contender: asyncio.Task, what: str) -> None: + """Return once ``contender`` is parked on ``lock``, the barrier a "still waiting" check needs. + + Without it a contender that has not yet REACHED the lock satisfies the negative as + well as one the lock is holding back, so the check passes with the lock removed. A + contender that finishes instead of parking fails here by name; one that never + parks fails at ``until_parked``'s deadline, quoting how many waiters it saw. + """ + parked = asyncio.ensure_future(until_parked(lock, describe=lambda: what)) + try: + await asyncio.wait({parked, contender}, return_when=asyncio.FIRST_COMPLETED) + finally: + if not parked.done(): + parked.cancel() + with contextlib.suppress(asyncio.CancelledError): + await parked + assert not contender.done(), f"{what} finished without ever waiting on the lock" + parked.result() + + @pytest.mark.asyncio async def test_a_sync_while_the_temp_is_still_staging_never_commits_it( fixtures, @@ -3870,9 +3891,11 @@ def parked_stage(tmp, content): # type: ignore[no-untyped-def] ) # Requested while staging is parked: the Sync must wait on the - # vault write lock the staging call holds, not run beside it. + # vault write lock the staging call holds, not run beside it. It is + # seen parked there first: a Sync that has not reached the lock yet + # would pass this check as well, lock or no lock. sync = real_asyncio.create_task(client.post("/api/sync")) - await real_asyncio.sleep(0.2) + await _parked_behind(_mod.vault_write_lock(vault["localPath"]), sync, "the Sync") assert not sync.done(), "the Sync ran while a note was still staging" finally: release_stage.set() @@ -4061,16 +4084,17 @@ def __getattr__(self, name): # type: ignore[no-untyped-def] str(target) ).locked(), "the per-note lock was released during an unguarded retry" - task_b = real_asyncio.create_task( - client.put("/api/note", {"path": "One.md", "content": "B"}) - ) - # Yield generously rather than sleeping: if B could interleave it - # would have completed by now. - for _ in range(100): - await real_asyncio.sleep(0) - assert not task_b.done(), "a later save interleaved into the retry gap" - - release_a.set() + try: + task_b = real_asyncio.create_task( + client.put("/api/note", {"path": "One.md", "content": "B"}) + ) + # B is seen parked on the note's lock before the check: a B whose + # request has not reached the lock yet would pass it as well, lock + # or no lock. + await _parked_behind(_mod._save_lock(str(target)), task_b, "the later save") + assert not task_b.done(), "a later save interleaved into the retry gap" + finally: + release_a.set() status_a, body_a = await real_asyncio.wait_for(task_a, timeout=5) status_b, body_b = await real_asyncio.wait_for(task_b, timeout=5) @@ -4142,19 +4166,20 @@ def parked_stage(tmp, content): # type: ignore[no-untyped-def] str(target) ).locked(), "a tokenless save staged its temp outside the per-note lock" - task_b = real_asyncio.create_task( - client.put("/api/note", {"path": "One.md", "content": "B"}) - ) - # Yield generously rather than sleeping: were the lock free, B would - # have staged and published by now. - for _ in range(100): - await real_asyncio.sleep(0) - assert staged == [ - "A" - ], f"a later save staged while the earlier one held the lock: {staged}" - assert not task_b.done(), "a later save completed while the earlier one was staging" - - release_a.set() + try: + task_b = real_asyncio.create_task( + client.put("/api/note", {"path": "One.md", "content": "B"}) + ) + # B is seen parked on the note's lock before the checks: a B whose + # request has not reached the lock yet would pass them as well, lock + # or no lock. + await _parked_behind(_mod._save_lock(str(target)), task_b, "the later save") + assert staged == [ + "A" + ], f"a later save staged while the earlier one held the lock: {staged}" + assert not task_b.done(), "a later save completed while the earlier one was staging" + finally: + release_a.set() status_a, body_a = await real_asyncio.wait_for(task_a, timeout=10) status_b, body_b = await real_asyncio.wait_for(task_b, timeout=10) diff --git a/test/test_overload_integration_glue.py b/test/test_overload_integration_glue.py index 097580ddf099..260efa64bbaf 100644 --- a/test/test_overload_integration_glue.py +++ b/test/test_overload_integration_glue.py @@ -53,6 +53,7 @@ from kiro_crew.taskq.adapters import runner as runner_mod from kiro_crew.taskq.store import TaskStore from kiro_crew.taskq.waits import WaitRecord +from kiro_crew.testing.wait import async_wait_until pytestmark = pytest.mark.usefixtures("healthy_host_memory") @@ -3227,6 +3228,11 @@ def _reenter(_claimed): # ── the pump sees a deferred row only after its defer landed ───────────────── +#: Lost-run bound for the gated defer and the spawn the test releases. The test's call +#: measured at most 0.04 s in repeats at -n0, so this only ends a broken run, by name, +#: well inside the suite's 120 s timeout. +_DEFER_LOST_RUN_SECS = 30.0 + @pytest.mark.asyncio async def test_admitting_id_is_held_until_the_posted_defer_landed( @@ -3241,26 +3247,41 @@ async def test_admitting_id_is_held_until_the_posted_defer_landed( monkeypatch.setattr(admission_mod.SpawnAdmissionCoordinator, "pump_off_loop", True) monkeypatch.setattr(subagent_mod, "check_memory_available", lambda min_gb: (False, 0.5)) gate = threading.Event() + defer_entered = threading.Event() real_defer = store.defer seen: dict[str, Any] = {} def _slow_defer(task_id, *, wait, reason): - assert gate.wait(5) + defer_entered.set() + assert gate.wait(_DEFER_LOST_RUN_SECS) seen["wait"] = wait return real_defer(task_id, wait=wait, reason=reason) monkeypatch.setattr(store, "defer", _slow_defer) task = asyncio.ensure_future(mgr.spawn_async("pressure", parent_session_key="web-1")) - for _ in range(20): - await asyncio.sleep(0.005) - admitting = getattr(mgr, "_admitting_ids") - assert ( - len(admitting) == 1 - ), "the row stays excluded from the refill while its defer is in flight" - (aid,) = tuple(admitting) - assert aid in mgr._admission.taskq_excluded_ids() - gate.set() - info = await task + try: + # The window under test opens when the posted defer is running and held by + # the gate. Wait for that, not for a count of loop turns: on a slow executor + # the spawn's off-loop hops outlast any count, and ``_admitting_ids`` does + # not exist until the spawn reaches the row it accepted. A spawn that + # returns without deferring ends the wait too, and fails below. + await async_wait_until( + lambda: defer_entered.is_set() or task.done(), + timeout=_DEFER_LOST_RUN_SECS, + describe=lambda: f"admitting={getattr(mgr, '_admitting_ids', None)!r}", + ) + assert defer_entered.is_set(), f"the spawn never posted its defer: {task!r}" + admitting = getattr(mgr, "_admitting_ids", set()) + assert ( + len(admitting) == 1 + ), "the row stays excluded from the refill while its defer is in flight" + (aid,) = tuple(admitting) + assert aid in mgr._admission.taskq_excluded_ids() + finally: + # Released on every path, so a failed assertion never leaves the store's + # writer thread parked on the gate. + gate.set() + info = await asyncio.wait_for(task, _DEFER_LOST_RUN_SECS) assert info is not None and info.queued assert not admitting, "cleared only once the defer landed" rec = store.get(info.id)