diff --git a/src/kiro_crew/apps/builtins/aws_control/crew/runtime/container_tests/test_supervisor_fakes.py b/src/kiro_crew/apps/builtins/aws_control/crew/runtime/container_tests/test_supervisor_fakes.py index 817b086ecf97..ffbe4c7a8701 100644 --- a/src/kiro_crew/apps/builtins/aws_control/crew/runtime/container_tests/test_supervisor_fakes.py +++ b/src/kiro_crew/apps/builtins/aws_control/crew/runtime/container_tests/test_supervisor_fakes.py @@ -38,8 +38,12 @@ p.add_argument("--leader-ttl", type=float, default=-1.0) a = p.parse_args() -if a.pidfile: - Path(a.pidfile).write_text(str(os.getpid())) +def _publish_pid(path): + # Written aside and renamed into place, so the pidfile never exists without its + # whole pid in it. + tmp = path + ".tmp" + Path(tmp).write_text(str(os.getpid())) + os.replace(tmp, path) # Holds the escaped child's pid so the SIGTERM handler can reap its group, # modelling the real backend, whose kiro-cli workers setsid into their own @@ -73,14 +77,22 @@ def _on_term(*_): # escaping the parent's process group exactly as a kiro-cli worker does. if a.child_setsid: os.setsid() - if a.child_pidfile: - Path(a.child_pidfile).write_text(str(os.getpid())) + # Disposition before pidfile, as for the leader. signal.signal(signal.SIGTERM, signal.SIG_IGN if a.ignore_sigterm else signal.SIG_DFL) + if a.child_pidfile: + _publish_pid(a.child_pidfile) time.sleep(a.ttl) os._exit(0) else: _child["pid"] = pid +# A test reads the pidfile as "this process is ready for SIGTERM", so it appears +# only after the disposition above, and after the forked child is recorded for the +# reap. Before the disposition, a SIGTERM meets the default action and the drain +# reads -15 instead of 0. +if a.pidfile: + _publish_pid(a.pidfile) + if a.port and a.run_dir: time.sleep(a.secret_delay) Path(a.run_dir).mkdir(parents=True, exist_ok=True) diff --git a/test/test_browser_cli_view.py b/test/test_browser_cli_view.py index 5122faf5d44b..698eea06cff4 100644 --- a/test/test_browser_cli_view.py +++ b/test/test_browser_cli_view.py @@ -11,6 +11,7 @@ import threading import time from collections.abc import Iterator +from types import SimpleNamespace import pytest @@ -942,9 +943,41 @@ def test_free_port_is_bindable_loopback() -> None: s.bind(("127.0.0.1", port)) -def test_free_port_is_not_hardcoded() -> None: - """A fixed port would collide with whatever else the operator runs.""" - assert mod._free_port() != mod._free_port() +def test_free_port_is_not_hardcoded(monkeypatch: pytest.MonkeyPatch) -> None: + """A fixed port would collide with whatever else the operator runs. + + So every number must be the kernel's: each call binds port 0 on loopback and + returns the port the kernel answered for that bind. That is asserted on the binds + and their answers, over two calls so a cached first answer fails too. The two + numbers are never compared: once the first socket is closed the kernel may legally + hand back the same port. + """ + binds: list[object] = [] + answers: list[int] = [] + + class _RecordingSocket(socket.socket): + def bind(self, address: object) -> None: + binds.append(address) + super().bind(address) + + def getsockname(self) -> object: + name = super().getsockname() + answers.append(name[1]) + return name + + # The module's own ``socket`` binding, so the stdlib module stays untouched. + monkeypatch.setattr( + mod, + "socket", + SimpleNamespace( + socket=_RecordingSocket, AF_INET=socket.AF_INET, SOCK_STREAM=socket.SOCK_STREAM + ), + ) + + first, second = mod._free_port(), mod._free_port() + + assert binds == [(mod.LOOPBACK_HOST, 0)] * 2, "a call did not leave the port to the kernel" + assert answers == [first, second], "a returned port is not the kernel's answer to that call" def test_ensure_running_returns_none_without_the_cli(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/test/test_design_tweak_relay_paths.py b/test/test_design_tweak_relay_paths.py index d7f33980e796..61f0309da234 100644 --- a/test/test_design_tweak_relay_paths.py +++ b/test/test_design_tweak_relay_paths.py @@ -885,6 +885,30 @@ def feed_then_break(): down_b.close() +#: Lost-run bound for each read and join the pump test unblocks itself. A passing run +#: moves a few dozen bytes across a socketpair, and its call measured under 0.01 s in +#: every repeat, so this only ends a broken run, by name, well inside the suite's 120 s +#: timeout. +_PUMP_LOST_RUN_SECS = 10.0 + + +def _recv_until(sock: socket.socket, complete, what: str) -> bytes: + """Read ``sock`` until ``complete(received)`` holds; fail by name if it never does.""" + received = b"" + sock.settimeout(_PUMP_LOST_RUN_SECS) + while not complete(received): + try: + chunk = sock.recv(4096) + except TimeoutError: + raise AssertionError( + f"{what}: nothing more arrived within {_PUMP_LOST_RUN_SECS}s; " + f"received {received!r}" + ) from None + assert chunk, f"{what}: the peer closed first; received {received!r}" + received += chunk + return received + + class TestRelayWsPump: """The bidirectional pump forwards frames and exits on hangup.""" @@ -898,52 +922,59 @@ def test_pump_forwards_and_exits_on_close(self): down_a, down_b = socket.socketpair() handshake = b"HTTP/1.1 101 OK\r\nUpgrade: websocket\r\n\r\n" + upstream_frame = b"hello from upstream" + client_frame = b"hello from client" p = _make_probe( headers={"Upgrade": "websocket", "Sec-WebSocket-Key": "k"}, connection=down_a, ) - def feed_handshake_then_frame(): - # Feed the handshake, wait for pump to start, then send a frame. - up_a.sendall(handshake) - time.sleep(0.1) - up_a.sendall(b"hello from upstream") - time.sleep(0.1) - up_a.close() - - feeder = threading.Thread(target=feed_handshake_then_frame, daemon=True) - feeder.start() - def run_relay(): with patch.object(server.socket, "create_connection", return_value=up_b): p._relay_ws() relay_thread = threading.Thread(target=run_relay, daemon=True) relay_thread.start() + try: + # Each step waits for the one before it to be observed, never for a + # duration. Upstream answers only the upgrade request it received, and + # sends its frame only once the client holds the whole relayed + # handshake, so the relay has finished reading the handshake and the + # frame can only reach the client through the pump. Sent any earlier, + # the frame can reach a late-starting relay in the same read as the + # handshake, and the client's handshake read then takes it too. + _recv_until(up_a, lambda got: b"\r\n\r\n" in got, "the relayed upgrade request") + up_a.sendall(handshake) + relayed = _recv_until(down_b, lambda got: b"\r\n\r\n" in got, "the relayed handshake") + # Bytes after the header are frame data, however the reads split them. + _head, _, early = relayed.partition(b"\r\n\r\n") + + up_a.sendall(upstream_frame) + forwarded = early + _recv_until( + down_b, + lambda got: len(early + got) >= len(upstream_frame), + "the upstream frame", + ) + assert forwarded == upstream_frame - # Read the handshake from the client side first. - down_b.settimeout(2) - handshake_received = b"" - while b"\r\n\r\n" not in handshake_received: - handshake_received += down_b.recv(4096) - - # Now read the forwarded frame. - frame_data = down_b.recv(4096) - assert frame_data == b"hello from upstream" - - # Send a frame from the "client" side back upstream. - down_b.sendall(b"hello from client") - # The relay forwards it to up_b, which comes out on up_a — but up_a - # may already be closed by the feeder. The point is the relay exits - # cleanly when up_a closes. - relay_thread.join(timeout=3) - feeder.join(timeout=1) - assert not relay_thread.is_alive() + down_b.sendall(client_frame) + assert ( + _recv_until(up_a, lambda got: len(got) >= len(client_frame), "the client frame") + == client_frame + ) - # Cleanup. - down_a.close() - down_b.close() + # Upstream hangs up: the pump must end. + up_a.close() + relay_thread.join(_PUMP_LOST_RUN_SECS) + assert ( + not relay_thread.is_alive() + ), f"the pump was still running {_PUMP_LOST_RUN_SECS}s after the upstream hangup" + finally: + # Closing both ends also ends a pump a failed assertion left running. + up_a.close() + down_a.close() + down_b.close() class TestRelayWsIdleCap: diff --git a/test/test_md_notebook.py b/test/test_md_notebook.py index 82528c02eade..71eb231454c0 100644 --- a/test/test_md_notebook.py +++ b/test/test_md_notebook.py @@ -37,6 +37,7 @@ from kiro_crew import platform_compat from kiro_crew.apps.builtins.md_notebook import git_ops from kiro_crew.loop_lock import LoopBoundLock +from kiro_crew.testing.wait import until_parked SECRET = "test-proxy-secret" @@ -3785,6 +3786,26 @@ def __getattr__(self, name): # type: ignore[no-untyped-def] assert not list(root.glob("One.md.*.tmp")), "a temp survived a successful save" +async def _parked_behind(lock: LoopBoundLock, contender: asyncio.Task, what: str) -> None: + """Return once ``contender`` is parked on ``lock``, the barrier a "still waiting" check needs. + + Without it a contender that has not yet REACHED the lock satisfies the negative as + well as one the lock is holding back, so the check passes with the lock removed. A + contender that finishes instead of parking fails here by name; one that never + parks fails at ``until_parked``'s deadline, quoting how many waiters it saw. + """ + parked = asyncio.ensure_future(until_parked(lock, describe=lambda: what)) + try: + await asyncio.wait({parked, contender}, return_when=asyncio.FIRST_COMPLETED) + finally: + if not parked.done(): + parked.cancel() + with contextlib.suppress(asyncio.CancelledError): + await parked + assert not contender.done(), f"{what} finished without ever waiting on the lock" + parked.result() + + @pytest.mark.asyncio async def test_a_sync_while_the_temp_is_still_staging_never_commits_it( fixtures, @@ -3870,9 +3891,11 @@ def parked_stage(tmp, content): # type: ignore[no-untyped-def] ) # Requested while staging is parked: the Sync must wait on the - # vault write lock the staging call holds, not run beside it. + # vault write lock the staging call holds, not run beside it. It is + # seen parked there first: a Sync that has not reached the lock yet + # would pass this check as well, lock or no lock. sync = real_asyncio.create_task(client.post("/api/sync")) - await real_asyncio.sleep(0.2) + await _parked_behind(_mod.vault_write_lock(vault["localPath"]), sync, "the Sync") assert not sync.done(), "the Sync ran while a note was still staging" finally: release_stage.set() @@ -4061,16 +4084,17 @@ def __getattr__(self, name): # type: ignore[no-untyped-def] str(target) ).locked(), "the per-note lock was released during an unguarded retry" - task_b = real_asyncio.create_task( - client.put("/api/note", {"path": "One.md", "content": "B"}) - ) - # Yield generously rather than sleeping: if B could interleave it - # would have completed by now. - for _ in range(100): - await real_asyncio.sleep(0) - assert not task_b.done(), "a later save interleaved into the retry gap" - - release_a.set() + try: + task_b = real_asyncio.create_task( + client.put("/api/note", {"path": "One.md", "content": "B"}) + ) + # B is seen parked on the note's lock before the check: a B whose + # request has not reached the lock yet would pass it as well, lock + # or no lock. + await _parked_behind(_mod._save_lock(str(target)), task_b, "the later save") + assert not task_b.done(), "a later save interleaved into the retry gap" + finally: + release_a.set() status_a, body_a = await real_asyncio.wait_for(task_a, timeout=5) status_b, body_b = await real_asyncio.wait_for(task_b, timeout=5) @@ -4142,19 +4166,20 @@ def parked_stage(tmp, content): # type: ignore[no-untyped-def] str(target) ).locked(), "a tokenless save staged its temp outside the per-note lock" - task_b = real_asyncio.create_task( - client.put("/api/note", {"path": "One.md", "content": "B"}) - ) - # Yield generously rather than sleeping: were the lock free, B would - # have staged and published by now. - for _ in range(100): - await real_asyncio.sleep(0) - assert staged == [ - "A" - ], f"a later save staged while the earlier one held the lock: {staged}" - assert not task_b.done(), "a later save completed while the earlier one was staging" - - release_a.set() + try: + task_b = real_asyncio.create_task( + client.put("/api/note", {"path": "One.md", "content": "B"}) + ) + # B is seen parked on the note's lock before the checks: a B whose + # request has not reached the lock yet would pass them as well, lock + # or no lock. + await _parked_behind(_mod._save_lock(str(target)), task_b, "the later save") + assert staged == [ + "A" + ], f"a later save staged while the earlier one held the lock: {staged}" + assert not task_b.done(), "a later save completed while the earlier one was staging" + finally: + release_a.set() status_a, body_a = await real_asyncio.wait_for(task_a, timeout=10) status_b, body_b = await real_asyncio.wait_for(task_b, timeout=10) diff --git a/test/test_overload_integration_glue.py b/test/test_overload_integration_glue.py index 097580ddf099..260efa64bbaf 100644 --- a/test/test_overload_integration_glue.py +++ b/test/test_overload_integration_glue.py @@ -53,6 +53,7 @@ from kiro_crew.taskq.adapters import runner as runner_mod from kiro_crew.taskq.store import TaskStore from kiro_crew.taskq.waits import WaitRecord +from kiro_crew.testing.wait import async_wait_until pytestmark = pytest.mark.usefixtures("healthy_host_memory") @@ -3227,6 +3228,11 @@ def _reenter(_claimed): # ── the pump sees a deferred row only after its defer landed ───────────────── +#: Lost-run bound for the gated defer and the spawn the test releases. The test's call +#: measured at most 0.04 s in repeats at -n0, so this only ends a broken run, by name, +#: well inside the suite's 120 s timeout. +_DEFER_LOST_RUN_SECS = 30.0 + @pytest.mark.asyncio async def test_admitting_id_is_held_until_the_posted_defer_landed( @@ -3241,26 +3247,41 @@ async def test_admitting_id_is_held_until_the_posted_defer_landed( monkeypatch.setattr(admission_mod.SpawnAdmissionCoordinator, "pump_off_loop", True) monkeypatch.setattr(subagent_mod, "check_memory_available", lambda min_gb: (False, 0.5)) gate = threading.Event() + defer_entered = threading.Event() real_defer = store.defer seen: dict[str, Any] = {} def _slow_defer(task_id, *, wait, reason): - assert gate.wait(5) + defer_entered.set() + assert gate.wait(_DEFER_LOST_RUN_SECS) seen["wait"] = wait return real_defer(task_id, wait=wait, reason=reason) monkeypatch.setattr(store, "defer", _slow_defer) task = asyncio.ensure_future(mgr.spawn_async("pressure", parent_session_key="web-1")) - for _ in range(20): - await asyncio.sleep(0.005) - admitting = getattr(mgr, "_admitting_ids") - assert ( - len(admitting) == 1 - ), "the row stays excluded from the refill while its defer is in flight" - (aid,) = tuple(admitting) - assert aid in mgr._admission.taskq_excluded_ids() - gate.set() - info = await task + try: + # The window under test opens when the posted defer is running and held by + # the gate. Wait for that, not for a count of loop turns: on a slow executor + # the spawn's off-loop hops outlast any count, and ``_admitting_ids`` does + # not exist until the spawn reaches the row it accepted. A spawn that + # returns without deferring ends the wait too, and fails below. + await async_wait_until( + lambda: defer_entered.is_set() or task.done(), + timeout=_DEFER_LOST_RUN_SECS, + describe=lambda: f"admitting={getattr(mgr, '_admitting_ids', None)!r}", + ) + assert defer_entered.is_set(), f"the spawn never posted its defer: {task!r}" + admitting = getattr(mgr, "_admitting_ids", set()) + assert ( + len(admitting) == 1 + ), "the row stays excluded from the refill while its defer is in flight" + (aid,) = tuple(admitting) + assert aid in mgr._admission.taskq_excluded_ids() + finally: + # Released on every path, so a failed assertion never leaves the store's + # writer thread parked on the gate. + gate.set() + info = await asyncio.wait_for(task, _DEFER_LOST_RUN_SECS) assert info is not None and info.queued assert not admitting, "cleared only once the defer landed" rec = store.get(info.id)