Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,12 @@
p.add_argument("--leader-ttl", type=float, default=-1.0)
a = p.parse_args()

if a.pidfile:
Path(a.pidfile).write_text(str(os.getpid()))
def _publish_pid(path):
# Written aside and renamed into place, so the pidfile never exists without its
# whole pid in it.
tmp = path + ".tmp"
Path(tmp).write_text(str(os.getpid()))
os.replace(tmp, path)

# Holds the escaped child's pid so the SIGTERM handler can reap its group,
# modelling the real backend, whose kiro-cli workers setsid into their own
Expand Down Expand Up @@ -73,14 +77,22 @@ def _on_term(*_):
# escaping the parent's process group exactly as a kiro-cli worker does.
if a.child_setsid:
os.setsid()
if a.child_pidfile:
Path(a.child_pidfile).write_text(str(os.getpid()))
# Disposition before pidfile, as for the leader.
signal.signal(signal.SIGTERM, signal.SIG_IGN if a.ignore_sigterm else signal.SIG_DFL)
if a.child_pidfile:
_publish_pid(a.child_pidfile)
time.sleep(a.ttl)
os._exit(0)
else:
_child["pid"] = pid

# A test reads the pidfile as "this process is ready for SIGTERM", so it appears
# only after the disposition above, and after the forked child is recorded for the
# reap. Before the disposition, a SIGTERM meets the default action and the drain
# reads -15 instead of 0.
if a.pidfile:
_publish_pid(a.pidfile)

if a.port and a.run_dir:
time.sleep(a.secret_delay)
Path(a.run_dir).mkdir(parents=True, exist_ok=True)
Expand Down
39 changes: 36 additions & 3 deletions test/test_browser_cli_view.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import threading
import time
from collections.abc import Iterator
from types import SimpleNamespace

import pytest

Expand Down Expand Up @@ -942,9 +943,41 @@ def test_free_port_is_bindable_loopback() -> None:
s.bind(("127.0.0.1", port))


def test_free_port_is_not_hardcoded() -> None:
"""A fixed port would collide with whatever else the operator runs."""
assert mod._free_port() != mod._free_port()
def test_free_port_is_not_hardcoded(monkeypatch: pytest.MonkeyPatch) -> None:
"""A fixed port would collide with whatever else the operator runs.

So every number must be the kernel's: each call binds port 0 on loopback and
returns the port the kernel answered for that bind. That is asserted on the binds
and their answers, over two calls so a cached first answer fails too. The two
numbers are never compared: once the first socket is closed the kernel may legally
hand back the same port.
"""
binds: list[object] = []
answers: list[int] = []

class _RecordingSocket(socket.socket):
def bind(self, address: object) -> None:
binds.append(address)
super().bind(address)

def getsockname(self) -> object:
name = super().getsockname()
answers.append(name[1])
return name

# The module's own ``socket`` binding, so the stdlib module stays untouched.
monkeypatch.setattr(
mod,
"socket",
SimpleNamespace(
socket=_RecordingSocket, AF_INET=socket.AF_INET, SOCK_STREAM=socket.SOCK_STREAM
),
)

first, second = mod._free_port(), mod._free_port()

assert binds == [(mod.LOOPBACK_HOST, 0)] * 2, "a call did not leave the port to the kernel"
assert answers == [first, second], "a returned port is not the kernel's answer to that call"


def test_ensure_running_returns_none_without_the_cli(monkeypatch: pytest.MonkeyPatch) -> None:
Expand Down
95 changes: 63 additions & 32 deletions test/test_design_tweak_relay_paths.py
Original file line number Diff line number Diff line change
Expand Up @@ -885,6 +885,30 @@ def feed_then_break():
down_b.close()


#: Lost-run bound for each read and join the pump test unblocks itself. A passing run
#: moves a few dozen bytes across a socketpair, and its call measured under 0.01 s in
#: every repeat, so this only ends a broken run, by name, well inside the suite's 120 s
#: timeout.
_PUMP_LOST_RUN_SECS = 10.0


def _recv_until(sock: socket.socket, complete, what: str) -> bytes:
"""Read ``sock`` until ``complete(received)`` holds; fail by name if it never does."""
received = b""
sock.settimeout(_PUMP_LOST_RUN_SECS)
while not complete(received):
try:
chunk = sock.recv(4096)
except TimeoutError:
raise AssertionError(
f"{what}: nothing more arrived within {_PUMP_LOST_RUN_SECS}s; "
f"received {received!r}"
) from None
assert chunk, f"{what}: the peer closed first; received {received!r}"
received += chunk
return received


class TestRelayWsPump:
"""The bidirectional pump forwards frames and exits on hangup."""

Expand All @@ -898,52 +922,59 @@ def test_pump_forwards_and_exits_on_close(self):
down_a, down_b = socket.socketpair()

handshake = b"HTTP/1.1 101 OK\r\nUpgrade: websocket\r\n\r\n"
upstream_frame = b"hello from upstream"
client_frame = b"hello from client"

p = _make_probe(
headers={"Upgrade": "websocket", "Sec-WebSocket-Key": "k"},
connection=down_a,
)

def feed_handshake_then_frame():
# Feed the handshake, wait for pump to start, then send a frame.
up_a.sendall(handshake)
time.sleep(0.1)
up_a.sendall(b"hello from upstream")
time.sleep(0.1)
up_a.close()

feeder = threading.Thread(target=feed_handshake_then_frame, daemon=True)
feeder.start()

def run_relay():
with patch.object(server.socket, "create_connection", return_value=up_b):
p._relay_ws()

relay_thread = threading.Thread(target=run_relay, daemon=True)
relay_thread.start()
try:
# Each step waits for the one before it to be observed, never for a
# duration. Upstream answers only the upgrade request it received, and
# sends its frame only once the client holds the whole relayed
# handshake, so the relay has finished reading the handshake and the
# frame can only reach the client through the pump. Sent any earlier,
# the frame can reach a late-starting relay in the same read as the
# handshake, and the client's handshake read then takes it too.
_recv_until(up_a, lambda got: b"\r\n\r\n" in got, "the relayed upgrade request")
up_a.sendall(handshake)
relayed = _recv_until(down_b, lambda got: b"\r\n\r\n" in got, "the relayed handshake")
# Bytes after the header are frame data, however the reads split them.
_head, _, early = relayed.partition(b"\r\n\r\n")

up_a.sendall(upstream_frame)
forwarded = early + _recv_until(
down_b,
lambda got: len(early + got) >= len(upstream_frame),
"the upstream frame",
)
assert forwarded == upstream_frame

# Read the handshake from the client side first.
down_b.settimeout(2)
handshake_received = b""
while b"\r\n\r\n" not in handshake_received:
handshake_received += down_b.recv(4096)

# Now read the forwarded frame.
frame_data = down_b.recv(4096)
assert frame_data == b"hello from upstream"

# Send a frame from the "client" side back upstream.
down_b.sendall(b"hello from client")
# The relay forwards it to up_b, which comes out on up_a — but up_a
# may already be closed by the feeder. The point is the relay exits
# cleanly when up_a closes.
relay_thread.join(timeout=3)
feeder.join(timeout=1)
assert not relay_thread.is_alive()
down_b.sendall(client_frame)
assert (
_recv_until(up_a, lambda got: len(got) >= len(client_frame), "the client frame")
== client_frame
)

# Cleanup.
down_a.close()
down_b.close()
# Upstream hangs up: the pump must end.
up_a.close()
relay_thread.join(_PUMP_LOST_RUN_SECS)
assert (
not relay_thread.is_alive()
), f"the pump was still running {_PUMP_LOST_RUN_SECS}s after the upstream hangup"
finally:
# Closing both ends also ends a pump a failed assertion left running.
up_a.close()
down_a.close()
down_b.close()


class TestRelayWsIdleCap:
Expand Down
75 changes: 50 additions & 25 deletions test/test_md_notebook.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
from kiro_crew import platform_compat
from kiro_crew.apps.builtins.md_notebook import git_ops
from kiro_crew.loop_lock import LoopBoundLock
from kiro_crew.testing.wait import until_parked

SECRET = "test-proxy-secret"

Expand Down Expand Up @@ -3785,6 +3786,26 @@ def __getattr__(self, name): # type: ignore[no-untyped-def]
assert not list(root.glob("One.md.*.tmp")), "a temp survived a successful save"


async def _parked_behind(lock: LoopBoundLock, contender: asyncio.Task, what: str) -> None:
"""Return once ``contender`` is parked on ``lock``, the barrier a "still waiting" check needs.

Without it a contender that has not yet REACHED the lock satisfies the negative as
well as one the lock is holding back, so the check passes with the lock removed. A
contender that finishes instead of parking fails here by name; one that never
parks fails at ``until_parked``'s deadline, quoting how many waiters it saw.
"""
parked = asyncio.ensure_future(until_parked(lock, describe=lambda: what))
try:
await asyncio.wait({parked, contender}, return_when=asyncio.FIRST_COMPLETED)
finally:
if not parked.done():
parked.cancel()
with contextlib.suppress(asyncio.CancelledError):
await parked
assert not contender.done(), f"{what} finished without ever waiting on the lock"
parked.result()


@pytest.mark.asyncio
async def test_a_sync_while_the_temp_is_still_staging_never_commits_it(
fixtures,
Expand Down Expand Up @@ -3870,9 +3891,11 @@ def parked_stage(tmp, content): # type: ignore[no-untyped-def]
)

# Requested while staging is parked: the Sync must wait on the
# vault write lock the staging call holds, not run beside it.
# vault write lock the staging call holds, not run beside it. It is
# seen parked there first: a Sync that has not reached the lock yet
# would pass this check as well, lock or no lock.
sync = real_asyncio.create_task(client.post("/api/sync"))
await real_asyncio.sleep(0.2)
await _parked_behind(_mod.vault_write_lock(vault["localPath"]), sync, "the Sync")
assert not sync.done(), "the Sync ran while a note was still staging"
finally:
release_stage.set()
Expand Down Expand Up @@ -4061,16 +4084,17 @@ def __getattr__(self, name): # type: ignore[no-untyped-def]
str(target)
).locked(), "the per-note lock was released during an unguarded retry"

task_b = real_asyncio.create_task(
client.put("/api/note", {"path": "One.md", "content": "B"})
)
# Yield generously rather than sleeping: if B could interleave it
# would have completed by now.
for _ in range(100):
await real_asyncio.sleep(0)
assert not task_b.done(), "a later save interleaved into the retry gap"

release_a.set()
try:
task_b = real_asyncio.create_task(
client.put("/api/note", {"path": "One.md", "content": "B"})
)
# B is seen parked on the note's lock before the check: a B whose
# request has not reached the lock yet would pass it as well, lock
# or no lock.
await _parked_behind(_mod._save_lock(str(target)), task_b, "the later save")
assert not task_b.done(), "a later save interleaved into the retry gap"
finally:
release_a.set()
status_a, body_a = await real_asyncio.wait_for(task_a, timeout=5)
status_b, body_b = await real_asyncio.wait_for(task_b, timeout=5)

Expand Down Expand Up @@ -4142,19 +4166,20 @@ def parked_stage(tmp, content): # type: ignore[no-untyped-def]
str(target)
).locked(), "a tokenless save staged its temp outside the per-note lock"

task_b = real_asyncio.create_task(
client.put("/api/note", {"path": "One.md", "content": "B"})
)
# Yield generously rather than sleeping: were the lock free, B would
# have staged and published by now.
for _ in range(100):
await real_asyncio.sleep(0)
assert staged == [
"A"
], f"a later save staged while the earlier one held the lock: {staged}"
assert not task_b.done(), "a later save completed while the earlier one was staging"

release_a.set()
try:
task_b = real_asyncio.create_task(
client.put("/api/note", {"path": "One.md", "content": "B"})
)
# B is seen parked on the note's lock before the checks: a B whose
# request has not reached the lock yet would pass them as well, lock
# or no lock.
await _parked_behind(_mod._save_lock(str(target)), task_b, "the later save")
assert staged == [
"A"
], f"a later save staged while the earlier one held the lock: {staged}"
assert not task_b.done(), "a later save completed while the earlier one was staging"
finally:
release_a.set()
status_a, body_a = await real_asyncio.wait_for(task_a, timeout=10)
status_b, body_b = await real_asyncio.wait_for(task_b, timeout=10)

Expand Down
43 changes: 32 additions & 11 deletions test/test_overload_integration_glue.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
from kiro_crew.taskq.adapters import runner as runner_mod
from kiro_crew.taskq.store import TaskStore
from kiro_crew.taskq.waits import WaitRecord
from kiro_crew.testing.wait import async_wait_until

pytestmark = pytest.mark.usefixtures("healthy_host_memory")

Expand Down Expand Up @@ -3227,6 +3228,11 @@ def _reenter(_claimed):

# ── the pump sees a deferred row only after its defer landed ─────────────────

#: Lost-run bound for the gated defer and the spawn the test releases. The test's call
#: measured at most 0.04 s in repeats at -n0, so this only ends a broken run, by name,
#: well inside the suite's 120 s timeout.
_DEFER_LOST_RUN_SECS = 30.0


@pytest.mark.asyncio
async def test_admitting_id_is_held_until_the_posted_defer_landed(
Expand All @@ -3241,26 +3247,41 @@ async def test_admitting_id_is_held_until_the_posted_defer_landed(
monkeypatch.setattr(admission_mod.SpawnAdmissionCoordinator, "pump_off_loop", True)
monkeypatch.setattr(subagent_mod, "check_memory_available", lambda min_gb: (False, 0.5))
gate = threading.Event()
defer_entered = threading.Event()
real_defer = store.defer
seen: dict[str, Any] = {}

def _slow_defer(task_id, *, wait, reason):
assert gate.wait(5)
defer_entered.set()
assert gate.wait(_DEFER_LOST_RUN_SECS)
seen["wait"] = wait
return real_defer(task_id, wait=wait, reason=reason)

monkeypatch.setattr(store, "defer", _slow_defer)
task = asyncio.ensure_future(mgr.spawn_async("pressure", parent_session_key="web-1"))
for _ in range(20):
await asyncio.sleep(0.005)
admitting = getattr(mgr, "_admitting_ids")
assert (
len(admitting) == 1
), "the row stays excluded from the refill while its defer is in flight"
(aid,) = tuple(admitting)
assert aid in mgr._admission.taskq_excluded_ids()
gate.set()
info = await task
try:
# The window under test opens when the posted defer is running and held by
# the gate. Wait for that, not for a count of loop turns: on a slow executor
# the spawn's off-loop hops outlast any count, and ``_admitting_ids`` does
# not exist until the spawn reaches the row it accepted. A spawn that
# returns without deferring ends the wait too, and fails below.
await async_wait_until(
lambda: defer_entered.is_set() or task.done(),
timeout=_DEFER_LOST_RUN_SECS,
describe=lambda: f"admitting={getattr(mgr, '_admitting_ids', None)!r}",
)
assert defer_entered.is_set(), f"the spawn never posted its defer: {task!r}"
admitting = getattr(mgr, "_admitting_ids", set())
assert (
len(admitting) == 1
), "the row stays excluded from the refill while its defer is in flight"
(aid,) = tuple(admitting)
assert aid in mgr._admission.taskq_excluded_ids()
finally:
# Released on every path, so a failed assertion never leaves the store's
# writer thread parked on the gate.
gate.set()
info = await asyncio.wait_for(task, _DEFER_LOST_RUN_SECS)
assert info is not None and info.queued
assert not admitting, "cleared only once the defer landed"
rec = store.get(info.id)
Expand Down
Loading