diff --git a/docs/system-specs/modules/acp-client.md b/docs/system-specs/modules/acp-client.md index 231649f5889a..5b06325f1307 100644 --- a/docs/system-specs/modules/acp-client.md +++ b/docs/system-specs/modules/acp-client.md @@ -852,11 +852,19 @@ on POSIX, where locks are advisory. Finalization releases the lock and removes both identity-verified sidecars, then prunes EAGERLY: a projection that ends -- `NativeSkillProjection.release()`, which the direct client calls when its process is dead, or the finalizer of a projection that was merely dropped -- runs the same -bounded prune, under the same lock and gates (lease scan, `_per_spawn_prune_due` -spacing, time budget), so this home's now-unleased views leave the agents -directory at once instead of waiting for the next spawn. The eager prune never -queues behind this process's own in-process lock (a start holding it runs its -own prune), and when the end runs on an event-loop thread the lease release and +bounded prune, under the same lock and gates (lease scan, time budget), so this +home's now-unleased views leave the agents directory at once instead of waiting +for the next spawn. A per-spawn eager prune honours the `_per_spawn_prune_due` +spacing and never queues behind this process's own in-process lock (a start +holding it runs its own prune). An end has no other party covering it, so it +skips the spacing and queues for the lock -- but ends are COALESCED per agents +directory and home (`_coalesced_end_prune`): while one end's prune runs, a later +end only records a request and returns, and once the running pass finishes one +daemon thread makes the follow-up passes, so the end that ran first is not held +for the length of a wave. Because an end releases its lease before it asks, that later +pass covers it. Without this a subagent fan-out queued one full prune per ended +projection on the lock set_mode re-preparation waits on, and starts ran out +their in-process wait and were refused. When the end runs on an event-loop thread the lease release and the prune both go to a daemon thread, so no session start is stalled behind a file-lock wait. A finalizer the cyclic collector runs does none of that work on the collecting thread: the collector fires on whatever thread allocated, possibly diff --git a/src/kiro_crew/acp/skill_projection.py b/src/kiro_crew/acp/skill_projection.py index 40568dfe347a..8208770ca64c 100644 --- a/src/kiro_crew/acp/skill_projection.py +++ b/src/kiro_crew/acp/skill_projection.py @@ -58,6 +58,9 @@ _DRAIN_BATCH_PAUSE_SECS, _DRAIN_IDLE_BATCHES, _DRAIN_MAX_BATCHES, + _END_PRUNE_GUARD, + _END_PRUNE_REQUESTED, + _END_PRUNE_RUNNING, _LEGACY_RECLAIM_MIN_AGE_SECS, _PROJECTION_PRUNE_WORK_LIMIT, _PRUNE_MAX_RECLAIMS_PER_RUN, @@ -65,10 +68,12 @@ _REAPER, _REAPER_LOCK, _SIDECAR_SWEEP_MAX_PER_RUN, + _coalesced_end_prune, _eager_prune, _eager_prune_in, _end_projection, _end_projection_off_loop, + _end_prune_more_requested, _ensure_projection_reaper, _migrate_legacy_side_files, _move_sidecar, @@ -81,6 +86,7 @@ _read_alias_spec_bounded, _reap_collected_projections, _reclaim_prune_candidate, + _run_requested_end_prunes, _sweep_orphan_alias_locks, _sweep_orphan_sidecars, _sweep_orphan_sidecars_in, diff --git a/src/kiro_crew/acp/skill_projection_parts/alias_reclaim.py b/src/kiro_crew/acp/skill_projection_parts/alias_reclaim.py index d42cc4aef42f..d1a5ce9651ce 100644 --- a/src/kiro_crew/acp/skill_projection_parts/alias_reclaim.py +++ b/src/kiro_crew/acp/skill_projection_parts/alias_reclaim.py @@ -158,9 +158,10 @@ def _eager_prune(directory: Path, crew_home_id: str, *, force: bool = False) -> ``kirocrew-skill-view-prune`` thread -- or when one ends inside the spacing window a recent spawn opened: a zero-wait end would fail the lock the first end holds and leave its own view on disk until the next spawn or boot drain, - the very "gateway stop" case this reclaim exists for. Queueing serialises the - ends behind one another, and because each end releases its lease before it - prunes, the walk that runs after the wait already sees this end's lease gone. + the very "gateway stop" case this reclaim exists for. Ends reach here through + :func:`_coalesced_end_prune`, so at most one end per target queues here at a + time, and because each end releases its lease before it asks, the walk that + runs after the wait already sees this end's lease gone. Every other gate still holds: the cross-process lock ceiling, the lease scan and the time budget. Never raises (the queue timeout is an ``OSError``). """ @@ -297,7 +298,83 @@ def _end_projection_off_loop( try: lease_stack.close() finally: + _coalesced_end_prune(side_dir, directory, crew_home_id) + + +# Ends whose prune is already covered by one running for the same target. An end +# releases its lease BEFORE it asks for a prune, so any prune pass that STARTS +# after that request sees the lease gone and covers it. Without coalescing, a +# subagent fan-out queues one forced prune per ended projection on the same +# in-process lock the session starts wait on, each spending its full walk budget, +# and the starts behind them run out their in-process wait and are refused. +_END_PRUNE_GUARD = threading.Lock() +_END_PRUNE_RUNNING: set[tuple[str, str, str]] = set() +_END_PRUNE_REQUESTED: set[tuple[str, str, str]] = set() + + +def _coalesced_end_prune(side_dir: Path, directory: Path, crew_home_id: str) -> None: + """Run the forced end prune, or hand it to the one already running for this target. + + At most one end prune per (side directory, agents directory, home) runs or + waits on the alias lock at a time. An end that arrives while one runs records + a request and returns. The end that ran the first pass runs it on its own + thread, as before; if requests arrived meanwhile, the follow-up passes go to + one daemon thread, which repeats while requests keep arriving, so the caller + is not held for the length of a wave. Every end is still followed by a prune + that started after its lease was released. Never raises. + """ + key = (side_dir.as_posix(), directory.absolute().as_posix(), crew_home_id) + with _END_PRUNE_GUARD: + if key in _END_PRUNE_RUNNING: + _END_PRUNE_REQUESTED.add(key) + return + _END_PRUNE_RUNNING.add(key) + try: _eager_prune_in(side_dir, directory, crew_home_id, force=True) + if not _end_prune_more_requested(key): + return + try: + threading.Thread( + target=_run_requested_end_prunes, + args=(key, side_dir, directory, crew_home_id), + name="kirocrew-skill-view-prune", + daemon=True, + ).start() + return + except RuntimeError: + logger.debug("skill projection: follow-up prune thread not started", exc_info=True) + _run_requested_end_prunes(key, side_dir, directory, crew_home_id) + except BaseException: + with _END_PRUNE_GUARD: + _END_PRUNE_RUNNING.discard(key) + _END_PRUNE_REQUESTED.discard(key) + raise + + +def _end_prune_more_requested(key: tuple[str, str, str]) -> bool: + """Consume a pending request for *key*, or end its run when there is none.""" + with _END_PRUNE_GUARD: + if key in _END_PRUNE_REQUESTED: + _END_PRUNE_REQUESTED.discard(key) + return True + _END_PRUNE_RUNNING.discard(key) + return False + + +def _run_requested_end_prunes( + key: tuple[str, str, str], side_dir: Path, directory: Path, crew_home_id: str +) -> None: + """The follow-up passes of one coalesced run, after its first request was consumed.""" + try: + while True: + _eager_prune_in(side_dir, directory, crew_home_id, force=True) + if not _end_prune_more_requested(key): + return + except BaseException: + with _END_PRUNE_GUARD: + _END_PRUNE_RUNNING.discard(key) + _END_PRUNE_REQUESTED.discard(key) + raise def _sweep_orphan_sidecars( diff --git a/test/test_native_skill_projection_rewrites.py b/test/test_native_skill_projection_rewrites.py index b80a4600e2b0..40fc0a0d8aa4 100644 --- a/test/test_native_skill_projection_rewrites.py +++ b/test/test_native_skill_projection_rewrites.py @@ -16,6 +16,7 @@ import threading import time import uuid +from contextlib import ExitStack from pathlib import Path import native_skill_projection_helpers @@ -1130,6 +1131,111 @@ def recording_lock(directory, *, in_process_wait=None): assert waits == [None], "an end skipped the lock instead of queueing for it" +def test_ends_that_arrive_while_one_prunes_share_its_next_pass(native_tree, monkeypatch): + """A subagent fan-out ends dozens of projections together, on the same + in-process alias lock that session starts wait on. One forced prune per end + would hold that lock long enough for the starts behind it to run out their + wait and be refused. Ends that arrive while one end's prune runs return at + once and are covered by ONE more pass, not one pass each.""" + monkeypatch.setattr(projection, "_eager_prune", _REAL_EAGER_PRUNE) + _home, agents, _project = native_tree + calls: list[str] = [] + entered = threading.Event() + gate = threading.Event() + follow_up_gate = threading.Event() + follow_up_done = threading.Event() + + def recording_prune(directory, crew_home_id, *, force=False): + calls.append(threading.current_thread().name) + if len(calls) == 1: + entered.set() + gate.wait(10) + elif len(calls) == 2: + follow_up_gate.wait(10) + _REAL_EAGER_PRUNE(directory, crew_home_id, force=force) + if len(calls) == 2: + follow_up_done.set() + + monkeypatch.setattr(projection, "_eager_prune", recording_prune) + side_dir = projection._projection_side_dir() + first = threading.Thread( + target=projection._end_projection_off_loop, + args=(ExitStack(), side_dir, agents, _crew_home_id()), + name="first-end", + ) + first.start() + assert entered.wait(10) + + late = [ + threading.Thread( + target=projection._end_projection_off_loop, + args=(ExitStack(), side_dir, agents, _crew_home_id()), + name=f"late-end-{index}", + ) + for index in range(10) + ] + for thread in late: + thread.start() + for thread in late: + thread.join(5) + assert not thread.is_alive(), "an end queued behind the running prune" + gate.set() + first.join(10) + # The follow-up pass is still held: the end that ran the first pass was + # not kept for the passes the later ends asked for. + assert not first.is_alive(), "the first end was held for the follow-up pass" + follow_up_gate.set() + assert follow_up_done.wait(10), "the later ends got no follow-up pass" + + assert calls == ["first-end", "kirocrew-skill-view-prune"], "each end ran a prune pass" + + +def test_an_end_covered_by_a_running_prune_still_loses_its_view(native_tree, monkeypatch, tmp_path): + """The coalescing must not drop an end: a view whose lease is released while + another end's prune is already past its lease scan is reclaimed by the extra + pass that prune makes, not left for the next spawn or boot.""" + monkeypatch.setattr(projection, "_eager_prune", _REAL_EAGER_PRUNE) + _home, agents, project = native_tree + source = agents / "custom.json" + source.write_text('{"name":"custom","prompt":"v1"}', encoding="utf-8") + first = projection.prepare_native_skill_projection(project) + first_alias = _alias_file(agents, first) + source.write_text('{"name":"custom","prompt":"v2"}', encoding="utf-8") + second = projection.prepare_native_skill_projection(tmp_path / "other-run") + second_alias = _alias_file(agents, second) + assert first_alias != second_alias + passes: list[bool] = [] + first_pass_done = threading.Event() + second_pass_done = threading.Event() + gate = threading.Event() + + def recording_prune(directory, crew_home_id, *, force=False): + _REAL_EAGER_PRUNE(directory, crew_home_id, force=force) + passes.append(second_alias.exists()) + if len(passes) == 1: + first_pass_done.set() + gate.wait(10) + else: + second_pass_done.set() + + monkeypatch.setattr(projection, "_eager_prune", recording_prune) + ender = threading.Thread(target=first.release, name="first-end") + ender.start() + assert first_pass_done.wait(10) + assert not first_alias.exists() + assert second_alias.exists(), "the second view went before its projection ended" + + second.release() # returns at once: the running end owes it one more pass + gate.set() + ender.join(10) + assert not ender.is_alive() + assert second_pass_done.wait(10), "the covered end got no follow-up pass" + + assert passes == [True, False] + assert not second_alias.exists(), "an end covered by a running prune kept its view" + assert not list(projection._projection_lease_dir().iterdir()) + + def test_a_projection_ending_later_prunes_the_home_it_published_into( native_tree, monkeypatch, tmp_path ):