Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 13 additions & 5 deletions docs/system-specs/modules/acp-client.md
Original file line number Diff line number Diff line change
Expand Up @@ -852,11 +852,19 @@ on POSIX, where locks are advisory. Finalization releases the lock and removes
both identity-verified sidecars, then prunes EAGERLY: a projection that ends --
`NativeSkillProjection.release()`, which the direct client calls when its process
is dead, or the finalizer of a projection that was merely dropped -- runs the same
bounded prune, under the same lock and gates (lease scan, `_per_spawn_prune_due`
spacing, time budget), so this home's now-unleased views leave the agents
directory at once instead of waiting for the next spawn. The eager prune never
queues behind this process's own in-process lock (a start holding it runs its
own prune), and when the end runs on an event-loop thread the lease release and
bounded prune, under the same lock and gates (lease scan, time budget), so this
home's now-unleased views leave the agents directory at once instead of waiting
for the next spawn. A per-spawn eager prune honours the `_per_spawn_prune_due`
spacing and never queues behind this process's own in-process lock (a start
holding it runs its own prune). An end has no other party covering it, so it
skips the spacing and queues for the lock -- but ends are COALESCED per agents
directory and home (`_coalesced_end_prune`): while one end's prune runs, a later
end only records a request and returns, and once the running pass finishes one
daemon thread makes the follow-up passes, so the end that ran first is not held
for the length of a wave. Because an end releases its lease before it asks, that later
pass covers it. Without this a subagent fan-out queued one full prune per ended
projection on the lock set_mode re-preparation waits on, and starts ran out
their in-process wait and were refused. When the end runs on an event-loop thread the lease release and
the prune both go to a daemon thread, so no session start is stalled behind a
file-lock wait. A finalizer the cyclic collector runs does none of that work on
the collecting thread: the collector fires on whatever thread allocated, possibly
Expand Down
6 changes: 6 additions & 0 deletions src/kiro_crew/acp/skill_projection.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,17 +58,22 @@
_DRAIN_BATCH_PAUSE_SECS,
_DRAIN_IDLE_BATCHES,
_DRAIN_MAX_BATCHES,
_END_PRUNE_GUARD,
_END_PRUNE_REQUESTED,
_END_PRUNE_RUNNING,
_LEGACY_RECLAIM_MIN_AGE_SECS,
_PROJECTION_PRUNE_WORK_LIMIT,
_PRUNE_MAX_RECLAIMS_PER_RUN,
_PRUNE_MAX_SECONDS_PER_RUN,
_REAPER,
_REAPER_LOCK,
_SIDECAR_SWEEP_MAX_PER_RUN,
_coalesced_end_prune,
_eager_prune,
_eager_prune_in,
_end_projection,
_end_projection_off_loop,
_end_prune_more_requested,
_ensure_projection_reaper,
_migrate_legacy_side_files,
_move_sidecar,
Expand All @@ -81,6 +86,7 @@
_read_alias_spec_bounded,
_reap_collected_projections,
_reclaim_prune_candidate,
_run_requested_end_prunes,
_sweep_orphan_alias_locks,
_sweep_orphan_sidecars,
_sweep_orphan_sidecars_in,
Expand Down
83 changes: 80 additions & 3 deletions src/kiro_crew/acp/skill_projection_parts/alias_reclaim.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,9 +158,10 @@ def _eager_prune(directory: Path, crew_home_id: str, *, force: bool = False) ->
``kirocrew-skill-view-prune`` thread -- or when one ends inside the spacing
window a recent spawn opened: a zero-wait end would fail the lock the first
end holds and leave its own view on disk until the next spawn or boot drain,
the very "gateway stop" case this reclaim exists for. Queueing serialises the
ends behind one another, and because each end releases its lease before it
prunes, the walk that runs after the wait already sees this end's lease gone.
the very "gateway stop" case this reclaim exists for. Ends reach here through
:func:`_coalesced_end_prune`, so at most one end per target queues here at a
time, and because each end releases its lease before it asks, the walk that
runs after the wait already sees this end's lease gone.
Every other gate still holds: the cross-process lock ceiling, the lease scan
and the time budget. Never raises (the queue timeout is an ``OSError``).
"""
Expand Down Expand Up @@ -297,7 +298,83 @@ def _end_projection_off_loop(
try:
lease_stack.close()
finally:
_coalesced_end_prune(side_dir, directory, crew_home_id)


# Ends whose prune is already covered by one running for the same target. An end
# releases its lease BEFORE it asks for a prune, so any prune pass that STARTS
# after that request sees the lease gone and covers it. Without coalescing, a
# subagent fan-out queues one forced prune per ended projection on the same
# in-process lock the session starts wait on, each spending its full walk budget,
# and the starts behind them run out their in-process wait and are refused.
_END_PRUNE_GUARD = threading.Lock()
_END_PRUNE_RUNNING: set[tuple[str, str, str]] = set()
_END_PRUNE_REQUESTED: set[tuple[str, str, str]] = set()


def _coalesced_end_prune(side_dir: Path, directory: Path, crew_home_id: str) -> None:
"""Run the forced end prune, or hand it to the one already running for this target.

At most one end prune per (side directory, agents directory, home) runs or
waits on the alias lock at a time. An end that arrives while one runs records
a request and returns. The end that ran the first pass runs it on its own
thread, as before; if requests arrived meanwhile, the follow-up passes go to
one daemon thread, which repeats while requests keep arriving, so the caller
is not held for the length of a wave. Every end is still followed by a prune
that started after its lease was released. Never raises.
"""
key = (side_dir.as_posix(), directory.absolute().as_posix(), crew_home_id)
with _END_PRUNE_GUARD:
if key in _END_PRUNE_RUNNING:
_END_PRUNE_REQUESTED.add(key)
return
_END_PRUNE_RUNNING.add(key)
try:
_eager_prune_in(side_dir, directory, crew_home_id, force=True)
if not _end_prune_more_requested(key):
return
try:
threading.Thread(
target=_run_requested_end_prunes,
args=(key, side_dir, directory, crew_home_id),
name="kirocrew-skill-view-prune",
daemon=True,
).start()
return
except RuntimeError:
logger.debug("skill projection: follow-up prune thread not started", exc_info=True)
_run_requested_end_prunes(key, side_dir, directory, crew_home_id)
except BaseException:
with _END_PRUNE_GUARD:
_END_PRUNE_RUNNING.discard(key)
_END_PRUNE_REQUESTED.discard(key)
raise


def _end_prune_more_requested(key: tuple[str, str, str]) -> bool:
"""Consume a pending request for *key*, or end its run when there is none."""
with _END_PRUNE_GUARD:
if key in _END_PRUNE_REQUESTED:
_END_PRUNE_REQUESTED.discard(key)
return True
_END_PRUNE_RUNNING.discard(key)
return False


def _run_requested_end_prunes(
key: tuple[str, str, str], side_dir: Path, directory: Path, crew_home_id: str
) -> None:
"""The follow-up passes of one coalesced run, after its first request was consumed."""
try:
while True:
_eager_prune_in(side_dir, directory, crew_home_id, force=True)
if not _end_prune_more_requested(key):
return
except BaseException:
with _END_PRUNE_GUARD:
_END_PRUNE_RUNNING.discard(key)
_END_PRUNE_REQUESTED.discard(key)
raise


def _sweep_orphan_sidecars(
Expand Down
106 changes: 106 additions & 0 deletions test/test_native_skill_projection_rewrites.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import threading
import time
import uuid
from contextlib import ExitStack
from pathlib import Path

import native_skill_projection_helpers
Expand Down Expand Up @@ -1130,6 +1131,111 @@ def recording_lock(directory, *, in_process_wait=None):
assert waits == [None], "an end skipped the lock instead of queueing for it"


def test_ends_that_arrive_while_one_prunes_share_its_next_pass(native_tree, monkeypatch):
"""A subagent fan-out ends dozens of projections together, on the same
in-process alias lock that session starts wait on. One forced prune per end
would hold that lock long enough for the starts behind it to run out their
wait and be refused. Ends that arrive while one end's prune runs return at
once and are covered by ONE more pass, not one pass each."""
monkeypatch.setattr(projection, "_eager_prune", _REAL_EAGER_PRUNE)
_home, agents, _project = native_tree
calls: list[str] = []
entered = threading.Event()
gate = threading.Event()
follow_up_gate = threading.Event()
follow_up_done = threading.Event()

def recording_prune(directory, crew_home_id, *, force=False):
calls.append(threading.current_thread().name)
if len(calls) == 1:
entered.set()
gate.wait(10)
elif len(calls) == 2:
follow_up_gate.wait(10)
_REAL_EAGER_PRUNE(directory, crew_home_id, force=force)
if len(calls) == 2:
follow_up_done.set()

monkeypatch.setattr(projection, "_eager_prune", recording_prune)
side_dir = projection._projection_side_dir()
first = threading.Thread(
target=projection._end_projection_off_loop,
args=(ExitStack(), side_dir, agents, _crew_home_id()),
name="first-end",
)
first.start()
assert entered.wait(10)

late = [
threading.Thread(
target=projection._end_projection_off_loop,
args=(ExitStack(), side_dir, agents, _crew_home_id()),
name=f"late-end-{index}",
)
for index in range(10)
]
for thread in late:
thread.start()
for thread in late:
thread.join(5)
assert not thread.is_alive(), "an end queued behind the running prune"
gate.set()
first.join(10)
# The follow-up pass is still held: the end that ran the first pass was
# not kept for the passes the later ends asked for.
assert not first.is_alive(), "the first end was held for the follow-up pass"
follow_up_gate.set()
assert follow_up_done.wait(10), "the later ends got no follow-up pass"

assert calls == ["first-end", "kirocrew-skill-view-prune"], "each end ran a prune pass"


def test_an_end_covered_by_a_running_prune_still_loses_its_view(native_tree, monkeypatch, tmp_path):
"""The coalescing must not drop an end: a view whose lease is released while
another end's prune is already past its lease scan is reclaimed by the extra
pass that prune makes, not left for the next spawn or boot."""
monkeypatch.setattr(projection, "_eager_prune", _REAL_EAGER_PRUNE)
_home, agents, project = native_tree
source = agents / "custom.json"
source.write_text('{"name":"custom","prompt":"v1"}', encoding="utf-8")
first = projection.prepare_native_skill_projection(project)
first_alias = _alias_file(agents, first)
source.write_text('{"name":"custom","prompt":"v2"}', encoding="utf-8")
second = projection.prepare_native_skill_projection(tmp_path / "other-run")
second_alias = _alias_file(agents, second)
assert first_alias != second_alias
passes: list[bool] = []
first_pass_done = threading.Event()
second_pass_done = threading.Event()
gate = threading.Event()

def recording_prune(directory, crew_home_id, *, force=False):
_REAL_EAGER_PRUNE(directory, crew_home_id, force=force)
passes.append(second_alias.exists())
if len(passes) == 1:
first_pass_done.set()
gate.wait(10)
else:
second_pass_done.set()

monkeypatch.setattr(projection, "_eager_prune", recording_prune)
ender = threading.Thread(target=first.release, name="first-end")
ender.start()
assert first_pass_done.wait(10)
assert not first_alias.exists()
assert second_alias.exists(), "the second view went before its projection ended"

second.release() # returns at once: the running end owes it one more pass
gate.set()
ender.join(10)
assert not ender.is_alive()
assert second_pass_done.wait(10), "the covered end got no follow-up pass"

assert passes == [True, False]
assert not second_alias.exists(), "an end covered by a running prune kept its view"
assert not list(projection._projection_lease_dir().iterdir())


def test_a_projection_ending_later_prunes_the_home_it_published_into(
native_tree, monkeypatch, tmp_path
):
Expand Down
Loading