diff --git a/docs/system-specs/modules/autopilot.md b/docs/system-specs/modules/autopilot.md index a27b584b7c09..f4eaff0b7b82 100644 --- a/docs/system-specs/modules/autopilot.md +++ b/docs/system-specs/modules/autopilot.md @@ -308,8 +308,12 @@ entry. (completed / execute-now / pending), previous stage results, the current stage's title and bullets, and an explicit "execute Stage N of M now" instruction. It is appended as a hidden user message (`auto-go` class) and - passed to `_run_chat`. An exception from `_run_chat` clears `_auto_run`, - posts a stage-error notice, logs `auto_run_stage_error`, and breaks. + passed to `_run_chat`, bounded by the stage timeout. Hitting that ceiling + clears `_auto_run`, posts a timed-out notice, logs `auto_run_timeout` + (`stage_turn_ceiling`), and breaks; any other exception from `_run_chat` does + the same with a stage-error notice and `auto_run_stage_error`. Both halts + cancel the stage's subagents first (see **Halts cancel the stage's + subagents** below). 7. **Wait for the stage's sub-agents.** Registers one `SubagentManager.completion_event(parent_key)` for every immutable parent key captured by the boundary, pulsed once per terminal report from @@ -337,6 +341,28 @@ entry. verification. Exhausting the ceiling stops auto-run with `auto_run_subagent_timeout`. + **Halts cancel the stage's subagents.** The wait-exhausted, stage-turn + timeout and stage-turn error halts switch `_auto_run` off and then call + `_cancel_exhausted_stage_subagents`, so the stage's children do not run on + to their own `subagent_timeout_secs` deadline. It cancels the same scope + explicit Cancel does, through the same per-parent sequence + (`_cancel_boundary_parents`): every captured parent is reserved first and + each `cancel_for_boundary` keeps its retained hold; when the reservation is + refused it falls back to `retain_scope=False`, with no hold. Every manager + call is guarded, so a failure is logged and counted, never raised into the + halt. The halt waits at most `_HALT_CANCEL_BUDGET_SECS` for the + cancel; a slower one keeps running on a tracked background task and is never + cancelled at the bound. The notice is worded from the real outcome + (`_halt_cancel_notice`: stopped, refused, still finishing, failed or + pending), and a halt that could not confirm every child stopped says to stop + them from the subagent panel before Go. Only a wait-exhausted halt whose + children were all confirmed stopped offers "Send Go to continue"; a plan + Cancel landing during the wait suppresses the Go line. If the controller is + itself cancelled during the bounded wait (Stop, slot close), the notice, + its audit event and, for the turn halts, stage preservation still land, + worded from the unfinished outcome, and the `CancelledError` then + propagates. + **Settlement invariants.** These ids are stable; code comments and review findings cite them bare, and the named tests decide if prose and behavior disagree. diff --git a/src/kiro_crew/dashboard/chat_orchestrator.py b/src/kiro_crew/dashboard/chat_orchestrator.py index b489cde50a21..dcf3064ef167 100644 --- a/src/kiro_crew/dashboard/chat_orchestrator.py +++ b/src/kiro_crew/dashboard/chat_orchestrator.py @@ -6,6 +6,8 @@ import logging import time import uuid +from collections.abc import Callable +from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path @@ -67,6 +69,12 @@ _SA_MAX_WAIT_SECS = 900 # How often the "waiting for N subagent(s)" status line is re-broadcast. _SA_STATUS_EVERY_SECS = 20.0 +# How long a halt waits for its own cancel of the stage's children. Cancelling a +# live child ends by awaiting that child's report delivery to the parent, which +# only the subagent layer's ``_ON_DONE_TIMEOUT`` (20 minutes) caps; the halt must +# not stand behind it. Past this bound the cancel keeps running on a tracked +# background task and the halt is published at once. +_HALT_CANCEL_BUDGET_SECS = 60.0 async def _build_stage_context( @@ -766,6 +774,81 @@ def _reserve_stage_cancellation_scopes( return reservation if isinstance(reservation, str) else "" +@dataclass +class _ParentCancelProgress: + """Running tally of one per-parent boundary-cancel sweep. + + Mutated in place so a caller that stops waiting mid-sweep (a halt past its + budget) can still read how far the sweep got. + """ + + stopped: int = 0 + failed: int = 0 + done: int = 0 + pending: list[str] = field(default_factory=list) + refused: list[str] = field(default_factory=list) + + +async def _cancel_boundary_parents( + manager: object, + owner: str, + parent_keys: tuple[str, ...], + *, + retain_scope: bool, + progress: _ParentCancelProgress, + guarded: bool, +) -> None: + """Cancel one stage scope parent by parent, then probe what each cancel left. + + The one per-parent sequence shared by explicit Cancel and the auto-run halts: + cancel under the boundary owner -- keeping the reserved hold (the manager's + default) unless the reservation was refused -- then read the pending + durable-queue reason and the refused-hold flag. With ``guarded`` false any + manager error propagates (explicit Cancel); with it true each failure is + logged and counted in ``progress.failed`` so the halt that called it still + lands. + """ + pending_reason = getattr(manager, "boundary_cancellation_pending_reason", None) + refused_probe = getattr(manager, "boundary_cancellation_refused", None) + for parent_key in parent_keys: + try: + if retain_scope: + cancellation = manager.cancel_for_boundary( # type: ignore[attr-defined] + parent_key, owner + ) + else: + cancellation = manager.cancel_for_boundary( # type: ignore[attr-defined] + parent_key, owner, retain_scope=False + ) + result = await cancellation if asyncio.iscoroutine(cancellation) else cancellation + except asyncio.CancelledError: + raise + except Exception: + if not guarded: + raise + logger.exception("Stage halt: cancelling children of %s failed", parent_key) + progress.failed += 1 + progress.done += 1 + continue + if isinstance(result, tuple): + progress.stopped += sum(n for n in result if isinstance(n, int)) + try: + reason = "" + if callable(pending_reason): + current = pending_reason(parent_key, owner) + if isinstance(current, str) and current: + reason = current + progress.pending.append(parent_key) + if callable(refused_probe) and refused_probe(parent_key, owner) is True: + progress.refused.append(reason) + except Exception: + if not guarded: + raise + logger.exception("Stage halt: reading the cancel state of %s failed", parent_key) + progress.failed += 1 + progress.done += 1 + + async def _cancel_stage_subagents( state: "DashboardState", slot: "_ChatSlot", @@ -778,33 +861,19 @@ async def _cancel_stage_subagents( if manager is None or scope is None: return True owner, parent_keys, stage_num = scope - pending: list[str] = [] - overflow: list[str] = [] - pending_reason = getattr(manager, "boundary_cancellation_pending_reason", None) - refused = getattr(manager, "boundary_cancellation_refused", None) if reservation_reason is None: reservation_reason = _reserve_stage_cancellation_scopes(state, scope) - if reservation_reason: - overflow.append(reservation_reason) - for parent_key in parent_keys: - if reservation_reason: - cancellation = manager.cancel_for_boundary( - parent_key, - owner, - retain_scope=False, - ) - else: - cancellation = manager.cancel_for_boundary(parent_key, owner) - if asyncio.iscoroutine(cancellation): - await cancellation - reason = "" - if callable(pending_reason): - current = pending_reason(parent_key, owner) - if isinstance(current, str) and current: - reason = current - pending.append(parent_key) - if callable(refused) and refused(parent_key, owner) is True: - overflow.append(reason) + progress = _ParentCancelProgress(refused=[reservation_reason] if reservation_reason else []) + await _cancel_boundary_parents( + manager, + owner, + parent_keys, + retain_scope=not reservation_reason, + progress=progress, + guarded=False, + ) + pending = progress.pending + overflow = progress.refused if pending or overflow: if overflow: latest = next((reason for reason in reversed(overflow) if reason), "scope cap reached") @@ -858,6 +927,207 @@ async def _queued_stage_work_pending(manager: object, slot: "_ChatSlot") -> bool return False +@dataclass(frozen=True) +class _HaltCancelOutcome: + """What cancelling a halted stage's children actually achieved. + + ``attempted`` is false when there was no armed boundary or no manager, so + there was nothing to cancel. ``pending`` names parents whose durable-queue + cancel is still retrying; ``refused`` names parents (or the reservation) + that could not enter the bounded cancellation hold, so their queued work + may still start. ``failed`` counts parents whose cancel raised. + ``still_cancelling`` counts parents whose cancel outlived the halt's budget + and is still finishing on a background task. + """ + + attempted: bool = False + stopped: int = 0 + pending: tuple[str, ...] = () + refused: tuple[str, ...] = () + failed: int = 0 + still_cancelling: int = 0 + + +def _halt_cancel_notice(outcome: _HaltCancelOutcome) -> str: + """One sentence for the halt message, worded from what really happened.""" + if not outcome.attempted: + return "" + if outcome.refused: + reason = next((r for r in reversed(outcome.refused) if r), "scope cap reached") + return ( + f" Stopped {outcome.stopped} of this stage's subagent run(s), but " + f"cancellation could not enter the bounded durable-queue hold ({reason}), " + "so some queued stage work may still start." + ) + if outcome.still_cancelling: + return ( + f" Stopped {outcome.stopped} of this stage's subagent run(s) so far; " + "cancellation is still finishing in the background for " + f"{outcome.still_cancelling} parent session(s)." + ) + if outcome.failed: + return ( + f" Stopped {outcome.stopped} of this stage's subagent run(s); cancelling " + f"the rest failed for {outcome.failed} parent session(s) (see the log), " + "so some may still be running." + ) + if outcome.pending: + return ( + f" Stopped {outcome.stopped} of this stage's subagent run(s); its queued " + "work stays blocked while cancellation finishes writing to the durable " + "task queue, and that retries automatically." + ) + if outcome.stopped: + return f" Stopped {outcome.stopped} unfinished subagent run(s) from this stage." + return " No unfinished subagents from this stage were left to stop." + + +def _halt_cancel_left_work(outcome: _HaltCancelOutcome) -> bool: + """Whether the halt could not confirm this stage's children are all stopped.""" + return bool(outcome.pending or outcome.refused or outcome.failed or outcome.still_cancelling) + + +def _halt_residual_warning(outcome: _HaltCancelOutcome) -> str: + """Warn that Go can wait again when the halt left this stage's work unstopped.""" + if not _halt_cancel_left_work(outcome): + return "" + return ( + " Stop this stage's remaining subagents from the subagent panel " + "before sending Go: until they stop, Go can wait on them again." + ) + + +def _halt_go_guidance(outcome: _HaltCancelOutcome) -> str: + """What Go does after a wait-exhausted halt, worded from the cancel outcome.""" + if _halt_cancel_left_work(outcome): + return ( + " Stop this stage's remaining subagents from the subagent panel " + "before sending Go: until they stop, Go can wait on them again. Go " + "then continues to the next stage with this stage's partial " + "results; subagent results that had finished but not yet been " + "delivered are discarded." + ) + return ( + " Send Go to continue to the next stage with this stage's partial " + "results; subagent results that had finished but not yet been " + "delivered are discarded." + ) + + +async def _cancel_exhausted_stage_subagents( + state: "DashboardState", + slot: "_ChatSlot", + *, + publish: Callable[[_HaltCancelOutcome], None] | None = None, +) -> _HaltCancelOutcome: + """Cancel a halted stage's children instead of leaving them to their deadline. + + Called when auto-run halts a stage it will not settle: the subagent wait is + exhausted, the stage turn hits its ceiling, or the stage turn errors. + Without it the children keep running until their own + ``subagent_timeout_secs`` deadline (three hours by default): the plan says + it stopped while the slot still carries live work whose completions land in + a plan that has stopped. + + The scope is the one explicit Cancel uses, and so is the ordering: every + captured parent is RESERVED before any cancel runs, and each cancel keeps + its hold (``retain_scope=True``). The hold is what stops the dispatcher + starting a queued stage row while the durable-queue cancel is suspended in + its store write; without it such a row escapes the cancel and runs to its + own deadline. A settled cancel releases its hold, so a later Go is not + blocked. When the reservation itself is refused (scope cap reached), fall + back to ``retain_scope=False`` exactly as explicit Cancel does. + + Best-effort: every manager call -- the scope capture, the reservation, each + cancel and each pending / refused probe -- is guarded, so a failure is + logged and counted and never raises into the halt that called this. The + returned outcome carries the stopped count and any pending / refused / + failed parents, so the halt message can say what really happened -- the + same signals ``_cancel_stage_subagents`` surfaces for explicit Cancel. + + ``publish`` is the halt's own notice + audit, called exactly once with the + outcome. It runs here rather than after the return because the bounded wait + below is the one await on the halt path: when the controller is cancelled + during it, ``publish`` still runs with the incomplete outcome before the + ``CancelledError`` is re-raised, so a Stop never swallows the halt notice. + """ + manager = state.subagents + try: + scope = _capture_stage_cancellation_scope(slot) + except Exception: + logger.exception("Stage halt: capturing the cancellation scope failed") + outcome = _HaltCancelOutcome(attempted=True, failed=1) + if publish is not None: + publish(outcome) + return outcome + cancel = getattr(manager, "cancel_for_boundary", None) + if scope is None or not callable(cancel): + outcome = _HaltCancelOutcome() + if publish is not None: + publish(outcome) + return outcome + owner, parent_keys, _stage_num = scope + try: + reservation_reason = _reserve_stage_cancellation_scopes(state, scope) + except Exception: + logger.exception("Stage halt: reserving the cancellation scope failed") + reservation_reason = "cancellation reservation failed" + progress = _ParentCancelProgress(refused=[reservation_reason] if reservation_reason else []) + + def _outcome(task: "asyncio.Future[None]") -> _HaltCancelOutcome: + return _HaltCancelOutcome( + attempted=True, + stopped=progress.stopped, + pending=tuple(progress.pending), + refused=tuple(progress.refused), + failed=progress.failed, + still_cancelling=0 if task.done() else len(parent_keys) - progress.done, + ) + + # Tracked, waited on with a bound, and never cancelled at the bound: a cancel + # cut mid-reap would abandon the kill it started, so a slow one finishes in + # the background while the halt is published now. + task = asyncio.ensure_future( + _cancel_boundary_parents( + manager, + owner, + parent_keys, + retain_scope=not reservation_reason, + progress=progress, + guarded=True, + ) + ) + state._background_tasks.add(task) + task.add_done_callback(state._background_tasks.discard) + task.add_done_callback(_log_task_exception) + try: + await asyncio.wait({task}, timeout=_HALT_CANCEL_BUDGET_SECS) + except asyncio.CancelledError: + # The controller is being torn down (Stop / slot close) mid-wait. The + # halt notice and its audit event were unconditional before this wait + # existed, so they still land -- worded from the unfinished outcome -- + # and the cancellation then propagates. ``asyncio.wait`` never cancels + # the tracked cancel task, so it keeps running in the background. + outcome = _outcome(task) + if publish is not None: + try: + publish(outcome) + except Exception: + logger.exception("Stage halt: publishing the interrupted halt failed") + raise + outcome = _outcome(task) + if outcome.still_cancelling: + logger.warning( + "Stage halt: cancelling %d parent session(s) outlived %.0fs; it " + "continues in the background", + outcome.still_cancelling, + _HALT_CANCEL_BUDGET_SECS, + ) + if publish is not None: + publish(outcome) + return outcome + + async def _settle_stage_delivery( state: "DashboardState", slot: "_ChatSlot", @@ -944,16 +1214,28 @@ def _delivery_failed(reason: str = "") -> bool: now = time.monotonic() elapsed = now - wait_started if elapsed >= max_wait: - _halt_plan( - state, - slot, - f"⚠️ Stage {stage_num}: subagent wait exhausted after " - f"{int(elapsed) // 60} minutes. Auto-run stopped — some " - "results may be incomplete.", - event_type="auto_run_subagent_timeout", - operation="subagent_wait_exhausted", - stage_num=stage_num, - ) + # Stop auto-run BEFORE the cancel awaits, so nothing advances the + # plan while it runs. Cancel FIRST so the notice states what + # actually happened. + slot._auto_run = False + exhausted_minutes = int(elapsed) // 60 + + def _publish_exhausted(outcome: _HaltCancelOutcome) -> None: + # An explicit Cancel pressed while the cancel was running has + # revoked the plan; offering Go would contradict it. + go_guidance = "" if slot._plan_cancelled else _halt_go_guidance(outcome) + _halt_plan( + state, + slot, + f"⚠️ Stage {stage_num}: subagent wait exhausted after " + f"{exhausted_minutes} minutes. Auto-run stopped." + f"{_halt_cancel_notice(outcome)}{go_guidance}", + event_type="auto_run_subagent_timeout", + operation="subagent_wait_exhausted", + stage_num=stage_num, + ) + + await _cancel_exhausted_stage_subagents(state, slot, publish=_publish_exhausted) return False if last_status_at is None or now - last_status_at >= _SA_STATUS_EVERY_SECS: last_status_at = now @@ -1580,59 +1862,74 @@ def _record_stage_turn_consumed(consumed: bool) -> None: tracker.stage_timeout_seconds, slot.key, ) - _timeout_msg = ( - f"⏱️ Stage {stage_num} timed out after {tracker.timeout_human}. " - "Auto-run stopped." - ) slot._auto_run = False - slot.append("assistant", _timeout_msg, "msg msg-a") - state.broadcast_ws( - "chat_append", - {"slot": slot.key, "html": _timeout_msg, "cls": "msg msg-a"}, - ) - sel().log( - SecurityEvent( - event_id=uuid.uuid4().hex, - timestamp=datetime.now(tz=timezone.utc).isoformat(), - event_type="auto_run_timeout", - caller_identity=f"dashboard:{slot.key}", - agent=getattr(slot, "agent", ""), - source="dashboard", - operation="stage_turn_ceiling", - outcome="stopped", - resources=f"slot={slot.key},stage={stage_num}", + + # Cancel FIRST (the scope is read off the still-armed boundary, + # before preserve) so the notice states what actually happened. + # The notice, audit and preserve run inside the cancel helper so + # a Stop landing during its bounded wait cannot skip them. + def _publish_turn_timeout(outcome: _HaltCancelOutcome) -> None: + _timeout_msg = ( + f"⏱️ Stage {stage_num} timed out after {tracker.timeout_human}. " + f"Auto-run stopped.{_halt_cancel_notice(outcome)}" + f"{_halt_residual_warning(outcome)}" ) - ) - _preserve_interrupted_stage() + slot.append("assistant", _timeout_msg, "msg msg-a") + state.broadcast_ws( + "chat_append", + {"slot": slot.key, "html": _timeout_msg, "cls": "msg msg-a"}, + ) + sel().log( + SecurityEvent( + event_id=uuid.uuid4().hex, + timestamp=datetime.now(tz=timezone.utc).isoformat(), + event_type="auto_run_timeout", + caller_identity=f"dashboard:{slot.key}", + agent=getattr(slot, "agent", ""), + source="dashboard", + operation="stage_turn_ceiling", + outcome="stopped", + resources=f"slot={slot.key},stage={stage_num}", + ) + ) + _preserve_interrupted_stage() + + await _cancel_exhausted_stage_subagents(state, slot, publish=_publish_turn_timeout) # stage-boundary-exit: stage-turn-timeout owned break except Exception: logger.exception( "_run_chat failed during stage %d for slot %s", stage_num, slot.key ) - _err_msg = ( - f"❌ Stage {stage_num} failed due to an internal error. Auto-run stopped." - ) slot._auto_run = False - slot.append("assistant", _err_msg, "msg msg-a") - state.broadcast_ws( - "chat_append", - {"slot": slot.key, "html": _err_msg, "cls": "msg msg-a"}, - ) - sel().log( - SecurityEvent( - event_id=uuid.uuid4().hex, - timestamp=datetime.now(tz=timezone.utc).isoformat(), - event_type="auto_run_stage_error", - caller_identity=f"dashboard:{slot.key}", - agent=getattr(slot, "agent", ""), - source="dashboard", - operation="stage_error", - outcome="error", - resources=f"slot={slot.key},stage={stage_num}", + + def _publish_turn_error(outcome: _HaltCancelOutcome) -> None: + _err_msg = ( + f"❌ Stage {stage_num} failed due to an internal error. " + f"Auto-run stopped.{_halt_cancel_notice(outcome)}" + f"{_halt_residual_warning(outcome)}" ) - ) - _preserve_interrupted_stage() + slot.append("assistant", _err_msg, "msg msg-a") + state.broadcast_ws( + "chat_append", + {"slot": slot.key, "html": _err_msg, "cls": "msg msg-a"}, + ) + sel().log( + SecurityEvent( + event_id=uuid.uuid4().hex, + timestamp=datetime.now(tz=timezone.utc).isoformat(), + event_type="auto_run_stage_error", + caller_identity=f"dashboard:{slot.key}", + agent=getattr(slot, "agent", ""), + source="dashboard", + operation="stage_error", + outcome="error", + resources=f"slot={slot.key},stage={stage_num}", + ) + ) + _preserve_interrupted_stage() + + await _cancel_exhausted_stage_subagents(state, slot, publish=_publish_turn_error) # stage-boundary-exit: stage-turn-error owned break diff --git a/test/test_autopilot_halt_cancels_real_manager.py b/test/test_autopilot_halt_cancels_real_manager.py new file mode 100644 index 000000000000..a25650d085f8 --- /dev/null +++ b/test/test_autopilot_halt_cancels_real_manager.py @@ -0,0 +1,164 @@ +"""A halted autopilot stage cancels its children through a REAL SubagentManager. + +The dashboard-side tests pin the call shape with mock managers. This one drives +``_cancel_exhausted_stage_subagents`` against a real ``SubagentManager`` on top +of the durable task queue, so the reservation, the held boundary cancel and the +queue settle all actually run. The one faked seam is the run itself (no +kiro-cli) and the reap (no provider to tear down). +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from chat_test_helpers import _make_state + +from kiro_crew.subagent import SubagentManager +from kiro_crew.taskq import model + +pytestmark = pytest.mark.usefixtures("healthy_host_memory") + + +@pytest.fixture(autouse=True) +def _isolate_config_dir(tmp_path, monkeypatch): + for module in ("state", "chat", "chat_orchestrator"): + monkeypatch.setattr(f"kiro_crew.dashboard.{module}.config_dir", lambda: tmp_path) + + +@pytest.fixture +def quiet(): + with patch("kiro_crew.subagent.Stats"), patch("kiro_crew.subagent.sel"): + yield + + +def _sessions() -> MagicMock: + sessions = MagicMock() + sessions.get_pid = MagicMock(return_value=None) + sessions.get_approval_policy = MagicMock(return_value="auto") + sessions.get_agent = MagicMock(return_value="") + sessions.get_agent_selection = MagicMock(return_value=("template", "")) + sessions.has_session = MagicMock(return_value=True) + sessions.release = MagicMock() + sessions.reset = AsyncMock() + return sessions + + +@asynccontextmanager +async def _manager() -> AsyncIterator[SubagentManager]: + ctx = MagicMock() + ctx.hooks.auto_approve_subagent_spawn = True + mgr = SubagentManager(sessions=_sessions(), ctx_builder=ctx, max_concurrent=1) + try: + await mgr.wait_taskq_ready() + mgr._spawn_stagger_secs = 0.0 + mgr._last_spawn_ts = 0.0 + mgr._taskq._window = 1 + yield mgr + finally: + try: + await mgr.cancel_all() + finally: + mgr.close() + + +@pytest.mark.asyncio +async def test_halt_cancels_running_and_queued_stage_children_only(tmp_path, quiet): + from kiro_crew.dashboard.chat_orchestrator import _cancel_exhausted_stage_subagents + + async with _manager() as mgr: + state = _make_state(tmp_path) + slot = state.get_or_create_slot("halt-real-manager", mode="orchestrator") + slot.stage_boundary.arm(1, consumed=False) + owner = slot.stage_boundary.owner + assert owner + parent = f"dashboard:{slot.key}" + + with patch.object(SubagentManager, "_run", new=AsyncMock()): + running = mgr.spawn( + "stage work", parent_session_key=parent, _stage_boundary_owner=owner + ) + queued = mgr.spawn( + "queued stage work", parent_session_key=parent, _stage_boundary_owner=owner + ) + foreign = mgr.spawn( + "another stage's work", + parent_session_key=parent, + _stage_boundary_owner="some-other-owner", + ) + assert mgr._taskq.state_of(queued.id) == model.QUEUED + assert mgr._taskq.state_of(foreign.id) == model.QUEUED + mgr._report_queued_stop = MagicMock() # type: ignore[method-assign] + mgr._force_reap = AsyncMock() # type: ignore[method-assign] + state.subagents = mgr + + outcome = await _cancel_exhausted_stage_subagents(state, slot) + + assert outcome.attempted is True + assert outcome.failed == 0 and outcome.pending == () and outcome.refused == () + # The queued child never starts: its durable row is cancelled. + assert mgr._taskq.state_of(queued.id) == model.CANCELLED + # The live child was revoked and sent down the reap path. + assert running._stage_boundary_cancelled is True + reaped = {call.args[0] for call in mgr._force_reap.await_args_list} + assert running.id in reaped + assert outcome.stopped >= 2 + # Another owner's work under the SAME parent is untouched. + assert mgr._taskq.state_of(foreign.id) == model.QUEUED + assert foreign.id not in reaped + # A settled cancel leaves no hold behind, so Go can still resume the plan. + assert (parent, owner) not in mgr._pending_boundary_cancellations + + assert mgr._taskq is None + + +@pytest.mark.asyncio +async def test_halt_blocks_queued_stage_rows_while_the_store_cancel_is_suspended(tmp_path, quiet): + """The hold is what keeps a queued stage row from dispatching mid-cancel. + + ``taskq_cancel_boundary_async`` suspends on the store write. While it is + suspended, the dispatcher's own gate (``_boundary_cancellation_pending``) + must refuse this stage's queued rows -- otherwise a pump pass in that gap + starts the row and it runs to its own deadline. Without the reservation + and the retained hold, the gate reads False here and this test fails. + """ + from kiro_crew.dashboard.chat_orchestrator import _cancel_exhausted_stage_subagents + + async with _manager() as mgr: + state = _make_state(tmp_path) + slot = state.get_or_create_slot("halt-real-hold", mode="orchestrator") + slot.stage_boundary.arm(1, consumed=False) + owner = slot.stage_boundary.owner + parent = f"dashboard:{slot.key}" + + with patch.object(SubagentManager, "_run", new=AsyncMock()): + mgr.spawn("stage work", parent_session_key=parent, _stage_boundary_owner=owner) + queued = mgr.spawn( + "queued stage work", parent_session_key=parent, _stage_boundary_owner=owner + ) + assert mgr._taskq.state_of(queued.id) == model.QUEUED + mgr._report_queued_stop = MagicMock() # type: ignore[method-assign] + mgr._force_reap = AsyncMock() # type: ignore[method-assign] + state.subagents = mgr + + stage_row = {"parent_session_key": parent, "_stage_boundary_owner": owner} + gate_during_write: list[bool] = [] + coordinator = type(mgr._admission) + real_cancel = coordinator.taskq_cancel_boundary_async + + async def suspended_cancel(self, parent_key, boundary_owner): + gate_during_write.append(mgr._boundary_cancellation_pending(stage_row)) + return await real_cancel(self, parent_key, boundary_owner) + + with patch.object(coordinator, "taskq_cancel_boundary_async", suspended_cancel): + outcome = await _cancel_exhausted_stage_subagents(state, slot) + + assert gate_during_write == [True] + assert mgr._taskq.state_of(queued.id) == model.CANCELLED + assert outcome.pending == () and outcome.refused == () + # Settling released the hold, so Go is not blocked afterwards. + assert mgr._boundary_cancellation_pending(stage_row) is False + + assert mgr._taskq is None diff --git a/test/test_autopilot_stage_completion_handoff.py b/test/test_autopilot_stage_completion_handoff.py index 2f26da566f6b..708bcbd6137b 100644 --- a/test/test_autopilot_stage_completion_handoff.py +++ b/test/test_autopilot_stage_completion_handoff.py @@ -2342,3 +2342,226 @@ async def test_settlement_wait_exhaustion_halts_the_plan(tmp_path): assert len(halts) == 1 assert "Auto-run stopped" in halts[0] assert slot._auto_run is False + + +@pytest.mark.asyncio +async def test_settlement_wait_exhaustion_cancels_the_stage_children(tmp_path): + """A halted stage must not leave its children running to their own deadline. + + Flipping ``_auto_run`` alone would leave the children running for up to + ``subagent_timeout_secs`` while the plan claims it has stopped. The halt + cancels exactly the boundary's scope -- every + captured parent key, under the boundary's owner token -- reserving every + parent first and holding the scope while each cancel runs. A settled cancel + releases that hold, so a later Go can still resume the plan. + """ + from kiro_crew.context_management import OrchestrationTracker + from kiro_crew.dashboard.chat_orchestrator import _settle_stage_delivery + + state = _make_state(tmp_path) + slot = state.get_or_create_slot("settle-wait-cancels", mode="orchestrator") + slot._in_stage_execution = True + slot._auto_run = True + slot.stage_boundary.arm(1, consumed=False) + slot.stage_boundary.parent_session_keys.add("subagent:parent-2") + owner = slot.stage_boundary.owner + assert owner + manager = MagicMock() + manager.running_agents_for.return_value = [{"id": "sa-1", "status": "running"}] + manager.has_pending_work_for_async = AsyncMock(return_value=False) + manager.wait_for_parent_reports = AsyncMock(return_value=False) + manager.cancel_for_boundary = AsyncMock(return_value=(1, 0)) + state.subagents = manager + + tracker = OrchestrationTracker(stage_timeout_seconds=2) + assert await _settle_stage_delivery(state, slot, tracker, 1) is False + + calls = sorted(c.args for c in manager.cancel_for_boundary.await_args_list) + assert calls == sorted([(f"dashboard:{slot.key}", owner), ("subagent:parent-2", owner)]) + for call in manager.cancel_for_boundary.await_args_list: + # The retained hold is ``cancel_for_boundary``'s default; the halt shares + # explicit Cancel's call shape, which omits the kwarg when it holds. + assert call.kwargs.get("retain_scope", True) is True + manager.reserve_boundary_cancellation_scopes.assert_called_once() + halts = [ + message["content"] + for message in slot.messages + if "subagent wait exhausted" in message.get("content", "") + ] + assert len(halts) == 1 + assert "Stopped 2 unfinished subagent run(s)" in halts[0] + assert "Send Go to continue to the next stage" in halts[0] + assert slot._auto_run is False + + +@pytest.mark.asyncio +async def test_settlement_wait_exhaustion_cancel_failure_still_halts(tmp_path): + """A failing cancel is logged, never raised: the halt itself must still land.""" + from kiro_crew.context_management import OrchestrationTracker + from kiro_crew.dashboard.chat_orchestrator import _settle_stage_delivery + + state = _make_state(tmp_path) + slot = state.get_or_create_slot("settle-wait-cancel-fails", mode="orchestrator") + slot._in_stage_execution = True + slot._auto_run = True + slot.stage_boundary.arm(1, consumed=False) + manager = MagicMock() + manager.running_agents_for.return_value = [{"id": "sa-1", "status": "running"}] + manager.has_pending_work_for_async = AsyncMock(return_value=False) + manager.wait_for_parent_reports = AsyncMock(return_value=False) + manager.cancel_for_boundary = AsyncMock(side_effect=RuntimeError("store down")) + state.subagents = manager + + tracker = OrchestrationTracker(stage_timeout_seconds=2) + assert await _settle_stage_delivery(state, slot, tracker, 1) is False + assert manager.cancel_for_boundary.await_count == 1 + assert slot._auto_run is False + halts = [ + message["content"] + for message in slot.messages + if "subagent wait exhausted" in message.get("content", "") + ] + assert len(halts) == 1 + # Children may still be running, so the halt must not promise a clean Go. + assert "Send Go to continue" not in halts[0] + assert "before sending Go" in halts[0] + + +def _exhausting_manager() -> MagicMock: + manager = MagicMock() + manager.running_agents_for.return_value = [{"id": "sa-1", "status": "running"}] + manager.has_pending_work_for_async = AsyncMock(return_value=False) + manager.wait_for_parent_reports = AsyncMock(return_value=False) + manager.boundary_cancellation_pending_reason.return_value = "" + manager.boundary_cancellation_refused.return_value = False + manager.reserve_boundary_cancellation_scopes.return_value = "" + manager.cancel_for_boundary = AsyncMock(return_value=(1, 0)) + return manager + + +@pytest.mark.asyncio +async def test_settlement_wait_exhaustion_guards_reservation_and_probe_failures(tmp_path): + """A raising reservation or state probe must not skip the halt itself.""" + from kiro_crew.context_management import OrchestrationTracker + from kiro_crew.dashboard.chat_orchestrator import _settle_stage_delivery + + state = _make_state(tmp_path) + slot = state.get_or_create_slot("settle-wait-probe-fails", mode="orchestrator") + slot._in_stage_execution = True + slot._auto_run = True + slot.stage_boundary.arm(1, consumed=False) + manager = _exhausting_manager() + manager.reserve_boundary_cancellation_scopes.side_effect = RuntimeError("cap store") + manager.boundary_cancellation_pending_reason.side_effect = RuntimeError("probe") + state.subagents = manager + + tracker = OrchestrationTracker(stage_timeout_seconds=2) + assert await _settle_stage_delivery(state, slot, tracker, 1) is False + + # The reservation failed, so the cancel ran without a retained hold. + assert manager.cancel_for_boundary.await_args.kwargs == {"retain_scope": False} + assert slot._auto_run is False + halts = [ + message["content"] + for message in slot.messages + if "subagent wait exhausted" in message.get("content", "") + ] + assert len(halts) == 1 + assert "cancellation reservation failed" in halts[0] + assert "Send Go to continue" not in halts[0] + + +@pytest.mark.asyncio +async def test_settlement_wait_exhaustion_stops_auto_run_before_cancel_and_honors_cancel( + tmp_path, +): + """Auto-run is off while the cancel awaits, and a Cancel in that window wins. + + A plan Cancel pressed while the halt's cancel is still running revokes the + plan; the halt notice that follows must not then offer Go. + """ + from kiro_crew.context_management import OrchestrationTracker + from kiro_crew.dashboard.chat_orchestrator import _settle_stage_delivery + + state = _make_state(tmp_path) + slot = state.get_or_create_slot("settle-wait-cancel-race", mode="orchestrator") + slot._in_stage_execution = True + slot._auto_run = True + slot.stage_boundary.arm(1, consumed=False) + manager = _exhausting_manager() + auto_run_during_cancel: list[bool] = [] + + async def cancel_while_user_cancels(*_args, **_kwargs): + auto_run_during_cancel.append(slot._auto_run) + slot._plan_cancelled = True # the user's Cancel lands mid-await + return (1, 0) + + manager.cancel_for_boundary = AsyncMock(side_effect=cancel_while_user_cancels) + state.subagents = manager + + tracker = OrchestrationTracker(stage_timeout_seconds=2) + assert await _settle_stage_delivery(state, slot, tracker, 1) is False + + assert auto_run_during_cancel and not any(auto_run_during_cancel) + halts = [ + message["content"] + for message in slot.messages + if "subagent wait exhausted" in message.get("content", "") + ] + assert len(halts) == 1 + assert "Go" not in halts[0] + + +@pytest.mark.asyncio +async def test_settlement_wait_exhaustion_halt_is_not_held_by_a_slow_cancel(tmp_path, monkeypatch): + """A cancel stuck behind a child's report delivery must not delay the halt. + + Cancelling a live child ends by awaiting that child's delivery to the + parent, which the subagent layer caps only at 20 minutes. The halt waits at + most ``_HALT_CANCEL_BUDGET_SECS`` for it, then publishes, and leaves the + cancel running (not cancelled) on a tracked background task. + """ + from kiro_crew.context_management import OrchestrationTracker + from kiro_crew.dashboard import chat_orchestrator + from kiro_crew.dashboard.chat_orchestrator import _settle_stage_delivery + + monkeypatch.setattr(chat_orchestrator, "_HALT_CANCEL_BUDGET_SECS", 0.05) + state = _make_state(tmp_path) + slot = state.get_or_create_slot("settle-wait-slow-cancel", mode="orchestrator") + slot._in_stage_execution = True + slot._auto_run = True + slot.stage_boundary.arm(1, consumed=False) + manager = _exhausting_manager() + release = asyncio.Event() + cancel_was_cancelled = False + + async def never_resolving_cancel(*_args, **_kwargs): + nonlocal cancel_was_cancelled + try: + await release.wait() + except asyncio.CancelledError: + cancel_was_cancelled = True + raise + return (1, 0) + + manager.cancel_for_boundary = AsyncMock(side_effect=never_resolving_cancel) + state.subagents = manager + + tracker = OrchestrationTracker(stage_timeout_seconds=2) + assert await asyncio.wait_for(_settle_stage_delivery(state, slot, tracker, 1), 10) is False + + halts = [ + message["content"] + for message in slot.messages + if "subagent wait exhausted" in message.get("content", "") + ] + assert len(halts) == 1 + assert "still finishing in the background for 1 parent session(s)" in halts[0] + assert "Send Go to continue" not in halts[0] + assert slot._auto_run is False + # The cancel was left running, not cut mid-reap. + assert cancel_was_cancelled is False + release.set() + for _ in range(20): + await asyncio.sleep(0) + assert manager.cancel_for_boundary.await_count == 1 diff --git a/test/test_autopilot_wave_wait_event.py b/test/test_autopilot_wave_wait_event.py index 731c65722b09..a5ee31bf7db3 100644 --- a/test/test_autopilot_wave_wait_event.py +++ b/test/test_autopilot_wave_wait_event.py @@ -338,3 +338,227 @@ async def test_the_table_is_fused_against_a_leak(self): assert len(mgr._completion_waiters) == subagent_mod._MAX_COMPLETION_WAITERS mgr.signal_completion("one-too-many") assert overflow.is_set() is False + + +class TestGoAfterAWaitExhaustionHalt: + """What the halt notice promises Go does, pinned against the real stage loop.""" + + @pytest.mark.asyncio + async def test_go_after_exhaustion_advances_to_the_next_stage(self, monkeypatch): + """Exhaustion cancels the stage's children; Go then runs the NEXT stage. + + The halt notice tells the user "Send Go to continue to the next stage with + this stage's partial results". If Go instead re-ran the halted stage, or + waited on the cancelled children again, that notice would be a lie. + """ + from kiro_crew.dashboard import chat_orchestrator + from kiro_crew.dashboard.chat import _stage_loop + + monkeypatch.setattr(chat_orchestrator, "_SA_FALLBACK_SECS", 0.02) + + subagents = _Subagents(pending=[{"id": "a1"}]) # never finishes on its own + cancelled: list[tuple[str, str, bool]] = [] + + async def _cancel_for_boundary(parent_key, owner, *, retain_scope=True): + cancelled.append((parent_key, owner, retain_scope)) + subagents.pending = [] # the cancel is what empties the wave + return (1, 0) + + subagents.cancel_for_boundary = _cancel_for_boundary + stage_turns: list[str] = [] + + async def _mock_run_chat(state, slot, message, **kwargs): + stage_turns.append(message) + callback = kwargs.get("_on_consumed") + if callable(callback): + callback(True) + slot.append("assistant", "stage output", "msg msg-a") + + monkeypatch.setattr("kiro_crew.dashboard.chat_orchestrator._run_chat", _mock_run_chat) + slot = _make_slot(stage_timeout=2, titles=("Collect", "Verify")) + state = _make_state(subagents) + + await asyncio.wait_for(_stage_loop(state, slot, auto_run=True), 20) + + text = _assistant_text(slot) + assert "subagent wait exhausted" in text + assert "Stopped 1 unfinished subagent run(s)" in text + assert "Send Go to continue to the next stage" in text + assert slot._auto_run is False + assert len(cancelled) == 1 and cancelled[0][2] is True + assert len(stage_turns) == 1 and "Execute Stage 1 of 2 now" in stage_turns[0] + + # Go: the same controller entry the plan-action handler starts. + await asyncio.wait_for(_stage_loop(state, slot, auto_run=False), 20) + + assert len(stage_turns) == 2, stage_turns + assert ( + "Execute Stage 2 of 2 now" in stage_turns[1] + ), "Go re-ran the halted stage instead of advancing" + + @pytest.mark.parametrize( + ("raised", "notice"), + [ + (TimeoutError("stage ceiling"), "timed out after"), + (RuntimeError("provider blew up"), "failed due to an internal error"), + ], + ids=["stage-turn-timeout", "stage-turn-error"], + ) + @pytest.mark.asyncio + async def test_turn_timeout_and_error_halts_cancel_the_stage_children( + self, monkeypatch, raised, notice + ): + """The stage-turn timeout and error halts cancel the stage's children. + + The cancel runs BEFORE the notice is written and before the boundary is + preserved, so it reads the still-armed scope and the notice reports the + real outcome. + """ + from kiro_crew.dashboard.chat import _stage_loop + + subagents = _Subagents(pending=[{"id": "a1"}]) + cancelled: list[tuple[str, str, bool]] = [] + + async def _cancel_for_boundary(parent_key, owner, *, retain_scope=True): + assert owner, "cancel ran after the boundary lost its owner" + cancelled.append((parent_key, owner, retain_scope)) + return (1, 0) + + subagents.cancel_for_boundary = _cancel_for_boundary + + async def _mock_run_chat(state, slot, message, **kwargs): + raise raised + + monkeypatch.setattr("kiro_crew.dashboard.chat_orchestrator._run_chat", _mock_run_chat) + slot = _make_slot(titles=("Collect", "Verify")) + + await asyncio.wait_for(_stage_loop(_make_state(subagents), slot, auto_run=True), 20) + + text = _assistant_text(slot) + assert notice in text + assert "Stopped 1 unfinished subagent run(s) from this stage." in text + assert slot._auto_run is False + assert [c[0] for c in cancelled] == ["dashboard:wave-wait-slot"] + assert cancelled[0][2] is True + + @pytest.mark.parametrize( + "raised", + [TimeoutError("stage ceiling"), RuntimeError("provider blew up")], + ids=["stage-turn-timeout", "stage-turn-error"], + ) + @pytest.mark.asyncio + async def test_turn_halts_warn_when_children_were_not_confirmed_stopped( + self, monkeypatch, raised + ): + """A timeout/error halt whose cancel failed warns that Go can wait again.""" + from kiro_crew.dashboard.chat import _stage_loop + + subagents = _Subagents(pending=[{"id": "a1"}]) + + async def _cancel_for_boundary(parent_key, owner, *, retain_scope=True): + raise RuntimeError("cancel exploded") + + subagents.cancel_for_boundary = _cancel_for_boundary + + async def _mock_run_chat(state, slot, message, **kwargs): + raise raised + + monkeypatch.setattr("kiro_crew.dashboard.chat_orchestrator._run_chat", _mock_run_chat) + slot = _make_slot(titles=("Collect", "Verify")) + + await asyncio.wait_for(_stage_loop(_make_state(subagents), slot, auto_run=True), 20) + + text = _assistant_text(slot) + assert "cancelling the rest failed" in text + assert "until they stop, Go can wait on them again" in text + assert slot._auto_run is False + + +class TestAStopDuringTheHaltCancelKeepsTheNotice: + """A Stop landing while a halt waits on its cancel must not eat the halt. + + Before the halts cancelled their stage's children there was no await + between deciding to halt and publishing the notice, so the notice and its + audit event always landed. The bounded cancel wait is now that await: a + controller cancelled inside it (the user's Stop, a slot close) must still + publish the notice -- worded from the unfinished outcome -- and then let + the ``CancelledError`` propagate rather than swallow it. + """ + + @pytest.mark.parametrize( + ("path", "notice", "event_type"), + [ + ("wait-exhausted", "subagent wait exhausted", "auto_run_subagent_timeout"), + ("stage-turn-timeout", "timed out after", "auto_run_timeout"), + ("stage-turn-error", "failed due to an internal error", "auto_run_stage_error"), + ], + ) + @pytest.mark.asyncio + async def test_cancel_during_halt_cancel_still_publishes_and_propagates( + self, monkeypatch, path, notice, event_type + ): + from kiro_crew.dashboard import chat_orchestrator + from kiro_crew.dashboard.chat import _stage_loop + + monkeypatch.setattr(chat_orchestrator, "_SA_FALLBACK_SECS", 0.02) + audited: list[str] = [] + monkeypatch.setattr( + chat_orchestrator, + "sel", + lambda: MagicMock(log=lambda event: audited.append(event.event_type)), + ) + + subagents = _Subagents(pending=[{"id": "a1"}]) # never finishes on its own + cancel_entered = asyncio.Event() + release = asyncio.Event() + cancel_was_cancelled = False + + async def _stuck_cancel(parent_key, owner, *, retain_scope=True): + nonlocal cancel_was_cancelled + cancel_entered.set() + try: + await release.wait() + except asyncio.CancelledError: + cancel_was_cancelled = True + raise + return (1, 0) + + subagents.cancel_for_boundary = _stuck_cancel + + if path == "wait-exhausted": + _stage_turn(monkeypatch) + slot = _make_slot(stage_timeout=2, titles=("Collect", "Verify")) + else: + raised = ( + TimeoutError("stage ceiling") + if path == "stage-turn-timeout" + else (RuntimeError("provider blew up")) + ) + + async def _mock_run_chat(state, slot, message, **kwargs): + raise raised + + monkeypatch.setattr("kiro_crew.dashboard.chat_orchestrator._run_chat", _mock_run_chat) + slot = _make_slot(titles=("Collect", "Verify")) + + state = _make_state(subagents) + controller = asyncio.ensure_future(_stage_loop(state, slot, auto_run=True)) + await asyncio.wait_for(cancel_entered.wait(), 20) + assert not _assistant_text(slot).count(notice), "halt published before its cancel" + + controller.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(controller, 20) + + text = _assistant_text(slot) + assert text.count(notice) == 1, text + assert "still finishing in the background for 1 parent session(s)" in text + assert "Send Go to continue" not in text + assert audited.count(event_type) == 1, audited + assert slot._auto_run is False + # The halt's own cancel keeps running; the Stop did not cut it mid-reap. + assert cancel_was_cancelled is False + release.set() + for _ in range(20): + await asyncio.sleep(0) + assert cancel_was_cancelled is False