diff --git a/src/kiro_crew/dashboard/channel_slots.py b/src/kiro_crew/dashboard/channel_slots.py index 3178d8276dbc..926d18beddaf 100644 --- a/src/kiro_crew/dashboard/channel_slots.py +++ b/src/kiro_crew/dashboard/channel_slots.py @@ -670,6 +670,18 @@ def surface_channel_session( # the frozen prefix a save never rewrites, and redact assistant content at # the read boundary. _rebuild_window(slot, messages) + # Channel transcripts bypass both dashboard restore drivers, so the local-turn + # marker a dashboard turn on this slot left behind is reconciled here, once + # the whole window exists. Local import: chat_persistence imports this module. + from kiro_crew.dashboard.chat_persistence import ( + _local_turn_generation, + _local_turn_prompt, + _reconcile_local_turn_marker, + ) + + _reconcile_local_turn_marker( + slot, _local_turn_generation(meta), _local_turn_prompt(meta), persisted=messages + ) # The window corresponds to the file as of this listing, so the refresh pass # has nothing to do until the channel writes again. slot._channel_window_mtime = float(session_info.get("modified", 0) or 0) diff --git a/src/kiro_crew/dashboard/chat_handlers.py b/src/kiro_crew/dashboard/chat_handlers.py index 5f6c9309cd0f..689db226a111 100644 --- a/src/kiro_crew/dashboard/chat_handlers.py +++ b/src/kiro_crew/dashboard/chat_handlers.py @@ -72,6 +72,9 @@ _TRANSIENT_ROLES, COLOR_HEX_RE, _attach_variants, + _local_turn_generation, + _local_turn_prompt, + _reconcile_local_turn_marker, _rehydrate_slot_title, _remember_reasoning_effort_for_restore, _restored_agent_name, @@ -11873,7 +11876,12 @@ def _hydrate_slot_from_history( # by variant count). for m in messages: role = m.get("role", "assistant") - cls = "msg msg-u" if role == "user" else "msg msg-a" + # Carry the persisted ``cls`` like the startup restore paths do: a Stop + # card's discriminator lives only in that JSON string (see + # ``is_stop_event_row``), so a resumed session dropping it reads the + # user's deliberate Stop as an interrupted turn. Import rows carry none + # and fall through to the role default as before. + cls = m.get("cls") or ("msg msg-u" if role == "user" else "msg msg-a") content = m.get("content", "") slot.append( role, @@ -11896,6 +11904,12 @@ def _hydrate_slot_from_history( # _disk_older_count above) are the frozen prefix saves never rewrite, # so older on-disk turns are preserved. slot._disk_window_len = len(slot.messages) + # After the window boundary, like the startup restore: a local turn the + # previous process admitted and never tore down becomes the interruption + # row here, so a session pulled up off disk and one restored at boot agree. + _reconcile_local_turn_marker( + slot, _local_turn_generation(meta), _local_turn_prompt(meta), persisted=all_messages + ) async def api_chat_slot_resume(request: web.Request) -> web.Response: @@ -12297,7 +12311,10 @@ async def api_chat_slot_resume(request: web.Request) -> web.Response: # Restore the protected choice read before construction, not the # editable transcript's provisional agent name. slot.agent = restored_agent - total = len(all_messages) + # Hydrated length, not the raw disk count: materialisation may append one + # unsaved interruption row, and the paging cursor below has to account for + # it or the next older page repeats a row. + total = slot._disk_older_count + len(slot.messages) recent = slot.messages[-200:] if len(slot.messages) > 200 else slot.messages # The slot was registered throughout hydration (so a concurrent same-key # resume resolved it and hit the idempotency guard) but hidden from the @@ -12311,8 +12328,9 @@ async def api_chat_slot_resume(request: web.Request) -> web.Response: { "ok": True, "key": slot.key, - # `total` is the full on-disk length here, so this already is the - # raw index the next older page starts from. + # `total` is the effective hydrated length (durable rows plus a + # recovered interruption row when one was appended), so this is + # the raw index the next older page starts from. "next_before": total - len(recent), "messages": _prepare_messages( recent, diff --git a/src/kiro_crew/dashboard/chat_persistence.py b/src/kiro_crew/dashboard/chat_persistence.py index ed6c99f8af5f..80415d73e8e2 100644 --- a/src/kiro_crew/dashboard/chat_persistence.py +++ b/src/kiro_crew/dashboard/chat_persistence.py @@ -34,6 +34,7 @@ config_dir, ) from kiro_crew.dashboard.channel_slots import slot_closed_since +from kiro_crew.dashboard.chat_delivery import ATTACHMENT_LIST_MAX_ITEMS, ATTACHMENT_PATH_MAX_LEN from kiro_crew.dashboard.chat_utils import ( _normalize_model, _redact_meta_for_role, @@ -65,6 +66,8 @@ _normalize_slot_key, _note_authorized_elsewhere, durable_row_count, + is_stop_event_row, + is_turn_interrupted, row_mid, ) from kiro_crew.effort import EFFORT_LEVELS, EFFORT_VALUES @@ -123,6 +126,232 @@ # the persistence module's load path). _TITLE_ORIGINS = ("auto", "user") +_RESTART_INTERRUPTION_KIND = "gateway_restart_interruption" +# "app", not "gateway": the sibling relay-interruption row in +# ``_rehydrate_slot_from_history`` names the same event the same way, and the +# process name means nothing to the person reading the transcript. +_RESTART_INTERRUPTION_MSG = ( + "This turn was interrupted when the app restarted. " + "Review the partial output above, then resume to continue." +) + + +def _local_turn_generation(meta: Mapping[str, object]) -> int: + """The persisted local-turn generation, or zero when none is recorded. + + ``bool`` is an ``int`` subclass, so it is refused explicitly: the metadata + line is an ordinary writable file, and a malformed or hand-edited value + must never manufacture an interruption row. + """ + stored = meta.get("turn_in_flight_generation") + return stored if type(stored) is int and stored > 0 else 0 + + +_LOCAL_TURN_PROMPT_ROLES = frozenset({"user", "nudge", "inject"}) +_LOCAL_TURN_PROMPT_META_KEYS = ("mid", "files", "dirs", "injectKind") +#: Bounds on the opening-row copy the marker retains. The attachment bounds are +#: the send path's own (``chat_delivery.ATTACHMENT_LIST_MAX_ITEMS`` entries of +#: ``ATTACHMENT_PATH_MAX_LEN`` chars), so every row the send path accepts fits +#: the copy and a narrower local bound cannot drop an accepted opener. The +#: total is sized for ``MAX_PROMPT_BYTES`` (100 KB) plus two full attachment +#: lists plus the identity fields, so it only rejects a line that no writer of +#: this gateway produced. An out-of-bounds copy is dropped whole, never +#: truncated: the restore re-appends it as the user's own transcript row, and a +#: shortened row would misstate what they sent. A dropped copy leaves the +#: generation alone to flag the turn. +_LOCAL_TURN_PROMPT_MAX_ATTACHMENTS = ATTACHMENT_LIST_MAX_ITEMS +_LOCAL_TURN_PROMPT_MAX_FIELD_CHARS = ATTACHMENT_PATH_MAX_LEN +_LOCAL_TURN_PROMPT_MAX_BYTES = 128 * 1024 + 2 * ATTACHMENT_LIST_MAX_ITEMS * ( + ATTACHMENT_PATH_MAX_LEN + 4 +) + + +def local_turn_prompt_within_bounds(prompt: Mapping[str, object]) -> bool: + """Whether an opening-row copy fits the bounds the marker retains. + + Checked at retention (``_local_turn_opening_row``) so an oversize copy is + never written, and again at restore (:func:`_local_turn_prompt`) because + the metadata line is a writable file. ``content`` is bounded only through + the serialized total; ``ts``, ``mid``, ``injectKind`` and each + attachment path are bounded per field, and the attachment lists per count. + """ + ts = prompt.get("ts") + if isinstance(ts, str) and len(ts) > _LOCAL_TURN_PROMPT_MAX_FIELD_CHARS: + return False + meta = prompt.get("meta") + if isinstance(meta, Mapping): + for key in ("mid", "injectKind"): + value = meta.get(key) + if isinstance(value, str) and len(value) > _LOCAL_TURN_PROMPT_MAX_FIELD_CHARS: + return False + for key in ("files", "dirs"): + value = meta.get(key) + if isinstance(value, list): + if len(value) > _LOCAL_TURN_PROMPT_MAX_ATTACHMENTS: + return False + if any( + isinstance(item, str) and len(item) > _LOCAL_TURN_PROMPT_MAX_FIELD_CHARS + for item in value + ): + return False + try: + size = len(json.dumps(prompt, ensure_ascii=False, separators=(",", ":"))) + except (TypeError, ValueError): + return False + return size <= _LOCAL_TURN_PROMPT_MAX_BYTES + + +def _local_turn_prompt(meta: Mapping[str, object]) -> dict | None: + """The persisted copy of the in-flight turn's opening row, validated. + + Same trust boundary as :func:`_local_turn_generation`: the line is an + ordinary writable file, so every field is re-checked rather than trusted. + A copy that is not a dict, names a role that never opens a turn, or has + no content is dropped. ``meta`` is reduced to the row's identity, its + attachment lists and the inject kind; a restored row must not arrive + wearing a flag that changes what a later reader does with it. + """ + stored = meta.get("turn_in_flight_prompt") + if not isinstance(stored, dict) or not local_turn_prompt_within_bounds(stored): + return None + role = stored.get("role") + content = stored.get("content") + if role not in _LOCAL_TURN_PROMPT_ROLES or not isinstance(content, str) or not content: + return None + raw_meta = stored.get("meta") + kept_meta: dict = {} + if isinstance(raw_meta, dict): + mid = raw_meta.get("mid") + if isinstance(mid, str) and mid: + kept_meta["mid"] = mid + for key in ("files", "dirs"): + value = raw_meta.get(key) + if isinstance(value, list) and all(isinstance(item, str) for item in value): + kept_meta[key] = list(value) + kind = raw_meta.get("injectKind") + if isinstance(kind, str) and kind: + kept_meta["injectKind"] = kind + ts = stored.get("ts") + # No ``cls`` is read: the transcript never persists one for these roles, + # and a JSON ``cls`` on the re-appended row would be parsed into ``meta`` + # on emit, over the identity this copy exists to carry. + return { + "role": role, + "content": content, + "ts": ts if isinstance(ts, str) else "", + "meta": kept_meta, + } + + +def _window_holds_row( + messages: Iterable[Mapping[str, object]], prompt: Mapping[str, object] +) -> bool: + """Whether *messages* (a window or the whole on-disk list) holds the marker's opening row. + + Matched by ``mid`` when the copy has one, which is the identity every + other dual-writer uses; a copy without an id falls back to the row's + ordering ``ts`` plus role, the pair ``_ChatSlot.append`` makes unique + within one transcript. + """ + meta = prompt.get("meta") + mid = meta.get("mid") if isinstance(meta, dict) else None + ts = prompt.get("ts") + for row in messages: + row_meta = row.get("meta") + if mid and isinstance(row_meta, dict) and row_meta.get("mid") == mid: + return True + if not mid and ts and row.get("ts") == ts and row.get("role") == prompt.get("role"): + return True + return False + + +def _latest_turn_was_deliberately_stopped(messages: list[dict]) -> bool: + """Whether the newest turn ends in the user's own Stop card. + + Same tail walk as :func:`state.is_turn_interrupted`: tool, notice and + status rows are looked through, and the newest stop or conversational row + decides. Read only when a local-turn marker outlived its process. The stop + handler cancels the runner, and a shutdown landing in that same instant can + leave the marker on disk, so the Stop card -- the user's recorded intent -- + must win over a restart-interruption row. + """ + for message in reversed(messages): + if is_stop_event_row(message): + return True + if message.get("role") in ("user", "assistant") and message.get("content"): + return False + return False + + +def _reconcile_local_turn_marker( + slot: _ChatSlot, + generation: int, + prompt: Mapping[str, object] | None = None, + *, + persisted: Iterable[Mapping[str, object]] | None = None, +) -> bool: + """Turn a local-turn marker that outlived its process into the interruption row. + + Returns whether a marker was present, so the startup restore can avoid + layering the remote-relay notice on top of metadata that claims both kinds + of execution. Call it only after the whole window is loaded and + ``_disk_window_len`` is set: rows are appended past that boundary so the + next save writes them rather than counting them as already on disk. + + The marker says a turn was admitted and never reached teardown, which the + transcript alone cannot show -- partial assistant prose followed by + finished tool rows is shape-identical to a clean answer. Rows ride the + periodic flush, so the opening row itself may be missing: when the marker + carries a copy (*prompt*) and no row with its identity is on disk, the + copy is appended first. *persisted* is EVERY on-disk row of the session, + not the loaded window: a long turn can push its own opener past the + window bound into the frozen prefix, and a check against the window alone + would append a duplicate that the next save then writes for good. Callers + that hold only the window pass nothing and the window is searched. Only + then is the tail judged: when the transcript already proves the + interruption (an unanswered user row, a trailing error) no second row is + needed; when the newest turn ends in the user's own Stop card, that intent + wins. Otherwise one ``error`` row lands at the tail so the classifier, + composer and sidebar agree. + + Every window save rewrites the window from its first row, so a lost + opener implies every row after it is lost too and the tail is where it + belongs. The runtime marker starts clear and the slot is marked dirty even + when no row was appended, so the next save omits the slot-owned keys. A + second restart before that save re-runs this decision from the same bytes + and cannot accumulate rows. + """ + if generation <= 0: + return False + if prompt is not None and not _window_holds_row( + slot.messages if persisted is None else persisted, prompt + ): + prompt_meta = prompt.get("meta") + role = str(prompt["role"]) + slot.append( + role, + str(prompt["content"]), + # The loader's own default for a row whose ``cls`` is not on disk. + "msg msg-u" if role == "user" else "msg msg-a", + str(prompt.get("ts") or ""), + broadcast=False, + meta=dict(prompt_meta) if isinstance(prompt_meta, dict) and prompt_meta else None, + ) + if not is_turn_interrupted(slot.messages) and not _latest_turn_was_deliberately_stopped( + slot.messages + ): + slot.append( + "error", + _RESTART_INTERRUPTION_MSG, + "msg msg-err", + broadcast=False, + meta={"kind": _RESTART_INTERRUPTION_KIND}, + ) + slot._turn_in_flight_generation = 0 + slot._turn_in_flight_prompt = None + slot._dirty = True + return True + def _rehydrate_title_origin(titled: bool, stored: object) -> str: """Resolve a rehydrated slot's title origin from persisted metadata. @@ -1713,6 +1942,8 @@ def _rehydrate_slot_from_history( # not slot.is_remote`` -> 409 ``remote_binding_incomplete``) plus the # ``_run_chat`` chokepoint (keyed on ``executor``, not ``is_remote``) refuse # the send with a message the user can act on, rather than run local. + _local_turn_was_in_flight = _local_turn_generation(meta) + _local_turn_prompt_copy = _local_turn_prompt(meta) _relay_was_in_flight = False _executor_meta = meta.get("executor") _instance_meta = meta.get("instance_id") @@ -1977,7 +2208,14 @@ def _rehydrate_slot_from_history( # turns counted above) is never rewritten. slot._disk_window_len = len(slot.messages) slot._dirty = False - if _relay_was_in_flight: + # A local turn admitted by the previous process and never torn down. + # Placed with the relay notice below for the same window-boundary + # reasons; when both markers are present the metadata is inconsistent + # (a slot runs locally OR on a peer), and one row is enough. + _had_local_turn_marker = _reconcile_local_turn_marker( + slot, _local_turn_was_in_flight, _local_turn_prompt_copy, persisted=messages + ) + if _relay_was_in_flight and not _had_local_turn_marker: # The gateway crashed while this slot's turn was executing on the peer # (flagged in the binding block above). The relay reader died with it # and the turn's tail was never mirrored here, so the loaded window @@ -2513,6 +2751,9 @@ def _apply_recent_session( # _disk_older_count above) are the frozen prefix saves never rewrite. slot._disk_window_len = len(slot.messages) slot._dirty = False + _reconcile_local_turn_marker( + slot, _local_turn_generation(meta), _local_turn_prompt(meta), persisted=messages + ) logger.info("Restored session %s (%s)", slot_name, slot.title) @@ -4056,6 +4297,14 @@ def _fresh_fields(meta: dict) -> dict: fields["channel_origin"] = True if slot.forked_from is not None: fields["forked_from"] = slot.forked_from + # CLEARABLE: the merge cannot delete a key and rehydrate reads + # zero as "no local turn outstanding", so the current value is + # written either way -- a forced save after a turn's teardown + # must not leave the admitted generation on disk. + fields["turn_in_flight_generation"] = slot._turn_in_flight_generation + # Same rule for the opening-row copy: None is the cleared value + # (rehydrate reads anything but a dict as "no copy"). + fields["turn_in_flight_prompt"] = slot._turn_in_flight_prompt if slot.executor == "remote" and slot.instance_id and slot.remote_slot: # All three or none, exactly like the full save: a newborn # bound to a peer has an EMPTY window until the first relayed @@ -4506,6 +4755,13 @@ def _refresh_under_lock(meta: dict) -> bool: # in-flight, so a True read back on reload is the crash signal # that triggers the interrupted-turn row. meta_line["relay_in_flight"] = True + if slot._turn_in_flight_generation > 0: + # Written while a local turn is between admission and teardown + # and omitted otherwise; ``SLOT_OWNED_META_KEYS`` makes that + # omission the durable clear. + meta_line["turn_in_flight_generation"] = slot._turn_in_flight_generation + if slot._turn_in_flight_prompt is not None: + meta_line["turn_in_flight_prompt"] = slot._turn_in_flight_prompt if slot.folder_id: meta_line["folder_id"] = slot.folder_id if slot._channel_folder_filed or existing_meta.get("channel_folder_filed"): diff --git a/src/kiro_crew/dashboard/chat_runner.py b/src/kiro_crew/dashboard/chat_runner.py index 2c8df57fc079..69540716115e 100644 --- a/src/kiro_crew/dashboard/chat_runner.py +++ b/src/kiro_crew/dashboard/chat_runner.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import functools import hashlib import inspect import json @@ -21,6 +22,7 @@ model_registry, resource_status, session_directive, + shutdown_event, ) from kiro_crew.acp.client import ( AcpAuthRequired, @@ -111,7 +113,11 @@ _resolve_folder_steering_dirs, slot_steering_principal, ) -from kiro_crew.dashboard.chat_persistence import save_slot_off_loop +from kiro_crew.dashboard.chat_persistence import ( + local_turn_prompt_within_bounds, + register_guarded_history_write, + save_slot_off_loop, +) from kiro_crew.dashboard.chat_summary import generate_session_summary from kiro_crew.dashboard.chat_tag_grants import refresh_cache as refresh_tag_grants_cache from kiro_crew.dashboard.chat_tags import resolve_board_tags @@ -155,6 +161,7 @@ parse_workflow_command, remember_slack_options, restore_replacement_if_handover_did_not_land, + run_to_completion, slack_mirror_is_paused, slot_history_key, tighten_live_slot_memory_mode, @@ -209,6 +216,7 @@ CrewLogPrevious, DashboardState, _ChatSlot, + _is_turn_inject, _mark_permission_resolved, append_and_surface, build_infra_retry_prompt, @@ -9287,6 +9295,226 @@ class _AppAgentNotLoaded(Exception): """ +def _local_turn_generation_for(slot: _ChatSlot) -> int: + """The generation the marker for this dispatch is written under. + + ``slot.task`` is installed before ``_run_chat`` gets its first loop step, so + ``_turn_generation`` already names this exact dispatch. Read into a local + BEFORE the marker save is awaited: a cancellation landing inside that await + (a tab close) must still leave the finally a generation it can retire, or + the save commits after the cancel and nothing ever clears it. + """ + return max(1, slot._turn_generation) + + +# Keys of the opening row copied into the marker: the row's identity, its +# attachment lists and, for an inject, the kind that makes the classifier +# count it as a turn opener. Everything else about the row is recomputed by +# ``_ChatSlot.append`` or belongs to the process that wrote it. +_LOCAL_TURN_PROMPT_META_KEYS = ("mid", "files", "dirs", "injectKind") +#: Roles whose row opens a turn by itself. ``inject`` opens one only with a +#: dispatching ``injectKind`` (``_is_turn_inject``). Mirrors +#: ``_LOCAL_TURN_PROMPT_ROLES`` in ``chat_persistence.py`` minus ``inject``. +_LOCAL_TURN_OPENER_ROLES = frozenset({"user", "nudge"}) + + +def _local_turn_opening_row(slot: _ChatSlot) -> "dict[str, Any] | None": + """A durable copy of the row that opened the turn being admitted. + + Walks back from the window tail to the newest row that opens a turn: a + ``user`` row, a ``nudge`` row (a monitor loop's cycle, which always + dispatches) or a dispatching ``inject`` (``_is_turn_inject``). Stops at + the first conversational assistant row, since a turn whose opener already + has an answer is not the one being admitted. Returns ``None`` when the + window holds no such row (a slot whose opener was consumed by an earlier + save-and-trim, or a test stub with an empty window). + + The copy carries what ``_ChatSlot.append`` needs to re-create the row as + the loader would: role, content, the ordering ``ts`` and the identity + ``mid`` plus the attachment lists. It does NOT carry the + generation, the turn actor or any directive flag -- the restored row is a + transcript row again, not an admission. A copy outside + ``local_turn_prompt_within_bounds`` (serialized size, attachment count, + per-field length) is not carried at all. + """ + for row in reversed(slot.messages): + role = row.get("role") + if role == "assistant" and row.get("content"): + return None + meta = row.get("meta") + if role in _LOCAL_TURN_OPENER_ROLES or (role == "inject" and _is_turn_inject(meta)): + kept_meta = { + key: meta[key] + for key in _LOCAL_TURN_PROMPT_META_KEYS + if isinstance(meta, dict) and key in meta + } + # No ``cls``: the transcript never persists one for a ``user`` or + # ``inject`` row, and a cron inject's in-memory ``cls`` is JSON that + # the emit path would parse into ``meta`` over the row's real + # ``mid`` / ``injectKind``. The restore assigns the loader's default. + copy = { + "role": role, + "content": str(row.get("content") or ""), + "ts": str(row.get("ts") or ""), + "meta": kept_meta, + } + # Bounded as a whole, never truncated: a shortened copy would come + # back as the user's own transcript row. Over the bound, the + # generation alone marks the turn. + return copy if local_turn_prompt_within_bounds(copy) else None + return None + + +async def _begin_local_turn_marker(state: DashboardState, slot: _ChatSlot, generation: int) -> None: + """Persist this turn's generation on the slot's metadata line before dispatch. + + The transcript cannot record a process death: a turn cut by a force exit + leaves partial assistant prose and finished tool rows, the same shape as a + clean answer, and the restore heuristic reads it as finished. Writing the + generation BEFORE any provider work begins means no output can outrun it, + and every restore path turns a leftover value into the interruption row. + + A metadata-only merge, not a window save: the rows stay with the periodic + flush, so admitting a turn changes nothing about what the prompt builder + reads off disk. Only a transcript that does not exist yet (a newborn slot's + first turn) takes the full forced save, which is the one writer that can + create it. Both writes are awaited to completion under cancellation, however + many times it is delivered (``run_to_completion``: a graceful shutdown + escalating after its timeout cancels the runner more than once), so the + teardown clear that follows a cancelled turn is always ordered after them. + Best-effort like the queue's own durable copy: a + failure leaves the value in memory for the next save and never refuses the + turn -- the marker is a recovery hint, not the user's words. + + Both writes are fenced to THIS slot incarnation. A tab closed mid-turn and + reopened under the same key (``close_slot`` pops the slot, then waits up to + 2 s for the cancelled runner) resumes the same transcript, so the routing + pin alone cannot tell the two apart; the merge's guard and the forced + save's ``expected_slot_name`` recheck ``state._slots`` under the owner lock + and refuse once the map holds a different object, so a cancelled runner + that outlives its replacement never writes over the replacement's line or + rebuilds the window over its rows. + A restricted session (``memory_mode`` other than ``persistent``) gets the + marker like any other: the transcript save writes every mode's rows and + records the strictest ``memory_mode`` it can see, so the teardown clear + lands and the restart reads the marker. The prompt copy on the metadata + line adds no exposure the transcript rows do not already have -- every + reader that learns gates on the line's ``memory_mode``. + """ + slot._turn_in_flight_generation = generation + # The opening row travels with the generation. The row itself waits for the + # periodic flush (an immediate window save would put it on disk before the + # prompt build, where the builder's recent-context projection would read + # the current prompt as history), so a death inside that window loses it + # and the restore would judge the previous turn's tail instead. With the + # copy on the metadata line the restore puts the row back first. + slot._turn_in_flight_prompt = _local_turn_opening_row(slot) + log = state.conversation_log + if log is None: + return + history_key = slot_history_key(slot) + slot_name = slot.key + # Both keys every time. The merge cannot delete a key, so a copy the + # previous turn's clear failed to remove would otherwise stay paired with + # this generation and come back as this turn's opener; ``None`` is the + # cleared value the restore reads as "no copy" (same rule as the merge + # save in ``_save_slot_to_history``). + marker_fields: dict[str, Any] = { + "turn_in_flight_generation": generation, + "turn_in_flight_prompt": slot._turn_in_flight_prompt, + } + + def _still_this_incarnation(_meta: dict) -> bool: + return state._slots.get(slot_name) is slot + + # Same fence and same registry as every truncating save. A close raises + # ``is_closing``, waits for the futures in ``_guarded_history_writes``, and + # only then pops the name; a write dispatched after the fence has nothing + # left to order against it, and a write not registered is invisible to the + # wait, so a close-and-same-key-reopen could land this line onto the + # replacement's transcript. Read the fence here with no suspension before + # the registration so the two interleavings are the only ones: fence first + # and this refuses, or registration first and the retraction waits. + if getattr(slot, "is_closing", False): + slot._dirty = True + return + merge = asyncio.get_running_loop().run_in_executor( + None, + functools.partial( + log.update_metadata_if, + history_key, + marker_fields, + _still_this_incarnation, + require_existing=True, + ), + ) + register_guarded_history_write(slot, merge) + try: + merged = await run_to_completion(merge) + except asyncio.CancelledError: + raise + except Exception: + logger.warning("Could not persist the turn marker for slot %s", slot.key, exc_info=True) + slot._dirty = True + return + if not merged and state._slots.get(slot_name) is slot: + await run_to_completion( + save_slot_off_loop( + state, + slot, + force=True, + expected_history_key=history_key, + expected_slot_name=slot_name, + ) + ) + + +def _retire_local_turn_marker(slot: _ChatSlot, generation: int) -> bool: + """Clear the in-memory marker for one turn without touching a successor's. + + Returns whether this call cleared it. A successor can be admitted in the + same slot before this turn's teardown finishes (the queue drain installs a + new task with its own generation), so only the generation that wrote the + marker may retire it. + """ + if generation <= 0 or slot._turn_in_flight_generation != generation: + return False + slot._turn_in_flight_generation = 0 + slot._turn_in_flight_prompt = None + return True + + +async def _clear_local_turn_marker(state: DashboardState, slot: _ChatSlot, generation: int) -> None: + """Durably clear one turn's marker on an exit that wrote no other save. + + The landed path retires the marker in memory right before its own transcript + save, so the omission rides that write. Every other exit -- an error card, + a Stop, a recovery re-queue -- reaches here and pays one forced save, since + a restart before the periodic flush would otherwise read the stale value + and flag a turn that ended in plain sight as interrupted. + """ + if not _retire_local_turn_marker(slot, generation): + return + # ``best_effort`` re-arms ``_dirty`` on failure so the flush retries the + # omission; until then a stale on-disk value fails safe to one extra + # recovery prompt rather than a missed interruption. The identity pins + # refuse the write when a same-key replacement now owns the transcript + # (see ``_begin_local_turn_marker``): the replacement's own saves carry + # its marker state, and the live window is the replacement's, not this one. + await save_slot_off_loop( + state, + slot, + force=True, + expected_history_key=slot_history_key(slot), + expected_slot_name=slot.key, + ) + + +def _gateway_shutdown_requested() -> bool: + """Whether the process is shutting down, read without retaining the event.""" + return shutdown_event.is_set() + + async def _run_chat( state: DashboardState, slot: _ChatSlot, @@ -10769,6 +10997,9 @@ def _wt_note_approved(event) -> None: # noqa: ANN001 -- ACP event union _mirror_thread: str | None = "" _mirror_task_counter = 0 _memory_preparation_admitted = False + # Zero until the marker below is written, so a cancellation that lands during + # startup admission has nothing to clear. + _local_turn_marker_generation = 0 # Bound before the try because cancellation may land while this turn waits # for shared memory preparation, before any provider is allocated. client: Any = None @@ -10807,6 +11038,15 @@ def _wt_note_approved(event) -> None: # noqa: ANN001 -- ACP event union if _prompt_depth == 0: await expire_slack_options(state, session_key) + # Durable "turn in flight" marker, written at the same boundary as the + # active-turn identity: every local command has returned, no provider + # work has begun. Not gated on ``_prompt_depth``: the expanded re-entry + # is the only call that reaches here for a /prompts mention. The + # generation is bound before the save is awaited so a cancellation + # inside it still reaches the finally with something to retire. + _local_turn_marker_generation = _local_turn_generation_for(slot) + await _begin_local_turn_marker(state, slot, _local_turn_marker_generation) + # Resolve agent bindings early so we pass the correct kiro-cli # agent name (e.g. "kirocrew") instead of the KiroCrew slot name # (e.g. "default") which has no matching ~/.kiro/agents/ config. @@ -17552,6 +17792,14 @@ def _emit_error(msg: str, *, will_retry: bool = False) -> None: ) # Attach accumulated file changes to last assistant message before persist _flush_file_changes(slot) + # The reply is in the window, so this save is the durable clear of + # the in-flight marker: retire it first and the omission rides the + # same write instead of costing a second one in the finally. Not + # for a provider-side cancel: a shutdown cancels the provider the + # same way, persists the partial reply as an ordinary row, and the + # finally's shutdown check must still see the marker to preserve it. + if _stop_reason != STOP_REASON_CANCELLED: + _retire_local_turn_marker(slot, _local_turn_marker_generation) # Save to history and trigger memory consolidation await save_slot_off_loop(state, slot) # Reset ALL retry budgets once the cycle completes (success OR the @@ -19589,6 +19837,20 @@ def _emit_error(msg: str, *, will_retry: bool = False) -> None: _unclaimed_markers, _identity_markers, ) + # Retire the in-flight marker once every transcript mutation is complete + # and the session permit above is released (an await here cannot skip + # that release), and only while the process is going to live on: during + # a shutdown the cancellation that ended this turn came from the process + # dying, which is exactly the interruption the marker exists to record -- + # the graceful save writes it and the next start converts it. A turn + # that landed inside the shutdown grace keeps its clear. Before the queue + # drain below, so the successor's own marker is never overtaken by this + # omission. Guarded so a failing save cannot skip the steer requeue. + if not (_gateway_shutdown_requested() and not _turn_landed): + try: + await _clear_local_turn_marker(state, slot, _local_turn_marker_generation) + except Exception: + logger.debug("_clear_local_turn_marker failed", exc_info=True) # End-of-turn fallback: catches set_project and reset_conversation calls # that fired mid-turn, after the start-of-turn consume already ran. This # is the ONLY caller that may consume a queued conversation discard — diff --git a/src/kiro_crew/dashboard/state.py b/src/kiro_crew/dashboard/state.py index fc280d84a539..2b6f2b5e1af4 100644 --- a/src/kiro_crew/dashboard/state.py +++ b/src/kiro_crew/dashboard/state.py @@ -994,13 +994,42 @@ def is_stop_event_row(m: dict) -> bool: return bool(parsed and parsed.get("kind") == "stop_event") +#: ``meta.injectKind`` values stamped on an ``inject`` row that DISPATCHED a +#: turn (the queue drain, the cron injectors, the synthesis kick-off, an app +#: message's delivery). Every other inject row -- a ``/note`` breadcrumb, a +#: Stop-hook halt card, a policy refusal notice -- is appended without one and +#: opens nothing. Mirrors ``TURN_INJECT_KINDS`` in +#: ``website/src/store/chatSlice.ts``, which is keyed by the ``InjectKind`` +#: type so a new kind cannot be stamped without being classified there. +#: Wider than ``_TURN_OPENING_INJECT_KINDS`` in ``chat_handlers.py`` on +#: purpose: that set counts turns for the session-start failure streak and +#: walks past ``recovery`` / ``user_replay`` because they resume the SAME +#: turn; here the question is whether a dispatch happened that got no reply, +#: and a recovery or replay dispatch that died is exactly such a turn. +_TURN_INJECT_KINDS: frozenset[str] = frozenset( + {"cron", "mcp_app", "recovery", "user_replay", "synthesis"} +) + + +def _is_turn_inject(meta: object) -> bool: + """Whether an ``inject`` row's meta says it dispatched a turn.""" + return isinstance(meta, dict) and meta.get("injectKind") in _TURN_INJECT_KINDS + + def is_turn_interrupted(messages: list[dict]) -> bool: """True when the transcript shows a turn that ended without a reply. - Two shapes qualify: the last conversational row is the USER's (nothing came - back at all — a gateway restart mid-turn leaves exactly this), or it is the + Two shapes qualify: the last turn-opening row is the USER's, a monitor + loop's NUDGE, or a runner-authored INJECT's (nothing came back at all — a gateway restart + mid-turn leaves exactly this), or the last conversational row is the ASSISTANT's but an error row follows it (the turn streamed partway then died, - which is otherwise shape-identical to a clean completion). + which is otherwise shape-identical to a clean completion). An inject counts + as an opener only when it carries a dispatching ``meta.injectKind`` (see + ``_TURN_INJECT_KINDS``): a queued continuation, a recovery or a synthesis + turn IS a turn, and without it the scan walks past an interrupted one and + can reach the previous turn's Stop card, which then hides the newer + interruption. An untagged inject -- a ``/note`` breadcrumb, a Stop-hook halt + card, a refusal notice -- dispatched nothing and is looked through. Two shapes are explicitly excluded. A trailing ``stop_event``: the user pressing Stop is a deliberate ending, not an interruption, and stopping @@ -1042,7 +1071,8 @@ def is_turn_interrupted(messages: list[dict]) -> bool: # "the gateway died before anything came back". See ``is_stop_event_row`` # for why the discriminator has to be resolved from three carriers. # Only the NEWEST turn's terminator reaches here -- an older stop card - # is never scanned, because a later user/assistant row returns first. + # is never scanned, because a later user/inject/assistant row returns + # first. if is_stop_event_row(m): return False if is_system_notice(role, meta): @@ -1061,6 +1091,12 @@ def is_turn_interrupted(messages: list[dict]) -> bool: ): saw_compaction_result = True continue + if role == "inject" and m.get("content") and _is_turn_inject(meta): + return True + # A monitor loop's cycle row always dispatches a turn; unanswered, it + # is the same shape as an unanswered user row. + if role == "nudge" and m.get("content"): + return True if role in ("user", "assistant") and m.get("content"): if role != "user": return saw_trailing_error @@ -1079,7 +1115,12 @@ def is_turn_interrupted(messages: list[dict]) -> bool: return True if role == "error": saw_trailing_error = True - return False + # The walk ran off the start of the window without meeting a conversational + # row: a long turn can push its own opener and reply into the frozen prefix, + # leaving only tool rows here. A trailing error row is still the evidence + # the assistant branch above honors -- the turn ended in it and nothing + # newer proves completion -- so it decides the same way. + return saw_trailing_error def _mark_permission_resolved( @@ -2651,6 +2692,8 @@ class _ChatSlot: "instance_id", "remote_slot", "_relay_in_flight", + "_turn_in_flight_generation", + "_turn_in_flight_prompt", "_active_turn_session_key", "_side", "_acp_client", @@ -2801,6 +2844,20 @@ def __init__( # and rehydration appends an "interrupted" row rather than leaving the # transcript silently stopped. Set/cleared in ``remote_relay.relay_remote_turn``. self._relay_in_flight: bool = False + # Generation of the LOCAL turn ``chat_runner._run_chat`` durably admitted; + # zero when no local turn is outstanding. Persisted on the metadata line + # before provider dispatch and omitted after teardown, so a process that + # dies mid-turn leaves it on disk and every restore path converts it into + # the interruption row the transcript shape alone cannot prove -- partial + # assistant text followed by completed tool rows and no error row looks + # exactly like a finished answer once the process is gone. + self._turn_in_flight_generation: int = 0 + # The row that opened the in-flight turn, persisted beside the + # generation. The row itself rides the periodic flush, so a process + # death inside that window loses it; the copy here lets the restore + # put it back before the interruption is judged. None when no turn is + # in flight. + self._turn_in_flight_prompt: dict[str, Any] | None = None self.created_at: str = datetime.now(timezone.utc).isoformat() self.messages: list[dict[str, Any]] = [] self._buffers = SlotBufferCoordinator() diff --git a/src/kiro_crew/history.py b/src/kiro_crew/history.py index 40c5d36860cf..35ecf32ba890 100644 --- a/src/kiro_crew/history.py +++ b/src/kiro_crew/history.py @@ -212,6 +212,17 @@ # written at relay start is carried forward past a clean completion, so # every later restart would append a false "interrupted" row. "relay_in_flight", + # Local-turn crash marker: the generation of the turn ``_run_chat`` + # admitted, written before provider dispatch and omitted once the turn + # reaches teardown. Owned for the same reason as ``relay_in_flight``: + # absence IS the clear, so a carried-forward value would flag every + # later restart as interrupted after one clean completion. + "turn_in_flight_generation", + # The row that opened that turn (role, content, cls, ts, meta), carried + # with the generation so a restart inside the periodic flush window + # can put the prompt back before flagging the interruption. Cleared by + # omission alongside the generation. + "turn_in_flight_prompt", "folder_id", "app", "artifact", diff --git a/temp-screenshots/restart-interruption/restart-interruption-inject-turn.png b/temp-screenshots/restart-interruption/restart-interruption-inject-turn.png new file mode 100644 index 000000000000..491e6f60bf53 Binary files /dev/null and b/temp-screenshots/restart-interruption/restart-interruption-inject-turn.png differ diff --git a/temp-screenshots/restart-interruption/restart-interruption-row.png b/temp-screenshots/restart-interruption/restart-interruption-row.png new file mode 100644 index 000000000000..9ca59905f026 Binary files /dev/null and b/temp-screenshots/restart-interruption/restart-interruption-row.png differ diff --git a/temp-screenshots/restart-interruption/restart-lost-opener-restored.png b/temp-screenshots/restart-interruption/restart-lost-opener-restored.png new file mode 100644 index 000000000000..fd25ffb42def Binary files /dev/null and b/temp-screenshots/restart-interruption/restart-lost-opener-restored.png differ diff --git a/test/test_channel_slots.py b/test/test_channel_slots.py index c7424ef65033..b8d4cb7a41fa 100644 --- a/test/test_channel_slots.py +++ b/test/test_channel_slots.py @@ -398,6 +398,30 @@ def test_a_slot_without_channel_provenance_is_never_rebound(self, dashboard_stat impostor.linked_session_key == "" ), "a slot with no channel provenance was bound to a channel conversation" + def test_reconciles_an_interrupted_local_turn_marker(self, dashboard_state: Any) -> None: + """A dashboard turn on a channel-surfaced slot leaves its in-flight marker + on the channel transcript's metadata line; surfacing is that slot's only + restore path, so it converts a leftover marker like the dashboard ones.""" + slot = channel_slots.surface_channel_session( + dashboard_state, + _session("slack:1.1", title="Interrupted channel turn"), + {"turn_in_flight_generation": 17}, + [ + {"role": "user", "content": "inspect it"}, + {"role": "assistant", "content": "partial result"}, + {"role": "tool", "content": "read complete", "meta": {"done": True}}, + ], + session_key="slack:1.1", + ) + + assert slot is not None + assert slot.to_dict()["interrupted"] is True + assert slot.messages[-1]["meta"]["kind"] == "gateway_restart_interruption" + assert slot._turn_in_flight_generation == 0 + assert slot._dirty is True + # Past the window boundary, so the next save writes the row. + assert slot._disk_window_len == len(slot.messages) - 1 + def test_an_unresolvable_session_key_surfaces_the_slot_unbound( self, dashboard_state: Any ) -> None: diff --git a/test/test_dashboard_chat.py b/test/test_dashboard_chat.py index dca305561496..9dfc02e0af39 100644 --- a/test/test_dashboard_chat.py +++ b/test/test_dashboard_chat.py @@ -20184,8 +20184,20 @@ def spy(self_slot, *a, **kw): notice_msgs = [m for m in slot.messages if m.get("role") == "notice"] assert not any("returned nothing this turn" in m.get("content", "") for m in notice_msgs) # Re-queue path must NOT persist/consolidate the spurious empty turn or - # record success (item 3 of the CR). - mock_save.assert_not_called() + # record success (item 3 of the CR). The only saves are the in-flight + # marker's: this slot has no transcript yet, so admission falls back to + # the forced save that creates it, and the finally clears the marker + # with a second one before the queued retry receives its own generation. + assert mock_save.await_count == 2 + for marker_save in mock_save.await_args_list: + assert marker_save.args == (state, slot) + # Both marker saves are pinned to this slot incarnation so a + # cancelled runner outliving a same-key replacement writes nothing. + assert marker_save.kwargs == { + "force": True, + "expected_history_key": "dashboard:empty-resp-slot", + "expected_slot_name": "empty-resp-slot", + } mock_consolidate.assert_not_called() state.sessions.record_success.assert_not_called() # _flush_file_changes is intentionally NOT skipped: the try-body call (inside @@ -25535,3 +25547,113 @@ def test_trailing_owed_rows_survive_the_idless_tail_filter(self): "stopped", "partial an", ], f"the owed chunk must keep its trailing window position; got {bodies}" + + +class TestHistoryResumeInterruptedTurnMarker: + """The History resume endpoint shares the startup restore's marker handling.""" + + @staticmethod + def _saved_state(tmp_path, monkeypatch, name: str, rows: list[tuple], generation: int): + from kiro_crew.dashboard.chat_persistence import _save_slot_to_history + + monkeypatch.setattr("kiro_crew.dashboard.state.config_dir", lambda: tmp_path) + state = _make_state(tmp_path) + slot = state.get_or_create_slot(name) + for row in rows: + slot.append(*row[:3], meta=row[3] if len(row) > 3 else None) + slot._turn_in_flight_generation = generation + assert _save_slot_to_history(state, slot, force=True) + del state._slots[slot.key] + return state + + @pytest.mark.asyncio + async def test_resume_reconciles_partial_assistant_and_tool_tail(self, tmp_path, monkeypatch): + state = self._saved_state( + tmp_path, + monkeypatch, + "resume-interrupted", + [ + ("user", "inspect it", "msg msg-u"), + ("assistant", "partial result", "msg msg-a"), + ("tool", "read complete", "tool", {"done": True}), + ], + generation=18, + ) + + async with TestClient(TestServer(_make_app(state))) as client: + response = await client.post( + "/api/chat/slots/resume-interrupted/resume", + json={"key": "dashboard:resume-interrupted"}, + ) + payload = await response.json() + + assert response.status == 200 + restored = state.get_slot("resume-interrupted") + assert restored is not None + assert restored.to_dict()["interrupted"] is True + assert restored.messages[-1]["meta"]["kind"] == "gateway_restart_interruption" + # The paging cursor counts the recovered row, so the next older page + # starts where this response's rows end. + assert payload["total"] == 4 + assert payload["messages"][-1]["role"] == "error" + assert payload["next_before"] == 0 + assert payload["has_more"] is False + + @pytest.mark.asyncio + async def test_resume_preserves_a_deliberate_stop(self, tmp_path, monkeypatch): + state = self._saved_state( + tmp_path, + monkeypatch, + "resume-stopped", + [ + ("user", "inspect it", "msg msg-u"), + ("system", "Stopped", json.dumps({"kind": "stop_event"})), + ], + generation=19, + ) + + async with TestClient(TestServer(_make_app(state))) as client: + response = await client.post( + "/api/chat/slots/resume-stopped/resume", + json={"key": "dashboard:resume-stopped"}, + ) + payload = await response.json() + + assert response.status == 200 + restored = state.get_slot("resume-stopped") + assert restored is not None + assert restored.to_dict()["interrupted"] is False + assert all( + (m.get("meta") or {}).get("kind") != "gateway_restart_interruption" + for m in restored.messages + ) + assert payload["total"] == 2 + + @pytest.mark.asyncio + async def test_resume_ignores_a_malformed_boolean_marker(self, tmp_path, monkeypatch): + state = self._saved_state( + tmp_path, + monkeypatch, + "resume-malformed", + [("user", "hi", "msg msg-u"), ("assistant", "complete", "msg msg-a")], + generation=0, + ) + assert state.conversation_log is not None + state.conversation_log.update_metadata( + "dashboard:resume-malformed", {"turn_in_flight_generation": True} + ) + + async with TestClient(TestServer(_make_app(state))) as client: + response = await client.post( + "/api/chat/slots/resume-malformed/resume", + json={"key": "dashboard:resume-malformed"}, + ) + + assert response.status == 200 + restored = state.get_slot("resume-malformed") + assert restored is not None + assert restored.to_dict()["interrupted"] is False + assert all( + (m.get("meta") or {}).get("kind") != "gateway_restart_interruption" + for m in restored.messages + ) diff --git a/test/test_is_interrupted.py b/test/test_is_interrupted.py index ca80320a68ea..96fc45584893 100644 --- a/test/test_is_interrupted.py +++ b/test/test_is_interrupted.py @@ -89,6 +89,13 @@ def test_superseded_error_is_not_interrupted(self): def test_empty_transcript_is_not_interrupted(self): assert _is_interrupted(slot()) is False + def test_window_of_only_tool_rows_reads_its_trailing_error(self): + # A long turn pushed its opener and reply into the frozen prefix; the + # loaded window is tool rows plus the restart's error row. + tool = {"role": "tool", "content": "read page", "meta": {"done": True}} + assert _is_interrupted(slot(tool, tool, error())) is True + assert _is_interrupted(slot(tool, tool)) is False + class TestDeliberateStop: """A user-initiated Stop is an ENDING, not an interruption.""" @@ -264,3 +271,70 @@ def test_borrowed_tag_notices_do_not_complete_a_compact(self): "meta": {"kind": "compaction", "notice": notice_kind}, } assert _is_interrupted(slot(user("/compact"), row)) is True + + +class TestInjectRowsOpenTurns: + """An ``inject`` row stamped with a dispatching ``meta.injectKind`` (the queue + drain's cron/recovery/user_replay, the synthesis kick-off) is a turn like a + user row. An untagged inject -- a ``/note`` breadcrumb, a Stop-hook halt card, + a policy refusal notice -- dispatched nothing and is looked through. + + Mirrors ``selectTurnInterrupted`` in ``chatSlice.ts``, which decides inject + rows before its CONTINUE_SCAN_SKIP set. + """ + + @staticmethod + def inject(kind: str | None = "user_replay", content: str = "go on", **meta) -> dict: + row = {"role": "inject", "content": content} + if kind is not None or meta: + row["meta"] = ({"injectKind": kind} if kind is not None else {}) | meta + return row + + def test_nudge_row_is_an_opener(self): + # A monitor loop's cycle dispatches a turn; unanswered it is interrupted, + # and an older Stop card behind it must not mask that. + nudge = {"role": "nudge", "content": "[auto-nudge cycle 3] check", "meta": {"nudge": {}}} + assert _is_interrupted(slot(user(), assistant(), nudge)) is True + assert _is_interrupted(slot(user(), stop_event(), nudge)) is True + assert _is_interrupted(slot(user(), assistant(), nudge, assistant())) is False + + def test_every_dispatching_kind_is_an_opener(self): + for kind in ("cron", "mcp_app", "recovery", "user_replay", "synthesis"): + assert _is_interrupted(slot(user(), assistant(), self.inject(kind))) is True + + def test_inject_followed_by_tool_rows_is_interrupted(self): + rows = ( + user(), + self.inject("recovery", "continue queued work"), + {"role": "tool", "content": "read complete", "meta": {"done": True}}, + ) + assert _is_interrupted(slot(*rows)) is True + + def test_answered_inject_is_finished(self): + assert _is_interrupted(slot(user(), self.inject(), assistant())) is False + + def test_older_stop_does_not_mask_a_newer_inject_turn(self): + rows = ( + user("first"), + stop_event(), + self.inject("synthesis", "continue queued work"), + {"role": "tool", "content": "read complete"}, + ) + assert _is_interrupted(slot(*rows)) is True + + def test_untagged_injects_are_looked_through(self): + # A halt card, a note (rehydrated and live shapes), an unknown or + # non-string kind: none dispatched a turn, so none is unanswered. + for row in ( + self.inject(None, "Stop hook halted #3"), + self.inject(None, "noted", noteSession="dashboard:s"), + {"role": "inject", "content": "noted", "cls": "reconcile-note"}, + self.inject("unknown"), + {"role": "inject", "content": "x", "meta": {"injectKind": 7}}, + {"role": "inject", "content": "x", "meta": "cron"}, + ): + assert _is_interrupted(slot(user(), assistant(), row)) is False + assert _is_interrupted(slot(user(), row)) is True + + def test_empty_inject_is_not_an_opener(self): + assert _is_interrupted(slot(user(), assistant(), self.inject("cron", ""))) is False diff --git a/test/test_local_turn_restart_marker.py b/test/test_local_turn_restart_marker.py new file mode 100644 index 000000000000..b307f5e1da6f --- /dev/null +++ b/test/test_local_turn_restart_marker.py @@ -0,0 +1,1404 @@ +"""The durable "turn in flight" marker that survives a gateway force exit. + +A turn cut by process death leaves ``user -> assistant(partial) -> tool rows`` +and no error row, which the transcript heuristic (``is_turn_interrupted``) +reads as a finished answer. ``_run_chat`` writes the turn's generation to the +slot-owned ``turn_in_flight_generation`` metadata key before provider dispatch +and omits it at teardown; every restore path converts a leftover value into the +existing interruption row. +""" + +from __future__ import annotations + +import asyncio +import json +from unittest.mock import AsyncMock, MagicMock + +import pytest +from chat_test_helpers import _make_state + +from kiro_crew.dashboard.chat_persistence import ( + _LOCAL_TURN_PROMPT_MAX_ATTACHMENTS, + _LOCAL_TURN_PROMPT_MAX_BYTES, + _LOCAL_TURN_PROMPT_MAX_FIELD_CHARS, + _local_turn_prompt, + _rehydrate_slot_from_history, + _save_slot_to_history, + restore_recent_sessions, +) +from kiro_crew.dashboard.chat_runner import ( + _begin_local_turn_marker, + _clear_local_turn_marker, + _local_turn_generation_for, + _local_turn_opening_row, + _retire_local_turn_marker, +) +from kiro_crew.history import SLOT_OWNED_META_KEYS + +_RESTART_KIND = "gateway_restart_interruption" + + +def _restart_rows(messages: list[dict]) -> list[dict]: + return [m for m in messages if (m.get("meta") or {}).get("kind") == _RESTART_KIND] + + +def _provider_mock() -> AsyncMock: + # Same shape as test_dashboard_chat._provider_mock: the telemetry accessors + # the runner reads after a turn are SYNCHRONOUS on the real provider, so they + # are pinned as MagicMock or each call hands back a never-awaited coroutine. + client = AsyncMock() + client.context_usage_pct = MagicMock(return_value=10.0) + client.context_window_tokens = MagicMock(return_value=0) + client.context_used_tokens = MagicMock(return_value=0) + client.mcp_session_report = MagicMock(return_value=None) + client.available_models = MagicMock(return_value=[]) + client.client.pop_pending_oauth_requests = MagicMock(return_value=[]) + return client + + +def _client_streaming(events: list) -> AsyncMock: + client = _provider_mock() + + async def _stream(msg): + for ev in events: + yield ev + + client.stream = _stream + client.stream_command = _stream + return client + + +def _state_for_run_chat(tmp_path, monkeypatch): + monkeypatch.setattr("kiro_crew.dashboard.state.config_dir", lambda: tmp_path) + state = _make_state(tmp_path) + state.broadcast_ws = MagicMock() + state.push_slots_update = MagicMock() + state.context_builder = None + state.consolidator = MagicMock() + state._hook_store = None + state._yolo = False + return state + + +# ── marker lifecycle ─────────────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_marker_is_durable_before_dispatch_and_omitted_after_clear(tmp_path): + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "do something long", "msg msg-u") + slot._turn_generation = 7 + + generation = _local_turn_generation_for(slot) + await _begin_local_turn_marker(state, slot, generation) + + assert generation == 7 + assert slot._turn_in_flight_generation == 7 + assert state.conversation_log is not None + assert state.conversation_log.get_metadata("dashboard:chat-1")["turn_in_flight_generation"] == 7 + + slot.append("assistant", "done", "msg msg-a") + await _clear_local_turn_marker(state, slot, generation) + + assert slot._turn_in_flight_generation == 0 + assert "turn_in_flight_generation" not in state.conversation_log.get_metadata( + "dashboard:chat-1" + ) + + +def test_marker_key_is_slot_owned(): + # Omission on the teardown save IS the durable clear. An unowned key would be + # carried forward by carry_unowned_metadata and flag every later restart. + assert "turn_in_flight_generation" in SLOT_OWNED_META_KEYS + + +def test_retire_is_generation_checked_so_a_successor_keeps_its_marker(): + from kiro_crew.dashboard.state import _ChatSlot + + slot = _ChatSlot("chat-1") + slot._turn_in_flight_generation = 9 + # A stale predecessor's clear must not erase the generation a successor wrote. + assert _retire_local_turn_marker(slot, 8) is False + assert slot._turn_in_flight_generation == 9 + assert _retire_local_turn_marker(slot, 0) is False + assert _retire_local_turn_marker(slot, 9) is True + assert slot._turn_in_flight_generation == 0 + assert _retire_local_turn_marker(slot, 9) is False + + +@pytest.mark.asyncio +async def test_admission_writes_only_metadata_when_the_transcript_exists(tmp_path): + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "first", "msg msg-u") + slot.append("assistant", "answered", "msg msg-a") + assert _save_slot_to_history(state, slot, force=True) + slot.append("user", "second", "msg msg-u") + assert state.conversation_log is not None + before = state.conversation_log.read_messages("dashboard:chat-1") + + await _begin_local_turn_marker(state, slot, 4) + + # The unflushed user row stays with the periodic flush: admitting a turn + # must not change what the prompt builder reads off disk. + assert state.conversation_log.read_messages("dashboard:chat-1") == before + assert state.conversation_log.get_metadata("dashboard:chat-1")["turn_in_flight_generation"] == 4 + + +@pytest.mark.asyncio +async def test_admission_creates_the_transcript_for_a_newborn_slot(tmp_path): + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "first ever", "msg msg-u") + assert state.conversation_log is not None + assert state.conversation_log.get_metadata("dashboard:chat-1") == {} + + await _begin_local_turn_marker(state, slot, 1) + + # No line to merge into, so the full save is the one writer that can make + # the marker durable; it carries the user row along. + assert state.conversation_log.get_metadata("dashboard:chat-1")["turn_in_flight_generation"] == 1 + assert [m["role"] for m in state.conversation_log.read_messages("dashboard:chat-1")] == ["user"] + + +def _replace_slot_under_same_key(state, old_slot): + """Mimic a tab close + same-key reopen that resumes the same transcript.""" + from kiro_crew.dashboard.state import _ChatSlot + + replacement = _ChatSlot(old_slot.key) + replacement._tab_id = old_slot._tab_id + state._slots[old_slot.key] = replacement + return replacement + + +@pytest.mark.asyncio +async def test_admission_merge_refuses_once_a_replacement_owns_the_key(tmp_path): + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "first", "msg msg-u") + slot.append("assistant", "answered", "msg msg-a") + assert _save_slot_to_history(state, slot, force=True) + replacement = _replace_slot_under_same_key(state, slot) + replacement.append("user", "the replacement's turn", "msg msg-u") + assert _save_slot_to_history(state, replacement, force=True) + assert state.conversation_log is not None + before = state.conversation_log.read_messages("dashboard:chat-1") + + # The cancelled predecessor's admission lands after the replacement took the key. + await _begin_local_turn_marker(state, slot, 3) + + meta = state.conversation_log.get_metadata("dashboard:chat-1") + assert "turn_in_flight_generation" not in meta + assert state.conversation_log.read_messages("dashboard:chat-1") == before + # In memory the stale object still carries its value; nothing reads it again. + assert slot._turn_in_flight_generation == 3 + + +@pytest.mark.asyncio +async def test_admission_merge_refuses_once_the_slot_is_closing(tmp_path): + # The close raises its fence before it waits on the guarded writes; a merge + # dispatched after the fence has nothing left to order against it. + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "first", "msg msg-u") + slot.append("assistant", "answered", "msg msg-a") + assert _save_slot_to_history(state, slot, force=True) + assert state.conversation_log is not None + before = dict(state.conversation_log.get_metadata("dashboard:chat-1")) + slot.append("user", "second", "msg msg-u") + slot.begin_close() + + await _begin_local_turn_marker(state, slot, 3) + + assert state.conversation_log.get_metadata("dashboard:chat-1") == before + assert slot._dirty is True + + +@pytest.mark.asyncio +async def test_admission_merge_is_a_guarded_history_write(tmp_path): + # The merge's future is in the registry a close waits on while the worker + # runs, and leaves it when the worker returns; the retraction therefore + # orders itself after the write instead of racing it. + import threading + + from kiro_crew.dashboard.chat_handlers import _pending_guarded_history_writes + + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "first", "msg msg-u") + slot.append("assistant", "answered", "msg msg-a") + assert _save_slot_to_history(state, slot, force=True) + slot.append("user", "second", "msg msg-u") + assert state.conversation_log is not None + log = state.conversation_log + entered = threading.Event() + release = threading.Event() + real = log.update_metadata_if + seen: list[set] = [] + + def slow_merge(*args, **kwargs): + entered.set() + release.wait(5) + return real(*args, **kwargs) + + log.update_metadata_if = slow_merge # type: ignore[method-assign] + task = asyncio.ensure_future(_begin_local_turn_marker(state, slot, 3)) + await asyncio.get_running_loop().run_in_executor(None, entered.wait, 5) + seen.append(_pending_guarded_history_writes(slot)) + release.set() + await task + + assert len(seen[0]) == 1 + assert _pending_guarded_history_writes(slot) == set() + assert log.get_metadata("dashboard:chat-1")["turn_in_flight_generation"] == 3 + + +@pytest.mark.asyncio +async def test_newborn_forced_save_refuses_once_a_replacement_owns_the_key(tmp_path): + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "predecessor's first row", "msg msg-u") + replacement = _replace_slot_under_same_key(state, slot) + replacement.append("user", "the replacement's turn", "msg msg-u") + assert _save_slot_to_history(state, replacement, force=True) + assert state.conversation_log is not None + + # ``require_existing`` is satisfied by the replacement's file, so the merge + # path is taken and the guard refuses; the newborn fallback must not then + # rebuild the window from the stale object's rows either. + await _begin_local_turn_marker(state, slot, 2) + + rows = state.conversation_log.read_messages("dashboard:chat-1") + assert [m["content"] for m in rows] == ["the replacement's turn"] + assert "turn_in_flight_generation" not in state.conversation_log.get_metadata( + "dashboard:chat-1" + ) + + +@pytest.mark.asyncio +async def test_teardown_clear_refuses_to_overwrite_a_replacement_slot(tmp_path): + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "first", "msg msg-u") + await _begin_local_turn_marker(state, slot, 5) + replacement = _replace_slot_under_same_key(state, slot) + replacement.append("user", "first", "msg msg-u") + replacement.append("assistant", "partial from the replacement", "msg msg-a") + replacement.append("tool", "tool output", "msg msg-tool") + await _begin_local_turn_marker(state, replacement, 1) + assert state.conversation_log is not None + before = state.conversation_log.read_messages("dashboard:chat-1") + assert state.conversation_log.get_metadata("dashboard:chat-1")["turn_in_flight_generation"] == 1 + + # The old runner's finally fires after close_slot's 2 s grace has admitted a + # same-key replacement whose own turn is in flight. An unfenced forced save + # from the stale object would republish ITS slot-owned metadata (no marker) + # over the replacement's line and erase the replacement's in-flight marker. + await _clear_local_turn_marker(state, slot, 5) + + assert state.conversation_log.get_metadata("dashboard:chat-1")["turn_in_flight_generation"] == 1 + assert state.conversation_log.read_messages("dashboard:chat-1") == before + assert slot._turn_in_flight_generation == 0 + + +@pytest.mark.asyncio +async def test_a_cancelled_admission_write_lands_before_the_cancel_propagates( + tmp_path, monkeypatch +): + """A tab close while the metadata merge is on its thread must not let that + write land AFTER the teardown clear: the admission waits for the job.""" + import asyncio + import threading + + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "first", "msg msg-u") + assert _save_slot_to_history(state, slot, force=True) + assert state.conversation_log is not None + log = state.conversation_log + real_merge = log.update_metadata_if + entered = threading.Event() + release = threading.Event() + + def slow_merge(*args, **kwargs): + entered.set() + assert release.wait(10) + return real_merge(*args, **kwargs) + + monkeypatch.setattr(log, "update_metadata_if", slow_merge) + task = asyncio.ensure_future(_begin_local_turn_marker(state, slot, 6)) + await asyncio.to_thread(entered.wait, 10) + task.cancel() + # The cancellation is delivered only once the write has landed. + await asyncio.sleep(0) + assert not task.done() + release.set() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, 10) + assert log.get_metadata("dashboard:chat-1")["turn_in_flight_generation"] == 6 + # And a clear issued afterwards is ordered after it, so nothing resurrects it. + await _clear_local_turn_marker(state, slot, 6) + assert "turn_in_flight_generation" not in log.get_metadata("dashboard:chat-1") + + +@pytest.mark.asyncio +async def test_repeated_cancellation_still_waits_for_the_admission_write(tmp_path, monkeypatch): + """A graceful shutdown escalating after its timeout cancels the runner more + than once. The second cancel must not abandon the thread job either: a write + left running would land after the teardown clear and restore a stale marker.""" + import asyncio + import threading + + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "first", "msg msg-u") + assert _save_slot_to_history(state, slot, force=True) + assert state.conversation_log is not None + log = state.conversation_log + real_merge = log.update_metadata_if + entered = threading.Event() + release = threading.Event() + + def slow_merge(*args, **kwargs): + entered.set() + assert release.wait(10) + return real_merge(*args, **kwargs) + + monkeypatch.setattr(log, "update_metadata_if", slow_merge) + task = asyncio.ensure_future(_begin_local_turn_marker(state, slot, 7)) + await asyncio.to_thread(entered.wait, 10) + for _ in range(3): + task.cancel() + await asyncio.sleep(0) + assert not task.done(), "a cancel delivered mid-write must not end the admission" + release.set() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, 10) + assert log.get_metadata("dashboard:chat-1")["turn_in_flight_generation"] == 7 + await _clear_local_turn_marker(state, slot, 7) + assert "turn_in_flight_generation" not in log.get_metadata("dashboard:chat-1") + + +@pytest.mark.asyncio +async def test_a_failed_metadata_merge_never_refuses_the_turn(tmp_path, monkeypatch): + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "first", "msg msg-u") + assert _save_slot_to_history(state, slot, force=True) + slot._dirty = False + assert state.conversation_log is not None + monkeypatch.setattr( + state.conversation_log, + "update_metadata_if", + MagicMock(side_effect=OSError("disk unavailable")), + ) + + await _begin_local_turn_marker(state, slot, 5) + + assert slot._turn_in_flight_generation == 5 + assert slot._dirty is True + + +@pytest.mark.asyncio +async def test_a_failed_marker_save_never_refuses_the_turn(tmp_path, monkeypatch): + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "do something", "msg msg-u") + slot._turn_generation = 3 + slot._dirty = False + monkeypatch.setattr( + "kiro_crew.dashboard.chat_persistence._save_slot_to_history", + MagicMock(side_effect=OSError("disk unavailable")), + ) + + generation = _local_turn_generation_for(slot) + await _begin_local_turn_marker(state, slot, generation) + + # The marker is a recovery hint, not the user's words: it stays in memory + # for the periodic flush and the turn goes ahead. + assert generation == 3 + assert slot._turn_in_flight_generation == 3 + assert slot._dirty is True + + +def test_generation_is_bound_before_the_marker_save_is_awaited(): + from kiro_crew.dashboard.state import _ChatSlot + + # A cancellation inside the admission save must still leave the finally a + # generation to retire, so the generation is a plain read, not the save's + # return value. + slot = _ChatSlot("chat-1") + slot._turn_generation = 0 + assert _local_turn_generation_for(slot) == 1 + slot._turn_generation = 12 + assert _local_turn_generation_for(slot) == 12 + + +@pytest.mark.asyncio +async def test_a_cancel_inside_the_admission_save_still_clears_the_marker(tmp_path, monkeypatch): + """Tab close while the marker save is in flight: the save commits after the + cancel, and teardown must still retire the marker it wrote.""" + import asyncio + + from kiro_crew.dashboard.chat import _run_chat + + state = _state_for_run_chat(tmp_path, monkeypatch) + slot = state.get_or_create_slot("cancel-in-admission") + slot.append("user", "hello", "msg msg-u") + real_begin = __import__( + "kiro_crew.dashboard.chat_runner", fromlist=["x"] + )._begin_local_turn_marker + + async def cancelling_begin(st, sl, generation): + await real_begin(st, sl, generation) + raise asyncio.CancelledError() + + monkeypatch.setattr( + "kiro_crew.dashboard.chat_runner._begin_local_turn_marker", cancelling_begin + ) + + # _run_chat absorbs the cancellation (its except arm persists the partial + # reply) and reaches its finally, which owns the clear. + await _run_chat(state, slot, "hello") + + assert slot._turn_in_flight_generation == 0 + assert state.conversation_log is not None + assert "turn_in_flight_generation" not in state.conversation_log.get_metadata( + "dashboard:cancel-in-admission" + ) + + +@pytest.mark.asyncio +async def test_clear_for_another_generation_writes_nothing(tmp_path, monkeypatch): + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot._turn_in_flight_generation = 5 + save = AsyncMock(return_value=True) + monkeypatch.setattr("kiro_crew.dashboard.chat_runner.save_slot_off_loop", save) + + await _clear_local_turn_marker(state, slot, 4) + + save.assert_not_awaited() + assert slot._turn_in_flight_generation == 5 + + +def test_empty_window_forced_save_writes_the_marker_as_a_clearable_field(tmp_path): + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "hello", "msg msg-u") + assert _save_slot_to_history(state, slot, force=True) + assert state.conversation_log is not None + + # The merge writer cannot delete a key, so it writes the current value + # either way: the admitted generation while a turn runs, zero afterwards. + slot.messages.clear() + slot._turn_in_flight_generation = 6 + assert _save_slot_to_history(state, slot, force=True) + assert state.conversation_log.get_metadata("dashboard:chat-1")["turn_in_flight_generation"] == 6 + slot._turn_in_flight_generation = 0 + assert _save_slot_to_history(state, slot, force=True) + assert state.conversation_log.get_metadata("dashboard:chat-1")["turn_in_flight_generation"] == 0 + + +# ── restore reconciliation ───────────────────────────────────────────────── + + +def test_partial_assistant_and_finished_tools_are_flagged_after_restart(tmp_path): + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "inspect the page", "msg msg-u") + slot.append("assistant", "I am inspecting it now", "msg msg-a") + slot.append("tool", "read complete", "tool", meta={"done": True}) + slot._turn_in_flight_generation = 11 + assert _save_slot_to_history(state, slot, force=True) + # RED on main: this tail reads as a finished answer. + del state._slots["chat-1"] + + restored = _rehydrate_slot_from_history(state, "chat-1") + + assert restored is not None + assert restored.to_dict()["interrupted"] is True + assert restored.messages[-1]["role"] == "error" + assert restored.messages[-1]["meta"]["kind"] == _RESTART_KIND + assert restored._turn_in_flight_generation == 0 + assert restored._dirty is True + # Appended past the window boundary, so the next save writes it rather than + # counting it as already on disk. + assert restored._disk_window_len == len(restored.messages) - 1 + + +def test_reconciliation_is_idempotent_once_the_row_is_saved(tmp_path): + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "inspect the page", "msg msg-u") + slot.append("assistant", "partial result", "msg msg-a") + slot._turn_in_flight_generation = 12 + assert _save_slot_to_history(state, slot, force=True) + + del state._slots["chat-1"] + restored = _rehydrate_slot_from_history(state, "chat-1") + assert restored is not None + assert _save_slot_to_history(state, restored, force=True) + assert state.conversation_log is not None + assert "turn_in_flight_generation" not in state.conversation_log.get_metadata( + "dashboard:chat-1" + ) + + del state._slots["chat-1"] + restored_again = _rehydrate_slot_from_history(state, "chat-1") + assert restored_again is not None + assert len(_restart_rows(restored_again.messages)) == 1 + assert restored_again._turn_in_flight_generation == 0 + + +def test_a_second_restart_before_the_save_does_not_duplicate_the_row(tmp_path): + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "inspect the page", "msg msg-u") + slot.append("assistant", "partial result", "msg msg-a") + slot._turn_in_flight_generation = 12 + assert _save_slot_to_history(state, slot, force=True) + + # Two restores from the same unchanged bytes: the decision re-runs and the + # row is appended once per process, never accumulated on disk. + for _ in range(2): + del state._slots["chat-1"] + restored = _rehydrate_slot_from_history(state, "chat-1") + assert restored is not None + assert len(_restart_rows(restored.messages)) == 1 + + +def test_existing_interruption_evidence_does_not_gain_a_duplicate_error(tmp_path): + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "inspect the page", "msg msg-u") + slot.append("assistant", "partial result", "msg msg-a") + slot.append("error", "backend stopped", "msg msg-err") + slot._turn_in_flight_generation = 13 + assert _save_slot_to_history(state, slot, force=True) + + del state._slots["chat-1"] + restored = _rehydrate_slot_from_history(state, "chat-1") + + assert restored is not None + assert restored.to_dict()["interrupted"] is True + assert [m["role"] for m in restored.messages].count("error") == 1 + assert restored._turn_in_flight_generation == 0 + assert restored._dirty is True + + +def test_deliberate_stop_clears_a_stale_marker_without_flagging_interruption(tmp_path): + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "inspect the page", "msg msg-u") + slot.append("system", "Stopped", json.dumps({"kind": "stop_event"})) + slot._turn_in_flight_generation = 14 + assert _save_slot_to_history(state, slot, force=True) + + del state._slots["chat-1"] + restored = _rehydrate_slot_from_history(state, "chat-1") + + assert restored is not None + assert restored.to_dict()["interrupted"] is False + assert _restart_rows(restored.messages) == [] + assert restored._turn_in_flight_generation == 0 + assert restored._dirty is True + + +def test_a_stop_card_behind_tool_rows_still_counts_as_deliberate(tmp_path): + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "inspect the page", "msg msg-u") + slot.append("assistant", "working", "msg msg-a") + slot.append("system", "Stopped", json.dumps({"kind": "stop_event"})) + slot.append("tool", "late tool row", "tool", meta={"done": True}) + slot._turn_in_flight_generation = 14 + assert _save_slot_to_history(state, slot, force=True) + + del state._slots["chat-1"] + restored = _rehydrate_slot_from_history(state, "chat-1") + + assert restored is not None + assert _restart_rows(restored.messages) == [] + assert restored.to_dict()["interrupted"] is False + + +@pytest.mark.parametrize("malformed", [True, "11", -3, 0, 2.5, None]) +def test_malformed_marker_values_are_ignored(tmp_path, malformed): + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "hello", "msg msg-u") + slot.append("assistant", "complete", "msg msg-a") + assert _save_slot_to_history(state, slot, force=True) + assert state.conversation_log is not None + state.conversation_log.update_metadata( + "dashboard:chat-1", {"turn_in_flight_generation": malformed} + ) + + del state._slots["chat-1"] + restored = _rehydrate_slot_from_history(state, "chat-1") + + assert restored is not None + assert restored.to_dict()["interrupted"] is False + assert _restart_rows(restored.messages) == [] + # No marker means nothing to clear: the window is not dirtied for nothing. + assert restored._dirty is False + + +def test_recent_session_restore_also_reconciles_the_marker(tmp_path): + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "inspect the page", "msg msg-u") + slot.append("assistant", "partial result", "msg msg-a") + slot._turn_in_flight_generation = 15 + assert _save_slot_to_history(state, slot, force=True) + del state._slots["chat-1"] + + assert restore_recent_sessions(state, window_minutes=30) == 1 + + restored = state.get_slot("chat-1") + assert restored is not None + assert restored.to_dict()["interrupted"] is True + assert restored.messages[-1]["meta"]["kind"] == _RESTART_KIND + assert restored._turn_in_flight_generation == 0 + + +def test_local_marker_and_relay_marker_together_append_one_row(tmp_path): + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "inspect the page", "msg msg-u") + slot.append("assistant", "partial result", "msg msg-a") + assert _save_slot_to_history(state, slot, force=True) + assert state.conversation_log is not None + # Inconsistent metadata claiming both kinds of execution: one row, not two. + state.conversation_log.update_metadata( + "dashboard:chat-1", + { + "turn_in_flight_generation": 4, + "executor": "remote", + "instance_id": "peer-1", + "remote_slot": "chat-9", + "relay_in_flight": True, + }, + ) + + del state._slots["chat-1"] + restored = _rehydrate_slot_from_history(state, "chat-1") + + assert restored is not None + assert [m["role"] for m in restored.messages].count("error") == 1 + assert restored.messages[-1]["meta"]["kind"] == _RESTART_KIND + + +# ── _run_chat wiring ─────────────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_a_landed_turn_leaves_no_marker_on_disk(tmp_path, monkeypatch): + from kiro_crew.acp.types import STOP_REASON_END_TURN + from kiro_crew.dashboard.chat import _run_chat + from kiro_crew.providers.base import EVENT_COMPLETE, EVENT_TEXT_CHUNK, LLMEvent + + state = _state_for_run_chat(tmp_path, monkeypatch) + slot = state.get_or_create_slot("landed") + # The send handler appends the user row before dispatch; mirror it so the + # slot has a window (an empty newborn has no metadata line to carry a key). + slot.append("user", "hello", "msg msg-u") + client = _client_streaming( + [ + LLMEvent(kind=EVENT_TEXT_CHUNK, text="the answer"), + LLMEvent(kind=EVENT_COMPLETE, stop_reason=STOP_REASON_END_TURN), + ] + ) + state.sessions.get_or_create = AsyncMock(return_value=(client, True, False)) + + await _run_chat(state, slot, "hello") + + assert slot._turn_in_flight_generation == 0 + assert state.conversation_log is not None + assert "turn_in_flight_generation" not in state.conversation_log.get_metadata( + "dashboard:landed" + ) + # And a restart afterwards reads a finished answer, not an interruption. + del state._slots["landed"] + restored = _rehydrate_slot_from_history(state, "landed") + assert restored is not None + assert restored.to_dict()["interrupted"] is False + + +@pytest.mark.asyncio +async def test_a_cancelled_turn_clears_its_marker_while_the_process_lives(tmp_path, monkeypatch): + from kiro_crew.acp.types import STOP_REASON_CANCELLED + from kiro_crew.dashboard.chat import _run_chat + from kiro_crew.providers.base import EVENT_COMPLETE, EVENT_TEXT_CHUNK, LLMEvent + + state = _state_for_run_chat(tmp_path, monkeypatch) + slot = state.get_or_create_slot("cancelled") + # The send handler appends the user row before dispatch; mirror it so the + # slot has a window (an empty newborn has no metadata line to carry a key). + slot.append("user", "hello", "msg msg-u") + client = _client_streaming( + [ + LLMEvent(kind=EVENT_TEXT_CHUNK, text="partial"), + LLMEvent(kind=EVENT_COMPLETE, stop_reason=STOP_REASON_CANCELLED), + ] + ) + state.sessions.get_or_create = AsyncMock(return_value=(client, True, False)) + + await _run_chat(state, slot, "hello") + + # The process did not die, so whatever ended the turn is already in the + # transcript; the marker must not flag a later clean restart. + assert slot._turn_in_flight_generation == 0 + assert state.conversation_log is not None + assert "turn_in_flight_generation" not in state.conversation_log.get_metadata( + "dashboard:cancelled" + ) + + +@pytest.mark.asyncio +async def test_shutdown_preserves_the_marker_of_an_unlanded_turn(tmp_path, monkeypatch): + from kiro_crew import shutdown_event + from kiro_crew.acp.types import STOP_REASON_CANCELLED + from kiro_crew.dashboard.chat import _run_chat + from kiro_crew.providers.base import EVENT_COMPLETE, EVENT_TEXT_CHUNK, LLMEvent + + state = _state_for_run_chat(tmp_path, monkeypatch) + slot = state.get_or_create_slot("shutdown-cancelled") + # The send handler appends the user row before dispatch; mirror it so the + # slot has a window (an empty newborn has no metadata line to carry a key). + slot.append("user", "hello", "msg msg-u") + client = _client_streaming( + [ + LLMEvent(kind=EVENT_TEXT_CHUNK, text="partial"), + LLMEvent(kind=EVENT_COMPLETE, stop_reason=STOP_REASON_CANCELLED), + ] + ) + state.sessions.get_or_create = AsyncMock(return_value=(client, True, False)) + shutdown_event.set() + try: + await _run_chat(state, slot, "hello") + finally: + shutdown_event.clear() + + assert slot._turn_in_flight_generation > 0 + assert state.conversation_log is not None + meta = state.conversation_log.get_metadata("dashboard:shutdown-cancelled") + assert meta["turn_in_flight_generation"] == slot._turn_in_flight_generation + # The next process converts it into the interruption row. + del state._slots["shutdown-cancelled"] + restored = _rehydrate_slot_from_history(state, "shutdown-cancelled") + assert restored is not None + assert restored.to_dict()["interrupted"] is True + + +@pytest.mark.asyncio +async def test_a_turn_that_lands_during_shutdown_still_clears(tmp_path, monkeypatch): + from kiro_crew import shutdown_event + from kiro_crew.acp.types import STOP_REASON_END_TURN + from kiro_crew.dashboard.chat import _run_chat + from kiro_crew.providers.base import EVENT_COMPLETE, EVENT_TEXT_CHUNK, LLMEvent + + state = _state_for_run_chat(tmp_path, monkeypatch) + slot = state.get_or_create_slot("shutdown-landed") + # The send handler appends the user row before dispatch; mirror it so the + # slot has a window (an empty newborn has no metadata line to carry a key). + slot.append("user", "hello", "msg msg-u") + client = _client_streaming( + [ + LLMEvent(kind=EVENT_TEXT_CHUNK, text="the answer"), + LLMEvent(kind=EVENT_COMPLETE, stop_reason=STOP_REASON_END_TURN), + ] + ) + state.sessions.get_or_create = AsyncMock(return_value=(client, True, False)) + shutdown_event.set() + try: + await _run_chat(state, slot, "hello") + finally: + shutdown_event.clear() + + assert slot._turn_in_flight_generation == 0 + assert state.conversation_log is not None + assert "turn_in_flight_generation" not in state.conversation_log.get_metadata( + "dashboard:shutdown-landed" + ) + + +@pytest.mark.asyncio +async def test_the_marker_is_on_disk_before_the_provider_streams(tmp_path, monkeypatch): + from kiro_crew.acp.types import STOP_REASON_END_TURN + from kiro_crew.dashboard.chat import _run_chat + from kiro_crew.providers.base import EVENT_COMPLETE, EVENT_TEXT_CHUNK, LLMEvent + + state = _state_for_run_chat(tmp_path, monkeypatch) + slot = state.get_or_create_slot("ordered") + # The send handler appends the user row before dispatch; mirror it so the + # slot has a window (an empty newborn has no metadata line to carry a key). + slot.append("user", "hello", "msg msg-u") + seen: list[object] = [] + client = _provider_mock() + + async def _stream(msg): + assert state.conversation_log is not None + seen.append( + state.conversation_log.get_metadata("dashboard:ordered").get( + "turn_in_flight_generation" + ) + ) + yield LLMEvent(kind=EVENT_TEXT_CHUNK, text="partial") + yield LLMEvent(kind=EVENT_COMPLETE, stop_reason=STOP_REASON_END_TURN) + + client.stream = _stream + client.stream_command = _stream + state.sessions.get_or_create = AsyncMock(return_value=(client, True, False)) + + await _run_chat(state, slot, "hello") + + # A force exit at any point of the stream would have left this value behind. + assert seen and type(seen[0]) is int and seen[0] > 0 + + +# ── the opening row travels with the marker ───────────────────────────────── +# +# Rows ride the periodic flush; the marker is a metadata merge written before +# dispatch. A force exit inside that window loses the opening row, and a restore +# that judges the tail without it reads the PREVIOUS turn's ending (a Stop card, +# a finished answer) instead of the lost prompt. The marker therefore carries a +# copy of the row, and every restore path puts it back before deciding. + + +@pytest.mark.asyncio +async def test_admission_carries_the_opening_row_and_the_clear_drops_it(tmp_path): + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("assistant", "earlier answer", "msg msg-a") + assert _save_slot_to_history(state, slot, force=True) + user_row = slot.append("user", "the prompt that opens this turn", "msg msg-u") + slot._turn_generation = 3 + + await _begin_local_turn_marker(state, slot, _local_turn_generation_for(slot)) + + assert state.conversation_log is not None + stored = state.conversation_log.get_metadata("dashboard:chat-1")["turn_in_flight_prompt"] + assert stored["role"] == "user" + assert stored["content"] == "the prompt that opens this turn" + assert "cls" not in stored + assert stored["ts"] == user_row["ts"] + assert stored["meta"] == {"mid": user_row["meta"]["mid"]} + assert slot._turn_in_flight_prompt == stored + + slot.append("assistant", "done", "msg msg-a") + await _clear_local_turn_marker(state, slot, 3) + + assert slot._turn_in_flight_prompt is None + assert "turn_in_flight_prompt" not in state.conversation_log.get_metadata("dashboard:chat-1") + + +def test_prompt_copy_key_is_slot_owned(): + assert "turn_in_flight_prompt" in SLOT_OWNED_META_KEYS + + +def test_retire_clears_the_prompt_copy_with_the_generation(): + from kiro_crew.dashboard.state import _ChatSlot + + slot = _ChatSlot("chat-1") + slot._turn_in_flight_generation = 9 + slot._turn_in_flight_prompt = {"role": "user", "content": "x"} + assert _retire_local_turn_marker(slot, 8) is False + assert slot._turn_in_flight_prompt is not None + assert _retire_local_turn_marker(slot, 9) is True + assert slot._turn_in_flight_prompt is None + + +def test_opening_row_copy_picks_the_newest_unanswered_opener(): + from kiro_crew.dashboard.state import _ChatSlot + + slot = _ChatSlot("chat-1") + slot.append("user", "first", "msg msg-u") + slot.append("assistant", "answered", "msg msg-a") + # An answered opener is not the turn being admitted. + assert _local_turn_opening_row(slot) is None + + slot.append( + "user", + "second", + "msg msg-u", + meta={"files": ["f1"], "dirs": [], "sendId": "s-1", "_directive_user_origin": True}, + ) + slot.append("tool", "late tool row", "tool", meta={"done": True}) + copy = _local_turn_opening_row(slot) + assert copy is not None + assert copy["content"] == "second" + # Identity and attachments only: no send id, no directive flag. + assert set(copy["meta"]) == {"mid", "files", "dirs"} + + slot.append("inject", "[Cron notification] tick", "msg msg-u", meta={"injectKind": "cron"}) + copy = _local_turn_opening_row(slot) + assert copy is not None + assert copy["role"] == "inject" and copy["meta"]["injectKind"] == "cron" + + # An app message's delivery dispatches a whole new turn too; its opener is + # copied like the cron one so a restart mid app turn restores the request. + slot.append( + "inject", "app request", "msg msg-inject", meta={"injectKind": "mcp_app", "appLabel": "x"} + ) + copy = _local_turn_opening_row(slot) + assert copy is not None + assert copy["role"] == "inject" and copy["meta"]["injectKind"] == "mcp_app" + + # An untagged inject dispatches nothing and is looked through. + slot.append("inject", "hook halted", "msg msg-u") + copy = _local_turn_opening_row(slot) + assert copy is not None and copy["content"] == "app request" + + # A monitor loop's cycle row opens a turn like a user row. + slot.append( + "nudge", "[auto-nudge cycle 4] check the PR", "msg msg-nudge", meta={"nudge": {"cycle": 4}} + ) + copy = _local_turn_opening_row(slot) + assert copy is not None and copy["role"] == "nudge" + assert set(copy["meta"]) == {"mid"} + + +def test_a_lost_nudge_opener_behind_a_stop_card_is_restored(tmp_path): + # An armed loop survives a Stop; its next cycle's row rides the periodic + # flush. A force exit inside that window leaves the Stop card as the + # newest row on disk, and without the copy the restore would read that + # Stop as intent and offer nothing. + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "watch the PR", "msg msg-u") + slot.append("assistant", "watching", "msg msg-a") + slot.append("system", "Stopped", json.dumps({"kind": "stop_event"})) + assert _save_slot_to_history(state, slot, force=True) + nudge = slot.append( + "nudge", "[auto-nudge cycle 2] check the PR", "msg msg-nudge", meta={"nudge": {"cycle": 2}} + ) + copy = _local_turn_opening_row(slot) + assert copy is not None and copy["role"] == "nudge" + assert state.conversation_log is not None + state.conversation_log.update_metadata( + "dashboard:chat-1", {"turn_in_flight_generation": 7, "turn_in_flight_prompt": copy} + ) + + del state._slots["chat-1"] + restored = _rehydrate_slot_from_history(state, "chat-1") + + assert restored is not None + tail = restored.messages[-1] + assert tail["role"] == "nudge" and tail["content"] == nudge["content"] + assert tail["meta"]["mid"] == nudge["meta"]["mid"] + assert restored.to_dict()["interrupted"] is True + + +def _prompt_copy(row: dict) -> dict: + return { + "role": row["role"], + "content": row["content"], + "ts": row["ts"], + "meta": {"mid": row["meta"]["mid"]}, + } + + +def test_a_lost_opening_row_is_restored_before_the_tail_is_judged(tmp_path): + # The chat-434 shape with the previous turn ended by Stop and the new prompt + # still in the flush window when the process died: on disk the tail is the + # Stop card, the marker names a row the window does not hold. + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "first", "msg msg-u") + slot.append("assistant", "working", "msg msg-a") + slot.append("system", "Stopped", json.dumps({"kind": "stop_event"})) + assert _save_slot_to_history(state, slot, force=True) + lost = slot.append("user", "second prompt, never flushed", "msg msg-u") + assert state.conversation_log is not None + state.conversation_log.update_metadata( + "dashboard:chat-1", + {"turn_in_flight_generation": 15, "turn_in_flight_prompt": _prompt_copy(lost)}, + ) + + del state._slots["chat-1"] + restored = _rehydrate_slot_from_history(state, "chat-1") + + assert restored is not None + # RED before the copy: the Stop tail read as deliberate, no row, no flag. + tail = restored.messages[-1] + assert tail["role"] == "user" + assert tail["content"] == "second prompt, never flushed" + assert tail["ts"] == lost["ts"] + assert tail["meta"]["mid"] == lost["meta"]["mid"] + assert restored.to_dict()["interrupted"] is True + # The unanswered prompt is the interruption evidence; no second row. + assert _restart_rows(restored.messages) == [] + assert restored._turn_in_flight_generation == 0 + assert restored._turn_in_flight_prompt is None + assert restored._dirty is True + assert restored._disk_window_len == len(restored.messages) - 1 + + # The restored row is written by the next save and survives another restart. + assert _save_slot_to_history(state, restored, force=True) + del state._slots["chat-1"] + again = _rehydrate_slot_from_history(state, "chat-1") + assert again is not None + assert [m["content"] for m in again.messages if m["role"] == "user"] == [ + "first", + "second prompt, never flushed", + ] + assert "turn_in_flight_prompt" not in state.conversation_log.get_metadata("dashboard:chat-1") + + +def test_an_opening_row_already_on_disk_is_not_restored_twice(tmp_path): + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + opener = slot.append("user", "inspect the page", "msg msg-u") + slot.append("assistant", "I am inspecting it now", "msg msg-a") + slot.append("tool", "read complete", "tool", meta={"done": True}) + slot._turn_in_flight_generation = 11 + slot._turn_in_flight_prompt = _prompt_copy(opener) + assert _save_slot_to_history(state, slot, force=True) + + del state._slots["chat-1"] + restored = _rehydrate_slot_from_history(state, "chat-1") + + assert restored is not None + assert [m["content"] for m in restored.messages if m["role"] == "user"] == ["inspect the page"] + assert restored.messages[-1]["meta"]["kind"] == _RESTART_KIND + assert restored.to_dict()["interrupted"] is True + + +def test_an_opening_row_in_the_frozen_prefix_is_not_restored_twice(tmp_path): + # A long turn pushes its own opener past the 500-row window into the + # frozen prefix. The opener is on disk, just not in the loaded window; the + # restore must find it there rather than append the copy behind 500 rows + # of the turn's own output. + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + opener = slot.append("user", "inspect every page", "msg msg-u") + slot.append("assistant", "starting", "msg msg-a") + for i in range(520): + slot.append("tool", f"read page {i}", "tool", meta={"done": True}) + slot._turn_in_flight_generation = 12 + slot._turn_in_flight_prompt = _prompt_copy(opener) + assert _save_slot_to_history(state, slot, force=True) + + del state._slots["chat-1"] + restored = _rehydrate_slot_from_history(state, "chat-1") + + assert restored is not None + assert restored._disk_older_count > 0 + assert [m for m in restored.messages if m["role"] == "user"] == [] + assert restored.messages[-1]["meta"]["kind"] == _RESTART_KIND + assert restored.to_dict()["interrupted"] is True + # The next save writes the window; a second restart from those bytes still + # holds exactly one opener across the whole file. + assert _save_slot_to_history(state, restored, force=True) + del state._slots["chat-1"] + again = _rehydrate_slot_from_history(state, "chat-1") + assert again is not None + assert state.conversation_log is not None + on_disk = list(state.conversation_log.read_messages("dashboard:chat-1")) + assert [m["content"] for m in on_disk if m.get("role") == "user"] == ["inspect every page"] + assert sum(1 for m in on_disk if (m.get("meta") or {}).get("kind") == _RESTART_KIND) == 1 + + +def test_a_copy_without_an_id_matches_on_ts_and_role(tmp_path): + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + opener = slot.append("user", "inspect the page", "msg msg-u") + slot.append("assistant", "partial", "msg msg-a") + assert _save_slot_to_history(state, slot, force=True) + assert state.conversation_log is not None + copy = _prompt_copy(opener) + copy["meta"] = {} + state.conversation_log.update_metadata( + "dashboard:chat-1", {"turn_in_flight_generation": 16, "turn_in_flight_prompt": copy} + ) + + del state._slots["chat-1"] + restored = _rehydrate_slot_from_history(state, "chat-1") + + assert restored is not None + assert [m["content"] for m in restored.messages if m["role"] == "user"] == ["inspect the page"] + assert len(_restart_rows(restored.messages)) == 1 + + +@pytest.mark.parametrize( + "malformed", + [ + "a string", + ["user", "content"], + {"role": "assistant", "content": "an answer is not an opener"}, + {"role": "system", "content": "Stopped"}, + {"role": "user", "content": ""}, + {"role": "user"}, + {"content": "no role"}, + ], +) +def test_malformed_prompt_copies_are_ignored(tmp_path, malformed): + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "inspect the page", "msg msg-u") + slot.append("assistant", "partial", "msg msg-a") + assert _save_slot_to_history(state, slot, force=True) + assert state.conversation_log is not None + state.conversation_log.update_metadata( + "dashboard:chat-1", + {"turn_in_flight_generation": 17, "turn_in_flight_prompt": malformed}, + ) + + del state._slots["chat-1"] + restored = _rehydrate_slot_from_history(state, "chat-1") + + assert restored is not None + # The generation still counts; only the copy is dropped. + assert [m["role"] for m in restored.messages] == ["user", "assistant", "error"] + assert restored.to_dict()["interrupted"] is True + + +def test_a_restored_copy_carries_only_identity_and_attachments(tmp_path): + # The metadata line is a writable file: a flag planted on the copy must not + # come back as a row property another reader acts on. + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "first", "msg msg-u") + slot.append("assistant", "answered", "msg msg-a") + assert _save_slot_to_history(state, slot, force=True) + assert state.conversation_log is not None + state.conversation_log.update_metadata( + "dashboard:chat-1", + { + "turn_in_flight_generation": 18, + "turn_in_flight_prompt": { + "role": "user", + "content": "lost prompt", + "cls": 12, + "ts": None, + "meta": { + "mid": "m-lost", + "files": ["a.txt", 7], + "dirs": ["src"], + "_directive_user_origin": True, + "sendId": "s-9", + "injectKind": "cron", + }, + }, + }, + ) + + del state._slots["chat-1"] + restored = _rehydrate_slot_from_history(state, "chat-1") + + assert restored is not None + tail = restored.messages[-1] + assert tail["role"] == "user" and tail["content"] == "lost prompt" + # A planted ``cls`` is not read; the loader's default is assigned. + assert tail["cls"] == "msg msg-u" + # A mixed-type list is dropped whole; the clean one survives. + assert tail["meta"] == {"mid": "m-lost", "dirs": ["src"], "injectKind": "cron"} + assert restored.to_dict()["interrupted"] is True + + +def test_an_oversize_opener_is_not_copied_and_never_truncated(): + from kiro_crew.dashboard.state import _ChatSlot + + # The copy is bounded as a whole. Over the bound nothing is carried: a + # truncated copy would come back as the user's own words, shortened. + slot = _ChatSlot("chat-1") + slot.append("user", "x" * (_LOCAL_TURN_PROMPT_MAX_BYTES + 1), "msg msg-u") + assert _local_turn_opening_row(slot) is None + + slot = _ChatSlot("chat-1") + slot.append( + "user", + "many files", + "msg msg-u", + meta={ + "files": [f"f{i}" for i in range(_LOCAL_TURN_PROMPT_MAX_ATTACHMENTS + 1)], + "dirs": [], + }, + ) + assert _local_turn_opening_row(slot) is None + + slot = _ChatSlot("chat-1") + slot.append( + "user", + "long path", + "msg msg-u", + meta={"files": ["p" * (_LOCAL_TURN_PROMPT_MAX_FIELD_CHARS + 1)], "dirs": []}, + ) + assert _local_turn_opening_row(slot) is None + + # At the bound the copy is carried whole. + slot = _ChatSlot("chat-1") + slot.append( + "user", + "fits", + "msg msg-u", + meta={"files": [f"f{i}" for i in range(_LOCAL_TURN_PROMPT_MAX_ATTACHMENTS)], "dirs": []}, + ) + copy = _local_turn_opening_row(slot) + assert copy is not None and copy["content"] == "fits" + assert len(copy["meta"]["files"]) == _LOCAL_TURN_PROMPT_MAX_ATTACHMENTS + + +def test_an_oversize_stored_copy_is_dropped_at_restore(): + # The metadata line is a writable file, so the same bounds apply on read. + base = {"role": "user", "content": "lost prompt", "cls": "", "ts": "", "meta": {"mid": "m-1"}} + assert _local_turn_prompt({"turn_in_flight_prompt": base}) is not None + big = dict(base, content="x" * (_LOCAL_TURN_PROMPT_MAX_BYTES + 1)) + assert _local_turn_prompt({"turn_in_flight_prompt": big}) is None + crowded = dict( + base, + meta={ + "mid": "m-1", + "dirs": [f"d{i}" for i in range(_LOCAL_TURN_PROMPT_MAX_ATTACHMENTS + 1)], + }, + ) + assert _local_turn_prompt({"turn_in_flight_prompt": crowded}) is None + long_id = dict(base, meta={"mid": "m" * (_LOCAL_TURN_PROMPT_MAX_FIELD_CHARS + 1)}) + assert _local_turn_prompt({"turn_in_flight_prompt": long_id}) is None + long_ts = dict(base, ts="t" * (_LOCAL_TURN_PROMPT_MAX_FIELD_CHARS + 1)) + assert _local_turn_prompt({"turn_in_flight_prompt": long_ts}) is None + + +def test_a_cron_openers_json_cls_does_not_round_trip(tmp_path): + # A cron inject carries a JSON ``cls`` in memory that the transcript never + # persists. Had the copy carried it, the re-appended row would reach the + # emit path with that JSON, be parsed into ``meta`` over ``mid`` and + # ``injectKind``, and the frontend would stop reading the turn as + # interrupted. The copy carries no ``cls`` and the restore assigns the + # loader's default, so the emitted row keeps its identity. + from kiro_crew.dashboard.chat_utils import parse_cls_meta + + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "first", "msg msg-u") + slot.append("assistant", "answered", "msg msg-a") + assert _save_slot_to_history(state, slot, force=True) + opener = slot.append( + "inject", + "[Cron notification] tick", + json.dumps({"cronLabel": "nightly"}), + meta={"injectKind": "cron"}, + ) + copy = _local_turn_opening_row(slot) + assert copy is not None and "cls" not in copy + slot._turn_in_flight_generation = 9 + slot._turn_in_flight_prompt = copy + assert state.conversation_log is not None + state.conversation_log.update_metadata( + "dashboard:chat-1", {"turn_in_flight_generation": 9, "turn_in_flight_prompt": copy} + ) + # The window save has not run: the opener is not on disk. + state.conversation_log.update_metadata("dashboard:chat-1", {}) + del state._slots["chat-1"] + restored = _rehydrate_slot_from_history(state, "chat-1") + + assert restored is not None + row = next(m for m in restored.messages if m.get("role") == "inject") + assert row["meta"]["mid"] == opener["meta"]["mid"] + assert row["meta"]["injectKind"] == "cron" + assert parse_cls_meta(row["cls"]) is None + assert restored.to_dict()["interrupted"] is True + + +@pytest.mark.asyncio +async def test_admission_without_a_copy_clears_a_stale_copy_on_disk(tmp_path): + # The merge cannot delete a key. A previous turn whose clear failed left its + # copy on the line; the next admission carries no copy (oversize opener), so + # it must write None or a restart would restore the previous prompt as this + # turn's opener. + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.append("user", "first", "msg msg-u") + slot.append("assistant", "answered", "msg msg-a") + assert _save_slot_to_history(state, slot, force=True) + assert state.conversation_log is not None + state.conversation_log.update_metadata( + "dashboard:chat-1", + { + "turn_in_flight_generation": 2, + "turn_in_flight_prompt": { + "role": "user", + "content": "stale prompt", + "cls": "", + "ts": "", + "meta": {"mid": "m-stale"}, + }, + }, + ) + slot.append("user", "x" * (_LOCAL_TURN_PROMPT_MAX_BYTES + 1), "msg msg-u") + slot._turn_generation = 3 + + await _begin_local_turn_marker(state, slot, _local_turn_generation_for(slot)) + + meta = state.conversation_log.get_metadata("dashboard:chat-1") + assert meta["turn_in_flight_generation"] == 3 + assert meta["turn_in_flight_prompt"] is None + del state._slots["chat-1"] + restored = _rehydrate_slot_from_history(state, "chat-1") + assert restored is not None + assert all(m["content"] != "stale prompt" for m in restored.messages) + assert restored.messages[-1]["meta"]["kind"] == _RESTART_KIND + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["incognito", "temporary"]) +async def test_a_restricted_session_gets_and_clears_the_marker(tmp_path, mode): + # The transcript save writes every retention mode's rows, so a restricted + # session is restart-durable like any other: the admission writes the + # marker, the teardown clear lands, and neither state outlives the turn. + # The recorded memory_mode is what keeps the learning readers out, not the + # absence of the line. + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.memory_mode = mode + slot.append("user", "earlier, while restricted", "msg msg-u") + slot.append("assistant", "answered", "msg msg-a") + assert _save_slot_to_history(state, slot, force=True) + assert state.conversation_log is not None + slot.append("user", "a private prompt", "msg msg-u") + slot._turn_generation = 4 + + await _begin_local_turn_marker(state, slot, _local_turn_generation_for(slot)) + during = state.conversation_log.get_metadata("dashboard:chat-1") + assert during["turn_in_flight_generation"] == _local_turn_generation_for(slot) + assert during["turn_in_flight_prompt"]["content"] == "a private prompt" + assert during["memory_mode"] == mode + + await _clear_local_turn_marker(state, slot, 4) + after = state.conversation_log.get_metadata("dashboard:chat-1") + assert "turn_in_flight_generation" not in after + assert "turn_in_flight_prompt" not in after + assert after["memory_mode"] == mode + assert slot._turn_in_flight_generation == 0 + assert slot._turn_in_flight_prompt is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["incognito", "temporary"]) +async def test_a_restricted_session_killed_mid_turn_restores_resume(tmp_path, mode): + # A restricted turn admitted, the prompt row still in the unflushed window, + # then a force exit. The marker copy puts the row back and the restart + # offers Resume, mode preserved. + state = _make_state(tmp_path) + slot = state.get_or_create_slot("chat-1") + slot.memory_mode = mode + slot.append("user", "earlier, while restricted", "msg msg-u") + slot.append("assistant", "answered", "msg msg-a") + assert _save_slot_to_history(state, slot, force=True) + slot.append("user", "a private prompt", "msg msg-u") + slot._turn_generation = 5 + + await _begin_local_turn_marker(state, slot, _local_turn_generation_for(slot)) + + del state._slots["chat-1"] + restored = _rehydrate_slot_from_history(state, "chat-1") + + assert restored is not None + assert restored.memory_mode == mode + # The restored opener is itself the unanswered tail, so no extra error row. + assert [m["role"] for m in restored.messages] == ["user", "assistant", "user"] + assert restored.messages[-1]["content"] == "a private prompt" + assert restored.to_dict()["interrupted"] is True diff --git a/test/test_to_dict_board_fields.py b/test/test_to_dict_board_fields.py index 717f35e8ea9e..06dd162f28e0 100644 --- a/test/test_to_dict_board_fields.py +++ b/test/test_to_dict_board_fields.py @@ -280,6 +280,29 @@ def test_not_interrupted_while_running(): assert d["interrupted"] is False +def test_interrupted_inject_successor_outranks_older_stop(): + s = _slot( + {"role": "user", "content": "first", "ts": "t1"}, + {"role": "system", "content": "stopped", "cls": json.dumps({"kind": "stop_event"}), "ts": "t2"}, + {"role": "inject", "content": "continue queued work", "ts": "t3", "meta": {"injectKind": "recovery"}}, + {"role": "tool", "content": "read complete", "ts": "t4"}, + ) + + assert s.to_dict()["interrupted"] is True + + +def test_halted_hook_inject_is_not_interrupted(): + # A Stop-hook halt card is appended as ``inject`` but dispatched nothing, so + # the deliberately halted run must not read as interrupted. + s = _slot( + {"role": "user", "content": "first", "ts": "t1"}, + {"role": "assistant", "content": "done", "ts": "t2"}, + {"role": "inject", "content": "Stop hook halted #3", "ts": "t3"}, + ) + + assert s.to_dict()["interrupted"] is False + + def test_interrupted_scan_tolerates_non_string_cls(): # A row whose persisted `cls` is object-valued (foreign writer / corrupted # transcript) must not crash the summary scan — `to_dict()` runs on every diff --git a/website/src/store/chatSlice.ts b/website/src/store/chatSlice.ts index 1bc4a9b92299..8d01e2bc1c0c 100644 --- a/website/src/store/chatSlice.ts +++ b/website/src/store/chatSlice.ts @@ -22,6 +22,7 @@ import { safeSetItem } from '../utils/safeStorage' import { errMessage, isMissingSlotError, type StatusRejection } from '../utils/thunkError' import { jsonEqual } from '../utils/structuralEqual' import type { McpAppRenderPayload } from '../lib/mcpAppSrcdoc' +import type { InjectKind } from '../pages/chat/RecoveryCard' import { i18nT } from '../i18n/t' import { secureRandomId } from '../utils/secureId' import { mergeIntoDraft } from '../utils/chatDrafts' @@ -4278,6 +4279,25 @@ export const selectComposerBusy = (state: RootState, slot: string | null): boole * `user` / `assistant` / `error` rows. Keep them in sync — these predicates * decide whether to OFFER Continue and what to call it, those decide whether to * authorize it and what to tell the model. */ +/** `meta.injectKind` values the gateway stamps on an `inject` row that dispatched a + * turn. Every other inject row opens nothing. Mirrors `_TURN_INJECT_KINDS` in + * `dashboard/state.py`. Keyed by `InjectKind` (see `pages/chat/RecoveryCard.tsx`) + * so a new kind does not compile until it is classified here, the same guard + * `INJECT_KIND_OPENS_TURN` carries. Wider than that record on purpose: it + * answers "does this row start a turn the failure streak should count", and + * walks past `recovery` / `user_replay` because they resume the same turn; + * this one answers "did a dispatch happen that got no reply", and a recovery + * or replay dispatch that died is exactly such a turn. */ +const TURN_INJECT_DISPATCHED: Readonly> = { + cron: true, + mcp_app: true, + recovery: true, + synthesis: true, + user_replay: true, +} +const TURN_INJECT_KINDS: ReadonlySet = new Set( + (Object.keys(TURN_INJECT_DISPATCHED) as InjectKind[]).filter((k) => TURN_INJECT_DISPATCHED[k]), +) const CONTINUE_SCAN_SKIP = new Set(['queued', 'tool_call', 'tool_result', 'inject', 'subagent', 'permission', 'nudge']) /** @@ -4404,10 +4424,25 @@ export const selectTurnInterrupted = (state: RootState): boolean => { // segment had flushed first, i.e. on invisible timing the user cannot // predict. The user chose to stop; the floor is theirs, so the composer // shows Send. Reached only for the NEWEST turn's terminator — an older stop - // card deeper in history is never scanned, because a later user/assistant - // row returns first. + // card deeper in history is never scanned, because a later user/inject/ + // assistant row returns first. if (isStopEvent(m)) return false if (m.role === 'error') { sawTrailingError = true; continue } + // An inject row that DISPATCHED a turn (a queued continuation, a recovery, + // a synthesis, a cron prompt) opens it exactly as a user row does, so one + // with no reply after it is an interruption -- and an OLDER Stop card + // behind it must not be reached and mask it. Only the structurally tagged + // kinds qualify: a `/note` breadcrumb, a Stop-hook halt card or a refusal + // notice is appended as `inject` too but ran nothing, and offering Resume + // on a deliberately halted run would be wrong. Decided before + // CONTINUE_SCAN_SKIP, where `inject` stays for the selectors that look + // through continuations to the prior user floor. Mirrors + // `is_turn_interrupted` in `dashboard/state.py`. + if (m.role === 'inject' && m.content && TURN_INJECT_KINDS.has((m.meta as { injectKind?: unknown } | undefined)?.injectKind)) return true + // A monitor loop's cycle row always dispatches a turn; unanswered, it is + // the same shape as an unanswered user row. Decided before the skip set, + // where `nudge` stays for the selectors that look through it. + if (m.role === 'nudge' && m.content) return true if (CONTINUE_SCAN_SKIP.has(m.role)) continue if ((m.role === 'user' || m.role === 'assistant') && m.content) { const meta = m.meta as { kind?: string; notice?: string } | undefined @@ -4431,7 +4466,11 @@ export const selectTurnInterrupted = (state: RootState): boolean => { return true } } - return false + // Ran off the start of the loaded window with no conversational row: a long + // turn can push its opener and reply into the frozen prefix, leaving only + // tool rows here. A trailing error row is still the evidence the assistant + // branch honors, so it decides the same way. + return sawTrailingError } /** Monotonic tick, so an observation can be ordered against a request already in flight. diff --git a/website/src/test/ChatSliceCoverage.test.tsx b/website/src/test/ChatSliceCoverage.test.tsx index a9bca73286ce..83f39876a2ed 100644 --- a/website/src/test/ChatSliceCoverage.test.tsx +++ b/website/src/test/ChatSliceCoverage.test.tsx @@ -945,6 +945,16 @@ describe('chatSlice selectors', () => { expect(selectTurnInterrupted(root(store))).toBe(false) }) + it('treats a trailing dispatching inject as a new, unanswered turn floor', () => { + for (const injectKind of ['cron', 'mcp_app', 'recovery', 'user_replay', 'synthesis']) { + const store = makeStore() + store.dispatch(setActiveSlot('front')) + store.dispatch(appendMessage({ role: 'assistant', content: 'answered' } as ChatMessage)) + store.dispatch(appendMessage({ role: 'inject', content: 'dispatched prompt', meta: { injectKind } } as ChatMessage)) + expect(selectTurnInterrupted(root(store)), injectKind).toBe(true) + } + }) + it('reads an interruption from a trailing user row or an error after the answer', () => { const store = makeStore() store.dispatch(setActiveSlot('front')) @@ -959,6 +969,31 @@ describe('chatSlice selectors', () => { expect(selectTurnInterrupted(root(store))).toBe(true) }) + it('treats a trailing nudge row as a new, unanswered turn floor', () => { + const store = makeStore() + store.dispatch(setActiveSlot('front')) + store.dispatch(appendMessage({ role: 'user', content: 'watch it' } as ChatMessage)) + store.dispatch(appendMessage({ role: 'assistant', content: 'watching' } as ChatMessage)) + store.dispatch(appendMessage({ role: 'inject', content: 'stopped', meta: { kind: 'stop_event' } } as ChatMessage)) + expect(selectTurnInterrupted(root(store))).toBe(false) + store.dispatch(appendMessage({ role: 'nudge', content: '[auto-nudge cycle 2] check', meta: { nudge: { cycle: 2 } } } as ChatMessage)) + expect(selectTurnInterrupted(root(store))).toBe(true) + store.dispatch(appendMessage({ role: 'assistant', content: 'checked' } as ChatMessage)) + expect(selectTurnInterrupted(root(store))).toBe(false) + }) + + it('reads a trailing error when the loaded window holds only tool rows', () => { + // A long turn pushed its opener and reply past the window into the frozen + // prefix; only its tool rows and the restart's error row were loaded. + const store = makeStore() + store.dispatch(setActiveSlot('front')) + store.dispatch(appendMessage({ role: 'tool_result', content: 'read page 1' } as ChatMessage)) + store.dispatch(appendMessage({ role: 'tool_result', content: 'read page 2' } as ChatMessage)) + expect(selectTurnInterrupted(root(store))).toBe(false) + store.dispatch(appendMessage({ role: 'error', content: 'gateway died' } as ChatMessage)) + expect(selectTurnInterrupted(root(store))).toBe(true) + }) + it('treats a deliberate stop as the end of a turn, not an interruption', () => { const store = makeStore() store.dispatch(setActiveSlot('front')) diff --git a/website/src/test/chatSlice.continuable.test.ts b/website/src/test/chatSlice.continuable.test.ts index 6bf5c50880ee..d1bc2b900a42 100644 --- a/website/src/test/chatSlice.continuable.test.ts +++ b/website/src/test/chatSlice.continuable.test.ts @@ -363,11 +363,43 @@ describe('selectTurnInterrupted', () => { } }) - it('reads past an injected recovery row to the real floor beneath it', () => { + it('reads an injected recovery row as a turn opener', () => { expect(selectTurnInterrupted(state({ - messages: [msg('user'), msg('inject', '[Continue — requested by the user]\nresume')], + messages: [msg('user'), msg('inject', '[Continue — requested by the user]\nresume', { injectKind: 'recovery' })], }))).toBe(true) }) + + it('does not let an older Stop mask a newer interrupted inject turn', () => { + for (const injectKind of ['cron', 'recovery', 'user_replay', 'synthesis']) { + expect(selectTurnInterrupted(state({ + messages: [ + msg('user', 'first'), + msg('system', 'Stopped', { kind: 'stop_event' }), + msg('inject', 'continue queued work', { injectKind }), + msg('tool', 'read complete'), + ], + }))).toBe(true) + } + }) + + it('treats an answered dispatching inject as finished', () => { + expect(selectTurnInterrupted(state({ + messages: [msg('user'), msg('inject', 'go on', { injectKind: 'user_replay' }), msg('assistant', 'done')], + }))).toBe(false) + }) + + it('looks through an untagged inject (note, hook halt, refusal notice) to the real floor', () => { + // These rows dispatch nothing, so a trailing one is not an unanswered turn + // and a deliberately halted run must not offer Resume. + for (const meta of [undefined, { noteSession: 'dashboard:front' }, { injectKind: 'unknown' }, { injectKind: 7 }]) { + expect(selectTurnInterrupted(state({ + messages: [msg('user', 'first'), msg('assistant', 'answered'), msg('inject', 'halted', meta)], + }))).toBe(false) + expect(selectTurnInterrupted(state({ + messages: [msg('user', 'first'), msg('inject', 'halted', meta)], + }))).toBe(true) + } + }) }) /**