Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions src/kiro_crew/dashboard/channel_slots.py
Original file line number Diff line number Diff line change
Expand Up @@ -670,6 +670,18 @@ def surface_channel_session(
# the frozen prefix a save never rewrites, and redact assistant content at
# the read boundary.
_rebuild_window(slot, messages)
# Channel transcripts bypass both dashboard restore drivers, so the local-turn
# marker a dashboard turn on this slot left behind is reconciled here, once
# the whole window exists. Local import: chat_persistence imports this module.
from kiro_crew.dashboard.chat_persistence import (
_local_turn_generation,
_local_turn_prompt,
_reconcile_local_turn_marker,
)

_reconcile_local_turn_marker(
slot, _local_turn_generation(meta), _local_turn_prompt(meta), persisted=messages
)
# The window corresponds to the file as of this listing, so the refresh pass
# has nothing to do until the channel writes again.
slot._channel_window_mtime = float(session_info.get("modified", 0) or 0)
Expand Down
26 changes: 22 additions & 4 deletions src/kiro_crew/dashboard/chat_handlers.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,9 @@
_TRANSIENT_ROLES,
COLOR_HEX_RE,
_attach_variants,
_local_turn_generation,
_local_turn_prompt,
_reconcile_local_turn_marker,
_rehydrate_slot_title,
_remember_reasoning_effort_for_restore,
_restored_agent_name,
Expand Down Expand Up @@ -11873,7 +11876,12 @@ def _hydrate_slot_from_history(
# by variant count).
for m in messages:
role = m.get("role", "assistant")
cls = "msg msg-u" if role == "user" else "msg msg-a"
# Carry the persisted ``cls`` like the startup restore paths do: a Stop
# card's discriminator lives only in that JSON string (see
# ``is_stop_event_row``), so a resumed session dropping it reads the
# user's deliberate Stop as an interrupted turn. Import rows carry none
# and fall through to the role default as before.
cls = m.get("cls") or ("msg msg-u" if role == "user" else "msg msg-a")
content = m.get("content", "")
slot.append(
role,
Expand All @@ -11896,6 +11904,12 @@ def _hydrate_slot_from_history(
# _disk_older_count above) are the frozen prefix saves never rewrite,
# so older on-disk turns are preserved.
slot._disk_window_len = len(slot.messages)
# After the window boundary, like the startup restore: a local turn the
# previous process admitted and never tore down becomes the interruption
# row here, so a session pulled up off disk and one restored at boot agree.
_reconcile_local_turn_marker(
slot, _local_turn_generation(meta), _local_turn_prompt(meta), persisted=all_messages
)


async def api_chat_slot_resume(request: web.Request) -> web.Response:
Expand Down Expand Up @@ -12297,7 +12311,10 @@ async def api_chat_slot_resume(request: web.Request) -> web.Response:
# Restore the protected choice read before construction, not the
# editable transcript's provisional agent name.
slot.agent = restored_agent
total = len(all_messages)
# Hydrated length, not the raw disk count: materialisation may append one
# unsaved interruption row, and the paging cursor below has to account for
# it or the next older page repeats a row.
total = slot._disk_older_count + len(slot.messages)
recent = slot.messages[-200:] if len(slot.messages) > 200 else slot.messages
# The slot was registered throughout hydration (so a concurrent same-key
# resume resolved it and hit the idempotency guard) but hidden from the
Expand All @@ -12311,8 +12328,9 @@ async def api_chat_slot_resume(request: web.Request) -> web.Response:
{
"ok": True,
"key": slot.key,
# `total` is the full on-disk length here, so this already is the
# raw index the next older page starts from.
# `total` is the effective hydrated length (durable rows plus a
# recovered interruption row when one was appended), so this is
# the raw index the next older page starts from.
"next_before": total - len(recent),
"messages": _prepare_messages(
recent,
Expand Down
258 changes: 257 additions & 1 deletion src/kiro_crew/dashboard/chat_persistence.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
config_dir,
)
from kiro_crew.dashboard.channel_slots import slot_closed_since
from kiro_crew.dashboard.chat_delivery import ATTACHMENT_LIST_MAX_ITEMS, ATTACHMENT_PATH_MAX_LEN
from kiro_crew.dashboard.chat_utils import (
_normalize_model,
_redact_meta_for_role,
Expand Down Expand Up @@ -65,6 +66,8 @@
_normalize_slot_key,
_note_authorized_elsewhere,
durable_row_count,
is_stop_event_row,
is_turn_interrupted,
row_mid,
)
from kiro_crew.effort import EFFORT_LEVELS, EFFORT_VALUES
Expand Down Expand Up @@ -123,6 +126,232 @@
# the persistence module's load path).
_TITLE_ORIGINS = ("auto", "user")

_RESTART_INTERRUPTION_KIND = "gateway_restart_interruption"
# "app", not "gateway": the sibling relay-interruption row in
# ``_rehydrate_slot_from_history`` names the same event the same way, and the
# process name means nothing to the person reading the transcript.
_RESTART_INTERRUPTION_MSG = (
"This turn was interrupted when the app restarted. "
"Review the partial output above, then resume to continue."
)


def _local_turn_generation(meta: Mapping[str, object]) -> int:
"""The persisted local-turn generation, or zero when none is recorded.

``bool`` is an ``int`` subclass, so it is refused explicitly: the metadata
line is an ordinary writable file, and a malformed or hand-edited value
must never manufacture an interruption row.
"""
stored = meta.get("turn_in_flight_generation")
return stored if type(stored) is int and stored > 0 else 0


_LOCAL_TURN_PROMPT_ROLES = frozenset({"user", "nudge", "inject"})
_LOCAL_TURN_PROMPT_META_KEYS = ("mid", "files", "dirs", "injectKind")
#: Bounds on the opening-row copy the marker retains. The attachment bounds are
#: the send path's own (``chat_delivery.ATTACHMENT_LIST_MAX_ITEMS`` entries of
#: ``ATTACHMENT_PATH_MAX_LEN`` chars), so every row the send path accepts fits
#: the copy and a narrower local bound cannot drop an accepted opener. The
#: total is sized for ``MAX_PROMPT_BYTES`` (100 KB) plus two full attachment
#: lists plus the identity fields, so it only rejects a line that no writer of
#: this gateway produced. An out-of-bounds copy is dropped whole, never
#: truncated: the restore re-appends it as the user's own transcript row, and a
#: shortened row would misstate what they sent. A dropped copy leaves the
#: generation alone to flag the turn.
_LOCAL_TURN_PROMPT_MAX_ATTACHMENTS = ATTACHMENT_LIST_MAX_ITEMS
_LOCAL_TURN_PROMPT_MAX_FIELD_CHARS = ATTACHMENT_PATH_MAX_LEN
_LOCAL_TURN_PROMPT_MAX_BYTES = 128 * 1024 + 2 * ATTACHMENT_LIST_MAX_ITEMS * (
ATTACHMENT_PATH_MAX_LEN + 4
)


def local_turn_prompt_within_bounds(prompt: Mapping[str, object]) -> bool:
"""Whether an opening-row copy fits the bounds the marker retains.

Checked at retention (``_local_turn_opening_row``) so an oversize copy is
never written, and again at restore (:func:`_local_turn_prompt`) because
the metadata line is a writable file. ``content`` is bounded only through
the serialized total; ``ts``, ``mid``, ``injectKind`` and each
attachment path are bounded per field, and the attachment lists per count.
"""
ts = prompt.get("ts")
if isinstance(ts, str) and len(ts) > _LOCAL_TURN_PROMPT_MAX_FIELD_CHARS:
return False
meta = prompt.get("meta")
if isinstance(meta, Mapping):
for key in ("mid", "injectKind"):
value = meta.get(key)
if isinstance(value, str) and len(value) > _LOCAL_TURN_PROMPT_MAX_FIELD_CHARS:
return False
for key in ("files", "dirs"):
value = meta.get(key)
if isinstance(value, list):
if len(value) > _LOCAL_TURN_PROMPT_MAX_ATTACHMENTS:
return False
if any(
isinstance(item, str) and len(item) > _LOCAL_TURN_PROMPT_MAX_FIELD_CHARS
for item in value
):
return False
try:
size = len(json.dumps(prompt, ensure_ascii=False, separators=(",", ":")))
except (TypeError, ValueError):
return False
return size <= _LOCAL_TURN_PROMPT_MAX_BYTES


def _local_turn_prompt(meta: Mapping[str, object]) -> dict | None:
"""The persisted copy of the in-flight turn's opening row, validated.

Same trust boundary as :func:`_local_turn_generation`: the line is an
ordinary writable file, so every field is re-checked rather than trusted.
A copy that is not a dict, names a role that never opens a turn, or has
no content is dropped. ``meta`` is reduced to the row's identity, its
attachment lists and the inject kind; a restored row must not arrive
wearing a flag that changes what a later reader does with it.
"""
stored = meta.get("turn_in_flight_prompt")
if not isinstance(stored, dict) or not local_turn_prompt_within_bounds(stored):
return None
role = stored.get("role")
content = stored.get("content")
if role not in _LOCAL_TURN_PROMPT_ROLES or not isinstance(content, str) or not content:
return None
raw_meta = stored.get("meta")
kept_meta: dict = {}
if isinstance(raw_meta, dict):
mid = raw_meta.get("mid")
if isinstance(mid, str) and mid:
kept_meta["mid"] = mid
for key in ("files", "dirs"):
value = raw_meta.get(key)
if isinstance(value, list) and all(isinstance(item, str) for item in value):
kept_meta[key] = list(value)
kind = raw_meta.get("injectKind")
if isinstance(kind, str) and kind:
kept_meta["injectKind"] = kind
ts = stored.get("ts")
# No ``cls`` is read: the transcript never persists one for these roles,
# and a JSON ``cls`` on the re-appended row would be parsed into ``meta``
# on emit, over the identity this copy exists to carry.
return {
"role": role,
"content": content,
"ts": ts if isinstance(ts, str) else "",
"meta": kept_meta,
}


def _window_holds_row(
messages: Iterable[Mapping[str, object]], prompt: Mapping[str, object]
) -> bool:
"""Whether *messages* (a window or the whole on-disk list) holds the marker's opening row.

Matched by ``mid`` when the copy has one, which is the identity every
other dual-writer uses; a copy without an id falls back to the row's
ordering ``ts`` plus role, the pair ``_ChatSlot.append`` makes unique
within one transcript.
"""
meta = prompt.get("meta")
mid = meta.get("mid") if isinstance(meta, dict) else None
ts = prompt.get("ts")
for row in messages:
row_meta = row.get("meta")
if mid and isinstance(row_meta, dict) and row_meta.get("mid") == mid:
return True
if not mid and ts and row.get("ts") == ts and row.get("role") == prompt.get("role"):
return True
return False


def _latest_turn_was_deliberately_stopped(messages: list[dict]) -> bool:
"""Whether the newest turn ends in the user's own Stop card.

Same tail walk as :func:`state.is_turn_interrupted`: tool, notice and
status rows are looked through, and the newest stop or conversational row
decides. Read only when a local-turn marker outlived its process. The stop
handler cancels the runner, and a shutdown landing in that same instant can
leave the marker on disk, so the Stop card -- the user's recorded intent --
must win over a restart-interruption row.
"""
for message in reversed(messages):
if is_stop_event_row(message):
return True
if message.get("role") in ("user", "assistant") and message.get("content"):
return False
return False


def _reconcile_local_turn_marker(
slot: _ChatSlot,
generation: int,
prompt: Mapping[str, object] | None = None,
*,
persisted: Iterable[Mapping[str, object]] | None = None,
) -> bool:
"""Turn a local-turn marker that outlived its process into the interruption row.

Returns whether a marker was present, so the startup restore can avoid
layering the remote-relay notice on top of metadata that claims both kinds
of execution. Call it only after the whole window is loaded and
``_disk_window_len`` is set: rows are appended past that boundary so the
next save writes them rather than counting them as already on disk.

The marker says a turn was admitted and never reached teardown, which the
transcript alone cannot show -- partial assistant prose followed by
finished tool rows is shape-identical to a clean answer. Rows ride the
periodic flush, so the opening row itself may be missing: when the marker
carries a copy (*prompt*) and no row with its identity is on disk, the
copy is appended first. *persisted* is EVERY on-disk row of the session,
not the loaded window: a long turn can push its own opener past the
window bound into the frozen prefix, and a check against the window alone
would append a duplicate that the next save then writes for good. Callers
that hold only the window pass nothing and the window is searched. Only
then is the tail judged: when the transcript already proves the
interruption (an unanswered user row, a trailing error) no second row is
needed; when the newest turn ends in the user's own Stop card, that intent
wins. Otherwise one ``error`` row lands at the tail so the classifier,
composer and sidebar agree.

Every window save rewrites the window from its first row, so a lost
opener implies every row after it is lost too and the tail is where it
belongs. The runtime marker starts clear and the slot is marked dirty even
when no row was appended, so the next save omits the slot-owned keys. A
second restart before that save re-runs this decision from the same bytes
and cannot accumulate rows.
"""
if generation <= 0:
return False
if prompt is not None and not _window_holds_row(
slot.messages if persisted is None else persisted, prompt
):
prompt_meta = prompt.get("meta")
role = str(prompt["role"])
slot.append(
role,
str(prompt["content"]),
# The loader's own default for a row whose ``cls`` is not on disk.
"msg msg-u" if role == "user" else "msg msg-a",
str(prompt.get("ts") or ""),
broadcast=False,
meta=dict(prompt_meta) if isinstance(prompt_meta, dict) and prompt_meta else None,
)
if not is_turn_interrupted(slot.messages) and not _latest_turn_was_deliberately_stopped(
slot.messages
):
slot.append(
"error",
_RESTART_INTERRUPTION_MSG,
"msg msg-err",
broadcast=False,
meta={"kind": _RESTART_INTERRUPTION_KIND},
)
slot._turn_in_flight_generation = 0
slot._turn_in_flight_prompt = None
slot._dirty = True
return True


def _rehydrate_title_origin(titled: bool, stored: object) -> str:
"""Resolve a rehydrated slot's title origin from persisted metadata.
Expand Down Expand Up @@ -1713,6 +1942,8 @@ def _rehydrate_slot_from_history(
# not slot.is_remote`` -> 409 ``remote_binding_incomplete``) plus the
# ``_run_chat`` chokepoint (keyed on ``executor``, not ``is_remote``) refuse
# the send with a message the user can act on, rather than run local.
_local_turn_was_in_flight = _local_turn_generation(meta)
_local_turn_prompt_copy = _local_turn_prompt(meta)
_relay_was_in_flight = False
_executor_meta = meta.get("executor")
_instance_meta = meta.get("instance_id")
Expand Down Expand Up @@ -1977,7 +2208,14 @@ def _rehydrate_slot_from_history(
# turns counted above) is never rewritten.
slot._disk_window_len = len(slot.messages)
slot._dirty = False
if _relay_was_in_flight:
# A local turn admitted by the previous process and never torn down.
# Placed with the relay notice below for the same window-boundary
# reasons; when both markers are present the metadata is inconsistent
# (a slot runs locally OR on a peer), and one row is enough.
_had_local_turn_marker = _reconcile_local_turn_marker(
slot, _local_turn_was_in_flight, _local_turn_prompt_copy, persisted=messages
)
if _relay_was_in_flight and not _had_local_turn_marker:
# The gateway crashed while this slot's turn was executing on the peer
# (flagged in the binding block above). The relay reader died with it
# and the turn's tail was never mirrored here, so the loaded window
Expand Down Expand Up @@ -2513,6 +2751,9 @@ def _apply_recent_session(
# _disk_older_count above) are the frozen prefix saves never rewrite.
slot._disk_window_len = len(slot.messages)
slot._dirty = False
_reconcile_local_turn_marker(
slot, _local_turn_generation(meta), _local_turn_prompt(meta), persisted=messages
)
logger.info("Restored session %s (%s)", slot_name, slot.title)


Expand Down Expand Up @@ -4056,6 +4297,14 @@ def _fresh_fields(meta: dict) -> dict:
fields["channel_origin"] = True
if slot.forked_from is not None:
fields["forked_from"] = slot.forked_from
# CLEARABLE: the merge cannot delete a key and rehydrate reads
# zero as "no local turn outstanding", so the current value is
# written either way -- a forced save after a turn's teardown
# must not leave the admitted generation on disk.
fields["turn_in_flight_generation"] = slot._turn_in_flight_generation
# Same rule for the opening-row copy: None is the cleared value
# (rehydrate reads anything but a dict as "no copy").
fields["turn_in_flight_prompt"] = slot._turn_in_flight_prompt
if slot.executor == "remote" and slot.instance_id and slot.remote_slot:
# All three or none, exactly like the full save: a newborn
# bound to a peer has an EMPTY window until the first relayed
Expand Down Expand Up @@ -4506,6 +4755,13 @@ def _refresh_under_lock(meta: dict) -> bool:
# in-flight, so a True read back on reload is the crash signal
# that triggers the interrupted-turn row.
meta_line["relay_in_flight"] = True
if slot._turn_in_flight_generation > 0:
# Written while a local turn is between admission and teardown
# and omitted otherwise; ``SLOT_OWNED_META_KEYS`` makes that
# omission the durable clear.
meta_line["turn_in_flight_generation"] = slot._turn_in_flight_generation
if slot._turn_in_flight_prompt is not None:
meta_line["turn_in_flight_prompt"] = slot._turn_in_flight_prompt
if slot.folder_id:
meta_line["folder_id"] = slot.folder_id
if slot._channel_folder_filed or existing_meta.get("channel_folder_filed"):
Expand Down
Loading
Loading