Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions src/kiro_crew/history_index.py
Original file line number Diff line number Diff line change
Expand Up @@ -149,12 +149,18 @@ def cjk_inventory(folded: str) -> str:

The projection the ``unicode61`` column indexes. Order is stable only to keep
the value diffable while debugging; nothing depends on it.

``dict.fromkeys`` dedupes the whole string first — a C-level pass whose
result is bounded by the number of DISTINCT characters, never by transcript
length — and ``is_cjk_char`` then classifies only those survivors, a few
thousand calls at most. The naive orderings both lose on what may be
megabytes of transcript on the index write path: classifying per character
holds the GIL for a Python-level walk of the full text, and projecting with
``_CJK_PATTERN.findall`` materializes one list element per CJK occurrence,
which on a CJK-heavy transcript is a transient allocation proportional to
the whole document.
"""
seen: dict[str, None] = {}
for ch in folded:
if ch not in seen and is_cjk_char(ch):
seen[ch] = None
return " ".join(seen)
return " ".join(ch for ch in dict.fromkeys(folded) if is_cjk_char(ch))


def _sqlite_stat_identity(value: int) -> int | str:
Expand Down
32 changes: 30 additions & 2 deletions src/kiro_crew/history_search.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,14 @@
_TITLE_BOOST = 10 # field-boost multiplier for title matches in search_sessions
_PHRASE_BOOST = 4 # extra weight per exact whole-query hit in a multi-word search
_SEARCH_SCAN_WINDOW = 500 # cap files scanned per search to bound I/O

#: Seconds a session's transcript must have gone unmodified before the backfill
#: pass indexes it. A session that is being written changes on every turn, and
#: every change re-reads, re-folds and re-walks the whole file — O(n^2) work
#: over the session's life. Deferring until the file goes quiet indexes it once
#: instead. Until then the session is simply not vouched for, so search scans
#: it directly — the same superset guarantee un-indexed sessions already have.
_INDEX_QUIET_WINDOW_SECS = 120.0
# Recency boost bounds for search_sessions: a session modified now scores
# ×(1 + _RECENCY_MAX_BOOST); the extra weight halves every
# _RECENCY_HALF_WEIGHT_DAYS of age and decays toward ×1.0 — never a penalty
Expand Down Expand Up @@ -1314,14 +1322,22 @@ def backfill_index(self, *, budget_secs: float = 5.0) -> dict[str, int]:
half-indexed, and a row that describes half a file is exactly the kind of
lie this design refuses to store.

A session whose file changed within the last ``_INDEX_QUIET_WINDOW_SECS``
is deferred, not indexed: it is still being written, and indexing it now
buys a row the next turn invalidates. Deferred sessions are reported in
their own ``deferred`` count, NOT in ``remaining`` — the caller's pass
cadence keys on ``remaining``, and a deferral cannot be serviced by
coming straight back, only by waiting out the window. They are picked up
by the caller's idle-paced passes once quiet.

Rows for sessions that have left the search window are dropped in the
same pass. They are unreachable by search (the window is the only thing
scored) so keeping them would grow the index without bound while
answering nothing.
"""
index = self.search_index
if not index.available:
return {"indexed": 0, "dropped": 0, "remaining": 0}
return {"indexed": 0, "dropped": 0, "remaining": 0, "deferred": 0}
window = self._log.list_sessions()[: _facade_search_scan_window()]
window_keys = {meta["key"] for meta in window}
stats: dict[str, os.stat_result] = {}
Expand All @@ -1331,7 +1347,18 @@ def backfill_index(self, *, budget_secs: float = 5.0) -> dict[str, int]:
except OSError:
continue
fresh = index.fresh_keys(stats)
pending = [meta["key"] for meta in window if meta["key"] not in fresh]
quiet_cutoff_ns = _time.time_ns() - int(_INDEX_QUIET_WINDOW_SECS * 1e9)
pending: list[str] = []
deferred = 0
for meta in window:
key = meta["key"]
if key in fresh:
continue
st = stats.get(key)
if st is not None and st.st_mtime_ns > quiet_cutoff_ns:
deferred += 1
continue
pending.append(key)
departed = index.indexed_keys() - window_keys
index.drop(departed)
deadline = _time.monotonic() + budget_secs
Expand All @@ -1345,6 +1372,7 @@ def backfill_index(self, *, budget_secs: float = 5.0) -> dict[str, int]:
"indexed": indexed,
"dropped": len(departed),
"remaining": len(pending) - indexed,
"deferred": deferred,
}

def _index_shortlist(
Expand Down
21 changes: 21 additions & 0 deletions test/test_history_index_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,12 +89,30 @@ def _leave_no_children():
_POLL_SECS = 0.1


def _settle(sessions, *keys):
"""Backdate transcripts past the backfill quiet window.

``backfill_index`` defers a session whose file changed within
``history_search._INDEX_QUIET_WINDOW_SECS``: it is treated as still being
written. These tests append and expect the CHILD to index in the same
breath, and a monkeypatch does not reach a spawned interpreter, so the
file's own mtime is moved back instead -- the one signal the deferral reads
that crosses the process boundary.
"""
from kiro_crew import history_search

settled = time.time() - history_search._INDEX_QUIET_WINDOW_SECS - 600.0
for key in keys:
os.utime(sessions / f"{key}.jsonl", (settled, settled))


def _seeded_sessions(tmp_path):
"""A session directory with transcripts and no index yet."""
sessions = tmp_path / "sessions"
log = ConversationLog(base_dir=sessions)
log.append("alpha", "user", "a session about deployment contention")
log.append("beta", "user", "a session about astronomy and telescopes")
_settle(sessions, "alpha", "beta")
return sessions


Expand Down Expand Up @@ -800,6 +818,9 @@ def test_the_child_honours_the_cross_process_session_lock(tmp_path):

with log._locked("gamma"):
log.append("gamma", "user", "a session written while its lock is held")
# Settled while the lock is still held, so the only thing keeping
# the child off this session below is the lock itself.
_settle(sessions, "gamma")
# The child now sees an unindexed session it cannot lock. Give it
# several passes (its busy pause is 0.1 s here) to try and fail.
deadline = time.monotonic() + 2.5
Expand Down
153 changes: 153 additions & 0 deletions test/test_session_index_hot_path.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
"""The index write path's two hot-loop contracts, pinned.

``cjk_inventory`` is the projection the index's ``unicode61`` column stores, so
its output is on-disk contract: the dedupe-first scan must produce exactly what
the per-character reference walk produces — same characters, same
first-seen order — or existing rows and new rows disagree about the same text.
The reference walk lives HERE, as the regression guard: both classify through
``is_cjk_char``, so any divergence is a bug in the projection, not a drift in
the character set.

``backfill_index`` defers sessions whose file changed within the quiet window:
a session being written changes every turn, and re-indexing per turn re-reads
the whole transcript each time. The tests pin the deferral (a fresh file is
reported as ``deferred``, never indexed, and never as ``remaining`` — the
caller's busy cadence keys on ``remaining`` and coming straight back cannot
service a deferral), its end (a quiet file is indexed on the next pass), and
that going stale puts an indexed session back behind the window rather than
re-indexing it hot.

CJK text is written as escapes because this repository forbids literal Chinese
characters in test sources.
"""

from __future__ import annotations

import os

import pytest

from kiro_crew._sqlite_compat import _CJK_RANGES, fts5_available, is_cjk_char
from kiro_crew.history import ConversationLog
from kiro_crew.history_index import cjk_inventory

requires_fts5 = pytest.mark.skipif(not fts5_available(), reason="SQLite built without FTS5")


def _reference_inventory(folded: str) -> str:
"""The per-character walk ``cjk_inventory`` replaced, kept as its oracle."""
seen: dict[str, None] = {}
for ch in folded:
if ch not in seen and is_cjk_char(ch):
seen[ch] = None
return " ".join(seen)


#: Every range edge, its immediate outside neighbours, and representatives of
#: the scripts involved — the exact codepoints where a hand-built character
#: class would get an off-by-one wrong.
_BOUNDARY_CHARS = "".join(
chr(cp) for lo, hi in _CJK_RANGES for cp in (lo - 1, lo, lo + 1, hi - 1, hi, hi + 1)
)

_CASES = {
"empty": "",
"ascii_only": "the quick brown fox 0123456789 {}[]()",
"cjk_only": "\u4e00\u4e8c\u4e09\u3042\u30a2\u31f0\u3400\uf900",
"mixed": "error in \u4f1a\u8a71 module: \u30c6\u30b9\u30c8 failed at line 42",
"duplicates_keep_first_seen_order": "\u4e09\u4e00\u4e09\u4e8c\u4e00\u4e8c\u4e09",
"astral_extension_b": "before \U00020000\U0002a700\U0002ebef after",
"astral_non_cjk_emoji": "smile \U0001f600 flag \U0001f1ef\U0001f1f5 ok",
"range_boundaries": _BOUNDARY_CHARS,
"boundaries_interleaved_with_ascii": " a ".join(_BOUNDARY_CHARS),
"lone_surrogate_is_not_cjk": "x\ud800y\u4e00z",
}


@pytest.mark.parametrize("folded", list(_CASES.values()), ids=list(_CASES.keys()))
def test_cjk_inventory_matches_the_per_character_walk(folded):
assert cjk_inventory(folded) == _reference_inventory(folded)


def test_cjk_inventory_first_seen_order_survives_interleaving():
# Distinct characters arriving out of codepoint order: the inventory must
# report arrival order, which is what makes the value diffable.
folded = "\u9fff a \u4e00 b \u30a2 c \u4e00 d \u9fff"
assert cjk_inventory(folded) == "\u9fff \u4e00 \u30a2"


def test_cjk_inventory_of_pure_ascii_is_empty():
assert cjk_inventory("no ideographs here at all") == ""


def _quiet(log: ConversationLog, key: str, secs: float = 300.0) -> None:
"""Backdate *key*'s transcript so it reads as untouched for *secs*."""
path = log._path(key)
st = path.stat()
backdated = st.st_mtime_ns - int(secs * 1e9)
os.utime(path, ns=(st.st_atime_ns, backdated))


@pytest.fixture
def log(tmp_path):
log = ConversationLog(base_dir=tmp_path / "sessions")
try:
yield log
finally:
index = getattr(log._catalog_projection, "_index", None)
if index is not None:
index.close()


@requires_fts5
def test_a_freshly_written_session_is_deferred_not_indexed(log):
log.append("live", "user", "a session still being written")

report = log._catalog_projection.backfill_index(budget_secs=30)

assert report["indexed"] == 0
assert report["deferred"] == 1
# Deferred is NOT remaining: coming straight back cannot service a
# deferral, so it must not hold the caller at its busy cadence.
assert report["remaining"] == 0
assert "live" not in log._catalog_projection.search_index.indexed_keys()


@requires_fts5
def test_a_quiet_session_is_indexed(log):
log.append("settled", "user", "a session nobody is writing to")
_quiet(log, "settled")

report = log._catalog_projection.backfill_index(budget_secs=30)

assert report["indexed"] == 1
assert report["remaining"] == 0
assert report["deferred"] == 0
assert "settled" in log._catalog_projection.search_index.indexed_keys()


@requires_fts5
def test_a_deferred_session_is_indexed_once_it_goes_quiet(log):
log.append("live", "user", "first turn")
assert log._catalog_projection.backfill_index(budget_secs=30)["indexed"] == 0

_quiet(log, "live")
report = log._catalog_projection.backfill_index(budget_secs=30)

assert report["indexed"] == 1
assert report["remaining"] == 0
assert report["deferred"] == 0


@requires_fts5
def test_an_indexed_session_that_changes_goes_back_behind_the_window(log):
log.append("busy", "user", "first turn")
_quiet(log, "busy")
assert log._catalog_projection.backfill_index(budget_secs=30)["indexed"] == 1

log.append("busy", "user", "second turn") # fresh mtime again
report = log._catalog_projection.backfill_index(budget_secs=30)

assert report["indexed"] == 0
assert report["deferred"] == 1
assert report["remaining"] == 0
11 changes: 11 additions & 0 deletions test/test_session_search_index_equivalence.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,17 @@ def _tracking_init(self, *args, **kwargs):
pass


@pytest.fixture(autouse=True)
def _no_index_quiet_window(monkeypatch):
"""Zero the backfill quiet window: these corpora are written moments before
the backfill they feed, so the production deferral would skip every session
and the assertions below would be measuring the deferral, not the indexing.
"""
from kiro_crew import history_search

monkeypatch.setattr(history_search, "_INDEX_QUIET_WINDOW_SECS", 0.0)


# Written as escapes because this repository forbids literal Chinese characters in
# source. CJK is not incidental here: it is the script whose gate is made entirely
# of 1-character needles, so it exercises the distinct-character column rather than
Expand Down
11 changes: 11 additions & 0 deletions test/test_session_storage_search_index.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,17 @@
pytestmark = pytest.mark.skipif(not fts5_available(), reason="SQLite built without FTS5")


@pytest.fixture(autouse=True)
def _no_index_quiet_window(monkeypatch):
"""Zero the backfill quiet window: ``_seeded_log`` appends and indexes in the
same breath, so the production deferral would skip both sessions and the
``remaining == 0`` seed assertion would fail before any test ran.
"""
from kiro_crew import history_search

monkeypatch.setattr(history_search, "_INDEX_QUIET_WINDOW_SECS", 0.0)


def _seeded_log(tmp_path):
"""A log with two indexed sessions, and the index that vouches for them."""
log = ConversationLog(base_dir=tmp_path / "sessions")
Expand Down
Loading