diff --git a/src/kiro_crew/history_index.py b/src/kiro_crew/history_index.py index 3acf036bbff3..031813f371cd 100644 --- a/src/kiro_crew/history_index.py +++ b/src/kiro_crew/history_index.py @@ -149,12 +149,18 @@ def cjk_inventory(folded: str) -> str: The projection the ``unicode61`` column indexes. Order is stable only to keep the value diffable while debugging; nothing depends on it. + + ``dict.fromkeys`` dedupes the whole string first — a C-level pass whose + result is bounded by the number of DISTINCT characters, never by transcript + length — and ``is_cjk_char`` then classifies only those survivors, a few + thousand calls at most. The naive orderings both lose on what may be + megabytes of transcript on the index write path: classifying per character + holds the GIL for a Python-level walk of the full text, and projecting with + ``_CJK_PATTERN.findall`` materializes one list element per CJK occurrence, + which on a CJK-heavy transcript is a transient allocation proportional to + the whole document. """ - seen: dict[str, None] = {} - for ch in folded: - if ch not in seen and is_cjk_char(ch): - seen[ch] = None - return " ".join(seen) + return " ".join(ch for ch in dict.fromkeys(folded) if is_cjk_char(ch)) def _sqlite_stat_identity(value: int) -> int | str: diff --git a/src/kiro_crew/history_search.py b/src/kiro_crew/history_search.py index 3f07622bca9a..62c2027926d4 100644 --- a/src/kiro_crew/history_search.py +++ b/src/kiro_crew/history_search.py @@ -34,6 +34,14 @@ _TITLE_BOOST = 10 # field-boost multiplier for title matches in search_sessions _PHRASE_BOOST = 4 # extra weight per exact whole-query hit in a multi-word search _SEARCH_SCAN_WINDOW = 500 # cap files scanned per search to bound I/O + +#: Seconds a session's transcript must have gone unmodified before the backfill +#: pass indexes it. A session that is being written changes on every turn, and +#: every change re-reads, re-folds and re-walks the whole file — O(n^2) work +#: over the session's life. Deferring until the file goes quiet indexes it once +#: instead. Until then the session is simply not vouched for, so search scans +#: it directly — the same superset guarantee un-indexed sessions already have. +_INDEX_QUIET_WINDOW_SECS = 120.0 # Recency boost bounds for search_sessions: a session modified now scores # ×(1 + _RECENCY_MAX_BOOST); the extra weight halves every # _RECENCY_HALF_WEIGHT_DAYS of age and decays toward ×1.0 — never a penalty @@ -1314,6 +1322,14 @@ def backfill_index(self, *, budget_secs: float = 5.0) -> dict[str, int]: half-indexed, and a row that describes half a file is exactly the kind of lie this design refuses to store. + A session whose file changed within the last ``_INDEX_QUIET_WINDOW_SECS`` + is deferred, not indexed: it is still being written, and indexing it now + buys a row the next turn invalidates. Deferred sessions are reported in + their own ``deferred`` count, NOT in ``remaining`` — the caller's pass + cadence keys on ``remaining``, and a deferral cannot be serviced by + coming straight back, only by waiting out the window. They are picked up + by the caller's idle-paced passes once quiet. + Rows for sessions that have left the search window are dropped in the same pass. They are unreachable by search (the window is the only thing scored) so keeping them would grow the index without bound while @@ -1321,7 +1337,7 @@ def backfill_index(self, *, budget_secs: float = 5.0) -> dict[str, int]: """ index = self.search_index if not index.available: - return {"indexed": 0, "dropped": 0, "remaining": 0} + return {"indexed": 0, "dropped": 0, "remaining": 0, "deferred": 0} window = self._log.list_sessions()[: _facade_search_scan_window()] window_keys = {meta["key"] for meta in window} stats: dict[str, os.stat_result] = {} @@ -1331,7 +1347,18 @@ def backfill_index(self, *, budget_secs: float = 5.0) -> dict[str, int]: except OSError: continue fresh = index.fresh_keys(stats) - pending = [meta["key"] for meta in window if meta["key"] not in fresh] + quiet_cutoff_ns = _time.time_ns() - int(_INDEX_QUIET_WINDOW_SECS * 1e9) + pending: list[str] = [] + deferred = 0 + for meta in window: + key = meta["key"] + if key in fresh: + continue + st = stats.get(key) + if st is not None and st.st_mtime_ns > quiet_cutoff_ns: + deferred += 1 + continue + pending.append(key) departed = index.indexed_keys() - window_keys index.drop(departed) deadline = _time.monotonic() + budget_secs @@ -1345,6 +1372,7 @@ def backfill_index(self, *, budget_secs: float = 5.0) -> dict[str, int]: "indexed": indexed, "dropped": len(departed), "remaining": len(pending) - indexed, + "deferred": deferred, } def _index_shortlist( diff --git a/test/test_history_index_worker.py b/test/test_history_index_worker.py index e62da21a2389..31cccb06911f 100644 --- a/test/test_history_index_worker.py +++ b/test/test_history_index_worker.py @@ -89,12 +89,30 @@ def _leave_no_children(): _POLL_SECS = 0.1 +def _settle(sessions, *keys): + """Backdate transcripts past the backfill quiet window. + + ``backfill_index`` defers a session whose file changed within + ``history_search._INDEX_QUIET_WINDOW_SECS``: it is treated as still being + written. These tests append and expect the CHILD to index in the same + breath, and a monkeypatch does not reach a spawned interpreter, so the + file's own mtime is moved back instead -- the one signal the deferral reads + that crosses the process boundary. + """ + from kiro_crew import history_search + + settled = time.time() - history_search._INDEX_QUIET_WINDOW_SECS - 600.0 + for key in keys: + os.utime(sessions / f"{key}.jsonl", (settled, settled)) + + def _seeded_sessions(tmp_path): """A session directory with transcripts and no index yet.""" sessions = tmp_path / "sessions" log = ConversationLog(base_dir=sessions) log.append("alpha", "user", "a session about deployment contention") log.append("beta", "user", "a session about astronomy and telescopes") + _settle(sessions, "alpha", "beta") return sessions @@ -800,6 +818,9 @@ def test_the_child_honours_the_cross_process_session_lock(tmp_path): with log._locked("gamma"): log.append("gamma", "user", "a session written while its lock is held") + # Settled while the lock is still held, so the only thing keeping + # the child off this session below is the lock itself. + _settle(sessions, "gamma") # The child now sees an unindexed session it cannot lock. Give it # several passes (its busy pause is 0.1 s here) to try and fail. deadline = time.monotonic() + 2.5 diff --git a/test/test_session_index_hot_path.py b/test/test_session_index_hot_path.py new file mode 100644 index 000000000000..8a1b805cb4fb --- /dev/null +++ b/test/test_session_index_hot_path.py @@ -0,0 +1,153 @@ +"""The index write path's two hot-loop contracts, pinned. + +``cjk_inventory`` is the projection the index's ``unicode61`` column stores, so +its output is on-disk contract: the dedupe-first scan must produce exactly what +the per-character reference walk produces — same characters, same +first-seen order — or existing rows and new rows disagree about the same text. +The reference walk lives HERE, as the regression guard: both classify through +``is_cjk_char``, so any divergence is a bug in the projection, not a drift in +the character set. + +``backfill_index`` defers sessions whose file changed within the quiet window: +a session being written changes every turn, and re-indexing per turn re-reads +the whole transcript each time. The tests pin the deferral (a fresh file is +reported as ``deferred``, never indexed, and never as ``remaining`` — the +caller's busy cadence keys on ``remaining`` and coming straight back cannot +service a deferral), its end (a quiet file is indexed on the next pass), and +that going stale puts an indexed session back behind the window rather than +re-indexing it hot. + +CJK text is written as escapes because this repository forbids literal Chinese +characters in test sources. +""" + +from __future__ import annotations + +import os + +import pytest + +from kiro_crew._sqlite_compat import _CJK_RANGES, fts5_available, is_cjk_char +from kiro_crew.history import ConversationLog +from kiro_crew.history_index import cjk_inventory + +requires_fts5 = pytest.mark.skipif(not fts5_available(), reason="SQLite built without FTS5") + + +def _reference_inventory(folded: str) -> str: + """The per-character walk ``cjk_inventory`` replaced, kept as its oracle.""" + seen: dict[str, None] = {} + for ch in folded: + if ch not in seen and is_cjk_char(ch): + seen[ch] = None + return " ".join(seen) + + +#: Every range edge, its immediate outside neighbours, and representatives of +#: the scripts involved — the exact codepoints where a hand-built character +#: class would get an off-by-one wrong. +_BOUNDARY_CHARS = "".join( + chr(cp) for lo, hi in _CJK_RANGES for cp in (lo - 1, lo, lo + 1, hi - 1, hi, hi + 1) +) + +_CASES = { + "empty": "", + "ascii_only": "the quick brown fox 0123456789 {}[]()", + "cjk_only": "\u4e00\u4e8c\u4e09\u3042\u30a2\u31f0\u3400\uf900", + "mixed": "error in \u4f1a\u8a71 module: \u30c6\u30b9\u30c8 failed at line 42", + "duplicates_keep_first_seen_order": "\u4e09\u4e00\u4e09\u4e8c\u4e00\u4e8c\u4e09", + "astral_extension_b": "before \U00020000\U0002a700\U0002ebef after", + "astral_non_cjk_emoji": "smile \U0001f600 flag \U0001f1ef\U0001f1f5 ok", + "range_boundaries": _BOUNDARY_CHARS, + "boundaries_interleaved_with_ascii": " a ".join(_BOUNDARY_CHARS), + "lone_surrogate_is_not_cjk": "x\ud800y\u4e00z", +} + + +@pytest.mark.parametrize("folded", list(_CASES.values()), ids=list(_CASES.keys())) +def test_cjk_inventory_matches_the_per_character_walk(folded): + assert cjk_inventory(folded) == _reference_inventory(folded) + + +def test_cjk_inventory_first_seen_order_survives_interleaving(): + # Distinct characters arriving out of codepoint order: the inventory must + # report arrival order, which is what makes the value diffable. + folded = "\u9fff a \u4e00 b \u30a2 c \u4e00 d \u9fff" + assert cjk_inventory(folded) == "\u9fff \u4e00 \u30a2" + + +def test_cjk_inventory_of_pure_ascii_is_empty(): + assert cjk_inventory("no ideographs here at all") == "" + + +def _quiet(log: ConversationLog, key: str, secs: float = 300.0) -> None: + """Backdate *key*'s transcript so it reads as untouched for *secs*.""" + path = log._path(key) + st = path.stat() + backdated = st.st_mtime_ns - int(secs * 1e9) + os.utime(path, ns=(st.st_atime_ns, backdated)) + + +@pytest.fixture +def log(tmp_path): + log = ConversationLog(base_dir=tmp_path / "sessions") + try: + yield log + finally: + index = getattr(log._catalog_projection, "_index", None) + if index is not None: + index.close() + + +@requires_fts5 +def test_a_freshly_written_session_is_deferred_not_indexed(log): + log.append("live", "user", "a session still being written") + + report = log._catalog_projection.backfill_index(budget_secs=30) + + assert report["indexed"] == 0 + assert report["deferred"] == 1 + # Deferred is NOT remaining: coming straight back cannot service a + # deferral, so it must not hold the caller at its busy cadence. + assert report["remaining"] == 0 + assert "live" not in log._catalog_projection.search_index.indexed_keys() + + +@requires_fts5 +def test_a_quiet_session_is_indexed(log): + log.append("settled", "user", "a session nobody is writing to") + _quiet(log, "settled") + + report = log._catalog_projection.backfill_index(budget_secs=30) + + assert report["indexed"] == 1 + assert report["remaining"] == 0 + assert report["deferred"] == 0 + assert "settled" in log._catalog_projection.search_index.indexed_keys() + + +@requires_fts5 +def test_a_deferred_session_is_indexed_once_it_goes_quiet(log): + log.append("live", "user", "first turn") + assert log._catalog_projection.backfill_index(budget_secs=30)["indexed"] == 0 + + _quiet(log, "live") + report = log._catalog_projection.backfill_index(budget_secs=30) + + assert report["indexed"] == 1 + assert report["remaining"] == 0 + assert report["deferred"] == 0 + + +@requires_fts5 +def test_an_indexed_session_that_changes_goes_back_behind_the_window(log): + log.append("busy", "user", "first turn") + _quiet(log, "busy") + assert log._catalog_projection.backfill_index(budget_secs=30)["indexed"] == 1 + + log.append("busy", "user", "second turn") # fresh mtime again + report = log._catalog_projection.backfill_index(budget_secs=30) + + assert report["indexed"] == 0 + assert report["deferred"] == 1 + assert report["remaining"] == 0 diff --git a/test/test_session_search_index_equivalence.py b/test/test_session_search_index_equivalence.py index fd971d922806..089202bb1ae2 100644 --- a/test/test_session_search_index_equivalence.py +++ b/test/test_session_search_index_equivalence.py @@ -57,6 +57,17 @@ def _tracking_init(self, *args, **kwargs): pass +@pytest.fixture(autouse=True) +def _no_index_quiet_window(monkeypatch): + """Zero the backfill quiet window: these corpora are written moments before + the backfill they feed, so the production deferral would skip every session + and the assertions below would be measuring the deferral, not the indexing. + """ + from kiro_crew import history_search + + monkeypatch.setattr(history_search, "_INDEX_QUIET_WINDOW_SECS", 0.0) + + # Written as escapes because this repository forbids literal Chinese characters in # source. CJK is not incidental here: it is the script whose gate is made entirely # of 1-character needles, so it exercises the distinct-character column rather than diff --git a/test/test_session_storage_search_index.py b/test/test_session_storage_search_index.py index 9fbc144067d1..f3564e9dea5d 100644 --- a/test/test_session_storage_search_index.py +++ b/test/test_session_storage_search_index.py @@ -23,6 +23,17 @@ pytestmark = pytest.mark.skipif(not fts5_available(), reason="SQLite built without FTS5") +@pytest.fixture(autouse=True) +def _no_index_quiet_window(monkeypatch): + """Zero the backfill quiet window: ``_seeded_log`` appends and indexes in the + same breath, so the production deferral would skip both sessions and the + ``remaining == 0`` seed assertion would fail before any test ran. + """ + from kiro_crew import history_search + + monkeypatch.setattr(history_search, "_INDEX_QUIET_WINDOW_SECS", 0.0) + + def _seeded_log(tmp_path): """A log with two indexed sessions, and the index that vouches for them.""" log = ConversationLog(base_dir=tmp_path / "sessions")