diff --git a/skillopt_sleep/harvest.py b/skillopt_sleep/harvest.py index 7a9e2721..6b1708ff 100644 --- a/skillopt_sleep/harvest.py +++ b/skillopt_sleep/harvest.py @@ -63,6 +63,14 @@ def _iter_jsonl(path: str) -> Iterable[Dict[str, Any]]: return +def _safe_mtime(path: str) -> float: + """Return a sortable mtime without failing on a concurrently removed file.""" + try: + return os.path.getmtime(path) + except OSError: + return 0.0 + + def _text_from_content(content: Any) -> str: """Flatten a message.content (str or list of blocks) into text.""" if isinstance(content, str): @@ -354,7 +362,7 @@ def harvest( if fn.endswith(".jsonl") and not fn.startswith("agent-"): paths.append(os.path.join(root, fn)) # newest first by mtime - paths.sort(key=lambda p: os.path.getmtime(p), reverse=True) + paths.sort(key=_safe_mtime, reverse=True) for p in paths: d = digest_transcript(p) diff --git a/skillopt_sleep/harvest_codex.py b/skillopt_sleep/harvest_codex.py index c50a237c..51bc776b 100644 --- a/skillopt_sleep/harvest_codex.py +++ b/skillopt_sleep/harvest_codex.py @@ -15,6 +15,7 @@ _is_meta_prompt, _iter_jsonl, _project_matches, + _safe_mtime, ) from skillopt_sleep.staging import _SECRET_PATTERNS from skillopt_sleep.types import SessionDigest @@ -216,7 +217,7 @@ def harvest_codex( for fn in os.listdir(archived_sessions_dir) if fn.endswith(".jsonl") ] - paths.sort(key=lambda p: os.path.getmtime(p), reverse=True) + paths.sort(key=_safe_mtime, reverse=True) project_hint = invoked_project if scope == "invoked" else "" for path in paths: diff --git a/tests/test_harvest_race.py b/tests/test_harvest_race.py new file mode 100644 index 00000000..1b26064f --- /dev/null +++ b/tests/test_harvest_race.py @@ -0,0 +1,78 @@ +"""Regression tests for transcript files disappearing during discovery.""" +from __future__ import annotations + +import json +import os +import tempfile +import unittest +from unittest import mock + +from skillopt_sleep.harvest import harvest +from skillopt_sleep.harvest_codex import harvest_codex + + +class TestHarvestDiscoveryRace(unittest.TestCase): + def test_claude_harvest_skips_mtime_race(self): + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "session.jsonl") + with open(path, "w", encoding="utf-8") as handle: + handle.write(json.dumps({ + "timestamp": "2026-09-20T00:00:00Z", + "cwd": "/repo", + "message": {"role": "user", "content": "Fix the parser bug"}, + }) + "\n") + handle.write(json.dumps({ + "timestamp": "2026-09-20T00:00:10Z", + "cwd": "/repo", + "message": {"role": "assistant", "content": "The parser is fixed."}, + }) + "\n") + + real_mtime = os.path.getmtime + + def mtime_with_race(candidate): + if candidate == path: + raise FileNotFoundError(candidate) + return real_mtime(candidate) + + with mock.patch("skillopt_sleep.harvest.os.path.getmtime", side_effect=mtime_with_race): + digests = harvest(tmp, scope="all") + + self.assertEqual(len(digests), 1) + self.assertEqual(digests[0].session_id, "session") + + def test_codex_harvest_skips_mtime_race(self): + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "session.jsonl") + with open(path, "w", encoding="utf-8") as handle: + handle.write(json.dumps({ + "timestamp": "2026-09-20T00:00:00Z", + "payload": { + "type": "user_message", + "message": "Fix the parser bug", + "cwd": "/repo", + }, + }) + "\n") + handle.write(json.dumps({ + "timestamp": "2026-09-20T00:00:01Z", + "payload": { + "type": "agent_message", + "message": "The parser is fixed.", + }, + }) + "\n") + + real_mtime = os.path.getmtime + + def mtime_with_race(candidate): + if candidate == path: + raise FileNotFoundError(candidate) + return real_mtime(candidate) + + with mock.patch("skillopt_sleep.harvest.os.path.getmtime", side_effect=mtime_with_race): + digests = harvest_codex(tmp, scope="all") + + self.assertEqual(len(digests), 1) + self.assertEqual(digests[0].session_id, "session") + + +if __name__ == "__main__": + unittest.main()