fix: bound macOS watcher audits without retrieval changes (#756)

README.md

@@ -127,7 +127,8 @@ over stdio, with an optional localhost HTTP server.
   HCL, R, Dart/Flutter, and OCaml; more languages have lightweight outlines.
 - **Navigation:** task context, symbol definitions, callers, dependency graphs,
   trigram search, and portable snapshots.
-- **Live updates:** bounded OS watches with periodic content verification for overflow.
+- **Live updates:** recursive macOS events on local APFS/HFS volumes; bounded OS
+  watches and periodic content verification on fallback paths.
   Each MCP process owns its watcher; multiple clients can multiply background work.
 - **Distribution:** macOS, Linux, and Windows binaries; SHA256-verified downloads.
   macOS releases are signed and notarized.
@@ -429,7 +430,7 @@ The watcher stats `.git/HEAD` mtime before forking `git rev-parse HEAD`. These h
 
 - **Explorer** — structural index engine. Parses Zig, Python, TypeScript/JavaScript, Rust, Go, PHP, Ruby, HCL, R, and Dart. Maintains outlines, trigram index, inverted word index, content cache, and dependency graph behind a single mutex.
 - **Store** — append-only version log. Every recorded file change (snapshot, modification, deletion) gets a monotonically increasing sequence number. Version history capped at 100 per file.
-- **Watcher** — bounded OS watches, 100ms event coalescing, and content verification for overflow about every 2s. `FilteredWalker` prunes `.git`, `node_modules`, `zig-cache`, `__pycache__`, etc. before descending.
+- **Watcher** — recursive file events on local macOS APFS/HFS volumes, with bounded content probes about every 2s and a full directory audit every 60s. Ignore-policy events invalidate affected descendants immediately. Aliases, other volumes, unavailable streams, and other platforms retain bounded OS watches and two-second fallback audits. Set `CODEDB_NO_FSEVENTS=1` to exercise the macOS fallback. `FilteredWalker` prunes `.git`, `node_modules`, `zig-cache`, `__pycache__`, etc. before descending. See the [watcher measurements](docs/watcher-748.md).
 - **Agents** — first-class structs with cursors, heartbeats, and exclusive file locks. Stale agents reaped after 30s.
 
 ### Threading Model

docs/measurements/watcher-748-followup-fallback-final.json

@@ -0,0 +1,57 @@
+{
+  "mode": "forced-fallback",
+  "record_type": "summary_of_historical_stress_receipt",
+  "candidate_sha256": "c47e0224cce9e505fbb4b4622386890b00b9c904763c1f1aca183c4fbee17415",
+  "cases": [
+    {
+      "watch_budget": 32,
+      "clients": 3,
+      "directories": 1200,
+      "source_files": 4800,
+      "phases": [
+        {
+          "phase": "quiet",
+          "cpu_percent_per_client": [5.496953244966012, 5.397008640512084, 4.997230222696374],
+          "wall_seconds": 10.005542624974623
+        },
+        {
+          "phase": "churn",
+          "cpu_percent_per_client": [13.188560790897144, 13.088647451572166, 11.789774040347444],
+          "wall_seconds": 10.008673584088683
+        }
+      ],
+      "checks_passed": 61,
+      "checks_total": 61,
+      "unchanged_under_excluded_churn": true,
+      "churn_pairs": 11991,
+      "alive": true,
+      "rss_kib": [116592, 76192, 76256],
+      "failures": []
+    },
+    {
+      "watch_budget": 1024,
+      "clients": 1,
+      "directories": 1200,
+      "source_files": 4800,
+      "phases": [
+        {
+          "phase": "quiet",
+          "cpu_percent_per_client": [6.793338084503384],
+          "wall_seconds": 10.009806541958824
+        },
+        {
+          "phase": "churn",
+          "cpu_percent_per_client": [13.789510721876548],
+          "wall_seconds": 10.007606707978994
+        }
+      ],
+      "checks_passed": 61,
+      "checks_total": 61,
+      "unchanged_under_excluded_churn": true,
+      "churn_pairs": 10935,
+      "alive": true,
+      "rss_kib": [117296],
+      "failures": []
+    }
+  ]
+}

docs/measurements/watcher-748-followup-final.json

@@ -0,0 +1,57 @@
+{
+  "mode": "recursive-events",
+  "record_type": "summary_of_historical_stress_receipt",
+  "candidate_sha256": "c47e0224cce9e505fbb4b4622386890b00b9c904763c1f1aca183c4fbee17415",
+  "cases": [
+    {
+      "watch_budget": 32,
+      "clients": 3,
+      "directories": 1200,
+      "source_files": 4800,
+      "phases": [
+        {
+          "phase": "quiet",
+          "cpu_percent_per_client": [1.998991604590459, 1.4992437034428447, 1.34931933309856],
+          "wall_seconds": 20.010089040966704
+        },
+        {
+          "phase": "churn",
+          "cpu_percent_per_client": [1.7990985428938013, 1.6991486238441453, 1.7491235833689731],
+          "wall_seconds": 20.010021208785474
+        }
+      ],
+      "checks_passed": 61,
+      "checks_total": 61,
+      "unchanged_under_excluded_churn": true,
+      "churn_pairs": 12795,
+      "alive": true,
+      "rss_kib": [122896, 82512, 82576],
+      "failures": []
+    },
+    {
+      "watch_budget": 1024,
+      "clients": 1,
+      "directories": 1200,
+      "source_files": 4800,
+      "phases": [
+        {
+          "phase": "quiet",
+          "cpu_percent_per_client": [1.2494694102002888],
+          "wall_seconds": 20.00849304185249
+        },
+        {
+          "phase": "churn",
+          "cpu_percent_per_client": [3.3995172048054183],
+          "wall_seconds": 20.00284037506208
+        }
+      ],
+      "checks_passed": 61,
+      "checks_total": 61,
+      "unchanged_under_excluded_churn": true,
+      "churn_pairs": 10989,
+      "alive": true,
+      "rss_kib": [103728],
+      "failures": []
+    }
+  ]
+}

docs/watcher-748.md

@@ -122,3 +122,76 @@ Validation on Apple Silicon macOS:
 Whitespace checks passed. `zig fmt --check` passes changed Zig files except the
 pre-existing reference-array formatting in `src/mcp.zig`, unchanged from the
 base; ranking code was left untouched. No installed client binary was replaced.
+
+## Follow-up candidate after 0.2.5855
+
+The release still hashes overflow files periodically. Profiling the larger
+9,600-file fixture found approximately 40% churn CPU and 14% quiet CPU. Three
+independent clients can multiply that work. This follow-up uses recursive
+FSEvents on local APFS/HFS volumes, avoiding per-file overflow polling when the
+stream covers the entire admitted tree. It uses the macOS system frameworks;
+there is no additional package dependency.
+
+The callback only copies bounded path notifications into a private queue.
+Coalesced or dropped history triggers a full content audit. Root/mount changes,
+directory aliases, foreign volumes, unsupported filesystems or an unavailable
+stream use the existing bounded kqueue/polling path. Set
+`CODEDB_NO_FSEVENTS=1` to force that fallback. A healthy recursive stream
+verifies up to 64 known files every two seconds and runs a full directory audit
+every 60 seconds; explicit policy events invalidate affected descendants
+immediately. The fallback keeps its two-second directory audit and overflow-file
+verification. Missed metadata changes may wait for the audit; exact-metadata
+collisions rely on rotating probes and can take longer in large trees. Dropped
+event history still forces immediate recovery. Each MCP process still owns its
+watcher.
+
+Reconciliation also reuses bounded content scratch storage and retains unchanged
+subtrees when their identities and inherited ignore-policy metadata agree.
+Root and nested policy edits invalidate descendants, including when an ignore
+rule is later removed. A regression test uncovered an existing use-after-free
+in deletion bookkeeping; the mtime entry is now removed before its borrowed
+path is freed. Safe project-handle reads and sensitive-path exclusions remain
+in place.
+
+Use the stronger mutation harness alongside the CPU reproducer:
+
+```sh
+python3 scripts/stress_watcher_748.py \
+  --binary /absolute/path/to/stable/codedb \
+  --out "$HOME/tmp/watcher-events.json" --seconds 20
+CODEDB_NO_FSEVENTS=1 python3 scripts/stress_watcher_748.py \
+  --binary /absolute/path/to/stable/codedb \
+  --out "$HOME/tmp/watcher-fallback.json" --seconds 20
+```
+
+The harness uses 4,800 source files in 1,200 directories, then checks same-size
+edits with restored mtime, removal of old symbols, creation, rename, deletion
+and discovery of new nested directories under continuous excluded-file churn.
+The unit suite separately covers complete metadata collisions. Output paths
+must be new; the runner records the immutable binary hash.
+
+Final ARM64 candidate measurements (20-second windows, one core = 100%):
+
+| Clients / watch budget | Quiet CPU | Churn CPU | Mutation checks |
+| --- | ---: | ---: | ---: |
+| Three / 32 | 2.00%, 1.50%, 1.35% | 1.80%, 1.70%, 1.75% | 61/61 across all three |
+| One / 1,024 | 1.25% | 3.40% | 61/61 |
+
+Index counts and sequences stayed unchanged under excluded churn. All clients
+remained alive. These are short synthetic measurements, not general CPU or
+memory guarantees. The earlier 9,600-file recursive-event prototype measured
+3.15% churn / 2.65% quiet, but the table above is the final candidate's fixture.
+
+Historical receipt summaries: [recursive events](measurements/watcher-748-followup-final.json)
+and [forced fallback](measurements/watcher-748-followup-fallback-final.json).
+They retain CPU, stability, and mutation totals rather than the per-mutation log;
+rerun the harness for raw output.
+
+The final candidate passes the unit suite, 76/76 MCP scenarios and six lazy
+startup scenarios. Descriptor counts are 23 at both 128 and 2,048 files with a
+budget of 32. The Intel macOS build also starts its recursive stream and detects
+an edit under Rosetta. Forcing the fallback also passes all 122 mutation checks
+with stable counters (10-second CPU windows: three-client churn 13.19%, 13.09%,
+11.79%; single-client churn 13.79%). Linux and Windows cross-builds succeed; they have not
+been runtime-tested in this macOS session. This follow-up has not been released,
+notarized or installed over a client binary.

scripts/stress_watcher_748.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+"""Stress live watcher correctness under excluded-file churn and watch overflow.
+
+Run with a frozen ReleaseFast binary and a new --out report beneath an indexable
+scratch directory (for example ~/tmp). Same-size rewrites restore mtime, not
+ctime; the Zig regression suite separately covers fully colliding metadata.
+"""
+import sys, os, json, time, tempfile, subprocess, threading, hashlib, re, argparse
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).parent))
+from e2e_mcp_test import MCPProcess, do_initialize, all_tool_text
+from repro_watcher_748 import cpu_seconds
+
+parser = argparse.ArgumentParser(description=__doc__)
+parser.add_argument('--binary', required=True)
+parser.add_argument('--out', required=True)
+parser.add_argument('--seconds', type=float, default=20)
+args = parser.parse_args()
+if args.seconds <= 0: parser.error('--seconds must be positive')
+REPORT_PATH = Path(args.out).resolve()
+if REPORT_PATH.exists(): parser.error('Use a new output filename')
+OUT = REPORT_PATH.parent
+OUT.mkdir(parents=True, exist_ok=True)
+BINARY = str(Path(args.binary).resolve())
+report = {'binary': BINARY, 'sha256': hashlib.sha256(Path(BINARY).read_bytes()).hexdigest(), 'cases': []}
+
+def text(p, name, args):
+    r = p.call_tool(name, args, timeout=15)
+    if not r or r.get('error') or r.get('result', {}).get('isError'):
+        raise RuntimeError(str(r)[:400])
+    return all_tool_text(r)
+
+def run(budget, count):
+    case = {'budget': budget, 'clients': count, 'directories': 1200, 'source_files': 4800, 'checks': [], 'failures': []}
+    report['cases'].append(case)
+    clients = []
+    stop = threading.Event()
+    churn_count = [0]
+    with tempfile.TemporaryDirectory(prefix='codedb-brute-corpus-', dir=OUT) as tmp:
+        root = Path(tmp)
+        subprocess.run(['git', 'init', '-q', tmp], check=True)
+        (root / '.gitignore').write_text('codedb.snapshot\n.zigrep_archive\n.codedb*\n')
+        (root / '.codedbrc').write_text(f'max_watched = {budget}\n')
+        for d in range(1200):
+            folder = root / f'd{d:04}'
+            folder.mkdir()
+            for f in range(4):
+                (folder / f'f{f}.py').write_text(f'def original_{d:04}_{f}():\n    return 0\n' + '# padding\n' * 600)
+        env = dict(os.environ, CODEDB_NO_AUTO_UPDATE='1', CODEDB_NO_TELEMETRY='1', CODEDB_NO_CLI_DAEMON='1')
+        env.pop('CODEDB_LAZY_MCP', None)
+        try:
+            for _ in range(count):
+                p = MCPProcess(BINARY, [], tmp, command=[BINARY, tmp, 'mcp', f'--config-file={root / ".codedbrc"}', '--no-telemetry'], env=env)
+                clients.append(p)
+                assert do_initialize(p, with_roots=False)
+                deadline = time.monotonic() + 120
+                while 'scan: ready' not in text(p, 'codedb_status', {}).lower():
+                    assert time.monotonic() < deadline, 'scan timeout'
+                    time.sleep(.2)
+            time.sleep(4)
+            def noise():
+                try:
+                    while not stop.is_set():
+                        for folder in (root, root/'d0000', root/'d1199'):
+                            path = folder / 'ignored.lock'
+                            path.write_text('excluded churn')
+                            path.unlink()
+                            churn_count[0] += 1
+                        stop.wait(.005)
+                except Exception as e:
+                    case['failures'].append('churn thread: ' + repr(e))
+            before = [text(p, 'codedb_status', {}) for p in clients]
+            phases = []
+            for label in ('quiet', 'churn'):
+                if label == 'churn':
+                    worker = threading.Thread(target=noise)
+                    worker.start()
+                cpus = [cpu_seconds(p.proc.pid) for p in clients]
+                start = time.monotonic()
+                time.sleep(args.seconds)
+                wall = time.monotonic() - start
+                phases.append({'phase': label, 'cpu_percent_per_client': [100*(cpu_seconds(p.proc.pid)-c)/wall for p,c in zip(clients,cpus)], 'wall_seconds': wall})
+            case['phases'] = phases
+            after = [text(p, 'codedb_status', {}) for p in clients]
+            stable = lambda s: re.findall(r'(?:files|seq)\s*[:=]\s*\d+', s.lower())
+            case['unchanged_under_excluded_churn'] = all(bool(stable(a)) and stable(a)==stable(b) for a,b in zip(before, after))
+            case['status_before'] = before
+            case['status_after_churn'] = after
+            if not case['unchanged_under_excluded_churn']:
+                case['failures'].append('index changed during excluded churn')
+            def await_symbol(name, path, present=True):
+                started = time.monotonic()
+                pending = set(range(len(clients)))
+                while pending and time.monotonic()-started < 15:
+                    for i in list(pending):
+                        value = text(clients[i], 'codedb_symbol', {'name': name})
+                        found = path in value and name in value
+                        if found == present:
+                            pending.remove(i)
+                    if pending: time.sleep(.1)
+                check = {'symbol': name, 'path': path, 'present': present, 'latency_ms': round(1000*(time.monotonic()-started),2), 'passed': not pending}
+                case['checks'].append(check)
+                if pending: case['failures'].append(check)
+            for n in range(10):
+                folder = root / f'd{1190+n:04}'
+                path = folder / 'f0.py'
+                old = path.read_text()
+                name = f'mutation_{1190+n:04}_0'
+                changed = old.replace(f'original_{1190+n:04}_0', name)
+                assert len(changed) == len(old)
+                st = path.stat()
+                path.write_text(changed)
+                os.utime(path, ns=(st.st_atime_ns, st.st_mtime_ns))
+                await_symbol(name, str(path.relative_to(root)))
+                await_symbol(f'original_{1190+n:04}_0', str(path.relative_to(root)), False)
+                new = folder / f'created{n}.py'
+                symbol = f'new_symbol_{n}'
+                new.write_text(f'def {symbol}():\n    return {n}\n')
+                await_symbol(symbol, str(new.relative_to(root)))
+                renamed = folder / f'renamed{n}.py'
+                new.rename(renamed)
+                await_symbol(symbol, str(renamed.relative_to(root)))
+                await_symbol(symbol, str(new.relative_to(root)), False)
+                renamed.unlink()
+                await_symbol(symbol, str(renamed.relative_to(root)), False)
+            # Discovery of a new nested directory while root and overflow paths churn.
+            deep = root / 'fresh' / 'nested'
+            deep.mkdir(parents=True)
+            (deep / 'probe.py').write_text('def nested_created():\n    return 3\n')
+            await_symbol('nested_created', 'fresh/nested/probe.py')
+            stop.set()
+            worker.join()
+            case['churn_pairs'] = churn_count[0]
+            case['alive'] = all(p.proc.poll() is None for p in clients)
+            case['rss_kib'] = [int(subprocess.check_output(['ps','-p',str(p.proc.pid),'-o','rss='],text=True)) for p in clients]
+        except Exception as e:
+            case['failures'].append(repr(e))
+        finally:
+            stop.set()
+            if 'worker' in locals(): worker.join()
+            for p in clients: p.close()
+            REPORT_PATH.write_text(json.dumps(report, indent=2)+'\n')
+            print(json.dumps({k:v for k,v in case.items() if k not in ('status_before','status_after_churn','checks')}), flush=True)
+
+run(32, 3)
+run(1024, 1)
+sys.exit(int(any(c['failures'] for c in report['cases'])))