diff --git a/README.md b/README.md index ee55d495..2f75b2a0 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,8 @@ over stdio, with an optional localhost HTTP server. HCL, R, Dart/Flutter, and OCaml; more languages have lightweight outlines. - **Navigation:** task context, symbol definitions, callers, dependency graphs, trigram search, and portable snapshots. -- **Live updates:** bounded OS watches with periodic content verification for overflow. +- **Live updates:** recursive macOS events on local APFS/HFS volumes; bounded OS + watches and periodic content verification on fallback paths. Each MCP process owns its watcher; multiple clients can multiply background work. - **Distribution:** macOS, Linux, and Windows binaries; SHA256-verified downloads. macOS releases are signed and notarized. @@ -429,7 +430,7 @@ The watcher stats `.git/HEAD` mtime before forking `git rev-parse HEAD`. These h - **Explorer** — structural index engine. Parses Zig, Python, TypeScript/JavaScript, Rust, Go, PHP, Ruby, HCL, R, and Dart. Maintains outlines, trigram index, inverted word index, content cache, and dependency graph behind a single mutex. - **Store** — append-only version log. Every recorded file change (snapshot, modification, deletion) gets a monotonically increasing sequence number. Version history capped at 100 per file. -- **Watcher** — bounded OS watches, 100ms event coalescing, and content verification for overflow about every 2s. `FilteredWalker` prunes `.git`, `node_modules`, `zig-cache`, `__pycache__`, etc. before descending. +- **Watcher** — recursive file events on local macOS APFS/HFS volumes, with bounded content probes about every 2s and a full directory audit every 60s. Ignore-policy events invalidate affected descendants immediately. Aliases, other volumes, unavailable streams, and other platforms retain bounded OS watches and two-second fallback audits. Set `CODEDB_NO_FSEVENTS=1` to exercise the macOS fallback. `FilteredWalker` prunes `.git`, `node_modules`, `zig-cache`, `__pycache__`, etc. before descending. See the [watcher measurements](docs/watcher-748.md). - **Agents** — first-class structs with cursors, heartbeats, and exclusive file locks. Stale agents reaped after 30s. ### Threading Model diff --git a/docs/measurements/watcher-748-followup-fallback-final.json b/docs/measurements/watcher-748-followup-fallback-final.json new file mode 100644 index 00000000..14d3ab6d --- /dev/null +++ b/docs/measurements/watcher-748-followup-fallback-final.json @@ -0,0 +1,57 @@ +{ + "mode": "forced-fallback", + "record_type": "summary_of_historical_stress_receipt", + "candidate_sha256": "c47e0224cce9e505fbb4b4622386890b00b9c904763c1f1aca183c4fbee17415", + "cases": [ + { + "watch_budget": 32, + "clients": 3, + "directories": 1200, + "source_files": 4800, + "phases": [ + { + "phase": "quiet", + "cpu_percent_per_client": [5.496953244966012, 5.397008640512084, 4.997230222696374], + "wall_seconds": 10.005542624974623 + }, + { + "phase": "churn", + "cpu_percent_per_client": [13.188560790897144, 13.088647451572166, 11.789774040347444], + "wall_seconds": 10.008673584088683 + } + ], + "checks_passed": 61, + "checks_total": 61, + "unchanged_under_excluded_churn": true, + "churn_pairs": 11991, + "alive": true, + "rss_kib": [116592, 76192, 76256], + "failures": [] + }, + { + "watch_budget": 1024, + "clients": 1, + "directories": 1200, + "source_files": 4800, + "phases": [ + { + "phase": "quiet", + "cpu_percent_per_client": [6.793338084503384], + "wall_seconds": 10.009806541958824 + }, + { + "phase": "churn", + "cpu_percent_per_client": [13.789510721876548], + "wall_seconds": 10.007606707978994 + } + ], + "checks_passed": 61, + "checks_total": 61, + "unchanged_under_excluded_churn": true, + "churn_pairs": 10935, + "alive": true, + "rss_kib": [117296], + "failures": [] + } + ] +} diff --git a/docs/measurements/watcher-748-followup-final.json b/docs/measurements/watcher-748-followup-final.json new file mode 100644 index 00000000..1a50704d --- /dev/null +++ b/docs/measurements/watcher-748-followup-final.json @@ -0,0 +1,57 @@ +{ + "mode": "recursive-events", + "record_type": "summary_of_historical_stress_receipt", + "candidate_sha256": "c47e0224cce9e505fbb4b4622386890b00b9c904763c1f1aca183c4fbee17415", + "cases": [ + { + "watch_budget": 32, + "clients": 3, + "directories": 1200, + "source_files": 4800, + "phases": [ + { + "phase": "quiet", + "cpu_percent_per_client": [1.998991604590459, 1.4992437034428447, 1.34931933309856], + "wall_seconds": 20.010089040966704 + }, + { + "phase": "churn", + "cpu_percent_per_client": [1.7990985428938013, 1.6991486238441453, 1.7491235833689731], + "wall_seconds": 20.010021208785474 + } + ], + "checks_passed": 61, + "checks_total": 61, + "unchanged_under_excluded_churn": true, + "churn_pairs": 12795, + "alive": true, + "rss_kib": [122896, 82512, 82576], + "failures": [] + }, + { + "watch_budget": 1024, + "clients": 1, + "directories": 1200, + "source_files": 4800, + "phases": [ + { + "phase": "quiet", + "cpu_percent_per_client": [1.2494694102002888], + "wall_seconds": 20.00849304185249 + }, + { + "phase": "churn", + "cpu_percent_per_client": [3.3995172048054183], + "wall_seconds": 20.00284037506208 + } + ], + "checks_passed": 61, + "checks_total": 61, + "unchanged_under_excluded_churn": true, + "churn_pairs": 10989, + "alive": true, + "rss_kib": [103728], + "failures": [] + } + ] +} diff --git a/docs/watcher-748.md b/docs/watcher-748.md index c93d453e..d1fc0f08 100644 --- a/docs/watcher-748.md +++ b/docs/watcher-748.md @@ -122,3 +122,76 @@ Validation on Apple Silicon macOS: Whitespace checks passed. `zig fmt --check` passes changed Zig files except the pre-existing reference-array formatting in `src/mcp.zig`, unchanged from the base; ranking code was left untouched. No installed client binary was replaced. + +## Follow-up candidate after 0.2.5855 + +The release still hashes overflow files periodically. Profiling the larger +9,600-file fixture found approximately 40% churn CPU and 14% quiet CPU. Three +independent clients can multiply that work. This follow-up uses recursive +FSEvents on local APFS/HFS volumes, avoiding per-file overflow polling when the +stream covers the entire admitted tree. It uses the macOS system frameworks; +there is no additional package dependency. + +The callback only copies bounded path notifications into a private queue. +Coalesced or dropped history triggers a full content audit. Root/mount changes, +directory aliases, foreign volumes, unsupported filesystems or an unavailable +stream use the existing bounded kqueue/polling path. Set +`CODEDB_NO_FSEVENTS=1` to force that fallback. A healthy recursive stream +verifies up to 64 known files every two seconds and runs a full directory audit +every 60 seconds; explicit policy events invalidate affected descendants +immediately. The fallback keeps its two-second directory audit and overflow-file +verification. Missed metadata changes may wait for the audit; exact-metadata +collisions rely on rotating probes and can take longer in large trees. Dropped +event history still forces immediate recovery. Each MCP process still owns its +watcher. + +Reconciliation also reuses bounded content scratch storage and retains unchanged +subtrees when their identities and inherited ignore-policy metadata agree. +Root and nested policy edits invalidate descendants, including when an ignore +rule is later removed. A regression test uncovered an existing use-after-free +in deletion bookkeeping; the mtime entry is now removed before its borrowed +path is freed. Safe project-handle reads and sensitive-path exclusions remain +in place. + +Use the stronger mutation harness alongside the CPU reproducer: + +```sh +python3 scripts/stress_watcher_748.py \ + --binary /absolute/path/to/stable/codedb \ + --out "$HOME/tmp/watcher-events.json" --seconds 20 +CODEDB_NO_FSEVENTS=1 python3 scripts/stress_watcher_748.py \ + --binary /absolute/path/to/stable/codedb \ + --out "$HOME/tmp/watcher-fallback.json" --seconds 20 +``` + +The harness uses 4,800 source files in 1,200 directories, then checks same-size +edits with restored mtime, removal of old symbols, creation, rename, deletion +and discovery of new nested directories under continuous excluded-file churn. +The unit suite separately covers complete metadata collisions. Output paths +must be new; the runner records the immutable binary hash. + +Final ARM64 candidate measurements (20-second windows, one core = 100%): + +| Clients / watch budget | Quiet CPU | Churn CPU | Mutation checks | +| --- | ---: | ---: | ---: | +| Three / 32 | 2.00%, 1.50%, 1.35% | 1.80%, 1.70%, 1.75% | 61/61 across all three | +| One / 1,024 | 1.25% | 3.40% | 61/61 | + +Index counts and sequences stayed unchanged under excluded churn. All clients +remained alive. These are short synthetic measurements, not general CPU or +memory guarantees. The earlier 9,600-file recursive-event prototype measured +3.15% churn / 2.65% quiet, but the table above is the final candidate's fixture. + +Historical receipt summaries: [recursive events](measurements/watcher-748-followup-final.json) +and [forced fallback](measurements/watcher-748-followup-fallback-final.json). +They retain CPU, stability, and mutation totals rather than the per-mutation log; +rerun the harness for raw output. + +The final candidate passes the unit suite, 76/76 MCP scenarios and six lazy +startup scenarios. Descriptor counts are 23 at both 128 and 2,048 files with a +budget of 32. The Intel macOS build also starts its recursive stream and detects +an edit under Rosetta. Forcing the fallback also passes all 122 mutation checks +with stable counters (10-second CPU windows: three-client churn 13.19%, 13.09%, +11.79%; single-client churn 13.79%). Linux and Windows cross-builds succeed; they have not +been runtime-tested in this macOS session. This follow-up has not been released, +notarized or installed over a client binary. diff --git a/scripts/stress_watcher_748.py b/scripts/stress_watcher_748.py new file mode 100644 index 00000000..9b530126 --- /dev/null +++ b/scripts/stress_watcher_748.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Stress live watcher correctness under excluded-file churn and watch overflow. + +Run with a frozen ReleaseFast binary and a new --out report beneath an indexable +scratch directory (for example ~/tmp). Same-size rewrites restore mtime, not +ctime; the Zig regression suite separately covers fully colliding metadata. +""" +import sys, os, json, time, tempfile, subprocess, threading, hashlib, re, argparse +from pathlib import Path +sys.path.insert(0, str(Path(__file__).parent)) +from e2e_mcp_test import MCPProcess, do_initialize, all_tool_text +from repro_watcher_748 import cpu_seconds + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--binary', required=True) +parser.add_argument('--out', required=True) +parser.add_argument('--seconds', type=float, default=20) +args = parser.parse_args() +if args.seconds <= 0: parser.error('--seconds must be positive') +REPORT_PATH = Path(args.out).resolve() +if REPORT_PATH.exists(): parser.error('Use a new output filename') +OUT = REPORT_PATH.parent +OUT.mkdir(parents=True, exist_ok=True) +BINARY = str(Path(args.binary).resolve()) +report = {'binary': BINARY, 'sha256': hashlib.sha256(Path(BINARY).read_bytes()).hexdigest(), 'cases': []} + +def text(p, name, args): + r = p.call_tool(name, args, timeout=15) + if not r or r.get('error') or r.get('result', {}).get('isError'): + raise RuntimeError(str(r)[:400]) + return all_tool_text(r) + +def run(budget, count): + case = {'budget': budget, 'clients': count, 'directories': 1200, 'source_files': 4800, 'checks': [], 'failures': []} + report['cases'].append(case) + clients = [] + stop = threading.Event() + churn_count = [0] + with tempfile.TemporaryDirectory(prefix='codedb-brute-corpus-', dir=OUT) as tmp: + root = Path(tmp) + subprocess.run(['git', 'init', '-q', tmp], check=True) + (root / '.gitignore').write_text('codedb.snapshot\n.zigrep_archive\n.codedb*\n') + (root / '.codedbrc').write_text(f'max_watched = {budget}\n') + for d in range(1200): + folder = root / f'd{d:04}' + folder.mkdir() + for f in range(4): + (folder / f'f{f}.py').write_text(f'def original_{d:04}_{f}():\n return 0\n' + '# padding\n' * 600) + env = dict(os.environ, CODEDB_NO_AUTO_UPDATE='1', CODEDB_NO_TELEMETRY='1', CODEDB_NO_CLI_DAEMON='1') + env.pop('CODEDB_LAZY_MCP', None) + try: + for _ in range(count): + p = MCPProcess(BINARY, [], tmp, command=[BINARY, tmp, 'mcp', f'--config-file={root / ".codedbrc"}', '--no-telemetry'], env=env) + clients.append(p) + assert do_initialize(p, with_roots=False) + deadline = time.monotonic() + 120 + while 'scan: ready' not in text(p, 'codedb_status', {}).lower(): + assert time.monotonic() < deadline, 'scan timeout' + time.sleep(.2) + time.sleep(4) + def noise(): + try: + while not stop.is_set(): + for folder in (root, root/'d0000', root/'d1199'): + path = folder / 'ignored.lock' + path.write_text('excluded churn') + path.unlink() + churn_count[0] += 1 + stop.wait(.005) + except Exception as e: + case['failures'].append('churn thread: ' + repr(e)) + before = [text(p, 'codedb_status', {}) for p in clients] + phases = [] + for label in ('quiet', 'churn'): + if label == 'churn': + worker = threading.Thread(target=noise) + worker.start() + cpus = [cpu_seconds(p.proc.pid) for p in clients] + start = time.monotonic() + time.sleep(args.seconds) + wall = time.monotonic() - start + phases.append({'phase': label, 'cpu_percent_per_client': [100*(cpu_seconds(p.proc.pid)-c)/wall for p,c in zip(clients,cpus)], 'wall_seconds': wall}) + case['phases'] = phases + after = [text(p, 'codedb_status', {}) for p in clients] + stable = lambda s: re.findall(r'(?:files|seq)\s*[:=]\s*\d+', s.lower()) + case['unchanged_under_excluded_churn'] = all(bool(stable(a)) and stable(a)==stable(b) for a,b in zip(before, after)) + case['status_before'] = before + case['status_after_churn'] = after + if not case['unchanged_under_excluded_churn']: + case['failures'].append('index changed during excluded churn') + def await_symbol(name, path, present=True): + started = time.monotonic() + pending = set(range(len(clients))) + while pending and time.monotonic()-started < 15: + for i in list(pending): + value = text(clients[i], 'codedb_symbol', {'name': name}) + found = path in value and name in value + if found == present: + pending.remove(i) + if pending: time.sleep(.1) + check = {'symbol': name, 'path': path, 'present': present, 'latency_ms': round(1000*(time.monotonic()-started),2), 'passed': not pending} + case['checks'].append(check) + if pending: case['failures'].append(check) + for n in range(10): + folder = root / f'd{1190+n:04}' + path = folder / 'f0.py' + old = path.read_text() + name = f'mutation_{1190+n:04}_0' + changed = old.replace(f'original_{1190+n:04}_0', name) + assert len(changed) == len(old) + st = path.stat() + path.write_text(changed) + os.utime(path, ns=(st.st_atime_ns, st.st_mtime_ns)) + await_symbol(name, str(path.relative_to(root))) + await_symbol(f'original_{1190+n:04}_0', str(path.relative_to(root)), False) + new = folder / f'created{n}.py' + symbol = f'new_symbol_{n}' + new.write_text(f'def {symbol}():\n return {n}\n') + await_symbol(symbol, str(new.relative_to(root))) + renamed = folder / f'renamed{n}.py' + new.rename(renamed) + await_symbol(symbol, str(renamed.relative_to(root))) + await_symbol(symbol, str(new.relative_to(root)), False) + renamed.unlink() + await_symbol(symbol, str(renamed.relative_to(root)), False) + # Discovery of a new nested directory while root and overflow paths churn. + deep = root / 'fresh' / 'nested' + deep.mkdir(parents=True) + (deep / 'probe.py').write_text('def nested_created():\n return 3\n') + await_symbol('nested_created', 'fresh/nested/probe.py') + stop.set() + worker.join() + case['churn_pairs'] = churn_count[0] + case['alive'] = all(p.proc.poll() is None for p in clients) + case['rss_kib'] = [int(subprocess.check_output(['ps','-p',str(p.proc.pid),'-o','rss='],text=True)) for p in clients] + except Exception as e: + case['failures'].append(repr(e)) + finally: + stop.set() + if 'worker' in locals(): worker.join() + for p in clients: p.close() + REPORT_PATH.write_text(json.dumps(report, indent=2)+'\n') + print(json.dumps({k:v for k,v in case.items() if k not in ('status_before','status_after_churn','checks')}), flush=True) + +run(32, 3) +run(1024, 1) +sys.exit(int(any(c['failures'] for c in report['cases']))) diff --git a/src/macos_events.zig b/src/macos_events.zig new file mode 100644 index 00000000..557ce3e8 --- /dev/null +++ b/src/macos_events.zig @@ -0,0 +1,293 @@ +//! Recursive notifications for local APFS/HFS projects. Other filesystems and +//! unavailable streams keep the existing bounded kqueue/polling watcher. +const std = @import("std"); +const builtin = @import("builtin"); +const cio = @import("cio.zig"); + +const Ref = *anyopaque; +const Context = extern struct { version: isize = 0, info: ?*anyopaque, retain: ?*anyopaque = null, release: ?*anyopaque = null, describe: ?*anyopaque = null }; +const Callback = *const fn (Ref, ?*anyopaque, usize, *anyopaque, [*]const u32, [*]const u64) callconv(.c) void; +const Api = struct { + CFStringCreateWithCString: *const fn (?Ref, [*:0]const u8, u32) callconv(.c) ?Ref, + CFArrayCreate: *const fn (?Ref, [*]const Ref, isize, ?*const anyopaque) callconv(.c) ?Ref, + CFRelease: *const fn (Ref) callconv(.c) void, + FSEventStreamCreate: *const fn (?Ref, Callback, *Context, Ref, u64, f64, u32) callconv(.c) ?Ref, + FSEventStreamSetDispatchQueue: *const fn (Ref, Ref) callconv(.c) void, + FSEventStreamStart: *const fn (Ref) callconv(.c) u8, + FSEventStreamStop: *const fn (Ref) callconv(.c) void, + FSEventStreamInvalidate: *const fn (Ref) callconv(.c) void, + FSEventStreamRelease: *const fn (Ref) callconv(.c) void, +}; +extern "c" fn dispatch_queue_create([*:0]const u8, ?Ref) ?Ref; +extern "c" fn dispatch_sync_f(Ref, ?*anyopaque, *const fn (?*anyopaque) callconv(.c) void) void; +extern "c" fn dispatch_release(Ref) void; + +// Darwin's 64-bit statfs ABI, from sys/mount.h. Restrict the optimization to +// local filesystems whose recursive notification support is known. +const Statfs = extern struct { + bsize: u32, + iosize: i32, + blocks: u64, + bfree: u64, + bavail: u64, + files: u64, + ffree: u64, + fsid: [2]i32, + owner: u32, + kind: u32, + flags: u32, + subtype: u32, + typename: [16]u8, + mounted_on: [1024]u8, + mounted_from: [1024]u8, + flags_ext: u32, + reserved: [7]u32, +}; +extern "c" fn fstatfs(c_int, *Statfs) c_int; +extern "c" fn @"fstatfs$INODE64"(c_int, *Statfs) c_int; +fn statVolume(fd: c_int, out: *Statfs) c_int { + return if (builtin.cpu.arch == .x86_64) @"fstatfs$INODE64"(fd, out) else fstatfs(fd, out); +} + +pub fn sameVolume(a: std.Io.Dir, b: std.Io.Dir) bool { + if (comptime builtin.os.tag != .macos) return true; + var first: Statfs = undefined; + var second: Statfs = undefined; + return statVolume(a.handle, &first) == 0 and statVolume(b.handle, &second) == 0 and + first.flags & 0x1000 != 0 and second.flags & 0x1000 != 0 and std.meta.eql(first.fsid, second.fsid); +} + +pub const Stream = struct { + alloc: std.mem.Allocator, + cf: std.DynLib, + fs: std.DynLib, + api: Api, + stream: Ref, + dispatch: Ref, + root_string: Ref, + roots: Ref, + root: [:0]u8, + mu: cio.Mutex = .{}, + pending: std.StringHashMap(void), + exclude_path: ?*const fn ([]const u8, bool) bool = null, + rescan: bool = false, + fallback: bool = false, + + pub fn create(alloc: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, exclude_path: *const fn ([]const u8, bool) bool) !?*Stream { + if (comptime builtin.os.tag != .macos) return null; + if (cio.posixGetenv("CODEDB_NO_FSEVENTS") != null) return null; + var stat: Statfs = undefined; + if (statVolume(dir.handle, &stat) != 0 or stat.flags & 0x1000 == 0) return null; // MNT_LOCAL + const kind = std.mem.sliceTo(&stat.typename, 0); + if (!std.mem.eql(u8, kind, "apfs") and !std.mem.eql(u8, kind, "hfs")) return null; + var root_buf: [std.fs.max_path_bytes]u8 = undefined; + const n = try dir.realPath(io, &root_buf); + const root = try alloc.dupeSentinel(u8, root_buf[0..n], 0); + errdefer alloc.free(root); + var cf = try std.DynLib.open("/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation"); + errdefer cf.close(); + var fs = try std.DynLib.open("/System/Library/Frameworks/CoreServices.framework/Frameworks/FSEvents.framework/FSEvents"); + errdefer fs.close(); + var api: Api = undefined; + inline for (@typeInfo(Api).@"struct".field_names, @typeInfo(Api).@"struct".field_types) |field_name, field_type| { + const lib = if (comptime std.mem.startsWith(u8, field_name, "CF")) &cf else &fs; + @field(api, field_name) = lib.lookup(field_type, field_name) orelse return error.MissingFSEventsSymbol; + } + const string = api.CFStringCreateWithCString(null, root, 0x08000100) orelse return error.OutOfMemory; + errdefer api.CFRelease(string); + // Keep string alive explicitly: this array intentionally has no callbacks. + const roots = api.CFArrayCreate(null, &.{string}, 1, null) orelse return error.OutOfMemory; + errdefer api.CFRelease(roots); + const queue = dispatch_queue_create("codedb.filesystem", null) orelse return error.OutOfMemory; + errdefer dispatch_release(queue); + const self = try alloc.create(Stream); + errdefer alloc.destroy(self); + self.* = .{ .alloc = alloc, .cf = cf, .fs = fs, .api = api, .stream = undefined, .dispatch = queue, .root_string = string, .roots = roots, .root = root, .pending = .init(alloc), .exclude_path = exclude_path }; + errdefer { + var keys = self.pending.keyIterator(); + while (keys.next()) |key| alloc.free(key.*); + self.pending.deinit(); + } + var context: Context = .{ .info = self }; + // SinceNow; NoDefer | WatchRoot | FileEvents. Coalesced/dropped history + // always requests a full content audit rather than assuming completeness. + const stream = api.FSEventStreamCreate(null, receive, &context, roots, std.math.maxInt(u64), 0.1, 0x16) orelse return error.FSEventsUnavailable; + self.stream = stream; + errdefer api.FSEventStreamRelease(stream); + api.FSEventStreamSetDispatchQueue(stream, queue); + if (api.FSEventStreamStart(stream) == 0) { + api.FSEventStreamInvalidate(stream); + dispatch_sync_f(queue, null, barrier); + return error.FSEventsUnavailable; + } + return self; + } + + fn barrier(_: ?*anyopaque) callconv(.c) void {} + + pub fn destroy(self: *Stream) void { + if (comptime builtin.os.tag != .macos) return; + self.api.FSEventStreamStop(self.stream); + self.api.FSEventStreamInvalidate(self.stream); + // Drain this private serial queue before freeing callback context. + dispatch_sync_f(self.dispatch, null, barrier); + self.api.FSEventStreamRelease(self.stream); + dispatch_release(self.dispatch); + var it = self.pending.keyIterator(); + while (it.next()) |key| self.alloc.free(key.*); + self.pending.deinit(); + self.api.CFRelease(self.roots); + self.api.CFRelease(self.root_string); + self.cf.close(); + self.fs.close(); + self.alloc.free(self.root); + const alloc = self.alloc; + alloc.destroy(self); + } + + fn receive(_: Ref, info: ?*anyopaque, count: usize, paths_raw: *anyopaque, flags: [*]const u32, _: [*]const u64) callconv(.c) void { + const self: *Stream = @ptrCast(@alignCast(info.?)); + const paths: [*]const [*:0]const u8 = @ptrCast(@alignCast(paths_raw)); + self.mu.lock(); + defer self.mu.unlock(); + for (0..count) |i| { + if (flags[i] & 0xe0 != 0) self.fallback = true; // root/mount changes + if (flags[i] & 0x0f != 0) self.rescan = true; // dropped/coalesced history + const full = std.mem.span(paths[i]); + if (!std.mem.startsWith(u8, full, self.root)) { + self.rescan = true; + continue; + } + const suffix = full[self.root.len..]; + if (suffix.len > 0 and suffix[0] != '/') { + self.rescan = true; + continue; + } + const rel = std.mem.trim(u8, suffix, "/"); + if (self.exclude_path) |excluded| { + // FileEvents reports item type. Unknown types and symlinks may + // be directories, so never filter them by file extension. + const is_file = flags[i] & 0x00010000 != 0 and flags[i] & 0x00060000 == 0; + if (excluded(rel, is_file)) continue; + } + if (self.pending.contains(rel)) continue; + if (self.pending.count() >= 2048) { + self.rescan = true; + continue; + } + const copy = self.alloc.dupe(u8, rel) catch { + self.rescan = true; + continue; + }; + self.pending.put(copy, {}) catch { + self.alloc.free(copy); + self.rescan = true; + }; + } + } + + pub const Drain = struct { rescan: bool, fallback: bool }; + pub fn drain(self: *Stream, dirty: *std.StringHashMap(void)) Drain { + self.mu.lock(); + defer self.mu.unlock(); + var it = self.pending.keyIterator(); + while (it.next()) |key| { + if (dirty.contains(key.*)) { + self.alloc.free(key.*); + continue; + } + const copy = dirty.allocator.dupe(u8, key.*) catch { + self.rescan = true; + self.alloc.free(key.*); + continue; + }; + dirty.put(copy, {}) catch { + dirty.allocator.free(copy); + self.rescan = true; + }; + self.alloc.free(key.*); + } + self.pending.clearRetainingCapacity(); + const result: Drain = .{ .rescan = self.rescan, .fallback = self.fallback }; + self.rescan = false; + return result; + } +}; + +test "recursive events coalesce paths and surface dropped history and root changes" { + const t = std.testing; + var stream: Stream = undefined; + stream.alloc = t.allocator; + stream.root = @constCast("/project"); + stream.mu = .{}; + stream.pending = .init(t.allocator); + stream.exclude_path = null; + stream.rescan = false; + stream.fallback = false; + defer { + var it = stream.pending.keyIterator(); + while (it.next()) |key| t.allocator.free(key.*); + stream.pending.deinit(); + } + var dirty = std.StringHashMap(void).init(t.allocator); + defer { + var it = dirty.keyIterator(); + while (it.next()) |key| t.allocator.free(key.*); + dirty.deinit(); + } + const paths = [_][*:0]const u8{ "/project/src/a.py", "/project/src/a.py" }; + Stream.receive(undefined, &stream, paths.len, @ptrCast(@constCast(&paths)), &.{ 0x1000, 0x1000 }, &.{ 1, 2 }); + try t.expectEqual(@as(u32, 1), stream.pending.count()); + const first = stream.drain(&dirty); + try t.expect(!first.rescan and !first.fallback); + try t.expect(dirty.contains("src/a.py")); + // Draining a duplicate into a nonempty dirty set must not leak its key. + Stream.receive(undefined, &stream, 1, @ptrCast(@constCast(&paths)), &.{0x1}, &.{3}); + try t.expect(stream.drain(&dirty).rescan); + Stream.receive(undefined, &stream, 1, @ptrCast(@constCast(&paths)), &.{0x20}, &.{4}); + try t.expect(stream.drain(&dirty).fallback); + stream.fallback = false; + const outside = [_][*:0]const u8{"/project-other/private.py"}; + Stream.receive(undefined, &stream, 1, @ptrCast(@constCast(&outside)), &.{0}, &.{5}); + try t.expect(stream.drain(&dirty).rescan); + try t.expectEqual(@as(u32, 1), dirty.count()); +} + +test "recursive events exclude ignored bursts before the pending cap" { + const t = std.testing; + var stream: Stream = undefined; + stream.alloc = t.allocator; + stream.root = @constCast("/project"); + stream.mu = .{}; + stream.pending = .init(t.allocator); + stream.exclude_path = &struct { + fn check(path: []const u8, is_file: bool) bool { + return std.mem.startsWith(u8, path, "node_modules/") or + (is_file and std.mem.endsWith(u8, path, ".lock")); + } + }.check; + stream.rescan = false; + stream.fallback = false; + defer { + var it = stream.pending.keyIterator(); + while (it.next()) |key| t.allocator.free(key.*); + stream.pending.deinit(); + } + + for (0..2050) |i| { + var path_buf: [80]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "/project/node_modules/file-{d}.js", .{i}); + path_buf[path.len] = 0; + const path_z: [:0]u8 = path_buf[0..path.len :0]; + const one = [_][*:0]const u8{path_z.ptr}; + Stream.receive(undefined, &stream, 1, @ptrCast(@constCast(&one)), &.{0}, &.{0}); + } + try t.expectEqual(@as(u32, 0), stream.pending.count()); + try t.expect(!stream.rescan); + const relevant = [_][*:0]const u8{ "/project/src/main.zig", "/project/generated.lock", "/project/ignored.lock", "/project/ambiguous.lock" }; + Stream.receive(undefined, &stream, relevant.len, @ptrCast(@constCast(&relevant)), &.{ 0x00010000, 0x00020000, 0x00010000, 0 }, &.{ 0, 0, 0, 0 }); + try t.expect(stream.pending.contains("src/main.zig")); + try t.expect(stream.pending.contains("generated.lock")); + try t.expect(!stream.pending.contains("ignored.lock")); + try t.expect(stream.pending.contains("ambiguous.lock")); +} diff --git a/src/test_index.zig b/src/test_index.zig index f4d9dc01..d4df60b7 100644 --- a/src/test_index.zig +++ b/src/test_index.zig @@ -4139,6 +4139,7 @@ test "issue-748: event reconciliation bounds quiet and skipped-file filesystem w var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); + try tmp.dir.writeFile(io, .{ .sub_path = ".gitignore", .data = ".gitignore\n*.tmp\n" }); try tmp.dir.createDirPath(io, "src/deep/nested"); try tmp.dir.createDirPath(io, "empty/nested"); try tmp.dir.writeFile(io, .{ .sub_path = "src/deep/nested/a.py", .data = "def a():\n return 1\n" }); @@ -4201,8 +4202,9 @@ test "issue-748: event reconciliation bounds quiet and skipped-file filesystem w defer arena.deinit(); try watcher.incrementalDiffEvents(io, &store, &explorer, queue, &known, &dirs, root, testing.allocator, arena.allocator(), &dirty); } - // Root and its immediate children are validated; deep descendants stay closed. - try testing.expectEqual(@as(usize, 3), watcher.debug_directory_opens); + // Stable child identities retain their indexed data without opening subtrees, + // including repositories with ignore rules. + try testing.expectEqual(@as(usize, 1), watcher.debug_directory_opens); try testing.expect(watcher.debug_content_reads <= 1); // only root's actual invalidation try testing.expectEqual(@as(u32, 3), known.count()); try testing.expect(known.contains("src/deep/nested/a.py")); @@ -4240,4 +4242,38 @@ test "issue-748: event reconciliation bounds quiet and skipped-file filesystem w try watcher.incrementalDiffEvents(io, &store, &explorer, queue, &known, &dirs, root, testing.allocator, arena.allocator(), &dirty); } try testing.expect(known.contains("empty/nested/first.py")); + // Root policy changes must invalidate retained descendants immediately. + try tmp.dir.writeFile(io, .{ .sub_path = ".gitignore", .data = ".gitignore\nsrc/\n" }); + dirty.clearRetainingCapacity(); + try dirty.put("", {}); + { + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + try watcher.incrementalDiffEvents(io, &store, &explorer, queue, &known, &dirs, root, testing.allocator, arena.allocator(), &dirty); + } + try testing.expect(!known.contains("src/deep/nested/a.py")); + try tmp.dir.writeFile(io, .{ .sub_path = ".gitignore", .data = ".gitignore\n*.tmp\n" }); + { + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + try watcher.incrementalDiffEvents(io, &store, &explorer, queue, &known, &dirs, root, testing.allocator, arena.allocator(), &dirty); + } + try testing.expect(known.contains("src/deep/nested/a.py")); + // A nested policy also applies to unchanged grandchildren. + try tmp.dir.writeFile(io, .{ .sub_path = "src/.gitignore", .data = "deep/\n" }); + dirty.clearRetainingCapacity(); + try dirty.put("src", {}); + { + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + try watcher.incrementalDiffEvents(io, &store, &explorer, queue, &known, &dirs, root, testing.allocator, arena.allocator(), &dirty); + } + try testing.expect(!known.contains("src/deep/nested/a.py")); + try tmp.dir.deleteFile(io, "src/.gitignore"); + { + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + try watcher.incrementalDiffEvents(io, &store, &explorer, queue, &known, &dirs, root, testing.allocator, arena.allocator(), &dirty); + } + try testing.expect(known.contains("src/deep/nested/a.py")); } diff --git a/src/watcher.zig b/src/watcher.zig index 532e8c3b..0c8a34b9 100644 --- a/src/watcher.zig +++ b/src/watcher.zig @@ -11,6 +11,7 @@ const git_mod = @import("git.zig"); const gitignore = @import("gitignore.zig"); const project_file = @import("project_file.zig"); const project_path = @import("project_path.zig"); +const macos_events = @import("macos_events.zig"); fn nsToMs(ns: i128) f64 { return @as(f64, @floatFromInt(ns)) / 1_000_000.0; @@ -89,6 +90,8 @@ const FileState = struct { pub const FileMap = std.StringHashMap(FileState); pub const DirState = struct { + policy_fingerprint: u64 = 0, + requires_polling: bool = false, mtime_ns: i128, ctime_ns: i128, inode: std.posix.ino_t, @@ -99,6 +102,8 @@ pub var debug_unchanged_full_scans: usize = 0; pub var debug_unchanged_file_stats: usize = 0; pub var debug_directory_opens: usize = 0; pub var debug_content_reads: usize = 0; +const recursive_probe_files_per_tick: usize = 64; +const recursive_audit_interval_ns: u64 = 60 * std.time.ns_per_s; fn parentRel(path: []const u8) []const u8 { return if (std.mem.lastIndexOfScalar(u8, path, '/')) |i| path[0..i] else ""; @@ -320,6 +325,18 @@ fn shouldSkip(path: []const u8) bool { return false; } +fn isPolicyPath(path: []const u8) bool { + return std.mem.eql(u8, path, ".gitignore") or + std.mem.endsWith(u8, path, "/.gitignore") or + std.mem.eql(u8, path, ".codedbignore") or + std.mem.eql(u8, path, ".git/info/exclude"); +} + +fn shouldIgnoreRecursiveEvent(path: []const u8, is_file: bool) bool { + if (std.mem.eql(u8, path, ".git/info/exclude")) return false; + return shouldSkip(path) or (is_file and !isPolicyPath(path) and shouldSkipFile(path)); +} + fn shouldSkipDir(name: []const u8) bool { for (skip_dirs) |skip| if (std.mem.eql(u8, name, skip)) return true; return false; @@ -440,11 +457,12 @@ fn resolvedFileTargetAllowed(io: std.Io, root_dir: std.Io.Dir, canonical_root: [ fn resolvedDirectoryTargetAllowed(io: std.Io, root_dir: std.Io.Dir, canonical_root: []const u8, alias_path: []const u8) bool { if (alias_path.len == 0) return canonical_root.len > 0; + if (shouldSkip(alias_path) or isSensitivePath(alias_path)) return false; var target_buf: [std.fs.max_path_bytes]u8 = undefined; const target_len = root_dir.realPathFile(io, alias_path, &target_buf) catch return false; var normalized_buf: [std.fs.max_path_bytes]u8 = undefined; const target_rel = canonicalTargetRelative(canonical_root, target_buf[0..target_len], &normalized_buf) orelse return false; - return !shouldSkip(target_rel) and !shouldSkipFile(target_rel); + return !shouldSkip(target_rel) and !isSensitivePath(target_rel); } /// Recursive directory walker that prunes skip_dirs before descending. @@ -604,7 +622,8 @@ pub const FilteredWalker = struct { const real_target = target_buf[0..target_len]; var normalized_buf: [std.fs.max_path_bytes]u8 = undefined; const target_rel = canonicalTargetRelative(self.real_root, real_target, &normalized_buf) orelse return null; - if (shouldSkip(target_rel) or shouldSkipFile(target_rel)) return null; + if (shouldSkip(visible_path) or isSensitivePath(visible_path) or + shouldSkip(target_rel) or isSensitivePath(target_rel)) return null; const opened_as_alias = !std.mem.eql(u8, target_rel, visible_path); // Ordinary directories may also be reachable through a deliberate // in-root alias. Only alias edges participate in cycle suppression; @@ -2456,6 +2475,68 @@ test "issue-709: macOS vnode descriptor admission is strictly bounded" { // .git/info/exclude lives outside the watched/indexed tree. Check its exact // identity along with root policies even when the OS reports no events. +fn needsPolling(dirs: *const DirMap) bool { + var states = dirs.valueIterator(); + while (states.next()) |state| if (state.requires_polling) return true; + return false; +} + +fn addRecursiveProbePaths(known: *const FileMap, cursor: *usize, dirty: *DirtySet) !void { + const count = known.count(); + if (count == 0) return; + const start = cursor.* % count; + const limit = @min(count, recursive_probe_files_per_tick); + var index: usize = 0; + var selected: usize = 0; + var it = known.keyIterator(); + while (it.next()) |path| : (index += 1) { + if (index >= start and selected < limit) { + try dirty.put(path.*, {}); + selected += 1; + } + } + if (selected < limit) { + it = known.keyIterator(); + while (it.next()) |path| { + if (selected == limit) break; + try dirty.put(path.*, {}); + selected += 1; + } + } + cursor.* = (start + limit) % count; +} + +// This is the recursive stream's scheduled dispatch path, not the fallback +// watcher's two-second full-tree polling path. Verify a bounded slice of +// content for same-metadata rewrites; audit all directory metadata less often +// to recover edits whose event was missed without hashing the whole tree. +fn runRecursiveScheduled( + io: std.Io, + store: *Store, + explorer: *Explorer, + queue: *EventQueue, + known: *FileMap, + dirs: *DirMap, + root: []const u8, + persistent: std.mem.Allocator, + tmp: std.mem.Allocator, + probe_due: bool, + audit_due: bool, + cursor: *usize, +) !void { + if (probe_due and known.count() > 0) { + var probe_dirty = DirtySet.init(tmp); + defer probe_dirty.deinit(); + try addRecursiveProbePaths(known, cursor, &probe_dirty); + try incrementalDiffEvents(io, store, explorer, queue, known, dirs, root, persistent, tmp, &probe_dirty); + } + if (audit_due) { + var audit_dirty = DirtySet.init(tmp); + defer audit_dirty.deinit(); + try incrementalDiffDirty(io, store, explorer, queue, known, dirs, root, persistent, tmp, &audit_dirty); + } +} + fn rootPolicyStates(io: std.Io, root: std.Io.Dir) [3]?DirState { var states: [3]?DirState = .{ dirState(io, root, ".gitignore"), dirState(io, root, ".codedbignore"), null }; const git_dir = root.openDir(io, ".git", .{ .follow_symlinks = false }) catch return states; @@ -2531,7 +2612,28 @@ pub fn incrementalLoop(io: std.Io, store: *Store, explorer: *Explorer, queue: *E var watch = FileChangeWatch.init(backing, max_watched); defer watch.deinit(io); - armAndCloseGap(io, &watch, stable_root, store, explorer, queue, &known, &dirs, root, backing); + var recursive: ?*macos_events.Stream = if (needsPolling(&dirs)) null else macos_events.Stream.create(std.heap.smp_allocator, io, stable_root, shouldIgnoreRecursiveEvent) catch null; + defer if (recursive) |stream| stream.destroy(); + if (recursive != null) { + // The stream already covers newly created descendants. Close the + // scan-to-subscribe gap without first opening thousands of vnodes. + var arena = std.heap.ArenaAllocator.init(backing); + defer arena.deinit(); + incrementalDiffInner(io, store, explorer, queue, &known, &dirs, root, backing, arena.allocator()) catch |err| { + std.log.err("watcher: post-stream reconcile failed: {}", .{err}); + recursive.?.destroy(); + recursive = null; + }; + if (recursive != null and needsPolling(&dirs)) { + recursive.?.destroy(); + recursive = null; + } + } + if (recursive == null) { + armAndCloseGap(io, &watch, stable_root, store, explorer, queue, &known, &dirs, root, backing); + } else { + std.log.info("watcher: recursive macOS file events", .{}); + } explorer.finishStartupReconcile(); startup_reconcile_finished = true; @@ -2546,12 +2648,48 @@ pub fn incrementalLoop(io: std.Io, store: *Store, explorer: *Explorer, queue: *E var last_policy_states = rootPolicyStates(io, stable_root); var polling_timer = cio.Timer.start() catch unreachable; + var recursive_audit_timer = cio.Timer.start() catch unreachable; + var recursive_probe_cursor: usize = 0; while (!shutdown.load(.acquire)) { var wait_arena = std.heap.ArenaAllocator.init(backing); defer wait_arena.deinit(); var dirty = DirtySet.init(wait_arena.allocator()); - if (watch.active) { + if (recursive) |stream| { + var attempts: usize = 0; + while (attempts < 20) : (attempts += 1) { + const state = stream.drain(&dirty); + if (state.fallback) { + stream.destroy(); + recursive = null; + armAndCloseGap(io, &watch, stable_root, store, explorer, queue, &known, &dirs, root, backing); + break; + } + if (state.rescan) { + var files = known.keyIterator(); + while (files.next()) |path| { + const copy = wait_arena.allocator().dupe(u8, path.*) catch continue; + dirty.put(copy, {}) catch {}; + } + var directories = dirs.keyIterator(); + while (directories.next()) |path| { + const copy = wait_arena.allocator().dupe(u8, path.*) catch continue; + dirty.put(copy, {}) catch {}; + } + break; + } + if (dirty.count() > 0 or shutdown.load(.acquire)) break; + cio.sleepMs(100); + } + // File-level events identify excluded activity precisely. Policy + // files remain eligible, and loss flags above still force audits. + var excluded: std.ArrayList([]const u8) = .empty; + var paths = dirty.keyIterator(); + while (paths.next()) |path| { + if (shouldIgnoreRecursiveEvent(path.*, false)) excluded.append(wait_arena.allocator(), path.*) catch {}; + } + for (excluded.items) |path| _ = dirty.remove(path); + } else if (watch.active) { watch.poll(&dirty, 2000); // A timeout is a maximum wait, not a debounce. Bound hot event // streams to one reconciliation per 100 ms; never wait for quiet. @@ -2632,40 +2770,50 @@ pub fn incrementalLoop(io: std.Io, store: *Store, explorer: *Explorer, queue: *E known.put(duped, .{ .mtime = mtime, .size = entry.size, .hash = 0, .seen = false }) catch backing.free(duped); } armAndCloseGap(io, &watch, stable_root, store, explorer, queue, &known, &dirs, root, backing); + if (recursive != null) watch.reset(io); continue; } var cycle_arena = std.heap.ArenaAllocator.init(backing); defer cycle_arena.deinit(); - if (watch.active) { + if (watch.active or recursive != null) { const had_events = dirty.count() > 0; // Hash overflow files on a fixed cadence, independently of native // events. Matching metadata alone is insufficient on coarse or // remote filesystems; bounded periodic verification keeps them live. const poll_due = polling_timer.read() >= 2 * std.time.ns_per_s; - if (poll_due) { - // Preserve the directory audit for nested ignore files and - // aliases, including policy files excluded from file watches. - // Fully watched files need no stats unless an event names them. + if (recursive != null) { + const audit_due = recursive_audit_timer.read() >= recursive_audit_interval_ns; + if (runRecursiveScheduled(io, store, explorer, queue, &known, &dirs, root, backing, cycle_arena.allocator(), poll_due, audit_due, &recursive_probe_cursor)) |_| { + if (audit_due) _ = recursive_audit_timer.lap(); + } else |err| { + std.log.err("watcher: recursive verification failed: {}", .{err}); + } + } else if (poll_due) { + // Fallback watchers still need the two-second directory audit + // and forced reads for files outside their vnode budget. var polling_dirty = DirtySet.init(cycle_arena.allocator()); watch.addUnwatched(&polling_dirty); incrementalDiffDirty(io, store, explorer, queue, &known, &dirs, root, backing, cycle_arena.allocator(), &polling_dirty) catch |err| { std.log.err("watcher: polling diff failed: {}", .{err}); }; - } - if (poll_due) { watch.retryUnwatchedDirs(io, stable_root, cycle_arena.allocator()); - _ = polling_timer.lap(); } + if (poll_due) _ = polling_timer.lap(); if (had_events) { incrementalDiffEvents(io, store, explorer, queue, &known, &dirs, root, backing, cycle_arena.allocator(), &dirty) catch |err| { std.log.err("watcher: event diff failed: {}", .{err}); }; } - if (known.count() != watch.armed_files or dirs.count() != watch.armed_dirs) { + if (recursive != null and needsPolling(&dirs)) { + recursive.?.destroy(); + recursive = null; + armAndCloseGap(io, &watch, stable_root, store, explorer, queue, &known, &dirs, root, backing); + } + if (recursive == null and (known.count() != watch.armed_files or dirs.count() != watch.armed_dirs)) { armAndCloseGap(io, &watch, stable_root, store, explorer, queue, &known, &dirs, root, backing); - } else if (had_events) { + } else if (recursive == null and had_events) { watch.rearmDirty(io, stable_root, &dirty); } } else { @@ -2794,7 +2942,10 @@ fn incrementalDiffSelected( var ignore = try FilteredWalker.init(io, dir, tmp); defer ignore.deinit(); + var content_arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); + defer content_arena.deinit(); var parents = try buildParentIndex(known, dirs, tmp); + parents.content_arena = &content_arena; if (events_only) { var affected = DirtySet.init(tmp); var it = dirty.?.keyIterator(); @@ -2808,7 +2959,7 @@ fn incrementalDiffSelected( } parents.affected = affected; } - try walkRel(io, store, explorer, queue, known, dirs, &ignore, dir, "", persistent, tmp, &parents, dirty, false); + try walkRel(io, store, explorer, queue, known, dirs, &ignore, dir, "", persistent, tmp, &parents, dirty, false, 0, false); // Detect deleted files var to_remove: std.ArrayList([]const u8) = .empty; @@ -2823,11 +2974,13 @@ fn incrementalDiffSelected( for (to_remove.items) |path| { const seq = store.recordDelete(path, 0) catch continue; explorer.removeFile(path); + // The removal list borrows known's keys; clear the mtime entry before + // freeing that key (policy changes can remove an entire subtree). + store.forgetMtime(path); if (known.fetchRemove(path)) |kv| { if (FsEvent.init(kv.key, .deleted, seq)) |ev| pushEventOrWait(queue, ev); persistent.free(kv.key); } - store.forgetMtime(path); } } @@ -2842,6 +2995,8 @@ fn applyKnownFile( read_path: []const u8, stat: std.Io.Dir.Stat, force_read: bool, + content_arena: *std.heap.ArenaAllocator, + canonical_root: []const u8, ) !void { const known_entry = known.getEntry(logical_path) orelse return; const old = known_entry.value_ptr; @@ -2863,11 +3018,12 @@ fn applyKnownFile( const stable_path = known_entry.key_ptr.*; var hash: u64 = 0; var content: ?[]const u8 = null; - var content_arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); - defer content_arena.deinit(); + // Reuse bounded scratch storage across the audit instead of mapping and + // unmapping pages for each unchanged overflow file. No content escapes. + defer _ = content_arena.reset(.{ .retain_with_limit = 256 * 1024 }); if (!shouldSkipFile(logical_path) and stat.size <= max_indexed_file_bytes) { if (builtin.is_test) debug_content_reads += 1; - if (project_file.readAllocNoFollow(io, dir, read_path, content_arena.allocator(), .limited(max_indexed_file_bytes))) |buf| { + if (project_file.readAllocNoFollowAtRoot(io, dir, canonical_root, read_path, content_arena.allocator(), .limited(max_indexed_file_bytes))) |buf| { content = buf; hash = std.hash.Wyhash.hash(0, buf); } else |_| { @@ -2898,6 +3054,7 @@ fn applyKnownFile( } const ParentIndex = struct { + content_arena: ?*std.heap.ArenaAllocator = null, files: std.StringHashMap(std.ArrayList([]const u8)), child_dirs: std.StringHashMap(std.StringHashMap(void)), affected: ?DirtySet = null, @@ -2974,6 +3131,43 @@ fn dirtyForcesFileRead(dirty: ?*const DirtySet, prefix: []const u8, path: []cons return set.get(prefix) != null or set.get(path) != null; } +fn policyEventDirty(dirty: ?*const DirtySet, prefix: []const u8, tmp: std.mem.Allocator) !bool { + const set = dirty orelse return false; + if (prefix.len == 0) { + return set.contains(".gitignore") or set.contains(".codedbignore") or set.contains(".git/info/exclude"); + } + return set.contains(try joinRel(tmp, prefix, ".gitignore")); +} + +// Retain only already-indexed data. No file is opened or newly trusted here. +// A changed identity or ignore policy takes the validated-handle walk below; +// periodic audits still examine every subtree, including dropped events. +fn retainUnchangedChild( + io: std.Io, + listing: std.Io.Dir, + name: []const u8, + child: []const u8, + known: *FileMap, + dirs: ?*DirMap, + parents: *const ParentIndex, + policy_fingerprint: u64, + through_symlink: bool, + tmp: std.mem.Allocator, +) !bool { + if (through_symlink) return false; + const affected = parents.affected orelse return false; + if (affected.contains(child)) return false; + const cached = (dirs orelse return false).get(child) orelse return false; + const stat = listing.statFile(io, name, .{ .follow_symlinks = false }) catch return false; + if (stat.kind != .directory or stat.inode != cached.inode or + stat.mtime.nanoseconds != cached.mtime_ns or stat.ctime.nanoseconds != cached.ctime_ns) return false; + var policy_hash = std.hash.Wyhash.init(policy_fingerprint); + std.hash.autoHash(&policy_hash, dirState(io, listing, try joinRel(tmp, name, ".gitignore"))); + if (policy_hash.final() != cached.policy_fingerprint) return false; + try markSubtreeSeen(known, parents, child, tmp); + return true; +} + fn walkRel( io: std.Io, store: *Store, @@ -2989,6 +3183,8 @@ fn walkRel( parents: *const ParentIndex, dirty: ?*const DirtySet, through_symlink: bool, + inherited_policy: u64, + inherited_policy_changed: bool, ) !void { // Open once, then validate the actual handle before observing its mtime, // identity, names, or sizes. Iterator kind and a pre-open realpath are @@ -3004,18 +3200,31 @@ fn walkRel( const opened_as_alias = ignore.claimOpenedDirectory(listing, prefix) orelse return; effective_through_symlink = effective_through_symlink or opened_as_alias; } - try ignore.loadNestedGitignore(listing, prefix); - - const current_dir_state = dirState(io, listing, ""); + // Include inherited and nested ignore-file identities in the cached view. + // A policy change invalidates descendants even when their directories + // have unchanged metadata; unrelated events can retain clean subtrees. + var policy_hash = std.hash.Wyhash.init(inherited_policy); + if (prefix.len == 0) { + std.hash.autoHash(&policy_hash, rootPolicyStates(io, listing)); + } else { + std.hash.autoHash(&policy_hash, dirState(io, listing, ".gitignore")); + } + const policy_fingerprint = policy_hash.final(); + var current_dir_state = dirState(io, listing, ""); + if (current_dir_state) |*state| { + state.policy_fingerprint = policy_fingerprint; + state.requires_polling = effective_through_symlink or !macos_events.sameVolume(dir, listing); + } const cached_dir_state = if (dirs) |d| d.get(prefix) else null; - const force_scan = dirtyForcesDirectoryScan(dirty, known, prefix); + const policy_changed = inherited_policy_changed or cached_dir_state == null or + cached_dir_state.?.policy_fingerprint != policy_fingerprint or + try policyEventDirty(dirty, prefix, tmp); + const force_scan = policy_changed or dirtyForcesDirectoryScan(dirty, known, prefix); const dir_unchanged = !force_scan and current_dir_state != null and cached_dir_state != null and std.meta.eql(current_dir_state.?, cached_dir_state.?); - // With no inherited ignore rules, a validated unchanged directory outside - // the event ancestry can retain its subtree without opening descendants. - // Ignore-bearing trees take the conservative walk so rule changes and - // directory aliases still pass through the existing safety checks. - if (dir_unchanged and !ignore.hasIgnoreRules() and !effective_through_symlink) { + // The opened directory has passed the same path/alias checks as a full + // walk. Matching inherited policy permits this shortcut in ignored trees. + if (dir_unchanged and !effective_through_symlink) { if (parents.affected) |affected| { if (!affected.contains(prefix)) { try markSubtreeSeen(known, parents, prefix, tmp); @@ -3023,6 +3232,7 @@ fn walkRel( } } } + try ignore.loadNestedGitignore(listing, prefix); if (dir_unchanged) { if (parents.files.get(prefix)) |file_list| { for (file_list.items) |path| { @@ -3038,7 +3248,7 @@ fn walkRel( if (effective_through_symlink and !resolvedFileTargetAllowed(io, listing, ignore.real_root, local_name)) continue; const stat = listing.statFile(io, local_name, .{ .follow_symlinks = false }) catch continue; if (stat.kind != .file) continue; - try applyKnownFile(io, store, explorer, queue, known, listing, path, local_name, stat, dirtyForcesFileRead(dirty, prefix, path)); + try applyKnownFile(io, store, explorer, queue, known, listing, path, local_name, stat, dirtyForcesFileRead(dirty, prefix, path), parents.content_arena.?, ignore.real_root); } } if (parents.child_dirs.get(prefix)) |children| { @@ -3047,10 +3257,11 @@ fn walkRel( if (shouldSkipDir(name.*)) continue; const child = try joinRel(tmp, prefix, name.*); if (ignore.hasIgnoreRules() and ignore.isIgnored(child, true)) continue; + if (!policy_changed and try retainUnchangedChild(io, listing, name.*, child, known, dirs, parents, policy_fingerprint, effective_through_symlink, tmp)) continue; const child_stat = listing.statFile(io, name.*, .{ .follow_symlinks = false }) catch continue; if (child_stat.kind != .directory and child_stat.kind != .sym_link) continue; const child_through_symlink = effective_through_symlink or child_stat.kind == .sym_link; - try walkRel(io, store, explorer, queue, known, dirs, ignore, dir, child, persistent, tmp, parents, dirty, child_through_symlink); + try walkRel(io, store, explorer, queue, known, dirs, ignore, dir, child, persistent, tmp, parents, dirty, child_through_symlink, policy_fingerprint, policy_changed); } } return; @@ -3064,7 +3275,8 @@ fn walkRel( if (shouldSkipDir(entry.name)) continue; const child = try joinRel(tmp, prefix, entry.name); if (ignore.hasIgnoreRules() and ignore.isIgnored(child, true)) continue; - try walkRel(io, store, explorer, queue, known, dirs, ignore, dir, child, persistent, tmp, parents, dirty, child_through_symlink); + if (!policy_changed and entry.kind == .directory and try retainUnchangedChild(io, listing, entry.name, child, known, dirs, parents, policy_fingerprint, effective_through_symlink, tmp)) continue; + try walkRel(io, store, explorer, queue, known, dirs, ignore, dir, child, persistent, tmp, parents, dirty, child_through_symlink, policy_fingerprint, policy_changed); continue; } @@ -3077,7 +3289,7 @@ fn walkRel( const stat = listing.statFile(io, entry.name, .{ .follow_symlinks = false }) catch continue; if (stat.kind != .file) continue; if (known.getEntry(rel)) |known_entry| { - try applyKnownFile(io, store, explorer, queue, known, listing, known_entry.key_ptr.*, entry.name, stat, dirtyForcesFileRead(dirty, prefix, rel)); + try applyKnownFile(io, store, explorer, queue, known, listing, known_entry.key_ptr.*, entry.name, stat, dirtyForcesFileRead(dirty, prefix, rel), parents.content_arena.?, ignore.real_root); } else { const mtime: i64 = @intCast(@divTrunc(stat.mtime.nanoseconds, std.time.ns_per_ms)); const duped = try persistent.dupe(u8, rel); @@ -3723,3 +3935,183 @@ test "benchmark watcher fixed cycles" { std.debug.print("\nwatcher_fixed_cycles event={} files={d} dirs={d} budget={d} cycles={d} diff_ns={d} rearm_ns={d}\n", .{ with_event, file_count, directory_count, budget, cycles, diff_ns, rearm_ns }); } } + +test "issue-756: recursive scheduled work bounds quiet walks and verifies content" { + const testing = std.testing; + const io = testing.io; + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + for (0..128) |i| { + var dir_buf: [16]u8 = undefined; + const dir_name = try std.fmt.bufPrint(&dir_buf, "d{d}", .{i}); + try tmp.dir.createDirPath(io, dir_name); + var path_buf: [32]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "d{d}/f.py", .{i}); + try tmp.dir.writeFile(io, .{ .sub_path = path, .data = "def original():\n return 1\n" }); + } + var root_buf: [std.fs.max_path_bytes]u8 = undefined; + const root_len = try tmp.dir.realPathFile(io, ".", &root_buf); + const root = root_buf[0..root_len]; + var store = Store.init(testing.allocator); + defer store.deinit(); + var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); + defer explorer.deinit(); + try explorer.setRoot(io, root); + var known = FileMap.init(testing.allocator); + defer { + var it = known.keyIterator(); + while (it.next()) |path| testing.allocator.free(path.*); + known.deinit(); + } + var dirs = DirMap.init(testing.allocator); + defer { + var it = dirs.keyIterator(); + while (it.next()) |path| testing.allocator.free(path.*); + dirs.deinit(); + } + const queue = try testing.allocator.create(EventQueue); + defer testing.allocator.destroy(queue); + queue.* = .{}; + for (0..2) |_| { + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + try incrementalDiffInner(io, &store, &explorer, queue, &known, &dirs, root, testing.allocator, arena.allocator()); + } + try testing.expectEqual(@as(u32, 128), known.count()); + + var cursor: usize = 0; + debug_directory_opens = 0; + debug_content_reads = 0; + { + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + try runRecursiveScheduled(io, &store, &explorer, queue, &known, &dirs, root, testing.allocator, arena.allocator(), true, false, &cursor); + } + try testing.expectEqual(recursive_probe_files_per_tick, debug_content_reads); + try testing.expect(debug_directory_opens <= recursive_probe_files_per_tick + 1); + + const before = "def original():\n return 1\n"; + const after = "def modified():\n return 2\n"; + try testing.expectEqual(before.len, after.len); + try tmp.dir.writeFile(io, .{ .sub_path = "d127/f.py", .data = after }); + const root_dir = explorer.root_dir orelse return error.TestUnexpectedResult; + const stat = try root_dir.statFile(io, "d127/f.py", .{ .follow_symlinks = false }); + const old = known.getPtr("d127/f.py").?; + // Model a rewrite whose exact metadata collides without a special mount. + old.mtime = @intCast(@divTrunc(stat.mtime.nanoseconds, std.time.ns_per_ms)); + old.mtime_ns = stat.mtime.nanoseconds; + old.ctime_ns = stat.ctime.nanoseconds; + old.inode = stat.inode; + old.size = stat.size; + old.hash = std.hash.Wyhash.hash(0, before); + for (0..2) |_| { + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + try runRecursiveScheduled(io, &store, &explorer, queue, &known, &dirs, root, testing.allocator, arena.allocator(), true, false, &cursor); + } + var outline: std.ArrayList(u8) = .empty; + defer outline.deinit(testing.allocator); + try testing.expect(try explorer.renderOutline("d127/f.py", testing.allocator, &outline, false)); + try testing.expect(std.mem.indexOf(u8, outline.items, "modified") != null); + + debug_directory_opens = 0; + debug_content_reads = 0; + { + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + try runRecursiveScheduled(io, &store, &explorer, queue, &known, &dirs, root, testing.allocator, arena.allocator(), false, true, &cursor); + } + try testing.expect(debug_directory_opens >= 129); + try testing.expectEqual(@as(usize, 0), debug_content_reads); + + try tmp.dir.createDirPath(io, "generated.lock"); + try tmp.dir.writeFile(io, .{ .sub_path = "generated.lock/main.py", .data = "def moved_directory_symbol():\n return 42\n" }); + try tmp.dir.createDirPath(io, "secret.key"); + try tmp.dir.writeFile(io, .{ .sub_path = "secret.key/private.py", .data = "def private_symbol():\n return 0\n" }); + var dirty = DirtySet.init(testing.allocator); + defer dirty.deinit(); + try dirty.put("generated.lock", {}); + try dirty.put("secret.key", {}); + { + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + try incrementalDiffEvents(io, &store, &explorer, queue, &known, &dirs, root, testing.allocator, arena.allocator(), &dirty); + } + try testing.expect(known.contains("generated.lock/main.py")); + try testing.expect(!known.contains("secret.key/private.py")); + var moved_outline: std.ArrayList(u8) = .empty; + defer moved_outline.deinit(testing.allocator); + try testing.expect(try explorer.renderOutline("generated.lock/main.py", testing.allocator, &moved_outline, false)); + try testing.expect(std.mem.indexOf(u8, moved_outline.items, "moved_directory_symbol") != null); +} + +test "issue-756: policy-file events invalidate descendants despite metadata collisions" { + const testing = std.testing; + const io = testing.io; + try testing.expect(!shouldIgnoreRecursiveEvent(".gitignore", true)); + try testing.expect(!shouldIgnoreRecursiveEvent("src/.gitignore", true)); + try testing.expect(!shouldIgnoreRecursiveEvent(".git/info/exclude", true)); + try testing.expect(shouldIgnoreRecursiveEvent("node_modules/.gitignore", true)); + try testing.expect(shouldIgnoreRecursiveEvent("generated.lock", true)); + try testing.expect(!shouldIgnoreRecursiveEvent("generated.lock", false)); + for ([_]bool{ false, true }) |nested| { + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + try tmp.dir.createDirPath(io, "src/deep"); + const policy_path = if (nested) "src/.gitignore" else ".gitignore"; + try tmp.dir.writeFile(io, .{ .sub_path = policy_path, .data = "*.md\n" }); + try tmp.dir.writeFile(io, .{ .sub_path = "src/deep/a.py", .data = "def a():\n return 1\n" }); + try tmp.dir.writeFile(io, .{ .sub_path = "src/deep/b.md", .data = "# B\n" }); + var root_buf: [std.fs.max_path_bytes]u8 = undefined; + const root_len = try tmp.dir.realPathFile(io, ".", &root_buf); + const root = root_buf[0..root_len]; + var store = Store.init(testing.allocator); + defer store.deinit(); + var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); + defer explorer.deinit(); + try explorer.setRoot(io, root); + var known = FileMap.init(testing.allocator); + defer { + var it = known.keyIterator(); + while (it.next()) |path| testing.allocator.free(path.*); + known.deinit(); + } + var dirs = DirMap.init(testing.allocator); + defer { + var it = dirs.keyIterator(); + while (it.next()) |path| testing.allocator.free(path.*); + dirs.deinit(); + } + const queue = try testing.allocator.create(EventQueue); + defer testing.allocator.destroy(queue); + queue.* = .{}; + { + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + try incrementalDiffInner(io, &store, &explorer, queue, &known, &dirs, root, testing.allocator, arena.allocator()); + } + try testing.expect(known.contains("src/deep/a.py")); + try testing.expect(!known.contains("src/deep/b.md")); + try tmp.dir.writeFile(io, .{ .sub_path = policy_path, .data = "*.py\n" }); + const root_dir = explorer.root_dir orelse return error.TestUnexpectedResult; + if (nested) { + const src_dir = try root_dir.openDir(io, "src", .{}); + defer src_dir.close(io); + var hash = std.hash.Wyhash.init(dirs.get("").?.policy_fingerprint); + std.hash.autoHash(&hash, dirState(io, src_dir, ".gitignore")); + dirs.getPtr("src").?.policy_fingerprint = hash.final(); + } else { + var hash = std.hash.Wyhash.init(0); + std.hash.autoHash(&hash, rootPolicyStates(io, root_dir)); + dirs.getPtr("").?.policy_fingerprint = hash.final(); + } + var dirty = DirtySet.init(testing.allocator); + defer dirty.deinit(); + try dirty.put(policy_path, {}); + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + try incrementalDiffEvents(io, &store, &explorer, queue, &known, &dirs, root, testing.allocator, arena.allocator(), &dirty); + try testing.expect(!known.contains("src/deep/a.py")); + try testing.expect(known.contains("src/deep/b.md")); + } +}