test: retain bounded default-root-build watcher failure diagnostics

.github/workflows/ci.yml

@@ -108,6 +108,16 @@ jobs:
       - name: Run tests with coverage
         run: npm run test:coverage
 
+      - name: Upload watcher failure diagnostics
+        if: failure()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: watcher-failures-node-${{ matrix.node }}-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/ocbi-watcher-failures/
+          if-no-files-found: ignore
+          include-hidden-files: true
+          retention-days: 14
+
       - name: Start mock embeddings server for eval smoke
         run: |
           node scripts/eval-mock-embeddings-server.mjs > /tmp/eval-mock-embeddings.log 2>&1 &

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- **Watcher failure diagnostics for contributors**: Retain bounded, test-owned observations and private fixture snapshots only when the default-root-build watcher rejection scenarios fail, with failure-only CI artifact pickup. Production watcher behavior, consumer assertions, and test deadlines are unchanged; unavailable internal observations are explicitly marked missing.
+
 ### Fixed
 
 - **Pi real-host prompt tests**: Use Pi 1.3's user-trigger/options runner API and raise only the development host floor to 1.3, retaining sibling section/append preservation and intentional override assertions in both extension orders. The optional Pi peer contract, production extension, legacy string-return fallback, and omp behavior are unchanged.

CONTRIBUTING.md

@@ -72,6 +72,40 @@ npm run test
 cd native && cargo test
 ```
 
+The default-root-build watcher rejection scenarios in
+`tests/include-excluded-paths-indexing.test.ts` retain diagnostics **only on
+failure**, after stopping watchers, settling handler tails, and closing indexers,
+before deleting their temporary fixtures. Successful runs retain nothing.
+
+Look for `[watcher failure capture]` in stderr. Locally the printed private
+directory is under `<os.tmpdir()>/ocbi-watcher-failures/`; CI uses
+`${RUNNER_TEMP}/ocbi-watcher-failures/` and uploads it on job failure as
+`watcher-failures-node-<version>-<run>-<attempt>` for 14 days.
+`capture.json` contains monotonic observations, first/last phase anchors,
+suppressed warnings, dropped-event and explicit detail-byte-cut counts,
+bounded numeric index statistics, runtime/backend information, and a snapshot
+inventory. Only genuine notifications from the initially available
+Chokidar emitter are observed; replacement/native notifications and internal
+filtering, queueing, and flushing are marked `MISSING`, not inferred.
+
+Limits per scenario: 512 tail events of at most 2 KiB each, fixed first/last
+phase anchors, 256 visited fixture entries, 16 MiB of combined source/index
+snapshots, and 18 MiB total including metadata. Caps, refused paths, capture
+errors, and unavailable evidence are reported as `MISSING`. Symlinks,
+hardlinks, special files, and replaced fixture roots are refused. Index
+snapshots require both all watcher stops and all indexer closes to succeed,
+after handler tails settle; otherwise only safe source fixtures are retained.
+These quiescence checks do not prove database consistency. POSIX mode and
+directory-FD guards may refuse Windows snapshots as `MISSING`; Windows
+snapshot coverage has not been verified. These checks are not an adversarial
+filesystem-race guarantee.
+
+Diagnostic text redacts absolute private paths, but **raw fixture index files
+can contain temporary paths**. Inspect artifacts privately; do not commit them
+or publish raw contents. A native cache observation and current-disk digest do
+not prove the bytes originally executed. A passing run is a non-reproduction,
+not evidence that a historical watcher failure is fixed.
+
 ### Linting
 
 ```bash

tests/include-excluded-paths-indexing.test.ts

@@ -10,6 +10,7 @@ import type { HostMode } from "../src/config/host.js";
 import { parseConfig, type ParsedCodebaseIndexConfig } from "../src/config/schema.js";
 import { Indexer } from "../src/indexer/index.js";
 import { FileWatcher, type FileChange, type FileWatcherBackend, type FileWatcherOptions } from "../src/watcher/file-watcher.js";
+import { DefaultRootBuildFailureCapture } from "./watcher/default-root-build-failure-capture.js";
 
 // Controlled filesystem boundaries only; successful reads always use the real filesystem.
 const snapshotFaults = vi.hoisted(() => ({
@@ -80,9 +81,20 @@ function writeSource(filePath: string, definitionName: string): void {
   fs.writeFileSync(filePath, `export function ${definitionName}() { return "${definitionName}"; }\n`);
 }
 
-async function hasDefinition(indexer: Indexer, definitionName: string, filePath: string): Promise<boolean> {
-  const results = await indexer.search(definitionName, 20, { definitionIntent: true });
-  return results.some((result) => result.name === definitionName && result.filePath === filePath);
+async function hasDefinition(
+  indexer: Indexer, definitionName: string, filePath: string, capture?: DefaultRootBuildFailureCapture,
+): Promise<boolean> {
+  capture?.record("query-start", { definitionName, filePath });
+  try {
+    const results = await indexer.search(definitionName, 20, { definitionIntent: true });
+    const found = results.some((result) => result.name === definitionName && result.filePath === filePath);
+    capture?.record("query-end", { definitionName, filePath, found,
+      results: results.map((result) => ({ name: result.name, filePath: result.filePath })) });
+    return found;
+  } catch (error) {
+    capture?.record("query-error", error);
+    throw error;
+  }
 }
 
 describe("include-excluded path indexing acceptance", () => {
@@ -92,6 +104,7 @@ describe("include-excluded path indexing acceptance", () => {
   let watchers: FileWatcher[];
   let handlerTails: Promise<void>[];
   let fetchSpy: ReturnType<typeof vi.spyOn>;
+  let failureCapture: DefaultRootBuildFailureCapture | undefined;
 
   const createIndexer = (
     config: ParsedCodebaseIndexConfig,
@@ -113,27 +126,45 @@ describe("include-excluded path indexing acceptance", () => {
     indexers = [];
     watchers = [];
     handlerTails = [];
+    failureCapture = undefined;
     fetchSpy = vi.spyOn(globalThis, "fetch").mockRejectedValue(new Error("network access is forbidden in this suite"));
   });
 
-  afterEach(async () => {
+  afterEach(async ({ task }) => {
+    const capture = failureCapture;
+    let watchersStopped = false;
+    let indexClosed = false;
     try {
       snapshotFaults.beforeRead = undefined;
       snapshotFaults.afterStat = undefined;
       const stopResults = await Promise.allSettled(watchers.map((watcher) => watcher.stop()));
+      watchersStopped = stopResults.every((result) => result.status === "fulfilled");
       const handlerResults = await Promise.allSettled(handlerTails);
       const closeResults = await Promise.allSettled(indexers.map((indexer) => indexer.close()));
+      indexClosed = closeResults.every((result) => result.status === "fulfilled");
+      capture?.record("cleanup", { watchersStopped, indexClosed, stop: stopResults, handlers: handlerResults, close: closeResults });
       const failures = [...stopResults, ...handlerResults, ...closeResults]
         .filter((result): result is PromiseRejectedResult => result.status === "rejected")
         .map((result) => result.reason);
       expect(fetchSpy).not.toHaveBeenCalled();
       if (failures.length > 0) {
         throw new AggregateError(failures, "Failed to close include-excluded acceptance test resources");
       }
+    } catch (error) {
+      capture?.fail(error);
+      throw error;
     } finally {
-      vi.restoreAllMocks();
-      fs.rmSync(projectDir, { recursive: true, force: true });
-      fs.rmSync(indexRoot, { recursive: true, force: true });
+      try {
+        const retained = capture?.retainFailure({ testFailed: task.result?.state === "fail", watchersStopped, indexClosed });
+        if (retained?.directory) console.error(`[watcher failure capture] ${retained.directory}`);
+        if (retained?.missing) console.error(`[watcher failure capture] ${retained.missing}`);
+      } catch (error) {
+        capture?.record("capture-error", { missing: "MISSING: retention/reporting failed", error });
+      } finally {
+        vi.restoreAllMocks();
+        fs.rmSync(projectDir, { recursive: true, force: true });
+        fs.rmSync(indexRoot, { recursive: true, force: true });
+      }
     }
   });
 
@@ -484,46 +515,99 @@ describe("include-excluded path indexing acceptance", () => {
   async function exerciseDefaultRootBuildWatcherRejection(
     backend: Exclude<FileWatcherBackend, "auto">,
   ): Promise<void> {
-    const config = structuralConfig();
-    const rootBuildHelperPath = path.join(projectDir, "build-helper.ts");
-    const ordinaryPath = path.join(projectDir, "src", "ordinary.ts");
-    const originalBuildDefinition = `${backend}DefaultOriginalBuildHelperDefinition`;
-    const updatedBuildDefinition = `${backend}DefaultUpdatedBuildHelperDefinition`;
-    const originalOrdinaryDefinition = `${backend}DefaultOriginalOrdinaryDefinition`;
-    const updatedOrdinaryDefinition = `${backend}DefaultUpdatedOrdinaryDefinition`;
-    writeSource(rootBuildHelperPath, originalBuildDefinition);
-    writeSource(ordinaryPath, originalOrdinaryDefinition);
+    const capture = new DefaultRootBuildFailureCapture(projectDir, indexRoot, backend);
+    failureCapture = capture;
+    try {
+      const config = structuralConfig();
+      const rootBuildHelperPath = path.join(projectDir, "build-helper.ts");
+      const ordinaryPath = path.join(projectDir, "src", "ordinary.ts");
+      const originalBuildDefinition = `${backend}DefaultOriginalBuildHelperDefinition`;
+      const updatedBuildDefinition = `${backend}DefaultUpdatedBuildHelperDefinition`;
+      const originalOrdinaryDefinition = `${backend}DefaultOriginalOrdinaryDefinition`;
+      const updatedOrdinaryDefinition = `${backend}DefaultUpdatedOrdinaryDefinition`;
+      writeSource(rootBuildHelperPath, originalBuildDefinition);
+      capture.record("write", { path: rootBuildHelperPath, definition: originalBuildDefinition });
+      writeSource(ordinaryPath, originalOrdinaryDefinition);
+      capture.record("write", { path: ordinaryPath, definition: originalOrdinaryDefinition });
 
-    const indexer = createIndexer(config);
-    await indexer.index();
-    expect(await hasDefinition(indexer, originalBuildDefinition, rootBuildHelperPath)).toBe(true);
+      const indexer = createIndexer(config);
+      capture.record("index-start", { initial: true });
+      try {
+        const stats = await indexer.index();
+        capture.record("index-end", { initial: true,
+          totalFiles: stats.totalFiles, totalChunks: stats.totalChunks, indexedChunks: stats.indexedChunks,
+          failedChunks: stats.failedChunks, existingChunks: stats.existingChunks, removedChunks: stats.removedChunks });
+      } catch (error) {
+        capture.record("index-error", error);
+        throw error;
+      }
+      expect(await hasDefinition(indexer, originalBuildDefinition, rootBuildHelperPath, capture)).toBe(true);
 
-    const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => undefined);
-    const watcher = new FileWatcher(projectDir, config, "jcode", { backend });
-    watchers.push(watcher);
-    const observedPaths: string[] = [];
-    let handlerTail = Promise.resolve();
-    watcher.start(async (changes) => {
-      observedPaths.push(...changes.map((change) => change.path));
-      handlerTail = handlerTail.then(async () => {
-        await indexer.index();
+      const warnSpy = vi.spyOn(console, "warn").mockImplementation((...args) => {
+        capture.record("warning", args);
       });
-      handlerTails.push(handlerTail);
-      await handlerTail;
-    });
-    await watcher.waitUntilReady();
+      const watcher = new FileWatcher(projectDir, config, "jcode", { backend });
+      watchers.push(watcher);
+      const observedPaths: string[] = [];
+      let handlerTail = Promise.resolve();
+      watcher.start(async (changes) => {
+        capture.record("callback-start", changes);
+        try {
+          observedPaths.push(...changes.map((change) => change.path));
+          handlerTail = handlerTail.then(async () => {
+            capture.record("index-start", { initial: false });
+            try {
+              const stats = await indexer.index();
+              capture.record("index-end", { initial: false,
+                totalFiles: stats.totalFiles, totalChunks: stats.totalChunks, indexedChunks: stats.indexedChunks,
+                failedChunks: stats.failedChunks, existingChunks: stats.existingChunks, removedChunks: stats.removedChunks });
+            } catch (error) {
+              capture.record("index-error", error);
+              throw error;
+            }
+          });
+          handlerTails.push(handlerTail);
+          await handlerTail;
+          capture.record("callback-end");
+        } catch (error) {
+          capture.record("callback-error", error);
+          throw error;
+        }
+      });
+      // Passive listeners on this actual emitter only; replacement/native layers remain MISSING.
+      const internals = watcher as unknown as { watcher: FSWatcher | null; nativeWatcher: object | null };
+      const emitter = internals.watcher;
+      if (emitter) {
+        capture.record("backend", { emitter: "initial Chokidar", options: emitter.options });
+        emitter.on("raw", (event, filePath, details) => capture.record("raw", { event, filePath, details }));
+        emitter.on("all", (event, filePath) => capture.record("backend-event", { event, filePath }));
+        emitter.on("ready", () => capture.record("backend-ready", { emitter: "initial Chokidar" }));
+        emitter.on("error", (error) => capture.record("backend-error", error));
+      } else {
+        capture.record("backend", { emitter: "MISSING: no Chokidar emitter at start", requested: backend });
+      }
+      capture.record("ready-start");
+      await watcher.waitUntilReady();
+      capture.record("ready-end", { running: watcher.isRunning(), nativeActive: internals.nativeWatcher !== null,
+        chokidar: internals.watcher === emitter ? "same captured emitter" : "MISSING: replacement emitter" });
 
-    writeSource(rootBuildHelperPath, updatedBuildDefinition);
-    writeSource(ordinaryPath, updatedOrdinaryDefinition);
-    await vi.waitFor(async () => {
-      expect(await hasDefinition(indexer, updatedOrdinaryDefinition, ordinaryPath)).toBe(true);
-      expect(observedPaths).toContain(ordinaryPath);
-    }, { timeout: WATCH_TIMEOUT_MS, interval: 50 });
+      writeSource(rootBuildHelperPath, updatedBuildDefinition);
+      capture.record("write", { path: rootBuildHelperPath, definition: updatedBuildDefinition });
+      writeSource(ordinaryPath, updatedOrdinaryDefinition);
+      capture.record("write", { path: ordinaryPath, definition: updatedOrdinaryDefinition });
+      await vi.waitFor(async () => {
+        expect(await hasDefinition(indexer, updatedOrdinaryDefinition, ordinaryPath, capture)).toBe(true);
+        expect(observedPaths).toContain(ordinaryPath);
+      }, { timeout: WATCH_TIMEOUT_MS, interval: 50 });
 
-    expect(observedPaths).not.toContain(rootBuildHelperPath);
-    expect(await hasDefinition(indexer, updatedBuildDefinition, rootBuildHelperPath)).toBe(true);
-    if (backend === "native") {
-      expect(warnSpy.mock.calls.some(([message]) => String(message).includes("Chokidar fallback"))).toBe(false);
+      expect(observedPaths).not.toContain(rootBuildHelperPath);
+      expect(await hasDefinition(indexer, updatedBuildDefinition, rootBuildHelperPath, capture)).toBe(true);
+      if (backend === "native") {
+        expect(warnSpy.mock.calls.some(([message]) => String(message).includes("Chokidar fallback"))).toBe(false);
+      }
+    } catch (error) {
+      capture.fail(error);
+      throw error;
     }
   }