From ae52cf95a81f533a918d3c756641fc3c04682403 Mon Sep 17 00:00:00 2001 From: Helweg Date: Mon, 19 Oct 2026 16:34:02 +0200 Subject: [PATCH 1/2] test: retain bounded watcher failure diagnostics --- .github/workflows/ci.yml | 10 + CHANGELOG.md | 4 + CONTRIBUTING.md | 29 ++ tests/include-excluded-paths-indexing.test.ts | 161 +++++++--- .../default-root-build-failure-capture.ts | 295 ++++++++++++++++++ 5 files changed, 457 insertions(+), 42 deletions(-) create mode 100644 tests/watcher/default-root-build-failure-capture.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7fc3acc2..6370ec64 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,6 +108,16 @@ jobs: - name: Run tests with coverage run: npm run test:coverage + - name: Upload watcher failure diagnostics + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: watcher-failures-node-${{ matrix.node }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/ocbi-watcher-failures/ + if-no-files-found: ignore + include-hidden-files: true + retention-days: 14 + - name: Start mock embeddings server for eval smoke run: | node scripts/eval-mock-embeddings-server.mjs > /tmp/eval-mock-embeddings.log 2>&1 & diff --git a/CHANGELOG.md b/CHANGELOG.md index 62a3b726..b84f814e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **Watcher failure diagnostics for contributors**: Retain bounded, test-owned observations and private fixture snapshots only when the default-root-build watcher rejection scenarios fail, with failure-only CI artifact pickup. Production watcher behavior, consumer assertions, and test deadlines are unchanged; unavailable internal observations are explicitly marked missing. + ### Fixed - **Pi real-host prompt tests**: Use Pi 1.3's user-trigger/options runner API and raise only the development host floor to 1.3, retaining sibling section/append preservation and intentional override assertions in both extension orders. The optional Pi peer contract, production extension, legacy string-return fallback, and omp behavior are unchanged. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cb13c44c..05081f4d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -72,6 +72,35 @@ npm run test cd native && cargo test ``` +The default-root-build watcher rejection scenarios in +`tests/include-excluded-paths-indexing.test.ts` retain diagnostics **only on +failure**, after stopping watchers, settling handler tails, and closing indexers, +before deleting their temporary fixtures. Successful runs retain nothing. + +Look for `[watcher failure capture]` in stderr. Locally the printed private +directory is under `/ocbi-watcher-failures/`; CI uses +`${RUNNER_TEMP}/ocbi-watcher-failures/` and uploads it on job failure as +`watcher-failures-node---` for 14 days. +`capture.json` contains monotonic observations, first/last phase anchors, +suppressed warnings, dropped-event counts, runtime/backend information, and a +snapshot inventory. Only genuine notifications from the initially available +Chokidar emitter are observed; replacement/native notifications and internal +filtering, queueing, and flushing are marked `MISSING`, not inferred. + +Limits per scenario: 512 tail events of at most 2 KiB each, fixed first/last +phase anchors, 256 visited fixture entries, 16 MiB of combined source/index +snapshots, and 18 MiB total including metadata. Caps, refused paths, capture +errors, and unavailable evidence are reported as `MISSING`. Symlinks, +hardlinks, special files, and replaced fixture roots are refused; an index +whose close failed or is unknown is not copied. These checks are not an +adversarial filesystem-race guarantee. + +Diagnostic text redacts absolute private paths, but **raw fixture index files +can contain temporary paths**. Inspect artifacts privately; do not commit them +or publish raw contents. A native cache observation and current-disk digest do +not prove the bytes originally executed. A passing run is a non-reproduction, +not evidence that a historical watcher failure is fixed. + ### Linting ```bash diff --git a/tests/include-excluded-paths-indexing.test.ts b/tests/include-excluded-paths-indexing.test.ts index b2778926..2d0285d8 100644 --- a/tests/include-excluded-paths-indexing.test.ts +++ b/tests/include-excluded-paths-indexing.test.ts @@ -10,6 +10,7 @@ import type { HostMode } from "../src/config/host.js"; import { parseConfig, type ParsedCodebaseIndexConfig } from "../src/config/schema.js"; import { Indexer } from "../src/indexer/index.js"; import { FileWatcher, type FileChange, type FileWatcherBackend, type FileWatcherOptions } from "../src/watcher/file-watcher.js"; +import { DefaultRootBuildFailureCapture } from "./watcher/default-root-build-failure-capture.js"; // Controlled filesystem boundaries only; successful reads always use the real filesystem. const snapshotFaults = vi.hoisted(() => ({ @@ -80,9 +81,20 @@ function writeSource(filePath: string, definitionName: string): void { fs.writeFileSync(filePath, `export function ${definitionName}() { return "${definitionName}"; }\n`); } -async function hasDefinition(indexer: Indexer, definitionName: string, filePath: string): Promise { - const results = await indexer.search(definitionName, 20, { definitionIntent: true }); - return results.some((result) => result.name === definitionName && result.filePath === filePath); +async function hasDefinition( + indexer: Indexer, definitionName: string, filePath: string, capture?: DefaultRootBuildFailureCapture, +): Promise { + capture?.record("query-start", { definitionName, filePath }); + try { + const results = await indexer.search(definitionName, 20, { definitionIntent: true }); + const found = results.some((result) => result.name === definitionName && result.filePath === filePath); + capture?.record("query-end", { definitionName, filePath, found, + results: results.map((result) => ({ name: result.name, filePath: result.filePath })) }); + return found; + } catch (error) { + capture?.record("query-error", error); + throw error; + } } describe("include-excluded path indexing acceptance", () => { @@ -92,6 +104,7 @@ describe("include-excluded path indexing acceptance", () => { let watchers: FileWatcher[]; let handlerTails: Promise[]; let fetchSpy: ReturnType; + let failureCapture: DefaultRootBuildFailureCapture | undefined; const createIndexer = ( config: ParsedCodebaseIndexConfig, @@ -113,16 +126,20 @@ describe("include-excluded path indexing acceptance", () => { indexers = []; watchers = []; handlerTails = []; + failureCapture = undefined; fetchSpy = vi.spyOn(globalThis, "fetch").mockRejectedValue(new Error("network access is forbidden in this suite")); }); - afterEach(async () => { + afterEach(async ({ task }) => { + let indexClosed = false; try { snapshotFaults.beforeRead = undefined; snapshotFaults.afterStat = undefined; const stopResults = await Promise.allSettled(watchers.map((watcher) => watcher.stop())); const handlerResults = await Promise.allSettled(handlerTails); const closeResults = await Promise.allSettled(indexers.map((indexer) => indexer.close())); + indexClosed = closeResults.every((result) => result.status === "fulfilled"); + failureCapture?.record("cleanup", { stop: stopResults, handlers: handlerResults, close: closeResults }); const failures = [...stopResults, ...handlerResults, ...closeResults] .filter((result): result is PromiseRejectedResult => result.status === "rejected") .map((result) => result.reason); @@ -130,10 +147,21 @@ describe("include-excluded path indexing acceptance", () => { if (failures.length > 0) { throw new AggregateError(failures, "Failed to close include-excluded acceptance test resources"); } + } catch (error) { + failureCapture?.fail(error); + throw error; } finally { - vi.restoreAllMocks(); - fs.rmSync(projectDir, { recursive: true, force: true }); - fs.rmSync(indexRoot, { recursive: true, force: true }); + try { + const retained = failureCapture?.retainFailure({ testFailed: task.result?.state === "fail", indexClosed }); + if (retained?.directory) console.error(`[watcher failure capture] ${retained.directory}`); + if (retained?.missing) console.error(`[watcher failure capture] ${retained.missing}`); + } catch (error) { + failureCapture?.record("capture-error", { missing: "MISSING: retention/reporting failed", error }); + } finally { + vi.restoreAllMocks(); + fs.rmSync(projectDir, { recursive: true, force: true }); + fs.rmSync(indexRoot, { recursive: true, force: true }); + } } }); @@ -484,46 +512,95 @@ describe("include-excluded path indexing acceptance", () => { async function exerciseDefaultRootBuildWatcherRejection( backend: Exclude, ): Promise { - const config = structuralConfig(); - const rootBuildHelperPath = path.join(projectDir, "build-helper.ts"); - const ordinaryPath = path.join(projectDir, "src", "ordinary.ts"); - const originalBuildDefinition = `${backend}DefaultOriginalBuildHelperDefinition`; - const updatedBuildDefinition = `${backend}DefaultUpdatedBuildHelperDefinition`; - const originalOrdinaryDefinition = `${backend}DefaultOriginalOrdinaryDefinition`; - const updatedOrdinaryDefinition = `${backend}DefaultUpdatedOrdinaryDefinition`; - writeSource(rootBuildHelperPath, originalBuildDefinition); - writeSource(ordinaryPath, originalOrdinaryDefinition); - - const indexer = createIndexer(config); - await indexer.index(); - expect(await hasDefinition(indexer, originalBuildDefinition, rootBuildHelperPath)).toBe(true); + const capture = new DefaultRootBuildFailureCapture(projectDir, indexRoot, backend); + failureCapture = capture; + try { + const config = structuralConfig(); + const rootBuildHelperPath = path.join(projectDir, "build-helper.ts"); + const ordinaryPath = path.join(projectDir, "src", "ordinary.ts"); + const originalBuildDefinition = `${backend}DefaultOriginalBuildHelperDefinition`; + const updatedBuildDefinition = `${backend}DefaultUpdatedBuildHelperDefinition`; + const originalOrdinaryDefinition = `${backend}DefaultOriginalOrdinaryDefinition`; + const updatedOrdinaryDefinition = `${backend}DefaultUpdatedOrdinaryDefinition`; + writeSource(rootBuildHelperPath, originalBuildDefinition); + capture.record("write", { path: rootBuildHelperPath, definition: originalBuildDefinition }); + writeSource(ordinaryPath, originalOrdinaryDefinition); + capture.record("write", { path: ordinaryPath, definition: originalOrdinaryDefinition }); - const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => undefined); - const watcher = new FileWatcher(projectDir, config, "jcode", { backend }); - watchers.push(watcher); - const observedPaths: string[] = []; - let handlerTail = Promise.resolve(); - watcher.start(async (changes) => { - observedPaths.push(...changes.map((change) => change.path)); - handlerTail = handlerTail.then(async () => { + const indexer = createIndexer(config); + capture.record("index-start", { initial: true }); + try { await indexer.index(); + capture.record("index-end", { initial: true }); + } catch (error) { + capture.record("index-error", error); + throw error; + } + expect(await hasDefinition(indexer, originalBuildDefinition, rootBuildHelperPath, capture)).toBe(true); + + const warnSpy = vi.spyOn(console, "warn").mockImplementation((...args) => { + capture.record("warning", args); }); - handlerTails.push(handlerTail); - await handlerTail; - }); - await watcher.waitUntilReady(); + const watcher = new FileWatcher(projectDir, config, "jcode", { backend }); + watchers.push(watcher); + const observedPaths: string[] = []; + let handlerTail = Promise.resolve(); + watcher.start(async (changes) => { + capture.record("callback-start", changes); + try { + observedPaths.push(...changes.map((change) => change.path)); + handlerTail = handlerTail.then(async () => { + capture.record("index-start", { initial: false }); + try { + await indexer.index(); + capture.record("index-end", { initial: false }); + } catch (error) { + capture.record("index-error", error); + throw error; + } + }); + handlerTails.push(handlerTail); + await handlerTail; + capture.record("callback-end"); + } catch (error) { + capture.record("callback-error", error); + throw error; + } + }); + // Passive listeners on this actual emitter only; replacement/native layers remain MISSING. + const internals = watcher as unknown as { watcher: FSWatcher | null; nativeWatcher: object | null }; + const emitter = internals.watcher; + if (emitter) { + capture.record("backend", { emitter: "initial Chokidar", options: emitter.options }); + emitter.on("raw", (event, filePath, details) => capture.record("raw", { event, filePath, details })); + emitter.on("all", (event, filePath) => capture.record("backend-event", { event, filePath })); + emitter.on("ready", () => capture.record("backend-ready", { emitter: "initial Chokidar" })); + emitter.on("error", (error) => capture.record("backend-error", error)); + } else { + capture.record("backend", { emitter: "MISSING: no Chokidar emitter at start", requested: backend }); + } + capture.record("ready-start"); + await watcher.waitUntilReady(); + capture.record("ready-end", { running: watcher.isRunning(), nativeActive: internals.nativeWatcher !== null, + chokidar: internals.watcher === emitter ? "same captured emitter" : "MISSING: replacement emitter" }); - writeSource(rootBuildHelperPath, updatedBuildDefinition); - writeSource(ordinaryPath, updatedOrdinaryDefinition); - await vi.waitFor(async () => { - expect(await hasDefinition(indexer, updatedOrdinaryDefinition, ordinaryPath)).toBe(true); - expect(observedPaths).toContain(ordinaryPath); - }, { timeout: WATCH_TIMEOUT_MS, interval: 50 }); + writeSource(rootBuildHelperPath, updatedBuildDefinition); + capture.record("write", { path: rootBuildHelperPath, definition: updatedBuildDefinition }); + writeSource(ordinaryPath, updatedOrdinaryDefinition); + capture.record("write", { path: ordinaryPath, definition: updatedOrdinaryDefinition }); + await vi.waitFor(async () => { + expect(await hasDefinition(indexer, updatedOrdinaryDefinition, ordinaryPath, capture)).toBe(true); + expect(observedPaths).toContain(ordinaryPath); + }, { timeout: WATCH_TIMEOUT_MS, interval: 50 }); - expect(observedPaths).not.toContain(rootBuildHelperPath); - expect(await hasDefinition(indexer, updatedBuildDefinition, rootBuildHelperPath)).toBe(true); - if (backend === "native") { - expect(warnSpy.mock.calls.some(([message]) => String(message).includes("Chokidar fallback"))).toBe(false); + expect(observedPaths).not.toContain(rootBuildHelperPath); + expect(await hasDefinition(indexer, updatedBuildDefinition, rootBuildHelperPath, capture)).toBe(true); + if (backend === "native") { + expect(warnSpy.mock.calls.some(([message]) => String(message).includes("Chokidar fallback"))).toBe(false); + } + } catch (error) { + capture.fail(error); + throw error; } } diff --git a/tests/watcher/default-root-build-failure-capture.ts b/tests/watcher/default-root-build-failure-capture.ts new file mode 100644 index 00000000..5dcc3e35 --- /dev/null +++ b/tests/watcher/default-root-build-failure-capture.ts @@ -0,0 +1,295 @@ +import * as crypto from "node:crypto"; +import * as fs from "node:fs"; +import { createRequire } from "node:module"; +import * as os from "node:os"; +import * as path from "node:path"; +import { performance } from "node:perf_hooks"; +import { fileURLToPath } from "node:url"; +import { inspect } from "node:util"; + +// Only the default-root-build acceptance scenario owns this diagnostic. +export const CAPTURE_LIMITS = { + events: 512, + eventBytes: 2048, + entries: 256, + snapshotBytes: 16 * 1024 * 1024, + totalBytes: 18 * 1024 * 1024, +} as const; + +const ANCHORED_PHASES: Record = { + scenario: true, "ready-start": true, "ready-end": true, write: true, + "callback-start": true, "callback-end": true, "callback-error": true, + "index-start": true, "index-end": true, "index-error": true, failure: true, + cleanup: true, warning: true, backend: true, +}; +const REPO_ROOT = fileURLToPath(new URL("../../", import.meta.url)); + +type Event = { ms: number; phase: string; detail: string }; +type Root = { label: "project" | "index"; directory: string; identity: fs.Stats }; +type Entry = { path: string; bytes?: number; missing?: string }; + +function sameIdentity(left: fs.Stats, right: fs.Stats): boolean { + return left.dev === right.dev && left.ino === right.ino; +} + +function checkedDirectory(directory: string): fs.Stats { + const stat = fs.lstatSync(directory); + if (!stat.isDirectory() || stat.isSymbolicLink() + || fs.realpathSync(directory) !== path.join(fs.realpathSync(path.dirname(directory)), path.basename(directory))) { + throw new Error("MISSING: unsafe directory boundary"); + } + return stat; +} + +export class DefaultRootBuildFailureCapture { + private readonly started = performance.now(); + private readonly roots: Root[] = []; + private readonly events: Event[] = []; + private readonly anchors: Record = {}; + private dropped = 0; + private failed = false; + private readonly missing: string[] = []; + + constructor(projectDir: string, indexRoot: string, private readonly backend: string) { + for (const [label, directory, prefix] of [ + ["project", projectDir, "ocbi-include-excluded-project-"], + ["index", indexRoot, "ocbi-include-excluded-index-"], + ] as const) { + try { + const identity = checkedDirectory(directory); + if (fs.realpathSync(path.dirname(directory)) !== fs.realpathSync(os.tmpdir()) + || !path.basename(directory).startsWith(prefix) + || (process.getuid && identity.uid !== process.getuid())) { + throw new Error("MISSING: not an owned scenario fixture root"); + } + this.roots.push({ label, directory, identity }); + } catch (error) { + this.missing.push(`${label}: ${this.text(error)}`); + } + } + this.record("scenario", { backend, host: "jcode", platform: process.platform, + arch: process.arch, osRelease: os.release(), node: process.version, versions: process.versions, + gaps: { filtering: "MISSING", queued: "MISSING", flush: "MISSING", + nativeNotifications: "MISSING: no public passive emitter", + chokidarReplacement: "MISSING: listeners cover only the captured emitter" } }); + } + + private text(value: unknown): string { + let text = inspect(value, { depth: 3, maxArrayLength: 20, maxStringLength: 1024, + customInspect: false, getters: false, breakLength: Infinity }); + for (const root of this.roots) text = text.replaceAll(root.directory, `<${root.label}>`); + text = text.replaceAll(REPO_ROOT.replace(/[\\/]$/, ""), "") + .replaceAll(os.homedir(), "") + .replaceAll(os.tmpdir(), ""); + // Raw backend errors may name unrelated absolute paths. Never retain those names. + text = text.replace(/(?])(?:[A-Za-z]:[\\/]|\/)[^\s'"<>),\]}]+/g, ""); + return Buffer.from(text).subarray(0, 1500).toString("utf8"); + } + + record(phase: string, detail?: unknown): void { + // Diagnostics must not change callback/query error propagation, even for unusual values. + try { + const event = { ms: performance.now() - this.started, phase: phase.slice(0, 64), detail: this.text(detail) }; + while (Buffer.byteLength(JSON.stringify(event)) > CAPTURE_LIMITS.eventBytes) { + event.detail = event.detail.slice(0, Math.floor(event.detail.length / 2)); + } + if (this.events.length === CAPTURE_LIMITS.events) { + this.events.shift(); + this.dropped += 1; + } + this.events.push(event); + if (Object.hasOwn(ANCHORED_PHASES, phase)) { + const anchor = this.anchors[phase]; + this.anchors[phase] = { first: anchor?.first ?? event, last: event }; + } + } catch { + this.dropped += 1; + } + } + + fail(error: unknown): void { + this.failed = true; + this.record("failure", error); + } + + private assertRoot(root: Root): void { + if (!sameIdentity(root.identity, checkedDirectory(root.directory))) { + throw new Error("MISSING: fixture root identity changed"); + } + } + + private nativeObservation(): unknown { + let suffix = ""; + if (process.platform === "linux") suffix = "-gnu"; + if (process.platform === "win32") suffix = "-msvc"; + const filename = `codebase-index-native.${process.platform}-${process.arch}${suffix}.node`; + const nativePath = path.join(REPO_ROOT, "native", filename); + // Observe only the ordinary cache; do not require, import, or reload the addon. + const cached = createRequire(import.meta.url).cache[nativePath]; + if (!cached) return { cache: "MISSING", executedBytes: "MISSING" }; + const observation = { cache: { filename: `native/${filename}`, loaded: cached.loaded }, + executedBytes: "MISSING: cache identity and current disk bytes do not prove executed bytes" }; + try { + const before = fs.lstatSync(nativePath); + if (!before.isFile() || before.isSymbolicLink() || before.nlink !== 1 || before.size > 64 * 1024 * 1024) { + throw new Error("unsafe native disk file"); + } + const fd = fs.openSync(nativePath, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + try { + if (!sameIdentity(before, fs.fstatSync(fd))) throw new Error("native disk identity changed"); + const hash = crypto.createHash("sha256"); + const buffer = Buffer.alloc(64 * 1024); + let remaining = before.size; + while (remaining > 0) { + const count = fs.readSync(fd, buffer, 0, Math.min(buffer.length, remaining), null); + if (!count) throw new Error("native disk read incomplete"); + hash.update(buffer.subarray(0, count)); + remaining -= count; + } + const after = fs.fstatSync(fd); + if (!sameIdentity(before, after) || after.size !== before.size || after.mtimeMs !== before.mtimeMs) { + throw new Error("native disk changed during hash"); + } + return { ...observation, currentDiskOnlySha256: hash.digest("hex") }; + } finally { + fs.closeSync(fd); + } + } catch (error) { + return { ...observation, currentDiskOnlySha256: this.text(error) }; + } + } + + // Called only after watcher stop, handler tails, and indexer close have settled. + // The caller still owns unconditional fixture deletion and the original test outcome. + retainFailure(options: { testFailed: boolean; indexClosed: boolean }): { directory?: string; missing?: string } { + if (!this.failed && !options.testFailed) return {}; + try { + const requestedBase = process.env.CI && process.env.RUNNER_TEMP ? process.env.RUNNER_TEMP : os.tmpdir(); + checkedDirectory(requestedBase); + const base = fs.realpathSync(requestedBase); + const parent = path.join(base, "ocbi-watcher-failures"); + try { + fs.mkdirSync(parent, { mode: 0o700 }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + } + const parentStat = checkedDirectory(parent); + if ((process.getuid && parentStat.uid !== process.getuid()) || (parentStat.mode & 0o077) !== 0) { + throw new Error("MISSING: artifact parent is not private/owned"); + } + const directory = fs.mkdtempSync(path.join(parent, `${this.backend}-`)); + fs.chmodSync(directory, 0o700); + if (!sameIdentity(parentStat, checkedDirectory(parent))) throw new Error("MISSING: artifact parent changed"); + checkedDirectory(directory); + const entries: Entry[] = []; + let bytes = 0; + let visited = 0; + const copy = (root: Root, relative: string): void => { + if (visited >= CAPTURE_LIMITS.entries) return; + visited += 1; + const source = path.join(root.directory, relative); + const destination = path.join(directory, root.label, relative); + const name = path.join(root.label, relative).replaceAll("\\", "/"); + try { + this.assertRoot(root); + if (fs.realpathSync(path.dirname(source)) !== path.join(fs.realpathSync(root.directory), path.dirname(relative))) { + throw new Error("MISSING: parent path escaped/replaced"); + } + const before = fs.lstatSync(source); + if (before.isSymbolicLink()) throw new Error("MISSING: symlink refused"); + if (before.isDirectory()) { + const fd = fs.openSync(source, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + try { + if (!sameIdentity(before, fs.fstatSync(fd))) throw new Error("MISSING: directory identity changed"); + fs.mkdirSync(destination, { mode: 0o700 }); + const dir = fs.opendirSync(source); + try { + let child: fs.Dirent | null; + while (visited < CAPTURE_LIMITS.entries && (child = dir.readSync())) { + copy(root, path.join(relative, child.name)); + } + } finally { + dir.closeSync(); + } + } finally { + fs.closeSync(fd); + } + } else { + if (!before.isFile() || before.nlink !== 1) throw new Error("MISSING: special/hardlinked file refused"); + if (before.size > CAPTURE_LIMITS.snapshotBytes - bytes) throw new Error("MISSING: snapshot byte cap"); + const fd = fs.openSync(source, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + try { + const opened = fs.fstatSync(fd); + if (!sameIdentity(before, opened) || opened.nlink !== 1 || opened.size !== before.size) { + throw new Error("MISSING: opened file identity/size changed"); + } + this.assertRoot(root); + const data = Buffer.alloc(before.size); + let offset = 0; + while (offset < data.length) { + const count = fs.readSync(fd, data, offset, data.length - offset, null); + if (!count) throw new Error("MISSING: incomplete fixture read"); + offset += count; + } + const after = fs.fstatSync(fd); + if (!sameIdentity(opened, after) || after.size !== opened.size || after.mtimeMs !== opened.mtimeMs) { + throw new Error("MISSING: fixture changed during capture"); + } + bytes += data.length; // Reserve even if a write partially fails. + fs.writeFileSync(destination, data, { flag: "wx", mode: 0o600 }); + entries.push({ path: name.slice(0, 512), bytes: data.length }); + } finally { + fs.closeSync(fd); + } + } + } catch (error) { + entries.push({ path: this.text(name), missing: this.text(error) }); + } + }; + for (const root of this.roots) { + if (root.label === "index" && !options.indexClosed) { + this.missing.push("index: MISSING: indexer close did not succeed; database snapshot refused"); + continue; + } + try { + this.assertRoot(root); + const fd = fs.openSync(root.directory, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + try { + if (!sameIdentity(root.identity, fs.fstatSync(fd))) throw new Error("MISSING: opened root changed"); + fs.mkdirSync(path.join(directory, root.label), { mode: 0o700 }); + const dir = fs.opendirSync(root.directory); + try { + let child: fs.Dirent | null; + while (visited < CAPTURE_LIMITS.entries && (child = dir.readSync())) copy(root, child.name); + } finally { + dir.closeSync(); + } + } finally { + fs.closeSync(fd); + } + } catch (error) { + this.missing.push(`${root.label}: ${this.text(error)}`); + } + } + let metadata = JSON.stringify({ schema: 1, backend: this.backend, host: "jcode", limits: CAPTURE_LIMITS, + runtime: { node: process.version, versions: process.versions, platform: process.platform, + arch: process.arch, osRelease: os.release(), pid: process.pid }, + elapsedMs: performance.now() - this.started, droppedEvents: this.dropped, + events: this.events, anchors: this.anchors, + missing: [...this.missing, "filtering: MISSING", "queued: MISSING", "flush: MISSING", + "native notifications: MISSING", "replacement Chokidar emitter coverage: MISSING"], + native: this.nativeObservation(), snapshot: { bytes, visited, entryCapReached: visited >= CAPTURE_LIMITS.entries, + indexClosed: options.indexClosed, entries }, + privacy: "Raw fixture index bytes may embed fixture paths; diagnostic text is redacted. Not an adversarial race-proof copier.", + }, null, 2); + if (Buffer.byteLength(metadata) + bytes > CAPTURE_LIMITS.totalBytes) { + metadata = JSON.stringify({ missing: "MISSING: shared metadata byte cap", limits: CAPTURE_LIMITS, + snapshotBytes: bytes, visited, droppedEvents: this.dropped, anchors: this.anchors }); + } + fs.writeFileSync(path.join(directory, "capture.json"), metadata, { flag: "wx", mode: 0o600 }); + return { directory }; + } catch (error) { + return { missing: this.text(error) }; + } + } +} From 8864f8a9f99fb2e4bc9df789a62018cc09f354c8 Mon Sep 17 00:00:00 2001 From: Helweg Date: Mon, 19 Oct 2026 16:49:23 +0200 Subject: [PATCH 2/2] test: refine bounded watcher failure capture --- CONTRIBUTING.md | 15 +++++--- tests/include-excluded-paths-indexing.test.ts | 23 +++++++---- .../default-root-build-failure-capture.ts | 38 ++++++++++++------- 3 files changed, 49 insertions(+), 27 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 05081f4d..e384ff4d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -82,8 +82,9 @@ directory is under `/ocbi-watcher-failures/`; CI uses `${RUNNER_TEMP}/ocbi-watcher-failures/` and uploads it on job failure as `watcher-failures-node---` for 14 days. `capture.json` contains monotonic observations, first/last phase anchors, -suppressed warnings, dropped-event counts, runtime/backend information, and a -snapshot inventory. Only genuine notifications from the initially available +suppressed warnings, dropped-event and explicit detail-byte-cut counts, +bounded numeric index statistics, runtime/backend information, and a snapshot +inventory. Only genuine notifications from the initially available Chokidar emitter are observed; replacement/native notifications and internal filtering, queueing, and flushing are marked `MISSING`, not inferred. @@ -91,9 +92,13 @@ Limits per scenario: 512 tail events of at most 2 KiB each, fixed first/last phase anchors, 256 visited fixture entries, 16 MiB of combined source/index snapshots, and 18 MiB total including metadata. Caps, refused paths, capture errors, and unavailable evidence are reported as `MISSING`. Symlinks, -hardlinks, special files, and replaced fixture roots are refused; an index -whose close failed or is unknown is not copied. These checks are not an -adversarial filesystem-race guarantee. +hardlinks, special files, and replaced fixture roots are refused. Index +snapshots require both all watcher stops and all indexer closes to succeed, +after handler tails settle; otherwise only safe source fixtures are retained. +These quiescence checks do not prove database consistency. POSIX mode and +directory-FD guards may refuse Windows snapshots as `MISSING`; Windows +snapshot coverage has not been verified. These checks are not an adversarial +filesystem-race guarantee. Diagnostic text redacts absolute private paths, but **raw fixture index files can contain temporary paths**. Inspect artifacts privately; do not commit them diff --git a/tests/include-excluded-paths-indexing.test.ts b/tests/include-excluded-paths-indexing.test.ts index 2d0285d8..649b67ee 100644 --- a/tests/include-excluded-paths-indexing.test.ts +++ b/tests/include-excluded-paths-indexing.test.ts @@ -131,15 +131,18 @@ describe("include-excluded path indexing acceptance", () => { }); afterEach(async ({ task }) => { + const capture = failureCapture; + let watchersStopped = false; let indexClosed = false; try { snapshotFaults.beforeRead = undefined; snapshotFaults.afterStat = undefined; const stopResults = await Promise.allSettled(watchers.map((watcher) => watcher.stop())); + watchersStopped = stopResults.every((result) => result.status === "fulfilled"); const handlerResults = await Promise.allSettled(handlerTails); const closeResults = await Promise.allSettled(indexers.map((indexer) => indexer.close())); indexClosed = closeResults.every((result) => result.status === "fulfilled"); - failureCapture?.record("cleanup", { stop: stopResults, handlers: handlerResults, close: closeResults }); + capture?.record("cleanup", { watchersStopped, indexClosed, stop: stopResults, handlers: handlerResults, close: closeResults }); const failures = [...stopResults, ...handlerResults, ...closeResults] .filter((result): result is PromiseRejectedResult => result.status === "rejected") .map((result) => result.reason); @@ -148,15 +151,15 @@ describe("include-excluded path indexing acceptance", () => { throw new AggregateError(failures, "Failed to close include-excluded acceptance test resources"); } } catch (error) { - failureCapture?.fail(error); + capture?.fail(error); throw error; } finally { try { - const retained = failureCapture?.retainFailure({ testFailed: task.result?.state === "fail", indexClosed }); + const retained = capture?.retainFailure({ testFailed: task.result?.state === "fail", watchersStopped, indexClosed }); if (retained?.directory) console.error(`[watcher failure capture] ${retained.directory}`); if (retained?.missing) console.error(`[watcher failure capture] ${retained.missing}`); } catch (error) { - failureCapture?.record("capture-error", { missing: "MISSING: retention/reporting failed", error }); + capture?.record("capture-error", { missing: "MISSING: retention/reporting failed", error }); } finally { vi.restoreAllMocks(); fs.rmSync(projectDir, { recursive: true, force: true }); @@ -530,8 +533,10 @@ describe("include-excluded path indexing acceptance", () => { const indexer = createIndexer(config); capture.record("index-start", { initial: true }); try { - await indexer.index(); - capture.record("index-end", { initial: true }); + const stats = await indexer.index(); + capture.record("index-end", { initial: true, + totalFiles: stats.totalFiles, totalChunks: stats.totalChunks, indexedChunks: stats.indexedChunks, + failedChunks: stats.failedChunks, existingChunks: stats.existingChunks, removedChunks: stats.removedChunks }); } catch (error) { capture.record("index-error", error); throw error; @@ -552,8 +557,10 @@ describe("include-excluded path indexing acceptance", () => { handlerTail = handlerTail.then(async () => { capture.record("index-start", { initial: false }); try { - await indexer.index(); - capture.record("index-end", { initial: false }); + const stats = await indexer.index(); + capture.record("index-end", { initial: false, + totalFiles: stats.totalFiles, totalChunks: stats.totalChunks, indexedChunks: stats.indexedChunks, + failedChunks: stats.failedChunks, existingChunks: stats.existingChunks, removedChunks: stats.removedChunks }); } catch (error) { capture.record("index-error", error); throw error; diff --git a/tests/watcher/default-root-build-failure-capture.ts b/tests/watcher/default-root-build-failure-capture.ts index 5dcc3e35..b43894f7 100644 --- a/tests/watcher/default-root-build-failure-capture.ts +++ b/tests/watcher/default-root-build-failure-capture.ts @@ -45,8 +45,10 @@ export class DefaultRootBuildFailureCapture { private readonly started = performance.now(); private readonly roots: Root[] = []; private readonly events: Event[] = []; + private nextEvent = 0; private readonly anchors: Record = {}; private dropped = 0; + private detailByteCuts = 0; private failed = false; private readonly missing: string[] = []; @@ -83,7 +85,9 @@ export class DefaultRootBuildFailureCapture { .replaceAll(os.tmpdir(), ""); // Raw backend errors may name unrelated absolute paths. Never retain those names. text = text.replace(/(?])(?:[A-Za-z]:[\\/]|\/)[^\s'"<>),\]}]+/g, ""); - return Buffer.from(text).subarray(0, 1500).toString("utf8"); + const bytes = Buffer.from(text); + if (bytes.length > 1500) this.detailByteCuts += 1; + return bytes.subarray(0, 1500).toString("utf8"); } record(phase: string, detail?: unknown): void { @@ -92,12 +96,11 @@ export class DefaultRootBuildFailureCapture { const event = { ms: performance.now() - this.started, phase: phase.slice(0, 64), detail: this.text(detail) }; while (Buffer.byteLength(JSON.stringify(event)) > CAPTURE_LIMITS.eventBytes) { event.detail = event.detail.slice(0, Math.floor(event.detail.length / 2)); + this.detailByteCuts += 1; } - if (this.events.length === CAPTURE_LIMITS.events) { - this.events.shift(); - this.dropped += 1; - } - this.events.push(event); + if (this.events.length === CAPTURE_LIMITS.events) this.dropped += 1; + this.events[this.nextEvent] = event; + this.nextEvent = (this.nextEvent + 1) % CAPTURE_LIMITS.events; if (Object.hasOwn(ANCHORED_PHASES, phase)) { const anchor = this.anchors[phase]; this.anchors[phase] = { first: anchor?.first ?? event, last: event }; @@ -161,7 +164,9 @@ export class DefaultRootBuildFailureCapture { // Called only after watcher stop, handler tails, and indexer close have settled. // The caller still owns unconditional fixture deletion and the original test outcome. - retainFailure(options: { testFailed: boolean; indexClosed: boolean }): { directory?: string; missing?: string } { + retainFailure(options: { + testFailed: boolean; watchersStopped: boolean; indexClosed: boolean; + }): { directory?: string; missing?: string } { if (!this.failed && !options.testFailed) return {}; try { const requestedBase = process.env.CI && process.env.RUNNER_TEMP ? process.env.RUNNER_TEMP : os.tmpdir(); @@ -247,8 +252,8 @@ export class DefaultRootBuildFailureCapture { } }; for (const root of this.roots) { - if (root.label === "index" && !options.indexClosed) { - this.missing.push("index: MISSING: indexer close did not succeed; database snapshot refused"); + if (root.label === "index" && (!options.watchersStopped || !options.indexClosed)) { + this.missing.push("index: MISSING: watcher stop or indexer close did not succeed; quiescence unverified; database snapshot refused"); continue; } try { @@ -271,20 +276,25 @@ export class DefaultRootBuildFailureCapture { this.missing.push(`${root.label}: ${this.text(error)}`); } } + const events = this.events.length === CAPTURE_LIMITS.events + ? [...this.events.slice(this.nextEvent), ...this.events.slice(0, this.nextEvent)] + : this.events; let metadata = JSON.stringify({ schema: 1, backend: this.backend, host: "jcode", limits: CAPTURE_LIMITS, runtime: { node: process.version, versions: process.versions, platform: process.platform, arch: process.arch, osRelease: os.release(), pid: process.pid }, - elapsedMs: performance.now() - this.started, droppedEvents: this.dropped, - events: this.events, anchors: this.anchors, + elapsedMs: performance.now() - this.started, droppedEvents: this.dropped, detailByteCuts: this.detailByteCuts, + events, anchors: this.anchors, missing: [...this.missing, "filtering: MISSING", "queued: MISSING", "flush: MISSING", - "native notifications: MISSING", "replacement Chokidar emitter coverage: MISSING"], + "native notifications: MISSING", "replacement Chokidar emitter coverage: MISSING", + "snapshot portability: POSIX mode/directory-FD guards; Windows coverage MISSING"], native: this.nativeObservation(), snapshot: { bytes, visited, entryCapReached: visited >= CAPTURE_LIMITS.entries, - indexClosed: options.indexClosed, entries }, + watchersStopped: options.watchersStopped, indexClosed: options.indexClosed, entries }, privacy: "Raw fixture index bytes may embed fixture paths; diagnostic text is redacted. Not an adversarial race-proof copier.", }, null, 2); if (Buffer.byteLength(metadata) + bytes > CAPTURE_LIMITS.totalBytes) { metadata = JSON.stringify({ missing: "MISSING: shared metadata byte cap", limits: CAPTURE_LIMITS, - snapshotBytes: bytes, visited, droppedEvents: this.dropped, anchors: this.anchors }); + snapshotBytes: bytes, visited, droppedEvents: this.dropped, detailByteCuts: this.detailByteCuts, + watchersStopped: options.watchersStopped, indexClosed: options.indexClosed, anchors: this.anchors }); } fs.writeFileSync(path.join(directory, "capture.json"), metadata, { flag: "wx", mode: 0o600 }); return { directory };