diff --git a/docs-site/src/content/docs/reference/configuration/server.md b/docs-site/src/content/docs/reference/configuration/server.md index 781c3b408f8..3779c93801f 100644 --- a/docs-site/src/content/docs/reference/configuration/server.md +++ b/docs-site/src/content/docs/reference/configuration/server.md @@ -52,6 +52,12 @@ runs helper features around provider requests. | `visionSidecar?` | `OcxVisionSidecarConfig` | on when usable | Image-description sidecar options. | | `images?` | `OcxImagesConfig` | automatic OpenAI selection | Standalone Images relay options for Codex `image_gen`. | +On Linux x64/arm64 with pidfd support, service health recovery binds termination to the proxy's +kernel process identity, so PID reuse cannot redirect it. The sentinel's watching log ends in +`(pidfd)`. Unsupported kernels or architectures retain the existing live-parent PID check and log +that fallback; other binding failures disable recovery for that proxy. macOS retains the live-parent +check, and Windows retains its process handle binding. + While the canonical ChatGPT upstream WebSocket waits for the first Responses event after sending the create frame, it watches for liveness rather than a fixed deadline. When the socket supports protocol pings, the proxy pings it every 15 seconds. Any inbound frame — quota, diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 5fbc7243f79..9925e111815 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1638,7 +1638,7 @@ "server-xai-responses-streaming.test.ts": "server", "service-auth-qualified-localhost.test.ts": "service", "service-child-ownership.test.ts": "service", - "service-handled-signal-exit.test.ts": "service", "service-health-sentinel-launch.test.ts": "service", "service-health-sentinel-policy.test.ts": "service", "service-health-sentinel-process.test.ts": "service", "service-health-sentinel-runner.test.ts": "service", + "service-handled-signal-exit.test.ts": "service", "service-health-sentinel-linux.test.ts": "service", "service-health-sentinel-launch.test.ts": "service", "service-health-sentinel-policy.test.ts": "service", "service-health-sentinel-process.test.ts": "service", "service-health-sentinel-runner.test.ts": "service", "service-claim.test.ts": "service", "service-ownership-compatibility.test.ts": "service", "service-ownership-handover.test.ts": "service", diff --git a/src/service/health-sentinel-linux.ts b/src/service/health-sentinel-linux.ts new file mode 100644 index 00000000000..6c29f82de69 --- /dev/null +++ b/src/service/health-sentinel-linux.ts @@ -0,0 +1,112 @@ +/** Linux pidfd ownership for the opt-in sentinel. Loaded only on Linux. */ +import { dlopen, FFIType, read } from "bun:ffi"; +import type { TerminateResult } from "./health-sentinel-main"; + +const ESRCH = 3; +const ENOSYS = 38; +// Syscalls from 424 onward share one number on every Linux architecture. Binding is limited to +// x64/arm64 because the FFI signature below assumes their LP64 integer-register calling convention. +const SYS_PIDFD_OPEN = 434; +const SYS_PIDFD_SEND_SIGNAL = 424; + +type CallResult = { value: number; errno: number }; +export type PidfdSyscalls = { + open(pid: number): CallResult; + signal(fd: number, signal: number): CallResult; + close(fd: number): void; + dispose(): void; +}; +export type ParentPidfd = { terminate(): TerminateResult; close(): void }; +export type ParentPidfdResult = + | { kind: "bound"; handle: ParentPidfd } + | { kind: "unsupported"; reason: string } + | { kind: "refused"; reason: string }; + +function nativeSyscalls(): PidfdSyscalls { + const symbols = { + // syscall is variadic, but these syscalls use integer/pointer arguments only. All supplied + // slots are machine-word integers (including zero for siginfo and flags), on these LP64 ABIs. + syscall: { args: [FFIType.i64, FFIType.i64, FFIType.i64, FFIType.i64, FFIType.i64], returns: FFIType.i64 }, + __errno_location: { args: [], returns: FFIType.ptr }, + close: { args: [FFIType.i32], returns: FFIType.i32 }, + } as const; + const load = (path: string) => dlopen(path, symbols); + let libc: ReturnType; + try { + libc = load("libc.so.6"); + } catch { + // musl exports libc symbols through its loader; it has no glibc soname or pidfd wrappers. + libc = load(`/lib/ld-musl-${process.arch === "x64" ? "x86_64" : "aarch64"}.so.1`); + } + const call = (nr: number, a: number, b: number, c = 0, d = 0): CallResult => { + const value = Number(libc.symbols.syscall(nr, a, b, c, d)); + return { value, errno: value === -1 ? read.i32(libc.symbols.__errno_location()!) : 0 }; + }; + return { + open: pid => call(SYS_PIDFD_OPEN, pid, 0), + signal: (fd, signal) => call(SYS_PIDFD_SEND_SIGNAL, fd, signal), + // Never retry close on Linux: even an EINTR has released the descriptor. + close: fd => { libc.symbols.close(fd); }, + dispose: () => libc.close(), + }; +} + +/** Acquire before arming; reparenting during acquisition invalidates even a successfully opened fd. */ +export function openParentPidfd( + pid: number, + isParent: () => boolean, + options: { arch?: string; load?: () => PidfdSyscalls } = {}, +): ParentPidfdResult { + if (!isParent()) return { kind: "refused", reason: "the sentinel is not the watched process's child" }; + const arch = options.arch ?? process.arch; + if (arch !== "x64" && arch !== "arm64") return { kind: "unsupported", reason: `pidfd syscall ABI is unsupported on ${arch}` }; + let api: PidfdSyscalls; + try { api = (options.load ?? nativeSyscalls)(); } + catch { return { kind: "refused", reason: "Linux pidfd libc binding is unavailable" }; } + let fd: number | null = null; + let closed = false; + let signalled = false; + const close = () => { + if (closed) return; + closed = true; + try { if (fd !== null) api.close(fd); } finally { api.dispose(); } + }; + const refuse = (reason: string): ParentPidfdResult => { close(); return { kind: "refused", reason }; }; + try { + const opened = api.open(pid); + if (opened.value < 0) { + close(); + return opened.errno === ENOSYS + ? { kind: "unsupported", reason: "pidfd_open is unsupported by this kernel" } + : { kind: "refused", reason: `pidfd_open failed (errno ${opened.errno})` }; + } + fd = opened.value; + if (!isParent()) return refuse("the watched parent exited during pidfd acquisition"); + // Check both syscall support and permission before arming, without sending a signal. + const probe = api.signal(fd, 0); + if (probe.value < 0) { + close(); + return probe.errno === ENOSYS + ? { kind: "unsupported", reason: "pidfd_send_signal is unsupported by this kernel" } + : { kind: "refused", reason: `pidfd signal check failed (errno ${probe.errno})` }; + } + if (!isParent()) return refuse("the watched parent exited during pidfd verification"); + return { + kind: "bound", + handle: { + close, + terminate: () => { + if (closed || signalled) return "not_target"; + // A throwing binding must settle the recorded intent as "failed", like process.kill does. + let sent: CallResult; + try { sent = api.signal(fd!, 9); } catch { return "failed"; } + if (sent.value < 0) return sent.errno === ESRCH ? "not_target" : "failed"; + signalled = true; + return "terminated"; + }, + }, + }; + } catch { + return refuse("Linux pidfd binding failed"); + } +} diff --git a/src/service/health-sentinel-main.ts b/src/service/health-sentinel-main.ts index ddcdd18f121..3099f5d34d7 100644 --- a/src/service/health-sentinel-main.ts +++ b/src/service/health-sentinel-main.ts @@ -12,11 +12,10 @@ * - Windows reuses ids quickly, so the sentinel opens a handle on its parent at startup, accepts it * only if that process was created before the sentinel, and terminates through the handle * (health-sentinel-windows.ts). While the handle is open the id cannot be reused. - * - POSIX allocates ids sequentially, and the sentinel's live `process.ppid` changes the moment - * its parent exits, before the exited parent can be reaped and its id freed. It is the last check - * before `kill`. Reuse inside that window needs the parent to exit, be reaped, and the id space to - * wrap completely (99,999 ids on macOS, `pid_max` on Linux) between two adjacent syscalls; that - * residual is accepted and documented in structure/ops/service-health-recovery.md. + * - Linux holds a pidfd after rechecking the live parent relationship, and signals that fd. + * Only unsupported kernels/architectures retain the live-ppid fallback used by macOS. + * - macOS checks the live parent id immediately before SIGKILL; PID reuse between those two + * syscalls remains a documented residual (structure/ops/service-health-recovery.md). * * SIGKILL rather than SIGTERM: a proxy whose event loop is stuck cannot run its own SIGTERM * handler, and systemd counts a SIGTERM death as a clean exit that `Restart=on-failure` does not @@ -80,6 +79,8 @@ export type HealthSentinelDeps = { isTargetGone(): boolean; /** Bind to the target once at startup; a failure stands the sentinel down. */ bindTarget(): { ok: true } | { ok: false; reason: string }; + /** Release the process owner on every exit, refusal, cancellation or exception. */ + closeTarget(): void; /** Terminate the bound target, re-proving it is still the target at the last moment. */ terminateTarget(): TerminateResult; /** Run under the ownership mutation lease, released on return; "busy" when it could not be taken promptly. */ @@ -149,6 +150,11 @@ function matchesProbeTarget(runtime: SentinelRuntimeRecord | null, pid: number, } export async function runHealthSentinel(deps: HealthSentinelDeps): Promise { + try { return await observeHealthSentinel(deps); } + finally { deps.closeTarget(); } +} + +async function observeHealthSentinel(deps: HealthSentinelDeps): Promise { const timing: HealthSentinelTiming = { ...DEFAULT_HEALTH_SENTINEL_TIMING, ...deps.timing }; const standDown = (reason: string): HealthSentinelResult => { deps.log(`standing down: ${reason} (pid ${deps.targetPid})`); @@ -407,59 +413,72 @@ export async function runHealthSentinelProcess(env: NodeJS.ProcessEnv = process. if (opened.ok) windowsHandle = opened.handle; else windowsBindFailure = opened.reason; } + let linuxHandle: import("./health-sentinel-linux").ParentPidfd | null = null; + let linuxBindFailure: string | null = null; + if (process.platform === "linux") { + const { openParentPidfd } = await import("./health-sentinel-linux"); + const opened = openParentPidfd(targetPid, () => !gone && process.ppid === targetPid); + if (opened.kind === "bound") linuxHandle = opened.handle; + else if (opened.kind === "refused") linuxBindFailure = opened.reason; + else appendBoundedLog(logPath, `pidfd unavailable: ${opened.reason}; using live parent PID check`); + } const lag = startLagMonitor(); - const result = await runHealthSentinel({ - targetPid, - now: Date.now, - sleep: ms => new Promise(resolve => setTimeout(resolve, ms)), - probe: async (target, timeoutMs) => { - lag.reset(); - const outcome = await probeHealth(target, timeoutMs); - return outcome.kind === "timeout" ? { ...outcome, sentinelLagMs: lag.read() } : outcome; - }, - readRuntime: () => readSentinelRuntime(runtimePath), - targetExited, - isTargetGone: () => gone || (windowsHandle?.exited() ?? false), - bindTarget: () => { - // POSIX: Bun reads the parent id live, and it stops naming the proxy once the proxy exits. - // Windows: it is the creating parent's id, so the handle below carries the identity. - if (process.ppid !== targetPid) return { ok: false, reason: "the sentinel is not the watched process's child" }; - if (process.platform !== "win32") return { ok: true }; - return windowsHandle ? { ok: true } : { ok: false, reason: windowsBindFailure ?? "the parent process cannot be opened" }; - }, - terminateTarget: () => { - if (process.platform === "win32") { - if (!windowsHandle) return "failed"; - return windowsHandle.terminate(1) ? "terminated" : "failed"; - } - if (gone || process.ppid !== targetPid) return "not_target"; - try { - process.kill(targetPid, "SIGKILL"); - return "terminated"; - } catch { - return "failed"; - } - }, - withOwnershipLease: run => { - let lease: { release(): void }; - try { - lease = acquireOwnershipMutationLease(statePaths, { waitMs: LEASE_WAIT_MS, processImage: () => null }); - } catch { - return "busy"; - } - try { return run(); } finally { lease.release(); } - }, - readHistory: () => { - const entries = readRecoveryEntries(historyPath); - return entries === null ? null : countedRecoveryTimes(entries); - }, - writeHistory: entry => writeRecoveryEntry(historyPath, entry), - log: line => appendBoundedLog(logPath, line), - timing: parseTimingOverride(env[HEALTH_SENTINEL_ENV.timing]), - }); - lag.stop(); - windowsHandle?.close(); - void result; + try { + await runHealthSentinel({ + targetPid, + now: Date.now, + sleep: ms => new Promise(resolve => setTimeout(resolve, ms)), + probe: async (target, timeoutMs) => { + lag.reset(); + const outcome = await probeHealth(target, timeoutMs); + return outcome.kind === "timeout" ? { ...outcome, sentinelLagMs: lag.read() } : outcome; + }, + readRuntime: () => readSentinelRuntime(runtimePath), + targetExited, + closeTarget: () => { linuxHandle?.close(); windowsHandle?.close(); }, + isTargetGone: () => gone || (windowsHandle?.exited() ?? false), + bindTarget: () => { + // POSIX: Bun reads the parent id live, and it stops naming the proxy once the proxy exits. + // Windows: it is the creating parent's id, so the handle below carries the identity. + if (process.ppid !== targetPid) return { ok: false, reason: "the sentinel is not the watched process's child" }; + if (linuxBindFailure) return { ok: false, reason: linuxBindFailure }; + if (process.platform !== "win32") return { ok: true }; + return windowsHandle ? { ok: true } : { ok: false, reason: windowsBindFailure ?? "the parent process cannot be opened" }; + }, + terminateTarget: () => { + if (process.platform === "win32") { + if (!windowsHandle) return "failed"; + return windowsHandle.terminate(1) ? "terminated" : "failed"; + } + if (gone || process.ppid !== targetPid) return "not_target"; + if (linuxHandle) return linuxHandle.terminate(); + try { + process.kill(targetPid, "SIGKILL"); + return "terminated"; + } catch { + return "failed"; + } + }, + withOwnershipLease: run => { + let lease: { release(): void }; + try { + lease = acquireOwnershipMutationLease(statePaths, { waitMs: LEASE_WAIT_MS, processImage: () => null }); + } catch { + return "busy"; + } + try { return run(); } finally { lease.release(); } + }, + readHistory: () => { + const entries = readRecoveryEntries(historyPath); + return entries === null ? null : countedRecoveryTimes(entries); + }, + writeHistory: entry => writeRecoveryEntry(historyPath, entry), + log: line => appendBoundedLog(logPath, linuxHandle && line.startsWith("watching pid ") ? `${line} (pidfd)` : line), + timing: parseTimingOverride(env[HEALTH_SENTINEL_ENV.timing]), + }); + } finally { + lag.stop(); + } return 0; } diff --git a/structure/INDEX.md b/structure/INDEX.md index f71e9bcfa45..83b95545d6d 100644 --- a/structure/INDEX.md +++ b/structure/INDEX.md @@ -234,6 +234,7 @@ A source area can be described by more than one doc, because these docs are orga | `src/service/` | [`runtime.md`](runtime.md) | | `src/service/desktop-startup.ts` | [`desktop-start-at-login.md`](desktop-start-at-login.md) | | `src/service/health-sentinel-launch.ts` | [`ops/service-health-recovery.md`](ops/service-health-recovery.md) | +| `src/service/health-sentinel-linux.ts` | [`ops/service-health-recovery.md`](ops/service-health-recovery.md) | | `src/service/health-sentinel-main.ts` | [`ops/service-health-recovery.md`](ops/service-health-recovery.md) | | `src/service/health-sentinel-policy.ts` | [`ops/service-health-recovery.md`](ops/service-health-recovery.md) | | `src/service/health-sentinel-probe.ts` | [`ops/service-health-recovery.md`](ops/service-health-recovery.md) | diff --git a/structure/manifest.json b/structure/manifest.json index 5ae90f7e07a..883dc88f14f 100644 --- a/structure/manifest.json +++ b/structure/manifest.json @@ -739,6 +739,7 @@ "documents": [ "src/service/health-sentinel-launch.ts", "src/service/health-sentinel-main.ts", + "src/service/health-sentinel-linux.ts", "src/service/health-sentinel-policy.ts", "src/service/health-sentinel-probe.ts", "src/service/health-sentinel-windows.ts" diff --git a/structure/ops/service-health-recovery.md b/structure/ops/service-health-recovery.md index baae6da325c..5535e5d551e 100644 --- a/structure/ops/service-health-recovery.md +++ b/structure/ops/service-health-recovery.md @@ -55,7 +55,7 @@ Probes accept only numeric IPv4/IPv6 destinations. Default and localhost binds u ## Binding the kill to the proxy -`src/service/health-sentinel-main.ts` never kills a bare number: +`src/service/health-sentinel-main.ts` selects the process binding before arming recovery: - Windows opens a kernel32 handle on the parent at startup through `bun:ffi` (`src/service/health-sentinel-windows.ts`), accepts it only if that process was created before the @@ -66,10 +66,24 @@ Probes accept only numeric IPv4/IPv6 destinations. Default and localhost binds u far enough that the new process appears to be older than the sentinel, the handle binds to that process and a later recovery would terminate it. A handle created by the proxy and inherited by the sentinel would close this gap; Bun's spawn does not offer one today. -- POSIX checks the live `process.ppid` immediately before SIGKILL. SIGTERM cannot run on a stuck - loop and is a clean exit to systemd. This binding is probabilistic: if the sentinel is descheduled - between that check and `kill` while the parent exits, is reaped, and a new process receives the - same id, the signal reaches that process. Bun exposes no pidfd or process handle on POSIX. +- Linux opens a pidfd through libc `syscall` using `bun:ffi` + (`src/service/health-sentinel-linux.ts`) on x64 and arm64, then checks that the watched PID is + still its live parent while holding the descriptor. A signal-zero call verifies signal support + and permissions before arming; the parent relationship is checked again afterward. Recovery + uses `pidfd_send_signal(SIGKILL)` on that descriptor, so PID reuse cannot redirect it. An exited + target (`ESRCH`) is skipped. A bound signal failure never falls back to a numeric PID. + `ENOSYS` from either startup syscall, or an unsupported syscall architecture, selects the existing + live-parent fallback and logs the limitation. Permission/resource/identity errors and unavailable + libc bindings refuse recovery. Both glibc and musl libc bindings are supported without new deps. + The watching line ends with `(pidfd)` when the binding succeeds. +- macOS, and Linux without pidfd support, check the live `process.ppid` immediately before + SIGKILL. This fallback is probabilistic: if the sentinel is descheduled between the check and + `kill` while the parent exits, is reaped, and its PID is reused, the signal can reach that process. + SIGTERM cannot run on a stuck loop and is a clean exit to systemd. + +The runner releases its process owner in `finally`, including binding refusal, EOF, failed +recovery and unexpected exceptions. Pidfd close is idempotent and prevents further signals; +Linux close is never retried because a reused descriptor could name an unrelated object. The lease-held runtime-record re-read and the durable `intent` write run under the service ownership mutation lease (`src/service/ownership-mutation-lease.mjs`, 500 ms wait, no subprocess); a busy lease @@ -90,3 +104,6 @@ replayed: requests the hung process held are lost as in a crash. phase-aligned recurring stalls), `-runner` (ordering, fail-closed paths, lease), `-launch` (gate, environment, drain hook, config admission) and `-process` (the real launcher and sentinel against a fixture that stops answering, on every CI OS). +`tests/service/service-health-sentinel-linux.test.ts` covers injected syscall errors, reparenting +during acquisition, PID reuse, already-exited targets, repeated recovery, teardown and descriptor +cleanup. Linux additionally exercises the real FFI syscalls against a disposable process. diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index e69df85fbc6..f5047535934 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -9,6 +9,7 @@ "codex-history-pending-wal.test.ts": "codex-integration", "compaction-history-framing.test.ts": "responses", "service-log.test.ts": "service", + "service-health-sentinel-linux.test.ts": "service", "service-state-isolation.test.ts": "service", "owner-registry-isolation.test.ts": "config", "service-health-sentinel-policy.test.ts": "service", diff --git a/tests/service/service-health-sentinel-linux.test.ts b/tests/service/service-health-sentinel-linux.test.ts new file mode 100644 index 00000000000..abed65ad5d9 --- /dev/null +++ b/tests/service/service-health-sentinel-linux.test.ts @@ -0,0 +1,230 @@ +import { describe, expect, test } from "bun:test"; +import { readdirSync, readlinkSync } from "node:fs"; +import { runHealthSentinel, type HealthSentinelDeps } from "../../src/service/health-sentinel-main"; +import { openParentPidfd, type PidfdSyscalls } from "../../src/service/health-sentinel-linux"; + +function fixture() { + const calls: unknown[][] = []; + let parent = true; + let openError = 0; + let descriptor = 17; + let signalError = 0; + let duringOpen = () => {}; + let duringProbe = () => {}; + const api: PidfdSyscalls = { + open: pid => { calls.push(["open", pid]); duringOpen(); return { value: openError ? -1 : descriptor, errno: openError }; }, + signal: (fd, signal) => { calls.push(["signal", fd, signal]); if (!signal) duringProbe(); return { value: signalError ? -1 : 0, errno: signalError }; }, + close: fd => { calls.push(["close", fd]); }, + dispose: () => { calls.push(["dispose"]); }, + }; + return { + calls, api, + open: (arch = "x64") => openParentPidfd(4242, () => parent, { arch, load: () => api }), + parent: (value: boolean) => { parent = value; }, + descriptor: (value: number) => { descriptor = value; }, + openError: (value: number) => { openError = value; }, + signalError: (value: number) => { signalError = value; }, + duringOpen: (fn: () => void) => { duringOpen = fn; }, + duringProbe: (fn: () => void) => { duringProbe = fn; }, + }; +} + +describe("Linux sentinel pidfd binding (#6987)", () => { + test.each(["x64", "arm64"])("holds identity on %s even after parent PID reuse, and signals only once", arch => { + const f = fixture(); + const opened = f.open(arch); + expect(opened.kind).toBe("bound"); + if (opened.kind !== "bound") throw new Error("not bound"); + // Simulate exit/reparenting and the numeric PID being reused after verification. + // The adapter still addresses the old kernel object (the runner also checks EOF/ppid). + f.parent(false); + + expect(opened.handle.terminate()).toBe("terminated"); + const count = f.calls.length; + expect(opened.handle.terminate()).toBe("not_target"); + expect(f.calls.length).toBe(count); + opened.handle.close(); + opened.handle.close(); + expect(f.calls.slice(-2)).toEqual([["close", 17], ["dispose"]]); + expect(opened.handle.terminate()).toBe("not_target"); + }); + + test("descriptor zero is valid and an already-exited target is never signalled by PID", () => { + const f = fixture(); + f.descriptor(0); + const opened = f.open(); + if (opened.kind !== "bound") throw new Error("not bound"); + f.signalError(3); + expect(opened.handle.terminate()).toBe("not_target"); + opened.handle.close(); + expect(f.calls).toEqual([["open", 4242], ["signal", 0, 0], ["signal", 0, 9], ["close", 0], ["dispose"]]); + }); + + test.each(["open", "probe"])("refuses reparenting during %s and releases the acquired descriptor", when => { + const f = fixture(); + if (when === "open") f.duringOpen(() => f.parent(false)); + else f.duringProbe(() => f.parent(false)); + expect(f.open().kind).toBe("refused"); + expect(f.calls.slice(-2)).toEqual([["close", 17], ["dispose"]]); + expect(f.calls).not.toContainEqual(["signal", 17, 9]); + }); + + test("wrong parent and unsupported architecture never call a syscall", () => { + const f = fixture(); + f.parent(false); + expect(f.open().kind).toBe("refused"); + f.parent(true); + expect(f.open("riscv64").kind).toBe("unsupported"); + expect(f.calls).toEqual([]); + }); + + test.each([1, 3, 12, 13, 22, 23, 24])("open errno %s refuses recovery instead of downgrading", errno => { + const f = fixture(); + f.openError(errno); + expect(f.open().kind).toBe("refused"); + expect(f.calls).toEqual([["open", 4242], ["dispose"]]); + }); + + test.each([1, 3, 9, 22, 24])("signal-check errno %s refuses recovery and closes", errno => { + const f = fixture(); + f.signalError(errno); + expect(f.open().kind).toBe("refused"); + expect(f.calls.slice(-2)).toEqual([["close", 17], ["dispose"]]); + }); + + test.each(["open", "signal"])("only ENOSYS at %s permits old-kernel fallback", when => { + const f = fixture(); + if (when === "open") f.openError(38); + else f.signalError(38); + expect(f.open().kind).toBe("unsupported"); + expect(f.calls.at(-1)).toEqual(["dispose"]); + expect(f.calls.filter(call => call[0] === "close")).toHaveLength(when === "open" ? 0 : 1); + }); + + test("loader and unexpected syscall exceptions refuse and release resources", () => { + expect(openParentPidfd(4242, () => true, { arch: "x64", load: () => { throw new Error("dlopen"); } }).kind).toBe("refused"); + const f = fixture(); + f.duringProbe(() => { throw new Error("ffi"); }); + expect(f.open().kind).toBe("refused"); + expect(f.calls.slice(-2)).toEqual([["close", 17], ["dispose"]]); + const bound = fixture(); + const opened = bound.open(); + if (opened.kind !== "bound") throw new Error("not bound"); + bound.api.signal = () => { throw new Error("ffi"); }; + expect(opened.handle.terminate()).toBe("failed"); + }); + + test.each([1, 38])("termination errno %s never falls back to a numeric PID", errno => { + const f = fixture(); + const opened = f.open(); + if (opened.kind !== "bound") throw new Error("not bound"); + f.signalError(errno); + expect(opened.handle.terminate()).toBe("failed"); + opened.handle.close(); + expect(f.calls).toEqual([["open", 4242], ["signal", 17, 0], ["signal", 17, 9], ["close", 17], ["dispose"]]); + }); + + test.each(["EOF", "refusal", "exception", "recovered", "failed", "not_target"])("runner releases the pidfd on %s", async mode => { + const f = fixture(); + const opened = f.open(); + if (opened.kind !== "bound") throw new Error("not bound"); + let resolveExit = () => {}; + const targetExited = new Promise(resolve => { resolveExit = resolve; }); + let clock = 1_000; + let probes = 0; + const deps: HealthSentinelDeps = { + targetPid: 4242, + now: () => clock, + sleep: async ms => { clock += ms; }, + probe: async () => { + if (mode === "exception") throw new Error("probe failed"); + if (mode === "EOF") { resolveExit(); return new Promise(() => {}); } + const startedAt = clock; + clock += 20; + return probes++ === 0 + ? { kind: "answered", pid: 4242, startedAt, endedAt: clock } + : { kind: "timeout", startedAt, endedAt: clock, sentinelLagMs: 0 }; + }, + readRuntime: () => ({ pid: 4242, port: 4100, sibling: false }), + targetExited, + isTargetGone: () => false, + bindTarget: () => mode === "refusal" ? { ok: false, reason: "refused" } : { ok: true }, + closeTarget: opened.handle.close, + terminateTarget: () => { + f.signalError(mode === "failed" ? 1 : mode === "not_target" ? 3 : 0); + const result = opened.handle.terminate(); + if (result === "terminated") resolveExit(); + return result; + }, + withOwnershipLease: run => run(), + readHistory: () => [], + writeHistory: () => true, + log: () => {}, + timing: { intervalMs: 0, startupGraceMs: 0, failureThreshold: 1, minUnhealthyMs: 0 }, + }; + if (mode === "exception") await expect(runHealthSentinel(deps)).rejects.toThrow("probe failed"); + else { + const result = await runHealthSentinel(deps); + expect(result.outcome).toBe(mode === "refusal" || mode === "failed" ? "stood_down" : mode === "recovered" ? "recovered" : "target_exited"); + } + expect(f.calls.slice(-2)).toEqual([["close", 17], ["dispose"]]); + const count = f.calls.length; + opened.handle.close(); + expect(opened.handle.terminate()).toBe("not_target"); + expect(f.calls.length).toBe(count); + }); +}); + +function pidfdCount(): number { + // Before Linux 6.9 a pidfd is an anonymous inode; pidfs (6.9+) names it pidfd:[]. + return readdirSync("/proc/self/fd").filter(fd => { + try { return /^(anon_inode:\[pidfd\]|pidfd:\[\d+\])$/.test(readlinkSync(`/proc/self/fd/${fd}`)); } catch { return false; } + }).length; +} + +test.if(process.platform === "linux")("native pidfd signals a disposable fixture and releases its descriptor", async () => { + // Bun may hold its own pidfd for a spawned child and release it on exit, so the leak check + // compares against the count taken before the fixture exists. + const baseline = pidfdCount(); + const child = Bun.spawn([process.execPath, "-e", "setInterval(() => {}, 1000)"], { stdin: "ignore", stdout: "ignore", stderr: "pipe" }); + const before = pidfdCount(); + const opened = openParentPidfd(child.pid, () => true); + try { + if (opened.kind === "unsupported") { + expect(opened.reason).toMatch(/unsupported/); + return; // Old kernels still exercise the real launcher's fallback in the process suite. + } + expect(opened.kind).toBe("bound"); + if (opened.kind !== "bound") throw new Error(JSON.stringify(opened)); + expect(pidfdCount()).toBe(before + 1); + expect(opened.handle.terminate()).toBe("terminated"); + await child.exited; + expect(child.signalCode).toBe("SIGKILL"); + } finally { + if (opened.kind === "bound") opened.handle.close(); + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + await child.exited; + } + expect(pidfdCount()).toBe(baseline); +}, 10_000); + + +test.if(process.platform === "linux")("native pidfd cannot signal an already-exited fixture", async () => { + const child = Bun.spawn([process.execPath, "-e", "await Bun.stdin.text()"], { stdin: "pipe", stdout: "ignore", stderr: "pipe" }); + const opened = openParentPidfd(child.pid, () => true); + try { + if (opened.kind === "unsupported") { + expect(opened.reason).toMatch(/unsupported/); + return; // Old kernels still exercise the real launcher's fallback in the process suite. + } + expect(opened.kind).toBe("bound"); + if (opened.kind !== "bound") throw new Error(JSON.stringify(opened)); + child.stdin.end(); + await child.exited; + expect(opened.handle.terminate()).toBe("not_target"); + } finally { + if (opened.kind === "bound") opened.handle.close(); + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + await child.exited; + } +}, 10_000); diff --git a/tests/service/service-health-sentinel-process.test.ts b/tests/service/service-health-sentinel-process.test.ts index 92960d1ac5b..baa57d29133 100644 --- a/tests/service/service-health-sentinel-process.test.ts +++ b/tests/service/service-health-sentinel-process.test.ts @@ -9,7 +9,7 @@ import { createTempHome, type TempHome } from "../helpers/temp-home"; * The sentinel against real processes (#6671): a fixture proxy spawns it through the real * launcher with a stdin pipe, and the sentinel must terminate the fixture once it stops answering, * or leave on its own once the fixture exits. On Windows this drives the kernel32 handle path; on - * POSIX the live-ppid path. Short timings come from the test-only override. + * Linux the pidfd path and macOS the live-ppid path. Short timings come from the test-only override. */ // Generous per-probe deadline and lag tolerance: a slow Windows runner must not miss arming or // discard every timeout as sentinel lag. @@ -91,7 +91,12 @@ describe("service health sentinel process (#6671)", () => { expect(entries![0]!.pid).toBe(proxy.pid); // Windows may end the sentinel together with the proxy before it settles the entry. expect(process.platform === "win32" ? ["intent", "terminated"] : ["terminated"]).toContain(entries![0]!.state); - expect(readFileSync(root.path("service-health-sentinel.log"), "utf8")).toContain(`recovering pid ${proxy.pid}`); + const log = readFileSync(root.path("service-health-sentinel.log"), "utf8"); + expect(log).toContain(`recovering pid ${proxy.pid}`); + if (process.platform === "linux") { + if (log.includes("pidfd unavailable:")) expect(log).toMatch(/pidfd unavailable: (pidfd_(open|send_signal) is unsupported by this kernel|pidfd syscall ABI is unsupported on \w+); using live parent PID check/); + else expect(log).toMatch(/watching pid \d+ on port \d+ \(pidfd\)/); + } }, 75_000); test("leaves without acting when the proxy exits on its own", async () => { diff --git a/tests/service/service-health-sentinel-runner.test.ts b/tests/service/service-health-sentinel-runner.test.ts index bec0f37f01f..c7ed3ca46cf 100644 --- a/tests/service/service-health-sentinel-runner.test.ts +++ b/tests/service/service-health-sentinel-runner.test.ts @@ -55,6 +55,7 @@ function world(script: Array<"answer" | "timeout" | "refused">, overrides: Parti targetExited, isTargetGone: () => gone, bindTarget: () => ({ ok: true }), + closeTarget: () => { events.push("close"); }, terminateTarget: () => { events.push(`terminate:${TARGET}:${leased ? "leased" : "unleased"}`); exitTarget(); return "terminated"; }, withOwnershipLease: run => { leased = true; try { return run(); } finally { leased = false; } }, readHistory: () => [],