From 1e55e8d86e364e3d19dc875cf2c8dfd9a2854b8b Mon Sep 17 00:00:00 2001 From: Yash Date: Mon, 14 Sep 2026 20:55:51 +0000 Subject: [PATCH] fix(harness): retry retained Terminal cleanup on shutdown --- .../session-manager-retirement-review.test.ts | 250 ++++++++++++++++++ packages/harness/src/core/session-manager.ts | 68 ++++- 2 files changed, 306 insertions(+), 12 deletions(-) create mode 100644 packages/harness/src/core/session-manager-retirement-review.test.ts diff --git a/packages/harness/src/core/session-manager-retirement-review.test.ts b/packages/harness/src/core/session-manager-retirement-review.test.ts new file mode 100644 index 000000000..4518e9d49 --- /dev/null +++ b/packages/harness/src/core/session-manager-retirement-review.test.ts @@ -0,0 +1,250 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { HarnessAdapter } from "../shared/types.js"; +import { IngestCredentialRegistry } from "./ingest-credentials.js"; +import { SessionManager, type PtySpawnFn } from "./session-manager.js"; + +function gate() { + let enter!: () => void, release!: () => void; + const entered = new Promise((done) => { + enter = done; + }); + const held = new Promise((done) => { + release = done; + }); + return { + entered, + release, + wait: async () => { + enter(); + await held; + }, + }; +} + +describe("retained Terminal cleanup and bound restart cancellation", () => { + let root: string; + let manager: SessionManager; + let spawns: { kill: ReturnType; exit: () => void }[]; + beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), "terminal-retirement-review-")); + spawns = []; + }); + afterEach(async () => { + spawns.forEach((spawn) => spawn.exit()); + await manager?.flush(); + vi.useRealTimers(); + await rm(root, { recursive: true, force: true }); + }); + + function fixture() { + const request = { cwd: root, harness: "claude-code" as const }; + const path = join(root, "sessions.json"); + const writeBinding = vi.fn(async (file: string, text: string) => { + await writeFile(file, text); + }); + const resolveIdentity = vi.fn(async (sessionId: string) => ({ + sessionId, + projectId: "project-1", + userId: "user-1", + })); + const isPidAlive = vi.fn(() => true); + const canResume = vi.fn(async () => false); + const spec = { command: "fake-terminal", args: [], env: {}, cwd: root }; + const adapter: HarnessAdapter = { + id: "claude-code", + eventSource: "hooks", + doctor: async () => [], + launch: vi.fn(() => spec), + resume: vi.fn(() => spec), + listPastSessions: async () => [], + canResume, + }; + const spawnPty: PtySpawnFn = vi.fn(() => { + const listeners: (() => void)[] = []; + const kill = vi.fn(); + spawns.push({ + kill, + exit: () => listeners.forEach((listener) => listener()), + }); + return { + pid: 900_000 + spawns.length, + kill, + write: vi.fn(), + resize: vi.fn(), + onData: () => ({ dispose: () => {} }), + onExit: (listener: (event: { exitCode: number }) => void) => { + listeners.push(() => listener({ exitCode: 0 })); + return { dispose: () => {} }; + }, + } as unknown as ReturnType; + }); + manager = new SessionManager({ + sessionsPath: path, + ingestUrl: "http://127.0.0.1:4100", + ingestCredentials: new IngestCredentialRegistry(() => "boot"), + adapters: { "claude-code": adapter }, + resolveAgentMapIdentity: resolveIdentity, + writeSubsessionBindingRegistry: writeBinding, + onSubsessionUserClosed: async () => {}, + spawnPty, + isPidAlive, + }); + return { + request, + path, + writeBinding, + resolveIdentity, + isPidAlive, + canResume, + spawnPty, + adapter, + }; + } + + it.each(["onExit", "liveness", "signal-failure"])( + "shutdown retries an exact retired handle and preserves %s evidence", + async (evidence) => { + vi.useFakeTimers(); + const { request, isPidAlive } = fixture(); + const session = await manager.create(request); + const epoch = manager.getRuntimeEpoch(session.id); + const closing = manager.closeWithResult(session.id); + await vi.advanceTimersByTimeAsync(2_500); + expect(await closing).toEqual({ + state: "unconfirmed", + runtimeEpoch: epoch, + }); + expect(manager.isLive(session.id)).toBe(false); + const kill = spawns[0]!.kill; + kill.mockImplementation(() => { + if (evidence === "onExit") spawns[0]!.exit(); + if (evidence === "liveness") isPidAlive.mockReturnValue(false); + if (evidence === "signal-failure") throw new Error("signal failed"); + }); + manager.beginShutdown(); + const shutdown = manager.killAll(), + duplicate = manager.killAll(); + expect(kill).toHaveBeenCalledTimes(3); + expect(kill).toHaveBeenLastCalledWith("SIGKILL"); + expect(manager.isLive(session.id)).toBe(false); + await vi.advanceTimersByTimeAsync(500); + await Promise.all([shutdown, duplicate]); + expect(await manager.closeWithResult(session.id)).toEqual({ + state: evidence === "signal-failure" ? "unconfirmed" : "confirmed", + runtimeEpoch: epoch, + }); + }, + ); + + it("a late retired-handle retry cannot touch a replacement runtime", async () => { + vi.useFakeTimers(); + const { request, canResume } = fixture(); + const session = await manager.create(request); + await manager.setAgentSessionId( + session.id, + "vendor-1", + "startup", + manager.getRuntimeEpoch(session.id)!, + ); + const closing = manager.closeWithResult(session.id); + await vi.advanceTimersByTimeAsync(2_500); + expect((await closing).state).toBe("unconfirmed"); + const retry = manager.killAll(); + spawns[0]!.exit(); + canResume.mockResolvedValue(true); + await manager.resume(session.id); + const epoch = manager.getRuntimeEpoch(session.id); + spawns[0]!.exit(); + await vi.advanceTimersByTimeAsync(500); + await retry; + expect(manager.getRuntimeEpoch(session.id)).toBe(epoch); + expect(manager.isLive(session.id)).toBe(true); + expect(spawns[1]!.kill).not.toHaveBeenCalled(); + }); + + it.each(["vendor-probe", "history-probe", "binding-write", "authority"])( + "End cannot be undone by a fresh restart paused in %s", + async (stage) => { + const { + request, + path, + writeBinding, + resolveIdentity, + canResume, + spawnPty, + adapter, + } = fixture(); + const id = "00000000-0000-4000-8000-000000000333"; + const marker = { + sessionId: id, + projectId: "project-1", + parentSessionId: "parent-1", + bindingId: "binding-1", + incarnation: 1, + spawnEpoch: 1, + }; + const trusted = { + agentMapIdentity: () => ({ + sessionId: id, + projectId: "project-1", + userId: "user-1", + }), + }; + await manager.createReserved(id, request, marker, trusted); + if (stage === "vendor-probe") + await manager.setAgentSessionId( + id, + "vendor-1", + "startup", + manager.getRuntimeEpoch(id)!, + ); + spawns[0]!.exit(); + await manager.flush(); + const held = gate(); + const recorded = vi.fn(async () => { + if (stage === "history-probe") await held.wait(); + return false; + }); + if (stage === "vendor-probe") + canResume.mockImplementationOnce(async () => { + await held.wait(); + return false; + }); + if (stage === "binding-write") + writeBinding.mockImplementationOnce(async (file, text) => { + await held.wait(); + await writeFile(file, text); + }); + if (stage === "authority") + resolveIdentity.mockImplementationOnce(async () => { + await held.wait(); + return trusted.agentMapIdentity(); + }); + const restarting = manager.restartFreshBound( + id, + marker, + { ...marker, incarnation: 2, spawnEpoch: 2 }, + trusted, + recorded, + ); + await held.entered; + await manager.close(id); + expect(manager.getSubsessionBinding(id)).toBeNull(); + held.release(); + await expect(restarting).rejects.toMatchObject({ + code: "SESSION_PREPARATION_CANCELLED", + }); + expect(manager.getSubsessionBinding(id)).toBeNull(); + expect( + JSON.parse(await readFile(`${path}.subsession-bindings.json`, "utf8")), + ).toEqual({ version: 1, markers: {}, closedSessionIds: [] }); + expect(spawnPty).toHaveBeenCalledOnce(); + expect(adapter.launch).toHaveBeenCalledOnce(); + expect(manager.get(id)?.status).toBe("exited"); + if (stage === "vendor-probe") expect(recorded).not.toHaveBeenCalled(); + }, + ); +}); diff --git a/packages/harness/src/core/session-manager.ts b/packages/harness/src/core/session-manager.ts index 70956830e..267d7e069 100644 --- a/packages/harness/src/core/session-manager.ts +++ b/packages/harness/src/core/session-manager.ts @@ -701,11 +701,13 @@ interface PtyHandle { cleanupConfirmed: boolean; } +/** Exact-runtime exit evidence; registry retirement alone remains unconfirmed. */ export type TerminalCleanupResult = Readonly<{ state: "confirmed" | "absent" | "unconfirmed"; runtimeEpoch: string | null; }>; +/** End invalidated an in-flight Terminal preparation before it could finish. */ export class SessionPreparationCancelledError extends Error { readonly code = "SESSION_PREPARATION_CANCELLED"; constructor() { @@ -714,6 +716,7 @@ export class SessionPreparationCancelledError extends Error { } } +/** A prior PTY still lacks positive exit evidence and blocks replacement. */ export class SessionCleanupUnconfirmedError extends Error { readonly code = "SESSION_CLEANUP_UNCONFIRMED"; constructor() { @@ -791,6 +794,7 @@ export class SessionInputIsolationError extends Error { } +/** Shutdown permanently closed Terminal admission for this manager instance. */ export class SessionManagerClosingError extends Error { readonly code = "SESSION_MANAGER_CLOSING"; @@ -1021,6 +1025,10 @@ export class SessionManager { private readonly confirmedCleanupEpochs = new Map(); private readonly dormantAllocations = new Map(); private dormantAllocationsLoaded = false; + private readonly retiringKillRetries = new WeakMap< + PtyHandle, + Promise + >(); private assertAdmission(id: string, generation: number): void { if (this.closing) throw new SessionManagerClosingError(); @@ -2059,9 +2067,9 @@ export class SessionManager { } /** - * Kills every currently-live pty and returns a Promise that resolves when - * all of their registry rows have exited (real or synthesized). Bounded by the - * same escalation window as `kill()` — never hangs. + * Retires current PTYs and retries retained unconfirmed handles, bounded by + * kill()'s escalation window. Completion is not proof every OS process exited; + * closeWithResult() retains each exact runtime's confirmed/unconfirmed result. * * Call this on server shutdown so the process actually exits instead of * waiting on orphaned claude/codex children. A bounded timeout can be layered @@ -2070,7 +2078,35 @@ export class SessionManager { */ async killAll(): Promise { const kills = [...this.ptys.keys()].map((id) => this.kill(id)); - await Promise.all(kills); + const retired = [...this.retiringPtys].map(([id, handle]) => + this.retryRetiringPty(id, handle), + ); + await Promise.all([...kills, ...retired]); + } + + private retryRetiringPty(id: string, handle: PtyHandle): Promise { + const existing = this.retiringKillRetries.get(handle); + if (existing) return existing; + const operation = async (): Promise => { + if (this.retiringPtys.get(id) !== handle) return; + if (!this.isPtyGone(handle)) { + try { + handle.pty.kill("SIGKILL"); + } catch { + /* Keep unresolved ownership. */ + } + await sleep(KILL_ESCALATION_CONFIRM_MS); + } + if (this.retiringPtys.get(id) === handle && this.isPtyGone(handle)) + this.markExited(id, handle, null); + }; + const retry = operation(); + this.retiringKillRetries.set(handle, retry); + const forget = (): void => { + this.retiringKillRetries.delete(handle); + }; + void retry.then(forget, forget); + return retry; } /** @@ -4142,20 +4178,27 @@ export class SessionManager { throw new SubsessionBindingMismatchError(); } const adapter = this.getAdapter(session.harness); - if ( - (session.agentSessionId !== null && - (await adapter.canResume(session.agentSessionId, session.cwd))) || - (await hasRecordedTurns(id)) - ) { - throw new SubsessionFreshRestartForbiddenError(); - } + const resumable = + session.agentSessionId !== null && + (await adapter.canResume(session.agentSessionId, session.cwd)); + this.assertAdmission(id, closeGeneration); + if (resumable) throw new SubsessionFreshRestartForbiddenError(); + const recordedTurns = await hasRecordedTurns(id); + this.assertAdmission(id, closeGeneration); + if (recordedTurns) throw new SubsessionFreshRestartForbiddenError(); if (!sameSubsessionBinding(current, next)) { this.subsessionBindings.set(id, next); try { await this.persistSubsessionBindings(); + this.assertAdmission(id, closeGeneration); } catch (error) { - this.subsessionBindings.set(id, current); + if ( + (this.closeGenerations.get(id) ?? 0) === closeGeneration && + this.subsessionBindings.get(id) === next + ) + this.subsessionBindings.set(id, current); + await this.persistSubsessionBindings().catch(() => {}); throw error; } } @@ -4164,6 +4207,7 @@ export class SessionManager { const agentMapIdentity = this.resolveAgentMapIdentity ? await this.resolveAgentMapIdentity(id, session.cwd, trustedIdentity) : trustedIdentity; + this.assertAdmission(id, closeGeneration); if ( !agentMapIdentity || agentMapIdentity.projectId !== next.projectId ||