Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
250 changes: 250 additions & 0 deletions packages/harness/src/core/session-manager-retirement-review.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,250 @@
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { HarnessAdapter } from "../shared/types.js";
import { IngestCredentialRegistry } from "./ingest-credentials.js";
import { SessionManager, type PtySpawnFn } from "./session-manager.js";

function gate() {
let enter!: () => void, release!: () => void;
const entered = new Promise<void>((done) => {
enter = done;
});
const held = new Promise<void>((done) => {
release = done;
});
return {
entered,
release,
wait: async () => {
enter();
await held;
},
};
}

describe("retained Terminal cleanup and bound restart cancellation", () => {
let root: string;
let manager: SessionManager;
let spawns: { kill: ReturnType<typeof vi.fn>; exit: () => void }[];
beforeEach(async () => {
root = await mkdtemp(join(tmpdir(), "terminal-retirement-review-"));
spawns = [];
});
afterEach(async () => {
spawns.forEach((spawn) => spawn.exit());
await manager?.flush();
vi.useRealTimers();
await rm(root, { recursive: true, force: true });
});

function fixture() {
const request = { cwd: root, harness: "claude-code" as const };
const path = join(root, "sessions.json");
const writeBinding = vi.fn(async (file: string, text: string) => {
await writeFile(file, text);
});
const resolveIdentity = vi.fn(async (sessionId: string) => ({
sessionId,
projectId: "project-1",
userId: "user-1",
}));
const isPidAlive = vi.fn(() => true);
const canResume = vi.fn(async () => false);
const spec = { command: "fake-terminal", args: [], env: {}, cwd: root };
const adapter: HarnessAdapter = {
id: "claude-code",
eventSource: "hooks",
doctor: async () => [],
launch: vi.fn(() => spec),
resume: vi.fn(() => spec),
listPastSessions: async () => [],
canResume,
};
const spawnPty: PtySpawnFn = vi.fn(() => {
const listeners: (() => void)[] = [];
const kill = vi.fn();
spawns.push({
kill,
exit: () => listeners.forEach((listener) => listener()),
});
return {
pid: 900_000 + spawns.length,
kill,
write: vi.fn(),
resize: vi.fn(),
onData: () => ({ dispose: () => {} }),
onExit: (listener: (event: { exitCode: number }) => void) => {
listeners.push(() => listener({ exitCode: 0 }));
return { dispose: () => {} };
},
} as unknown as ReturnType<PtySpawnFn>;
});
manager = new SessionManager({
sessionsPath: path,
ingestUrl: "http://127.0.0.1:4100",
ingestCredentials: new IngestCredentialRegistry(() => "boot"),
adapters: { "claude-code": adapter },
resolveAgentMapIdentity: resolveIdentity,
writeSubsessionBindingRegistry: writeBinding,
onSubsessionUserClosed: async () => {},
spawnPty,
isPidAlive,
});
return {
request,
path,
writeBinding,
resolveIdentity,
isPidAlive,
canResume,
spawnPty,
adapter,
};
}

it.each(["onExit", "liveness", "signal-failure"])(
"shutdown retries an exact retired handle and preserves %s evidence",
async (evidence) => {
vi.useFakeTimers();
const { request, isPidAlive } = fixture();
const session = await manager.create(request);
const epoch = manager.getRuntimeEpoch(session.id);
const closing = manager.closeWithResult(session.id);
await vi.advanceTimersByTimeAsync(2_500);
expect(await closing).toEqual({
state: "unconfirmed",
runtimeEpoch: epoch,
});
expect(manager.isLive(session.id)).toBe(false);
const kill = spawns[0]!.kill;
kill.mockImplementation(() => {
if (evidence === "onExit") spawns[0]!.exit();
if (evidence === "liveness") isPidAlive.mockReturnValue(false);
if (evidence === "signal-failure") throw new Error("signal failed");
});
manager.beginShutdown();
const shutdown = manager.killAll(),
duplicate = manager.killAll();
expect(kill).toHaveBeenCalledTimes(3);
expect(kill).toHaveBeenLastCalledWith("SIGKILL");
expect(manager.isLive(session.id)).toBe(false);
await vi.advanceTimersByTimeAsync(500);
await Promise.all([shutdown, duplicate]);
expect(await manager.closeWithResult(session.id)).toEqual({
state: evidence === "signal-failure" ? "unconfirmed" : "confirmed",
runtimeEpoch: epoch,
});
},
);

it("a late retired-handle retry cannot touch a replacement runtime", async () => {
vi.useFakeTimers();
const { request, canResume } = fixture();
const session = await manager.create(request);
await manager.setAgentSessionId(
session.id,
"vendor-1",
"startup",
manager.getRuntimeEpoch(session.id)!,
);
const closing = manager.closeWithResult(session.id);
await vi.advanceTimersByTimeAsync(2_500);
expect((await closing).state).toBe("unconfirmed");
const retry = manager.killAll();
spawns[0]!.exit();
canResume.mockResolvedValue(true);
await manager.resume(session.id);
const epoch = manager.getRuntimeEpoch(session.id);
spawns[0]!.exit();
await vi.advanceTimersByTimeAsync(500);
await retry;
expect(manager.getRuntimeEpoch(session.id)).toBe(epoch);
expect(manager.isLive(session.id)).toBe(true);
expect(spawns[1]!.kill).not.toHaveBeenCalled();
});

it.each(["vendor-probe", "history-probe", "binding-write", "authority"])(
"End cannot be undone by a fresh restart paused in %s",
async (stage) => {
const {
request,
path,
writeBinding,
resolveIdentity,
canResume,
spawnPty,
adapter,
} = fixture();
const id = "00000000-0000-4000-8000-000000000333";
const marker = {
sessionId: id,
projectId: "project-1",
parentSessionId: "parent-1",
bindingId: "binding-1",
incarnation: 1,
spawnEpoch: 1,
};
const trusted = {
agentMapIdentity: () => ({
sessionId: id,
projectId: "project-1",
userId: "user-1",
}),
};
await manager.createReserved(id, request, marker, trusted);
if (stage === "vendor-probe")
await manager.setAgentSessionId(
id,
"vendor-1",
"startup",
manager.getRuntimeEpoch(id)!,
);
spawns[0]!.exit();
await manager.flush();
const held = gate();
const recorded = vi.fn(async () => {
if (stage === "history-probe") await held.wait();
return false;
});
if (stage === "vendor-probe")
canResume.mockImplementationOnce(async () => {
await held.wait();
return false;
});
if (stage === "binding-write")
writeBinding.mockImplementationOnce(async (file, text) => {
await held.wait();
await writeFile(file, text);
});
if (stage === "authority")
resolveIdentity.mockImplementationOnce(async () => {
await held.wait();
return trusted.agentMapIdentity();
});
const restarting = manager.restartFreshBound(
id,
marker,
{ ...marker, incarnation: 2, spawnEpoch: 2 },
trusted,
recorded,
);
await held.entered;
await manager.close(id);
expect(manager.getSubsessionBinding(id)).toBeNull();
held.release();
await expect(restarting).rejects.toMatchObject({
code: "SESSION_PREPARATION_CANCELLED",
});
expect(manager.getSubsessionBinding(id)).toBeNull();
expect(
JSON.parse(await readFile(`${path}.subsession-bindings.json`, "utf8")),
).toEqual({ version: 1, markers: {}, closedSessionIds: [] });
expect(spawnPty).toHaveBeenCalledOnce();
expect(adapter.launch).toHaveBeenCalledOnce();
expect(manager.get(id)?.status).toBe("exited");
if (stage === "vendor-probe") expect(recorded).not.toHaveBeenCalled();
},
);
});
68 changes: 56 additions & 12 deletions packages/harness/src/core/session-manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -701,11 +701,13 @@ interface PtyHandle {
cleanupConfirmed: boolean;
}

/** Exact-runtime exit evidence; registry retirement alone remains unconfirmed. */
export type TerminalCleanupResult = Readonly<{
state: "confirmed" | "absent" | "unconfirmed";
runtimeEpoch: string | null;
}>;

/** End invalidated an in-flight Terminal preparation before it could finish. */
export class SessionPreparationCancelledError extends Error {
readonly code = "SESSION_PREPARATION_CANCELLED";
constructor() {
Expand All @@ -714,6 +716,7 @@ export class SessionPreparationCancelledError extends Error {
}
}

/** A prior PTY still lacks positive exit evidence and blocks replacement. */
export class SessionCleanupUnconfirmedError extends Error {
readonly code = "SESSION_CLEANUP_UNCONFIRMED";
constructor() {
Expand Down Expand Up @@ -791,6 +794,7 @@ export class SessionInputIsolationError extends Error {
}


/** Shutdown permanently closed Terminal admission for this manager instance. */
export class SessionManagerClosingError extends Error {
readonly code = "SESSION_MANAGER_CLOSING";

Expand Down Expand Up @@ -1021,6 +1025,10 @@ export class SessionManager {
private readonly confirmedCleanupEpochs = new Map<string, string>();
private readonly dormantAllocations = new Map<string, string>();
private dormantAllocationsLoaded = false;
private readonly retiringKillRetries = new WeakMap<
PtyHandle,
Promise<void>
>();

private assertAdmission(id: string, generation: number): void {
if (this.closing) throw new SessionManagerClosingError();
Expand Down Expand Up @@ -2059,9 +2067,9 @@ export class SessionManager {
}

/**
* Kills every currently-live pty and returns a Promise that resolves when
* all of their registry rows have exited (real or synthesized). Bounded by the
* same escalation window as `kill()` — never hangs.
* Retires current PTYs and retries retained unconfirmed handles, bounded by
* kill()'s escalation window. Completion is not proof every OS process exited;
* closeWithResult() retains each exact runtime's confirmed/unconfirmed result.
*
* Call this on server shutdown so the process actually exits instead of
* waiting on orphaned claude/codex children. A bounded timeout can be layered
Expand All @@ -2070,7 +2078,35 @@ export class SessionManager {
*/
async killAll(): Promise<void> {
const kills = [...this.ptys.keys()].map((id) => this.kill(id));
await Promise.all(kills);
const retired = [...this.retiringPtys].map(([id, handle]) =>
this.retryRetiringPty(id, handle),
);
await Promise.all([...kills, ...retired]);
}

private retryRetiringPty(id: string, handle: PtyHandle): Promise<void> {
const existing = this.retiringKillRetries.get(handle);
if (existing) return existing;
const operation = async (): Promise<void> => {
if (this.retiringPtys.get(id) !== handle) return;
if (!this.isPtyGone(handle)) {
try {
handle.pty.kill("SIGKILL");
} catch {
/* Keep unresolved ownership. */
}
await sleep(KILL_ESCALATION_CONFIRM_MS);
}
if (this.retiringPtys.get(id) === handle && this.isPtyGone(handle))
this.markExited(id, handle, null);
};
const retry = operation();
this.retiringKillRetries.set(handle, retry);
const forget = (): void => {
this.retiringKillRetries.delete(handle);
};
void retry.then(forget, forget);
return retry;
}

/**
Expand Down Expand Up @@ -4142,20 +4178,27 @@ export class SessionManager {
throw new SubsessionBindingMismatchError();
}
const adapter = this.getAdapter(session.harness);
if (
(session.agentSessionId !== null &&
(await adapter.canResume(session.agentSessionId, session.cwd))) ||
(await hasRecordedTurns(id))
) {
throw new SubsessionFreshRestartForbiddenError();
}
const resumable =
session.agentSessionId !== null &&
(await adapter.canResume(session.agentSessionId, session.cwd));
this.assertAdmission(id, closeGeneration);
if (resumable) throw new SubsessionFreshRestartForbiddenError();
const recordedTurns = await hasRecordedTurns(id);
this.assertAdmission(id, closeGeneration);
if (recordedTurns) throw new SubsessionFreshRestartForbiddenError();

if (!sameSubsessionBinding(current, next)) {
this.subsessionBindings.set(id, next);
try {
await this.persistSubsessionBindings();
this.assertAdmission(id, closeGeneration);
} catch (error) {
this.subsessionBindings.set(id, current);
if (
(this.closeGenerations.get(id) ?? 0) === closeGeneration &&
this.subsessionBindings.get(id) === next
)
this.subsessionBindings.set(id, current);
await this.persistSubsessionBindings().catch(() => {});

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 Failed repair resurrects ended binding

When End overtakes the marker write, persistSubsessionBindings ignores a failed repair. Restarting can reload the stale binding and recover the ended session.

Learn more

The delayed restart write can finish after close() has durably removed the binding. The cancellation path then writes the current in-memory state again to repair that stale disk state. If this repair fails, the catch suppresses the only evidence that disk still contains the obsolete open marker. A later process start loads that marker without the End tombstone and can treat the delegated session as recoverable.

Example: Restart writes incarnation 2 after End has persisted an empty registry. The repair then fails with ENOSPC. End already returned successfully, but the next launch reads incarnation 2 as open.

Recommended fix: Do not suppress the compensating persistSubsessionBindings() failure. Propagate it, or serialize binding writes so the stale restart write cannot complete after End's authoritative deletion. Preserve the cancellation error only after durable state is known correct.

Devin Review

Was this helpful? React with 👍 or 👎 to provide feedback.

throw error;
}
}
Expand All @@ -4164,6 +4207,7 @@ export class SessionManager {
const agentMapIdentity = this.resolveAgentMapIdentity
? await this.resolveAgentMapIdentity(id, session.cwd, trustedIdentity)
: trustedIdentity;
this.assertAdmission(id, closeGeneration);
if (
!agentMapIdentity ||
agentMapIdentity.projectId !== next.projectId ||
Expand Down
Loading