From d7113c7030f56b30fcae7207f87d9e9e4b12eb78 Mon Sep 17 00:00:00 2001 From: Yash Date: Mon, 14 Sep 2026 19:54:48 +0000 Subject: [PATCH] fix(harness): fence and join exact Assistant runtime retirement --- .changeset/assistant-runtime-retirement.md | 5 + .../harness/src/core/opencode-host.test.ts | 109 ++++++++++++- packages/harness/src/core/opencode-host.ts | 150 +++++++++++++++--- 3 files changed, 244 insertions(+), 20 deletions(-) create mode 100644 .changeset/assistant-runtime-retirement.md diff --git a/.changeset/assistant-runtime-retirement.md b/.changeset/assistant-runtime-retirement.md new file mode 100644 index 000000000..6ecb009aa --- /dev/null +++ b/.changeset/assistant-runtime-retirement.md @@ -0,0 +1,5 @@ +--- +"@sapiom/harness": patch +--- + +Fence pending Assistant launches during retirement, join cleanup by Studio session, and retain ownership when shutdown cannot be confirmed. diff --git a/packages/harness/src/core/opencode-host.test.ts b/packages/harness/src/core/opencode-host.test.ts index 4b5ccd661..6fc11ed79 100644 --- a/packages/harness/src/core/opencode-host.test.ts +++ b/packages/harness/src/core/opencode-host.test.ts @@ -141,7 +141,10 @@ describe("Studio-owned OpenCode lifecycle", () => { it("keeps accepted scope stable across credential rotation and runtime restart", async () => { const original = await host.ensure("studio-a"); expect(original.contextAuthorityScope).toMatch(/^[a-f0-9]{64}$/); - expect(original.model).toEqual({ providerID: "sapiom", modelID: "gpt-luna" }); + expect(original.model).toEqual({ + providerID: "sapiom", + modelID: "gpt-luna", + }); expect(Object.isFrozen(original.model)).toBe(true); expect(Object.isFrozen(original)).toBe(true); grant = { @@ -498,7 +501,7 @@ describe("Studio-owned OpenCode lifecycle", () => { }), ); const retiring = host.retire("studio-one"); - const pending = host.ensure("studio-two"); + const pending = host.ensure("studio-one"); const rejected = expect(pending).rejects.toThrow("workspace"); await vi.waitFor(() => expect(close).toHaveBeenCalledOnce()); authorize.mockResolvedValue(null); @@ -545,6 +548,108 @@ describe("Studio-owned OpenCode lifecycle", () => { await rejected; }); + it("joins repeated retirement for the same ID while other sessions start independently", async () => { + const a = await host.ensure("studio-a"); + let release!: () => void; + close.mockImplementationOnce( + () => + new Promise((resolve) => { + release = resolve; + }), + ); + const retiring = host.retire("studio-a"); + expect(host.retire("studio-a")).toBe(retiring); + expect(host.current("studio-a")).toBeNull(); + const b = await host.ensure("studio-b"); + expect(b.harnessSessionId).toBe("studio-b"); + expect(b.signal.aborted).toBe(false); + expect(a.signal.aborted).toBe(true); + expect(close).toHaveBeenCalledOnce(); + release(); + await retiring; + }); + + it("fences admission before asynchronous workspace authorization publishes an entry", async () => { + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + authorize.mockImplementationOnce(async (id) => { + await gate; + return { harnessSessionId: id, cwd }; + }); + const pending = host.ensure("studio-a"); + const rejected = expect(pending).rejects.toMatchObject({ + failure: { code: "transport_unavailable" }, + }); + await host.retire("studio-a"); + release(); + await rejected; + expect(start).not.toHaveBeenCalled(); + await host.ensure("studio-a"); + expect(start).toHaveBeenCalledOnce(); + }); + + it("retiring an old exact runtime leaves its replacement active", async () => { + const old = await host.ensure("studio-a"); + await host.retireExact(old); + const current = await host.ensure("studio-a"); + await host.retireExact(old); + expect(host.current("studio-a")).toBe(current); + expect(current.signal.aborted).toBe(false); + expect(close).toHaveBeenCalledOnce(); + }); + + it("reports bounded pending cleanup honestly and later joins positive completion", async () => { + await host.ensure("studio-a"); + let release!: () => void; + close.mockImplementationOnce( + () => + new Promise((resolve) => { + release = resolve; + }), + ); + expect(await host.retireWithResult("studio-a", 10)).toEqual({ + state: "unconfirmed", + }); + expect(close).toHaveBeenCalledOnce(); + const joined = host.retireWithResult("studio-a"); + release(); + expect(await joined).toEqual({ state: "confirmed" }); + expect(await host.retireWithResult("studio-a")).toEqual({ + state: "absent", + }); + expect(close).toHaveBeenCalledOnce(); + }); + + it("retains failed cleanup without repeating a memoized close or blocking another ID", async () => { + expectShutdownFailure = true; + await host.ensure("studio-a"); + close.mockRejectedValueOnce(new Error("not confirmed")); + expect(await host.retireWithResult("studio-a")).toEqual({ + state: "unconfirmed", + }); + expect(await host.retireWithResult("studio-a")).toEqual({ + state: "unconfirmed", + }); + await expect(host.ensure("studio-a")).rejects.toMatchObject({ + failure: { code: "transport_unavailable" }, + }); + await host.ensure("studio-b"); + expect(close).toHaveBeenCalledOnce(); + expect(start).toHaveBeenCalledTimes(2); + }); + + it("beginShutdown synchronously closes admission and aborts existing runtimes", async () => { + const a = await host.ensure("studio-a"); + host.beginShutdown(); + expect(a.signal.aborted).toBe(true); + expect(host.current("studio-a")).toBeNull(); + await expect(host.ensure("studio-b")).rejects.toThrow(); + await host.close(); + expect(start).toHaveBeenCalledOnce(); + }); + it("retires a confirmed exited process so retry can reopen the same persistent state", async () => { let exit!: () => void; const exited = new Promise((resolve) => { diff --git a/packages/harness/src/core/opencode-host.ts b/packages/harness/src/core/opencode-host.ts index 736f06ec9..8692e819b 100644 --- a/packages/harness/src/core/opencode-host.ts +++ b/packages/harness/src/core/opencode-host.ts @@ -60,6 +60,9 @@ export interface HostedOpenCode extends OpenCodeWorkspace { signal: AbortSignal; isCurrent: () => boolean; } +export interface OpenCodeRetirement { + state: "confirmed" | "absent" | "unconfirmed"; +} interface Managed { workspace: OpenCodeWorkspace; authority: string; @@ -129,9 +132,13 @@ export class OpenCodeHost { private readonly summaries = new Map(); private readonly assistantListeners = new Set<() => void>(); private entries = new Map(); - private closing = new Set>(); + private closing = new Map< + string, + { entry: Managed; promise: Promise } + >(); + private failedCleanup = new Map(); + private generations = new Map(); private closed = false; - private cleanupFailed = false; private workspaceTimer: ReturnType; private unsubscribe: () => void; constructor(private readonly options: Options) { @@ -298,7 +305,20 @@ export class OpenCodeHost { if (!owned()) throw failure(); } + /** Read an existing exact runtime without obtaining startup admission. */ + current(id: string): HostedOpenCode | null { + const hosted = this.entries.get(id)?.hosted; + return hosted?.isCurrent() ? hosted : null; + } + async ensure(id: string): Promise { + const generation = this.generations.get(id) ?? 0; + const assertAdmission = () => { + if (this.closed || (this.generations.get(id) ?? 0) !== generation) + throw new OpenCodeTransportError( + openCodeTransportFailure("transport_unavailable"), + ); + }; const grant = this.options.access.get(); if (this.closed || !grant) throw this.accessError("Assistant access is unavailable"); @@ -311,6 +331,7 @@ export class OpenCodeHost { ); throw error; }); + assertAdmission(); const { cwd } = workspace; const current = this.options.access.get(); if (this.closed || !current || authority(current) !== authority(grant)) @@ -326,10 +347,20 @@ export class OpenCodeHost { ) return existing.ready!; if (existing) - await this.retire(id, openCodeTransportFailure("access_denied")); + await this.retireEntry( + id, + existing, + openCodeTransportFailure("access_denied"), + ); // A prior retirement must finish before another process uses its database. - await Promise.all(this.closing); + if (this.failedCleanup.has(id)) + throw new OpenCodeTransportError( + openCodeTransportFailure("transport_unavailable"), + ); + await this.closing.get(id)?.promise; + assertAdmission(); const verified = await this.workspace(id); + assertAdmission(); if ( this.closed || this.options.access.get() !== current || @@ -352,8 +383,61 @@ export class OpenCodeHost { id: string, failure = openCodeTransportFailure("transport_unavailable"), ): Promise { + this.generations.set(id, (this.generations.get(id) ?? 0) + 1); const entry = this.entries.get(id); - if (!entry) return Promise.resolve(); + if (entry) return this.retireEntry(id, entry, failure); + const closing = this.closing.get(id); + if (closing) return closing.promise; + if (this.failedCleanup.has(id)) { + const rejected = Promise.reject(new OpenCodeShutdownError()); + void rejected.catch(() => {}); + return rejected; + } + return Promise.resolve(); + } + + /** A failed inspection must not retire a replacement belonging to a newer operation. */ + retireExact(hosted: HostedOpenCode): Promise { + const id = hosted.harnessSessionId; + const entry = this.entries.get(id); + if (entry?.hosted === hosted) return this.retire(id); + const closing = this.closing.get(id); + if (closing?.entry.hosted === hosted) return closing.promise; + if (this.failedCleanup.get(id)?.hosted === hosted) return this.retire(id); + return Promise.resolve(); + } + + /** Wait only a bounded time for reporting; timeout leaves ownership fenced. */ + async retireWithResult( + id: string, + waitMs = 3500, + ): Promise { + const owned = + this.entries.has(id) || + this.closing.has(id) || + this.failedCleanup.has(id); + const retiring = this.retire(id); + let timer: ReturnType | undefined; + try { + return await Promise.race([ + retiring.then( + (): OpenCodeRetirement => ({ state: owned ? "confirmed" : "absent" }), + (): OpenCodeRetirement => ({ state: "unconfirmed" }), + ), + new Promise((resolve) => { + timer = setTimeout(() => resolve({ state: "unconfirmed" }), waitMs); + }), + ]); + } finally { + clearTimeout(timer); + } + } + + private retireEntry( + id: string, + entry: Managed, + failure: OpenCodeTransportFailure, + ): Promise { this.entries.delete(id); const closing = (async () => { const hosted = await entry.ready?.catch(() => null); @@ -366,28 +450,50 @@ export class OpenCodeHost { // concurrently use a database whose previous owner may still be alive. await this.unlock(entry); })(); - this.closing.add(closing); - void closing.finally(() => this.closing.delete(closing)).catch(() => {}); + this.closing.set(id, { entry, promise: closing }); + void closing.then( + () => { + if (this.closing.get(id)?.entry === entry) this.closing.delete(id); + }, + () => { + this.failedCleanup.set(id, entry); + if (this.closing.get(id)?.entry === entry) this.closing.delete(id); + }, + ); // Own cleanup before removal notifies listeners that may reenter close(). // The first await above defers native cleanup until this retirement is fenced. const binding = entry.observation; entry.observation = undefined; - this.removeSummary(id); - binding?.observer?.dispose(); entry.abort.abort(new OpenCodeTransportError(failure)); entry.credential?.revoke(); + this.removeSummary(id); + try { + binding?.observer?.dispose(); + } catch { + /* Cleanup ownership is independent of observers. */ + } return closing; } - async close(): Promise { + /** Close launch admission synchronously before whole-server asynchronous cleanup. */ + beginShutdown(): void { + if (this.closed) return; this.closed = true; this.syncAssistantAccess(); clearInterval(this.workspaceTimer); this.unsubscribe(); for (const id of this.entries.keys()) void this.retire(id); - const results = await Promise.allSettled(this.closing); + } + + async close(): Promise { + this.beginShutdown(); + // A state listener may reenter close while beginShutdown is fencing siblings. + await Promise.resolve(); + const results = await Promise.allSettled( + [...this.closing.values()].map(({ promise }) => promise), + ); if ( - this.cleanupFailed || + this.failedCleanup.size || results.some((result) => result.status === "rejected") ) throw new OpenCodeShutdownError(); @@ -421,11 +527,16 @@ export class OpenCodeHost { private async unlock(entry: Managed): Promise { if (entry.cleanupFailed) { - this.cleanupFailed = true; + this.failedCleanup.set(entry.workspace.harnessSessionId, entry); + throw new OpenCodeShutdownError(); + } + try { + await entry.unlock?.(); + entry.unlock = undefined; + } catch { + this.failedCleanup.set(entry.workspace.harnessSessionId, entry); throw new OpenCodeShutdownError(); } - await entry.unlock?.(); - entry.unlock = undefined; } private async start( @@ -525,8 +636,6 @@ export class OpenCodeHost { entry.hosted = hosted; return hosted; } catch (error) { - if (this.entries.get(entry.workspace.harnessSessionId) === entry) - this.entries.delete(entry.workspace.harnessSessionId); entry.credential?.revoke(); if (error instanceof OpenCodeShutdownError) entry.cleanupFailed = true; if (server) { @@ -536,7 +645,12 @@ export class OpenCodeHost { entry.cleanupFailed = true; } } - await this.unlock(entry); + try { + await this.unlock(entry); + } finally { + if (this.entries.get(entry.workspace.harnessSessionId) === entry) + this.entries.delete(entry.workspace.harnessSessionId); + } if (error instanceof OpenCodeTransportError) throw error; if ( entry.abort.signal.aborted &&