Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/assistant-runtime-retirement.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@sapiom/harness": patch
---

Fence pending Assistant launches during retirement, join cleanup by Studio session, and retain ownership when shutdown cannot be confirmed.
109 changes: 107 additions & 2 deletions packages/harness/src/core/opencode-host.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,10 @@ describe("Studio-owned OpenCode lifecycle", () => {
it("keeps accepted scope stable across credential rotation and runtime restart", async () => {
const original = await host.ensure("studio-a");
expect(original.contextAuthorityScope).toMatch(/^[a-f0-9]{64}$/);
expect(original.model).toEqual({ providerID: "sapiom", modelID: "gpt-luna" });
expect(original.model).toEqual({
providerID: "sapiom",
modelID: "gpt-luna",
});
expect(Object.isFrozen(original.model)).toBe(true);
expect(Object.isFrozen(original)).toBe(true);
grant = {
Expand Down Expand Up @@ -498,7 +501,7 @@ describe("Studio-owned OpenCode lifecycle", () => {
}),
);
const retiring = host.retire("studio-one");
const pending = host.ensure("studio-two");
const pending = host.ensure("studio-one");
const rejected = expect(pending).rejects.toThrow("workspace");
await vi.waitFor(() => expect(close).toHaveBeenCalledOnce());
authorize.mockResolvedValue(null);
Expand Down Expand Up @@ -545,6 +548,108 @@ describe("Studio-owned OpenCode lifecycle", () => {
await rejected;
});

it("joins repeated retirement for the same ID while other sessions start independently", async () => {
const a = await host.ensure("studio-a");
let release!: () => void;
close.mockImplementationOnce(
() =>
new Promise<void>((resolve) => {
release = resolve;
}),
);
const retiring = host.retire("studio-a");
expect(host.retire("studio-a")).toBe(retiring);
expect(host.current("studio-a")).toBeNull();
const b = await host.ensure("studio-b");
expect(b.harnessSessionId).toBe("studio-b");
expect(b.signal.aborted).toBe(false);
expect(a.signal.aborted).toBe(true);
expect(close).toHaveBeenCalledOnce();
release();
await retiring;
});

it("fences admission before asynchronous workspace authorization publishes an entry", async () => {
let release!: () => void;
const gate = new Promise<void>((resolve) => {
release = resolve;
});
authorize.mockImplementationOnce(async (id) => {
await gate;
return { harnessSessionId: id, cwd };
});
const pending = host.ensure("studio-a");
const rejected = expect(pending).rejects.toMatchObject({
failure: { code: "transport_unavailable" },
});
await host.retire("studio-a");
release();
await rejected;
expect(start).not.toHaveBeenCalled();
await host.ensure("studio-a");
expect(start).toHaveBeenCalledOnce();
});

it("retiring an old exact runtime leaves its replacement active", async () => {
const old = await host.ensure("studio-a");
await host.retireExact(old);
const current = await host.ensure("studio-a");
await host.retireExact(old);
expect(host.current("studio-a")).toBe(current);
expect(current.signal.aborted).toBe(false);
expect(close).toHaveBeenCalledOnce();
});

it("reports bounded pending cleanup honestly and later joins positive completion", async () => {
await host.ensure("studio-a");
let release!: () => void;
close.mockImplementationOnce(
() =>
new Promise<void>((resolve) => {
release = resolve;
}),
);
expect(await host.retireWithResult("studio-a", 10)).toEqual({
state: "unconfirmed",
});
expect(close).toHaveBeenCalledOnce();
const joined = host.retireWithResult("studio-a");
release();
expect(await joined).toEqual({ state: "confirmed" });
expect(await host.retireWithResult("studio-a")).toEqual({
state: "absent",
});
expect(close).toHaveBeenCalledOnce();
});

it("retains failed cleanup without repeating a memoized close or blocking another ID", async () => {
expectShutdownFailure = true;
await host.ensure("studio-a");
close.mockRejectedValueOnce(new Error("not confirmed"));
expect(await host.retireWithResult("studio-a")).toEqual({
state: "unconfirmed",
});
expect(await host.retireWithResult("studio-a")).toEqual({
state: "unconfirmed",
});
await expect(host.ensure("studio-a")).rejects.toMatchObject({
failure: { code: "transport_unavailable" },
});
await host.ensure("studio-b");
expect(close).toHaveBeenCalledOnce();
expect(start).toHaveBeenCalledTimes(2);
});

it("beginShutdown synchronously closes admission and aborts existing runtimes", async () => {
const a = await host.ensure("studio-a");
host.beginShutdown();
expect(a.signal.aborted).toBe(true);
expect(host.current("studio-a")).toBeNull();
await expect(host.ensure("studio-b")).rejects.toThrow();
await host.close();
expect(start).toHaveBeenCalledOnce();
});

it("retires a confirmed exited process so retry can reopen the same persistent state", async () => {
let exit!: () => void;
const exited = new Promise<void>((resolve) => {
Expand Down
150 changes: 132 additions & 18 deletions packages/harness/src/core/opencode-host.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,9 @@ export interface HostedOpenCode extends OpenCodeWorkspace {
signal: AbortSignal;
isCurrent: () => boolean;
}
export interface OpenCodeRetirement {
state: "confirmed" | "absent" | "unconfirmed";
}
interface Managed {
workspace: OpenCodeWorkspace;
authority: string;
Expand Down Expand Up @@ -129,9 +132,13 @@ export class OpenCodeHost {
private readonly summaries = new Map<string, AssistantSessionSummary>();
private readonly assistantListeners = new Set<() => void>();
private entries = new Map<string, Managed>();
private closing = new Set<Promise<void>>();
private closing = new Map<
string,
{ entry: Managed; promise: Promise<void> }
>();
private failedCleanup = new Map<string, Managed>();
private generations = new Map<string, number>();
private closed = false;
private cleanupFailed = false;
private workspaceTimer: ReturnType<typeof setInterval>;
private unsubscribe: () => void;
constructor(private readonly options: Options) {
Expand Down Expand Up @@ -298,7 +305,20 @@ export class OpenCodeHost {
if (!owned()) throw failure();
}

/** Read an existing exact runtime without obtaining startup admission. */
current(id: string): HostedOpenCode | null {
const hosted = this.entries.get(id)?.hosted;
return hosted?.isCurrent() ? hosted : null;
}

async ensure(id: string): Promise<HostedOpenCode> {
const generation = this.generations.get(id) ?? 0;
const assertAdmission = () => {
if (this.closed || (this.generations.get(id) ?? 0) !== generation)
throw new OpenCodeTransportError(
openCodeTransportFailure("transport_unavailable"),
);
};
const grant = this.options.access.get();
if (this.closed || !grant)
throw this.accessError("Assistant access is unavailable");
Expand All @@ -311,6 +331,7 @@ export class OpenCodeHost {
);
throw error;
});
assertAdmission();
const { cwd } = workspace;
const current = this.options.access.get();
if (this.closed || !current || authority(current) !== authority(grant))
Expand All @@ -326,10 +347,20 @@ export class OpenCodeHost {
)
return existing.ready!;
if (existing)
await this.retire(id, openCodeTransportFailure("access_denied"));
await this.retireEntry(
id,
existing,
openCodeTransportFailure("access_denied"),
);
// A prior retirement must finish before another process uses its database.
await Promise.all(this.closing);
if (this.failedCleanup.has(id))
throw new OpenCodeTransportError(
openCodeTransportFailure("transport_unavailable"),
);
await this.closing.get(id)?.promise;
assertAdmission();
const verified = await this.workspace(id);
assertAdmission();
if (
this.closed ||
this.options.access.get() !== current ||
Expand All @@ -352,8 +383,61 @@ export class OpenCodeHost {
id: string,
failure = openCodeTransportFailure("transport_unavailable"),
): Promise<void> {
this.generations.set(id, (this.generations.get(id) ?? 0) + 1);
const entry = this.entries.get(id);
if (!entry) return Promise.resolve();
if (entry) return this.retireEntry(id, entry, failure);
const closing = this.closing.get(id);
if (closing) return closing.promise;
if (this.failedCleanup.has(id)) {
const rejected = Promise.reject<void>(new OpenCodeShutdownError());
void rejected.catch(() => {});
return rejected;
}
return Promise.resolve();
}

/** A failed inspection must not retire a replacement belonging to a newer operation. */
retireExact(hosted: HostedOpenCode): Promise<void> {
const id = hosted.harnessSessionId;
const entry = this.entries.get(id);
if (entry?.hosted === hosted) return this.retire(id);
const closing = this.closing.get(id);
if (closing?.entry.hosted === hosted) return closing.promise;
if (this.failedCleanup.get(id)?.hosted === hosted) return this.retire(id);
return Promise.resolve();
}

/** Wait only a bounded time for reporting; timeout leaves ownership fenced. */
async retireWithResult(
id: string,
waitMs = 3500,
): Promise<OpenCodeRetirement> {
const owned =
this.entries.has(id) ||
this.closing.has(id) ||
this.failedCleanup.has(id);
const retiring = this.retire(id);
let timer: ReturnType<typeof setTimeout> | undefined;
try {
return await Promise.race([
retiring.then(
(): OpenCodeRetirement => ({ state: owned ? "confirmed" : "absent" }),
(): OpenCodeRetirement => ({ state: "unconfirmed" }),
),
new Promise<OpenCodeRetirement>((resolve) => {
timer = setTimeout(() => resolve({ state: "unconfirmed" }), waitMs);
}),
]);
} finally {
clearTimeout(timer);
}
}

private retireEntry(
id: string,
entry: Managed,
failure: OpenCodeTransportFailure,
): Promise<void> {
this.entries.delete(id);
const closing = (async () => {
const hosted = await entry.ready?.catch(() => null);
Expand All @@ -366,28 +450,50 @@ export class OpenCodeHost {
// concurrently use a database whose previous owner may still be alive.
await this.unlock(entry);
})();
this.closing.add(closing);
void closing.finally(() => this.closing.delete(closing)).catch(() => {});
this.closing.set(id, { entry, promise: closing });
void closing.then(
() => {
if (this.closing.get(id)?.entry === entry) this.closing.delete(id);
},
() => {
this.failedCleanup.set(id, entry);
if (this.closing.get(id)?.entry === entry) this.closing.delete(id);
},
);
// Own cleanup before removal notifies listeners that may reenter close().
// The first await above defers native cleanup until this retirement is fenced.
const binding = entry.observation;
entry.observation = undefined;
this.removeSummary(id);
binding?.observer?.dispose();
entry.abort.abort(new OpenCodeTransportError(failure));
entry.credential?.revoke();
this.removeSummary(id);
try {
binding?.observer?.dispose();
} catch {
/* Cleanup ownership is independent of observers. */
}
return closing;
}

async close(): Promise<void> {
/** Close launch admission synchronously before whole-server asynchronous cleanup. */
beginShutdown(): void {
if (this.closed) return;
this.closed = true;
this.syncAssistantAccess();
clearInterval(this.workspaceTimer);
this.unsubscribe();
for (const id of this.entries.keys()) void this.retire(id);
const results = await Promise.allSettled(this.closing);
}

async close(): Promise<void> {
this.beginShutdown();
// A state listener may reenter close while beginShutdown is fencing siblings.
await Promise.resolve();
const results = await Promise.allSettled(
[...this.closing.values()].map(({ promise }) => promise),
);
if (
this.cleanupFailed ||
this.failedCleanup.size ||
results.some((result) => result.status === "rejected")
)
throw new OpenCodeShutdownError();
Expand Down Expand Up @@ -421,11 +527,16 @@ export class OpenCodeHost {

private async unlock(entry: Managed): Promise<void> {
if (entry.cleanupFailed) {
this.cleanupFailed = true;
this.failedCleanup.set(entry.workspace.harnessSessionId, entry);
throw new OpenCodeShutdownError();
}
try {
await entry.unlock?.();
entry.unlock = undefined;
} catch {
this.failedCleanup.set(entry.workspace.harnessSessionId, entry);
throw new OpenCodeShutdownError();
}
await entry.unlock?.();
entry.unlock = undefined;
}

private async start(
Expand Down Expand Up @@ -525,8 +636,6 @@ export class OpenCodeHost {
entry.hosted = hosted;
return hosted;
} catch (error) {
if (this.entries.get(entry.workspace.harnessSessionId) === entry)
this.entries.delete(entry.workspace.harnessSessionId);
entry.credential?.revoke();
if (error instanceof OpenCodeShutdownError) entry.cleanupFailed = true;
if (server) {
Expand All @@ -536,7 +645,12 @@ export class OpenCodeHost {
entry.cleanupFailed = true;
}
}
await this.unlock(entry);
try {
await this.unlock(entry);
} finally {
if (this.entries.get(entry.workspace.harnessSessionId) === entry)
this.entries.delete(entry.workspace.harnessSessionId);
}
if (error instanceof OpenCodeTransportError) throw error;
if (
entry.abort.signal.aborted &&
Expand Down
Loading