From b3e68bd90655330b8e0761d2ddbb0fae14769797 Mon Sep 17 00:00:00 2001 From: Yash Date: Mon, 14 Sep 2026 21:54:41 +0000 Subject: [PATCH] feat(harness): resume the exact saved Assistant runtime --- .changeset/same-assistant-resume.md | 5 + packages/harness/docs/assistant-history.md | 7 + .../harness/src/core/assistant-lifecycle.ts | 376 ++++++++++++++++-- .../harness/src/core/assistant-resume.test.ts | 295 ++++++++++++++ 4 files changed, 649 insertions(+), 34 deletions(-) create mode 100644 .changeset/same-assistant-resume.md create mode 100644 packages/harness/src/core/assistant-resume.test.ts diff --git a/.changeset/same-assistant-resume.md b/.changeset/same-assistant-resume.md new file mode 100644 index 000000000..e44a7b8a9 --- /dev/null +++ b/.changeset/same-assistant-resume.md @@ -0,0 +1,5 @@ +--- +"@sapiom/harness": patch +--- + +Resume the exact saved Assistant with paused restoration and durable operation retries. diff --git a/packages/harness/docs/assistant-history.md b/packages/harness/docs/assistant-history.md index 4e4b1beaa..5c2e65448 100644 --- a/packages/harness/docs/assistant-history.md +++ b/packages/harness/docs/assistant-history.md @@ -48,6 +48,13 @@ so a successful GET can return an older `capturedAt`. Disposal and authority los cancel capture; opening a retained record never resumes execution. # Selected native availability +The Resume coordinator claims the same operation slot as Attach and inspection. +It preserves a running exact lease and execution state; restored runtimes get a +new paused lease. Lifecycle publication and live-lease updates share a barrier +so in-flight execution cannot observe half of a Resume commit. End requests +process retirement immediately, while durable commit/rollback retains its lock +and publication ownership until its IO actually settles. + Resume commits its operation UUID and exact saved-binding digest atomically with the lifecycle header. The private proof is excluded from public state. An exact retry can reconcile a lost acknowledgement; a later lifecycle revision, End or diff --git a/packages/harness/src/core/assistant-lifecycle.ts b/packages/harness/src/core/assistant-lifecycle.ts index f21a9b9f2..5ac7aad98 100644 --- a/packages/harness/src/core/assistant-lifecycle.ts +++ b/packages/harness/src/core/assistant-lifecycle.ts @@ -1,13 +1,18 @@ import { randomUUID } from "node:crypto"; +import { basename } from "node:path"; import { clearTimeout, setTimeout } from "node:timers"; +import { z } from "zod"; import type { AssistantLifecycle } from "../shared/assistant-session.js"; import { openCodeTransportFailure } from "../shared/opencode-errors.js"; import { AssistantSessionRevisionError, + assistantResumeBindingDigest, + type AssistantAssociation, type AssistantSessionStore, } from "./assistant-session-store.js"; import { OpenCodeTransportError, + OpenCodeAccessError, type HostedOpenCode, type OpenCodeHost, } from "./opencode-host.js"; @@ -25,11 +30,21 @@ export interface AssistantEndFence { readonly id: string; readonly generation: number; } +export interface AssistantResumePreparation { + authorize(signal: AbortSignal): Promise; + read( + hosted: HostedOpenCode, + binding: AssistantAssociation, + signal: AbortSignal, + ): Promise; +} interface Options { inspectionTimeoutMs?: number; + resumeTimeoutMs?: number; /** Continue keeps an allocated child closed to ordinary Attach until seed preparation commits. */ canAttach?: (id: string) => Promise; - store: Pick; + store: Pick & + Partial>; host: Pick< OpenCodeHost, | "ensure" @@ -89,7 +104,16 @@ export class AssistantLifecycleCoordinator { private readonly states = new Map(); private readonly generations = new Map(); private readonly ending = new Set(); - private readonly inspections = new Map(); + private readonly preparations = new Map(); + private readonly resumes = new Map< + string, + { + operationId: string; + revision: number; + promise: Promise; + } + >(); + private readonly publications = new WeakMap>(); private readonly pending = new Map< string, { revision: number; promise: Promise } @@ -158,7 +182,7 @@ export class AssistantLifecycleCoordinator { attach(id: string, expectedRevision: number): Promise { if ( this.closed || - this.inspections.has(id) || + this.preparations.has(id) || !Number.isSafeInteger(expectedRevision) || expectedRevision < 0 ) @@ -240,6 +264,263 @@ export class AssistantLifecycleCoordinator { return { conversationId, lease, lifecycle: { ...lifecycle } }; } + /** Explicit same-ID Resume. Preparation may only read the authorized saved + * conversation; association creation is never a fallback here. */ + resume( + id: string, + expectedRevision: number, + operationId: string, + preparation: AssistantResumePreparation, + callerSignal?: AbortSignal, + ): Promise { + if ( + this.closed || + !Number.isSafeInteger(expectedRevision) || + expectedRevision < 0 || + !z.string().uuid().safeParse(operationId).success + ) + return Promise.reject(failure("lifecycle_changed")); + const pending = this.resumes.get(id); + if (pending) + return pending.operationId === operationId && + pending.revision === expectedRevision + ? pending.promise + : Promise.reject(failure("lifecycle_changed")); + if (this.pending.has(id) || this.preparations.has(id)) + return Promise.reject(failure("lifecycle_changed")); + const fence = { id, generation: this.generation(id) }, + controller = new AbortController(); + this.preparations.set(id, controller); + const promise = Promise.resolve() + .then(() => + this.resumeRuntime( + fence, + expectedRevision, + operationId, + preparation, + controller, + callerSignal, + ), + ) + .catch((error: unknown) => { + throw error instanceof AssistantSessionRevisionError + ? failure("lifecycle_changed") + : error; + }) + .finally(() => { + if (this.resumes.get(id)?.promise === promise) this.resumes.delete(id); + if (this.preparations.get(id) === controller) + this.preparations.delete(id); + }); + this.resumes.set(id, { operationId, revision: expectedRevision, promise }); + return promise; + } + + private async resumeRuntime( + fence: AssistantEndFence, + expectedRevision: number, + operationId: string, + preparation: AssistantResumePreparation, + controller: AbortController, + callerSignal?: AbortSignal, + ): Promise { + const { id } = fence, + { store, host } = this.options; + if (!store.resumeState || !store.commitResume) + throw inspectionFailure("transport_unavailable"); + let signal = callerSignal + ? AbortSignal.any([controller.signal, callerSignal]) + : controller.signal; + const timer = setTimeout( + () => controller.abort(inspectionFailure("transport_unavailable")), + this.options.resumeTimeoutMs ?? 15_000, + ); + const check = () => { + signal.throwIfAborted(); + this.check(fence); + }; + const wait = (work: Promise) => + awaitAssistantInspection(work, signal); + let hosted: HostedOpenCode | undefined, + provisional = false, + started = false, + complete = false; + try { + check(); + const binding = Object.freeze({ + ...(await wait(preparation.authorize(signal))), + }); + if (binding.harnessSessionId !== id || binding.conversationId === id) + throw new OpenCodeAccessError("Assistant binding changed"); + const bindingDigest = assistantResumeBindingDigest(binding); + const request = { operationId, expectedRevision, bindingDigest }; + check(); + const state = await wait(store.resumeState(id)); + check(); + const current = state.lifecycle ?? initial(id), + proof = state.resumeOperation; + const retry = proof?.operationId === operationId; + if ( + current.lifecycle === "ending" || + (retry + ? proof.expectedRevision !== expectedRevision || + proof.bindingDigest !== bindingDigest || + proof.committedRevision !== current.revision || + current.lifecycle !== "open" + : current.revision !== expectedRevision) + ) + throw failure("lifecycle_changed"); + if (this.options.canAttach) { + const allowed = await wait(this.options.canAttach(id)); + check(); + if (!allowed) throw failure("lifecycle_changed"); + } + hosted = host.current(id) ?? undefined; + provisional = !hosted; + if (!hosted) { + started = true; + const starting = host.ensure(id); + void starting.then( + (late) => { + if (signal.aborted || this.preparations.get(id) !== controller) + void host.retireExact(late).catch(() => {}); + }, + () => {}, + ); + hosted = await wait(starting); + } + check(); + signal = AbortSignal.any([signal, hosted.signal]); + if ( + hosted.harnessSessionId !== id || + hosted.cwd !== binding.cwd || + hosted.contextAuthorityScope !== binding.contextAuthorityScope || + basename(hosted.stateRoot) !== binding.nativeScope + ) + throw new OpenCodeAccessError("Assistant binding changed"); + await wait(host.assertCurrent(hosted)); + check(); + const candidate = this.leases.get(id); + if ( + candidate?.hosted === hosted && + candidate.conversationId !== binding.conversationId + ) + throw new OpenCodeAccessError("Assistant binding changed"); + const liveLease = () => { + const lease = this.leases.get(id); + return lease && + lease.hosted === hosted && + lease.lifecycle.revision === current.revision && + current.lifecycle === "open" + ? lease + : undefined; + }; + await wait(preparation.read(hosted, binding, signal)); + check(); + const authorized = async () => { + check(); + const fresh = await wait(preparation.authorize(signal)); + check(); + if (assistantResumeBindingDigest(fresh) !== bindingDigest) + throw new OpenCodeAccessError("Assistant binding changed"); + await wait(host.assertCurrent(hosted!)); + check(); + }; + await authorized(); + const before = await wait(store.resumeState(id)); + check(); + if ( + (before.lifecycle?.revision ?? 0) !== current.revision || + before.lifecycle?.lifecycle === "ending" + ) + throw failure("lifecycle_changed"); + const existing = liveLease(); + if (retry && existing) { + complete = true; + return this.publicLease(existing); + } + + // Publish the durable revision and update the same live lease as one + // coordinator operation. Existing execution waits through this boundary. + const runtime = hosted; + return await this.publishRuntime(runtime, signal, async () => { + check(); + await authorized(); + const live = liveLease(); + // Keep publication ownership until the exact write/rollback and lock + // release settle. Cancellation cannot make a still-pending write vanish. + const lifecycle = await store.commitResume!( + id, + current.revision, + request, + live?.lifecycle.execution ?? "paused", + signal, + ); + check(); + await authorized(); + const saved = await wait(store.lifecycle(id)); + check(); + if ( + saved?.revision !== lifecycle.revision || + saved.lifecycle !== "open" + ) + throw failure("lifecycle_changed"); + const lease: Lease = live ?? { + hosted: runtime, + conversationId: binding.conversationId, + lease: randomUUID(), + lifecycle, + }; + lease.lifecycle = lifecycle; + this.leases.set(id, lease); + this.remember(lifecycle); + check(); + if (!live) host.observe(runtime, binding.conversationId); + check(); + complete = true; + return this.publicLease(lease); + }); + } finally { + try { + if (!complete && provisional && started) { + if ( + this.preparations.get(id) === controller && + this.generation(id) === fence.generation + ) + await this.cleanInspection(id, hosted); + else if (hosted) void host.retireExact(hosted).catch(() => {}); + } + } finally { + clearTimeout(timer); + } + } + } + + private async publishRuntime( + hosted: HostedOpenCode, + signal: AbortSignal, + write: () => Promise, + ): Promise { + for (;;) { + const pending = this.publications.get(hosted); + if (!pending) break; + await awaitAssistantInspection(pending, signal); + } + signal.throwIfAborted(); + let release!: () => void; + const publication = new Promise((done) => { + release = done; + }); + this.publications.set(hosted, publication); + try { + return await write(); + } finally { + if (this.publications.get(hosted) === publication) + this.publications.delete(hosted); + release(); + } + } + /** Exclusive temporary ownership, shared with Attach, Continue and Resume. Never * creates an association or grants execution. Callback results stay private; * the caller revalidates its saved binding after provisional cleanup. */ @@ -260,7 +541,13 @@ export class AssistantLifecycleCoordinator { prepare: (hosted: HostedOpenCode, signal: AbortSignal) => Promise, callerSignal?: AbortSignal, ): Promise { - return this.prepareRuntime(id, expectedRevision, prepare, true, callerSignal); + return this.prepareRuntime( + id, + expectedRevision, + prepare, + true, + callerSignal, + ); } private async prepareRuntime( @@ -273,7 +560,7 @@ export class AssistantLifecycleCoordinator { if ( this.closed || this.pending.has(id) || - this.inspections.has(id) || + this.preparations.has(id) || !Number.isSafeInteger(expectedRevision) || expectedRevision < 0 ) @@ -281,7 +568,7 @@ export class AssistantLifecycleCoordinator { const fence = { id, generation: this.generation(id) }; this.check(fence); const controller = new AbortController(); - this.inspections.set(id, controller); + this.preparations.set(id, controller); const signal = callerSignal ? AbortSignal.any([controller.signal, callerSignal]) : controller.signal; @@ -319,7 +606,7 @@ export class AssistantLifecycleCoordinator { // exact old host. Never retire by ID from this late continuation. void starting.then( (late) => { - if (signal.aborted || this.inspections.get(id) !== controller) + if (signal.aborted || this.preparations.get(id) !== controller) void this.options.host.retireExact(late).catch(() => {}); }, () => {}, @@ -349,7 +636,7 @@ export class AssistantLifecycleCoordinator { try { if (provisional && started && !(complete && retain)) { if ( - this.inspections.get(id) === controller && + this.preparations.get(id) === controller && this.generation(id) === fence.generation ) { // With no returned host yet, this exact slot owns pending ensure. @@ -362,8 +649,8 @@ export class AssistantLifecycleCoordinator { if (complete && !retain) await revision(); } finally { clearTimeout(timer); - if (this.inspections.get(id) === controller) - this.inspections.delete(id); + if (this.preparations.get(id) === controller) + this.preparations.delete(id); } } } @@ -424,16 +711,28 @@ export class AssistantLifecycleCoordinator { throw failure("execution_paused"); }; check(); - await this.options.host.assertCurrent(hosted); - check(); - const saved = await this.options.store.lifecycle(id); - check(); - if ( - !saved || - saved.revision !== lease!.lifecycle.revision || - saved.lifecycle !== "open" - ) - throw failure("lifecycle_changed"); + for (;;) { + const publication = this.publications.get(hosted); + if (publication) + await awaitAssistantInspection(publication, hosted.signal); + check(); + await this.options.host.assertCurrent(hosted); + check(); + const snapshot = lease!.lifecycle; + const saved = await this.options.store.lifecycle(id); + check(); + // A read started before publication may have returned the old durable + // snapshot after the same exact lease advanced. Re-read after its commit. + if (this.publications.has(hosted) || lease!.lifecycle !== snapshot) + continue; + if ( + !saved || + saved.revision !== snapshot.revision || + saved.lifecycle !== "open" + ) + throw failure("lifecycle_changed"); + return; + } } /** Called inside the existing dispatch admission, after explicit user context preparation. */ @@ -443,16 +742,23 @@ export class AssistantLifecycleCoordinator { const lease = this.leases.get(id)!; if (lease.lifecycle.execution === "enabled") return; const fence = { id, generation: this.generation(id) }; - const state = await this.options.store.transition( - id, - lease.lifecycle.revision, - { lifecycle: "open", execution: "enabled" }, - hosted.signal, - ); - this.check(fence); - if (this.leases.get(id) !== lease) throw failure("lifecycle_changed"); - lease.lifecycle = state; - this.remember(state); + await this.publishRuntime(hosted, hosted.signal, async () => { + this.check(fence); + if (this.leases.get(id) !== lease) throw failure("lifecycle_changed"); + if (lease.lifecycle.execution === "enabled") return; + await this.options.host.assertCurrent(hosted); + this.check(fence); + const state = await this.options.store.transition( + id, + lease.lifecycle.revision, + { lifecycle: "open", execution: "enabled" }, + hosted.signal, + ); + this.check(fence); + if (this.leases.get(id) !== lease) throw failure("lifecycle_changed"); + lease.lifecycle = state; + this.remember(state); + }); } /** Fences execution and requests process termination before any persistence await. @@ -465,8 +771,9 @@ export class AssistantLifecycleCoordinator { this.ending.add(id); this.leases.delete(id); this.pending.delete(id); - this.inspections.get(id)?.abort(failure("lifecycle_changed")); - this.inspections.delete(id); + this.resumes.delete(id); + this.preparations.get(id)?.abort(failure("lifecycle_changed")); + this.preparations.delete(id); const native = this.options.host.retireWithResult(id); const persistence = beforePersistence ? beforePersistence.then(() => this.persistEnd(fence, "ending")) @@ -508,9 +815,10 @@ export class AssistantLifecycleCoordinator { this.closed = true; this.leases.clear(); this.pending.clear(); - for (const inspection of this.inspections.values()) + this.resumes.clear(); + for (const inspection of this.preparations.values()) inspection.abort(failure("lifecycle_changed")); - this.inspections.clear(); + this.preparations.clear(); this.options.host.beginShutdown(); this.changed(); } diff --git a/packages/harness/src/core/assistant-resume.test.ts b/packages/harness/src/core/assistant-resume.test.ts new file mode 100644 index 000000000..611634c9c --- /dev/null +++ b/packages/harness/src/core/assistant-resume.test.ts @@ -0,0 +1,295 @@ +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { AssistantLifecycleCoordinator } from "./assistant-lifecycle.js"; +import { AssistantSessionStore, type AssistantAssociation } from "./assistant-session-store.js"; +import type { HostedOpenCode } from "./opencode-host.js"; + +const id = "studio-a", operation = "f1872aaa-b7c0-44f1-a9bc-3f9613b4a52c"; +const otherOperation = "ac243472-d1ef-4926-bf47-2a32bd28e912"; +const binding: AssistantAssociation = { + version: 1, harnessSessionId: id, cwd: "/workspace", conversationId: "ses_saved", + contextAuthorityScope: "a".repeat(64), nativeScope: "b".repeat(64), createdAt: 1, +}; +const gate = () => { + let release!: () => void; + const promise = new Promise((done) => { release = done; }); + return { promise, release }; +}; +let root: string, store: AssistantSessionStore, coordinator: AssistantLifecycleCoordinator; +const hosts = new Map(), lifetimes = new WeakMap(); +const ensure = vi.fn(), observe = vi.fn(), associate = vi.fn(), retire = vi.fn(), retireExact = vi.fn(); +const assertCurrent = vi.fn(), canAttach = vi.fn(), authorize = vi.fn(), read = vi.fn(); +const preparation = { authorize, read }; +const fixture = (resumeTimeoutMs = 15_000) => new AssistantLifecycleCoordinator({ + store, canAttach, resumeTimeoutMs, associations: { ensure: associate }, + host: { + ensure, observe, assertCurrent, current: (key) => hosts.get(key) ?? null, + retireWithResult: retire, retireExact, beginShutdown: () => { + for (const key of hosts.keys()) void retire(key); + }, + }, +}); +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), "assistant-resume-")); + store = new AssistantSessionStore(root); + await store.transition(id, 0, { lifecycle: "ended", execution: "paused" }); + hosts.clear(); + ensure.mockReset().mockImplementation(async (key: string) => { + if (hosts.has(key)) return hosts.get(key)!; + const lifetime = new AbortController(); + const hosted: HostedOpenCode = { + harnessSessionId: key, cwd: binding.cwd, stateRoot: join(root, binding.nativeScope), + contextAuthorityScope: binding.contextAuthorityScope, model: { providerID: "sapiom", modelID: "test" }, + signal: lifetime.signal, isCurrent: () => hosts.get(key) === hosted && !lifetime.signal.aborted, + server: { pid: 1, exited: new Promise(() => {}), close: vi.fn(), fetch: vi.fn(), fetchJson: vi.fn() }, + }; + hosts.set(key, hosted); lifetimes.set(hosted, lifetime); return hosted; + }); + observe.mockReset(); + associate.mockReset().mockResolvedValue(binding.conversationId); + retire.mockReset().mockImplementation(async (key: string) => { + const hosted = hosts.get(key); + if (hosted) { lifetimes.get(hosted)!.abort(); hosts.delete(key); } + return { state: hosted ? "confirmed" : "absent" }; + }); + retireExact.mockReset().mockImplementation(async (hosted: HostedOpenCode) => { + if (hosts.get(hosted.harnessSessionId) === hosted) await retire(hosted.harnessSessionId); + }); + assertCurrent.mockReset().mockImplementation(async (hosted: HostedOpenCode) => { + if (!hosted.isCurrent()) throw new Error("not current"); + }); + canAttach.mockReset().mockResolvedValue(true); + authorize.mockReset().mockImplementation(async () => ({ ...binding })); + read.mockReset().mockResolvedValue(undefined); + coordinator = fixture(); +}); +afterEach(async () => { + vi.useRealTimers(); + for (const hosted of hosts.values()) lifetimes.get(hosted)?.abort(); + await rm(root, { recursive: true, force: true }); +}); +const resume = (expected = 1, op = operation) => coordinator.resume(id, expected, op, preparation); +async function live() { + await store.transition(id, 1, { lifecycle: "open", execution: "paused" }); + const lease = await coordinator.attach(id, 2); + await coordinator.enable(hosts.get(id)!); + return { ...lease, lifecycle: (await coordinator.describe(id)) }; +} + +it("resumes the exact saved native session paused without association creation or affecting same-folder B", async () => { + const b = await coordinator.attach("studio-b", 0), other = hosts.get("studio-b"); + associate.mockClear(); observe.mockClear(); + const attached = await resume(); + expect(attached).toMatchObject({ conversationId: binding.conversationId, lifecycle: { harnessSessionId: id, revision: 2, lifecycle: "open", execution: "paused" } }); + expect(associate).not.toHaveBeenCalled(); + expect(observe).toHaveBeenCalledExactlyOnceWith(hosts.get(id), binding.conversationId); + expect(read).toHaveBeenCalledExactlyOnceWith(hosts.get(id), binding, expect.any(AbortSignal)); + expect(await coordinator.attach(id, 2)).toEqual(attached); + await expect(coordinator.use(id, attached.lease, true)).rejects.toMatchObject({ failure: { code: "execution_paused" } }); + expect(hosts.get("studio-b")).toBe(other); + expect((await coordinator.use("studio-b", b.lease)).hosted).toBe(other); +}); + +it("coalesces pending operation retries and excludes Attach, inspection, Continue and competing Resume", async () => { + const held = gate(); + read.mockImplementationOnce(() => held.promise); + const first = resume(); + expect(resume()).toBe(first); + await vi.waitFor(() => expect(read).toHaveBeenCalledOnce()); + await expect(resume(1, otherOperation)).rejects.toThrow(); + await expect(coordinator.attach(id, 1)).rejects.toThrow(); + await expect(coordinator.inspect(id, 1, vi.fn())).rejects.toThrow(); + await expect(coordinator.prepareContinuation(id, 1, vi.fn())).rejects.toThrow(); + held.release(); + const attached = await first; + expect(await resume()).toEqual(attached); + expect(ensure).toHaveBeenCalledOnce(); + expect(observe).toHaveBeenCalledOnce(); + expect((await store.lifecycle(id))?.revision).toBe(2); +}); + +it("preserves an enabled exact live lease while execution waits across durable Resume publication", async () => { + const attached = await live(), hosted = hosts.get(id)!; + const published = gate(), release = gate(), commit = store.commitResume.bind(store); + vi.spyOn(store, "commitResume").mockImplementationOnce(async (...args) => { + const state = await commit(...args); published.release(); await release.promise; return state; + }); + const resumed = resume(attached.lifecycle.revision); + await published.promise; + const executing = coordinator.use(id, attached.lease, true); + let settled = false; void executing.then(() => { settled = true; }); + await new Promise((done) => setTimeout(done, 5)); + expect(settled).toBe(false); + expect(hosted.signal.aborted).toBe(false); + release.release(); + expect(await resumed).toMatchObject({ lease: attached.lease, lifecycle: { revision: 5, execution: "enabled" } }); + expect((await executing).hosted).toBe(hosted); + expect(observe).toHaveBeenCalledOnce(); +}); + +it("re-reads an old in-flight durable snapshot after the same live lease advances", async () => { + const attached = await live(), held = gate(), readStarted = gate(); + const lifecycle = store.lifecycle.bind(store); + vi.spyOn(store, "lifecycle").mockImplementationOnce(async (key) => { + const snapshot = await lifecycle(key); readStarted.release(); await held.promise; return snapshot; + }); + const executing = coordinator.use(id, attached.lease, true); + await readStarted.promise; + await resume(attached.lifecycle.revision); + held.release(); + expect((await executing).lease).toBe(attached.lease); +}); + +it("preserves a live lease when Resume overlaps an enable whose durable write already published", async () => { + await store.transition(id, 1, { lifecycle: "open", execution: "paused" }); + const attached = await coordinator.attach(id, 2), held = gate(), published = gate(); + const transition = store.transition.bind(store); + vi.spyOn(store, "transition").mockImplementationOnce(async (...args) => { + const state = await transition(...args); published.release(); await held.promise; return state; + }); + const enabling = coordinator.enable(hosts.get(id)!); + await published.promise; + const resumed = resume(4); + await vi.waitFor(() => expect(read).toHaveBeenCalledOnce()); + held.release(); await enabling; + expect(await resumed).toMatchObject({ lease: attached.lease, lifecycle: { revision: 5, execution: "enabled" } }); + await expect(coordinator.use(id, attached.lease, true)).resolves.toBeDefined(); +}); + +it("reconciles the same operation after host restart with a new paused lease and proved revision", async () => { + const first = await resume(); + await retire(id); coordinator = fixture(); + const next = await resume(); + expect(next.lease).not.toBe(first.lease); + expect(next).toMatchObject({ conversationId: binding.conversationId, lifecycle: { revision: 3, execution: "paused" } }); + await expect(coordinator.use(id, first.lease)).rejects.toThrow(); + expect(associate).not.toHaveBeenCalled(); + expect(read).toHaveBeenCalledTimes(2); +}); + +it("recovers a durable commit whose caller failed before its lease was granted", async () => { + const commit = store.commitResume.bind(store); + vi.spyOn(store, "commitResume").mockImplementationOnce(async (...args) => { + await commit(...args); throw new Error("lost acknowledgement"); + }); + await expect(resume()).rejects.toThrow("lost acknowledgement"); + expect(hosts.get(id)).toBeUndefined(); + coordinator = fixture(); + expect(await resume()).toMatchObject({ lifecycle: { revision: 3, execution: "paused" } }); + expect(associate).not.toHaveBeenCalled(); +}); + +it.each(["enable", "End", "binding"])("rejects an old operation after later %s without reopening", async (changed) => { + await resume(); + if (changed === "enable") await coordinator.enable(hosts.get(id)!); + if (changed === "End") { + const ending = coordinator.beginEnd(id); await ending.persistence; await coordinator.finishEnd(ending.fence); + } + if (changed === "binding") authorize.mockResolvedValue({ ...binding, createdAt: 2 }); + const before = await store.resumeState(id); + await expect(resume()).rejects.toMatchObject({ failure: { code: "lifecycle_changed" } }); + expect(await store.resumeState(id)).toEqual(before); + expect(ensure).toHaveBeenCalledOnce(); +}); + +it("denies pending Continue children before runtime startup", async () => { + canAttach.mockResolvedValue(false); + await expect(resume()).rejects.toThrow(); + expect(ensure).not.toHaveBeenCalled(); + expect(read).not.toHaveBeenCalled(); + expect(await store.lifecycle(id)).toMatchObject({ lifecycle: "ended", revision: 1 }); +}); + +it("keeps lifecycle and prior history unchanged when native preflight fails", async () => { + read.mockRejectedValueOnce(new Error("retained context unavailable")); + const before = await store.resumeState(id); + await expect(resume()).rejects.toThrow("retained context unavailable"); + expect(await store.resumeState(id)).toEqual(before); + expect(hosts.get(id)).toBeUndefined(); + expect(retireExact).toHaveBeenCalledOnce(); + expect(associate).not.toHaveBeenCalled(); +}); + +it.each(["authorization", "startup", "preflight", "before-commit", "after-commit"])("End preempts %s and late completion cannot reopen or retire a replacement", async (stage) => { + const held = gate(), reached = gate(); + const hold = async () => { reached.release(); await held.promise; }; + if (stage === "authorization") authorize.mockImplementationOnce(async () => { await hold(); return { ...binding }; }); + if (stage === "startup") { + const start = ensure.getMockImplementation()!; + ensure.mockImplementationOnce(async (...args) => { const hosted = await start(...args); await hold(); return hosted; }); + } + if (stage === "preflight") read.mockImplementationOnce(hold); + if (stage.endsWith("commit")) { + const commit = store.commitResume.bind(store); + vi.spyOn(store, "commitResume").mockImplementationOnce(async (...args) => { + if (stage === "before-commit") await hold(); + const state = await commit(...args); + if (stage === "after-commit") await hold(); + return state; + }); + } + const old = resume(), rejected = expect(old).rejects.toMatchObject({ failure: { code: "lifecycle_changed" } }); + await reached.promise; + const ending = coordinator.beginEnd(id); + expect(hosts.get(id)).toBeUndefined(); + if (!stage.endsWith("commit")) await rejected; + await ending.persistence; + const ended = await coordinator.finishEnd(ending.fence); + const next = await resume(ended.revision, otherOperation), replacement = hosts.get(id)!; + held.release(); await rejected; + await new Promise((done) => setTimeout(done, 5)); + expect(hosts.get(id)).toBe(replacement); + expect(replacement.signal.aborted).toBe(false); + expect((await coordinator.use(id, next.lease)).hosted).toBe(replacement); +}); + +it("bounds startup before any host exists and truthfully reports failed cleanup", async () => { + vi.useFakeTimers(); coordinator = fixture(50); + vi.spyOn(store, "resumeState").mockResolvedValue({ lifecycle: { version: 1, harnessSessionId: id, revision: 1, lifecycle: "ended", execution: "paused", updatedAt: 1 }, resumeOperation: null }); + ensure.mockImplementationOnce(() => new Promise(() => {})); + const pending = resume(), rejected = expect(pending).rejects.toMatchObject({ failure: { code: "transport_unavailable" } }); + await vi.advanceTimersByTimeAsync(50); await rejected; + expect(retire).toHaveBeenCalledExactlyOnceWith(id, 750); + retireExact.mockRejectedValueOnce(new Error("unconfirmed")); + read.mockRejectedValueOnce(new Error("failed preflight")); + await expect(resume()).rejects.toMatchObject({ failure: { code: "cleanup_unconfirmed" } }); +}); + +it("a timed-out live preflight preserves the enabled lease and runtime", async () => { + coordinator = fixture(50); + const attached = await live(), hosted = hosts.get(id)!; + read.mockImplementationOnce(() => new Promise(() => {})); + await expect(resume(attached.lifecycle.revision)).rejects.toMatchObject({ failure: { code: "transport_unavailable" } }); + expect(hosts.get(id)).toBe(hosted); + expect((await coordinator.use(id, attached.lease, true)).hosted).toBe(hosted); + expect(retire).not.toHaveBeenCalled(); +}); + +it("cancellation keeps the live publication barrier and operation slot until exact durable IO settles", async () => { + const cancellation = new AbortController(); + const attached = await live(), held = gate(), published = gate(), commit = store.commitResume.bind(store); + vi.spyOn(store, "commitResume").mockImplementationOnce(async (...args) => { + const state = await commit(...args); published.release(); await held.promise; return state; + }); + const resumed = coordinator.resume(id, attached.lifecycle.revision, operation, preparation, cancellation.signal); + const rejected = expect(resumed).rejects.toThrow("cancelled request"); + let settled = false; void resumed.then(() => { settled = true; }, () => { settled = true; }); + await published.promise; + cancellation.abort(new Error("cancelled request")); + await new Promise((done) => setTimeout(done, 5)); + expect(settled).toBe(false); + await expect(coordinator.inspect(id, 5, vi.fn())).rejects.toThrow(); + await expect(resume(5, otherOperation)).rejects.toThrow(); + const executing = coordinator.use(id, attached.lease, true); + let executionSettled = false; void executing.then(() => { executionSettled = true; }, () => { executionSettled = true; }); + const fenced = expect(executing).rejects.toMatchObject({ failure: { code: "lifecycle_changed" } }); + await new Promise((done) => setTimeout(done, 5)); + expect(executionSettled).toBe(false); + expect(hosts.get(id)?.signal.aborted).toBe(false); + held.release(); await rejected; await fenced; + const recovered = await resume(attached.lifecycle.revision); + expect(recovered).toMatchObject({ lifecycle: { revision: 6, execution: "paused" } }); + expect(recovered.lease).not.toBe(attached.lease); +});