diff --git a/src/node/services/attemptLedger.test.ts b/src/node/services/attemptLedger.test.ts new file mode 100644 index 00000000000..583524aca48 --- /dev/null +++ b/src/node/services/attemptLedger.test.ts @@ -0,0 +1,654 @@ +/* eslint-disable @typescript-eslint/await-thenable -- bun:test async matchers return thenables the rule cannot see */ +import * as fs from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; +import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test"; + +import type { TaskAttemptOutcome } from "@/common/types/tasks"; +import type { ProjectsConfig } from "@/node/config"; +import type { AgentTaskIndex } from "@/node/services/agentTaskIndex"; +import { AttemptLedger, type AttemptLedgerDeps } from "@/node/services/attemptLedger"; +import { readSubagentAttemptSettlementReceiptStrict } from "@/node/services/subagentAttemptSettlements"; +import type { + AdmittedSend, + AgentTaskReport, + OwnedTaskAttempt, + SettlementReceiptDecision, + StreamEndDecision, +} from "@/node/services/taskService"; +import { MutexMap } from "@/node/utils/concurrency/mutexMap"; + +const TASK = "task-a"; +const PARENT = "parent-a"; +const ROOT = "root-a"; +const ATTEMPT = "att_00000000000000a1"; +const OTHER = "att_00000000000000b2"; + +function send(attemptId: string): AdmittedSend { + return { taskId: TASK, attemptId, state: "admitted" } as unknown as AdmittedSend; +} + +/** A config whose only workspace is TASK, with this row. */ +function configWithRow(row: Record | undefined): ProjectsConfig { + const workspaces = row == null ? [] : [{ id: TASK, ...row }]; + return { projects: new Map([["/project", { workspaces }]]) } as unknown as ProjectsConfig; +} + +interface HarnessState { + config: ProjectsConfig | Error; + sessionsDir: string; + stopRecordAttemptId: string | undefined; + hasStopRecord: boolean; + sends: Set; + activeTurn: symbol | undefined; + liveRegistration: unknown; + streaming: boolean; + streamEnd: StreamEndDecision | undefined; + owners: string[]; + outcomes: Array>; +} + +/** + * Fake TaskService side: records the facade calls the ledger routes back through TaskService in + * `events`, in call order, and forwards them to the ledger as the real facades do. + */ +function createHarness() { + const events: string[] = []; + const state: HarnessState = { + config: configWithRow({ parentWorkspaceId: PARENT, taskAttemptId: ATTEMPT }), + sessionsDir: "/unused", + stopRecordAttemptId: undefined, + hasStopRecord: false, + sends: new Set(), + activeTurn: undefined, + liveRegistration: undefined, + streaming: false, + streamEnd: undefined, + owners: [PARENT, ROOT], + outcomes: [], + }; + const eventLocks = new MutexMap(); + const inspect = mock( + ( + _taskId: string, + _options?: { requestingWorkspaceId?: string } + ): Promise> => + Promise.resolve(state.outcomes.shift() ?? { kind: "live", executionId: TASK }) + ); + const deps: AttemptLedgerDeps = { + config: () => ({ + // Like the real Config: an unreadable file throws only for a strict read. + loadConfigOrDefault: (options?: { throwOnError?: boolean }) => { + if (!(state.config instanceof Error)) return state.config; + if (options?.throwOnError === true) throw state.config; + return configWithRow(undefined); + }, + sessionsDir: state.sessionsDir, + }), + aiService: () => ({ isStreaming: () => state.streaming }), + workspaceService: () => ({ getActiveTurnGeneration: () => state.activeTurn }), + turnManager: () => + ({ + getLiveWorkspaceTurnRegistration: () => state.liveRegistration, + }) as unknown as ReturnType, + workspaceEventLocks: () => eventLocks, + buildAgentTaskIndex: () => ({ parentById: new Map() }) as unknown as AgentTaskIndex, + listAncestorWorkspaceIdsUsingParentById: () => state.owners, + stopRecordAttemptId: () => state.stopRecordAttemptId, + hasStopRecord: () => state.hasStopRecord, + admittedSends: () => state.sends, + findStreamEndDecision: () => state.streamEnd, + discardStreamEndDecisions: (taskId) => events.push(`discardStreamEnds:${taskId}`), + inspectAttemptOutcome: (taskId, options) => { + events.push("inspect"); + return inspect(taskId, options); + }, + notifyAttemptSettlementListeners: (taskId) => { + events.push("notify"); + ledger.notifyAttemptSettlementListeners(taskId); + }, + closeAttemptAdmission: (taskId, attemptId, ownedAttempt, source) => { + events.push(`close:${attemptId ?? "none"}`); + ledger.closeAttemptAdmission(taskId, attemptId, ownedAttempt, source); + }, + settleOwnedTaskAttempt: (taskId, attempt, source) => { + events.push("settle"); + ledger.settleOwnedTaskAttempt(taskId, attempt, source); + }, + decideSettlementReceipt: (taskId, attempt, source) => { + events.push("decide"); + return ledger.decideSettlementReceipt(taskId, attempt, source); + }, + writeSettlementReceipt: (taskId, target, source) => { + events.push("write"); + return ledger.writeSettlementReceipt(taskId, target, source); + }, + }; + const ledger = new AttemptLedger(deps); + return { ledger, state, events, eventLocks, inspect }; +} + +function begin( + ledger: AttemptLedger, + attemptId: string | undefined = ATTEMPT, + receiptEligible = true +): OwnedTaskAttempt { + return ledger.beginOwnedTaskAttempt(TASK, "test", { attemptId, receiptEligible }); +} + +describe("AttemptLedger", () => { + describe("attempt identity", () => { + test("a new attempt replaces the owner, bumps its generation and drops the stream-end decisions", () => { + const { ledger, events } = createHarness(); + const first = begin(ledger, ATTEMPT); + const second = begin(ledger, OTHER); + + expect(first.generation).toBe(1); + expect(second.generation).toBe(2); + expect(ledger.ownedAttempt(TASK)).toBe(second); + expect(ledger.currentAttemptIdByTaskId.get(TASK)).toBe(OTHER); + expect(events).toEqual([`discardStreamEnds:${TASK}`, `discardStreamEnds:${TASK}`]); + }); + + test("a closure for the new id survives the begin; another attempt's closure is dropped", () => { + const { ledger } = createHarness(); + ledger.closeAttemptAdmission(TASK, OTHER, undefined, "early-close"); + begin(ledger, OTHER); + expect(ledger.isAttemptClosed(TASK, OTHER)).toBe(true); + + begin(ledger, ATTEMPT); + expect(ledger.attemptSettlement(TASK)).toBeUndefined(); + }); + + test("publishing a rotation keeps only a closure that names the new id", () => { + const { ledger } = createHarness(); + ledger.closeAttemptAdmission(TASK, OTHER, undefined, "early-close"); + ledger.publishAttemptRotation(TASK, OTHER); + expect(ledger.isAttemptClosed(TASK, OTHER)).toBe(true); + + ledger.publishAttemptRotation(TASK, ATTEMPT); + expect(ledger.attemptSettlement(TASK)).toBeUndefined(); + expect(ledger.currentAttemptIdByTaskId.get(TASK)).toBe(ATTEMPT); + }); + + test("releasing a reported attempt drops only that attempt's entries", () => { + const { ledger } = createHarness(); + const stale = begin(ledger, ATTEMPT); + const current = begin(ledger, OTHER); + ledger.settleOwnedTaskAttempt(TASK, current, "settled"); + + ledger.releaseReportedTaskAttempt(TASK, stale); + expect(ledger.ownedAttempt(TASK)).toBe(current); + + ledger.releaseReportedTaskAttempt(TASK, current); + expect(ledger.hasOwnedAttempt(TASK)).toBe(false); + expect(ledger.attemptSettlement(TASK)).toBeUndefined(); + }); + }); + + describe("attempt identity guards", () => { + test("an empty task id or a malformed attempt id is refused; a pre-identity owner is not", () => { + const { ledger } = createHarness(); + const begin1 = (taskId: string, attemptId: string | undefined) => + ledger.beginOwnedTaskAttempt(taskId, "test", { attemptId, receiptEligible: false }); + expect(() => begin1("", ATTEMPT)).toThrow(); + expect(() => begin1(TASK, "not-an-attempt-id")).toThrow(); + expect(begin1("t", undefined).attemptId).toBeUndefined(); + expect(ledger.currentAttemptIdByTaskId.has("t")).toBe(false); + }); + + test("the reservation's abort signal rides only with the attempt it was given to", () => { + const { ledger } = createHarness(); + const signal = new AbortController().signal; + const reserved = ledger.beginOwnedTaskAttempt(TASK, "test", { + attemptId: ATTEMPT, + receiptEligible: true, + abortSignal: signal, + }); + expect(reserved.abortSignal).toBe(signal); + expect("abortSignal" in begin(ledger, OTHER)).toBe(false); + }); + + test("entries that are absent are no-ops for release, rotation and the closure wait", async () => { + const { ledger } = createHarness(); + const owned = begin(ledger, ATTEMPT); + ledger.publishAttemptRotation(TASK, ATTEMPT); + ledger.releaseReportedTaskAttempt(TASK, owned); + expect(ledger.hasOwnedAttempt(TASK)).toBe(false); + await ledger.waitForAttemptClosureToSettle(TASK, 10_000); + expect(ledger.attemptSettlementListenersByTaskId.has(TASK)).toBe(false); + }); + }); + + describe("closure and settlement", () => { + test("closing records the owner only when it holds that id, and notifies once per new closure", () => { + const { ledger, events } = createHarness(); + const owned = begin(ledger, ATTEMPT); + events.length = 0; + + ledger.closeAttemptAdmission(TASK, OTHER, owned, "other"); + expect(ledger.attemptSettlement(TASK)).toEqual({ + attemptId: OTHER, + phase: "closing", + source: "other", + }); + ledger.closeAttemptAdmission(TASK, ATTEMPT, owned, "mine"); + expect(ledger.attemptSettlement(TASK)?.attempt).toBe(owned); + // Already closed for this id: no replacement, no second notification. + ledger.closeAttemptAdmission(TASK, ATTEMPT, undefined, "again"); + expect(ledger.attemptSettlement(TASK)?.source).toBe("mine"); + expect(events).toEqual(["notify", "notify"]); + }); + + test("closing unless another attempt is closed keeps that other closure", () => { + const { ledger } = createHarness(); + ledger.closeAttemptAdmission(TASK, OTHER, undefined, "other"); + ledger.closeAttemptAdmissionUnlessOtherClosed(TASK, ATTEMPT, undefined, "mine"); + expect(ledger.attemptSettlement(TASK)?.attemptId).toBe(OTHER); + }); + + test("only the current owner settles; every settlement notifies", () => { + const { ledger, events } = createHarness(); + const stale = begin(ledger, ATTEMPT); + const current = begin(ledger, OTHER); + events.length = 0; + + ledger.settleOwnedTaskAttempt(TASK, stale, "stale"); + expect(ledger.attemptSettlement(TASK)).toBeUndefined(); + ledger.settleOwnedTaskAttempt(TASK, current, "current"); + expect(ledger.attemptSettlement(TASK)).toEqual({ + attemptId: OTHER, + attempt: current, + phase: "settled", + source: "current", + }); + expect(events).toEqual(["notify", "notify"]); + // An unowned settlement is only announced. + ledger.settleOwnedTaskAttempt(TASK, undefined, "unowned"); + expect(events).toEqual(["notify", "notify", "notify"]); + }); + + test("notification iterates a copy, so a listener removing another still lets it run", () => { + const { ledger } = createHarness(); + const calls: string[] = []; + const second = () => calls.push("second"); + const first = () => { + calls.push("first"); + ledger.removeAttemptSettlementListener(TASK, second); + }; + ledger.addAttemptSettlementListener(TASK, first); + ledger.addAttemptSettlementListener(TASK, second); + + ledger.notifyAttemptSettlementListeners(TASK); + expect(calls).toEqual(["first", "second"]); + ledger.removeAttemptSettlementListener(TASK, first); + expect(ledger.attemptSettlementListenersByTaskId.has(TASK)).toBe(false); + }); + }); + + describe("no-report boundary", () => { + function closedOwner() { + const harness = createHarness(); + const owned = begin(harness.ledger, ATTEMPT) as OwnedTaskAttempt & { attemptId: string }; + harness.ledger.closeAttemptAdmission(TASK, ATTEMPT, owned, "closed"); + return { ...harness, owned }; + } + + test("an open attempt can still report unless a stop record latched it", () => { + const { ledger, state } = createHarness(); + const owned = begin(ledger, ATTEMPT) as OwnedTaskAttempt & { attemptId: string }; + expect(ledger.attemptCannotStillReport(TASK, owned)).toBe(false); + state.stopRecordAttemptId = ATTEMPT; + expect(ledger.attemptCannotStillReport(TASK, owned)).toBe(true); + }); + + test("an obligation of this attempt blocks; a successor's does not", () => { + const { ledger, state, owned } = closedOwner(); + state.sends.add(send(OTHER)); + expect(ledger.attemptCannotStillReport(TASK, owned)).toBe(true); + state.sends.add(send(ATTEMPT)); + expect(ledger.attemptCannotStillReport(TASK, owned)).toBe(false); + }); + + test.each(["turn", "registration", "stream"] as const)("a live %s blocks", (live) => { + const { ledger, state, owned } = closedOwner(); + if (live === "turn") state.activeTurn = Symbol("turn"); + if (live === "registration") state.liveRegistration = {}; + if (live === "stream") state.streaming = true; + expect(ledger.attemptCannotStillReport(TASK, owned)).toBe(false); + }); + + test.each([ + ["nonreport", true], + ["pending", false], + ["published", false], + ["indeterminate", false], + ] as const)("a %s stream-end decision gives %p", (outcome, expected) => { + const { ledger, state, owned } = closedOwner(); + const decision: StreamEndDecision = { + attempt: undefined, + attemptId: ATTEMPT, + messageId: "msg-1", + outcome, + turn: undefined, + }; + state.streamEnd = decision; + expect(ledger.attemptCannotStillReport(TASK, owned)).toBe(expected); + }); + }); + + describe("settlement receipts", () => { + let sessionsDir: string; + beforeEach(async () => { + sessionsDir = await fs.mkdtemp(path.join(os.tmpdir(), "attempt-ledger-")); + }); + afterEach(async () => { + await fs.rm(sessionsDir, { recursive: true, force: true }); + }); + + function eligibleOwner() { + const harness = createHarness(); + harness.state.sessionsDir = sessionsDir; + const owned = begin(harness.ledger, ATTEMPT); + harness.state.stopRecordAttemptId = ATTEMPT; + harness.events.length = 0; + return { ...harness, owned }; + } + + async function hasReceipt(ownerId: string): Promise { + const read = await readSubagentAttemptSettlementReceiptStrict( + path.join(sessionsDir, ownerId), + TASK, + ATTEMPT + ); + return read.kind === "found"; + } + + test("no receipt authority settles in memory only and closes nothing", () => { + const { ledger, events } = createHarness(); + const owned = begin(ledger, ATTEMPT, false); + events.length = 0; + expect(ledger.decideSettlementReceipt(TASK, owned, "src")).toEqual({ kind: "memory-only" }); + expect(ledger.decideSettlementReceipt(TASK, undefined, "src")).toEqual({ + kind: "memory-only", + }); + expect(events).toEqual([]); + }); + + test("a row naming the attempt closes it in the deciding call, before any await", () => { + const { ledger, events, owned } = eligibleOwner(); + const decision: SettlementReceiptDecision = ledger.decideSettlementReceipt( + TASK, + owned, + "src" + ); + expect(decision).toEqual({ + kind: "write", + attemptId: ATTEMPT, + parentWorkspaceId: PARENT, + owners: [PARENT, ROOT], + }); + // Synchronous: the closure is in place as the decision returns. + expect(events).toEqual([`close:${ATTEMPT}`, "notify"]); + expect(ledger.attemptSettlement(TASK)?.attempt).toBe(owned); + }); + + test("a row naming another attempt gives memory-only and closes nothing", () => { + const { ledger, events, state, owned } = eligibleOwner(); + state.config = configWithRow({ parentWorkspaceId: PARENT, taskAttemptId: OTHER }); + expect(ledger.decideSettlementReceipt(TASK, owned, "src")).toEqual({ kind: "memory-only" }); + expect(events).toEqual([]); + }); + + test("an unreadable config closes the attempt but keeps another attempt's closure", () => { + const { ledger, events, state, owned } = eligibleOwner(); + state.config = new Error("EIO"); + expect(ledger.decideSettlementReceipt(TASK, owned, "src")).toEqual({ kind: "unreadable" }); + expect(events).toEqual([`close:${ATTEMPT}`, "notify"]); + + ledger.closeAttemptAdmission(TASK, OTHER, undefined, "other"); + events.length = 0; + expect(ledger.decideSettlementReceipt(TASK, owned, "src")).toEqual({ kind: "unreadable" }); + expect(events).toEqual([]); + expect(ledger.attemptSettlement(TASK)?.attemptId).toBe(OTHER); + }); + + test.each(["missing", "parentless"] as const)( + "a %s row gives memory-only and closes nothing", + (variant) => { + const { ledger, events, state, owned } = eligibleOwner(); + state.config = configWithRow( + variant === "missing" ? undefined : { taskAttemptId: ATTEMPT } + ); + expect(ledger.decideSettlementReceipt(TASK, owned, "src")).toEqual({ kind: "memory-only" }); + expect(events).toEqual([]); + } + ); + + test("inconsistent ancestry reads as unreadable", () => { + const { ledger, state, owned } = eligibleOwner(); + state.owners = [ROOT]; + expect(ledger.decideSettlementReceipt(TASK, owned, "src")).toEqual({ kind: "unreadable" }); + }); + + test("the receipt reaches every owner, the parent last; a failure stops before the parent", async () => { + const { ledger } = eligibleOwner(); + const target = { attemptId: ATTEMPT, parentWorkspaceId: PARENT, owners: [PARENT, "mid"] }; + // The outer owner's session dir is a file: its write, the first, fails. + await fs.writeFile(path.join(sessionsDir, "mid"), "not a dir"); + expect(await ledger.writeSettlementReceipt(TASK, target, "idle-settled")).toBe(false); + expect(await hasReceipt(PARENT)).toBe(false); + + const ok = { attemptId: ATTEMPT, parentWorkspaceId: PARENT, owners: [PARENT, ROOT] }; + expect(await ledger.writeSettlementReceipt(TASK, ok, "idle-settled")).toBe(true); + expect(await hasReceipt(ROOT)).toBe(true); + expect(await hasReceipt(PARENT)).toBe(true); + }); + + test("persisting writes the receipt before settling", async () => { + const { ledger, events, owned } = eligibleOwner(); + await ledger.persistOwnedAttemptSettlement(TASK, owned, "idle-settled", "stop"); + expect(events).toEqual(["decide", `close:${ATTEMPT}`, "notify", "write", "settle", "notify"]); + expect(ledger.attemptSettlement(TASK)?.phase).toBe("settled"); + }); + + test.each(["unreadable", "write failed"] as const)( + "persisting leaves the attempt closing when %s", + async (variant) => { + const { ledger, events, state, owned } = eligibleOwner(); + if (variant === "unreadable") state.config = new Error("EIO"); + if (variant === "write failed") { + await fs.writeFile(path.join(sessionsDir, ROOT), "not a dir"); + } + await ledger.persistOwnedAttemptSettlement(TASK, owned, "idle-settled", "stop"); + expect(events).not.toContain("settle"); + expect(ledger.attemptSettlement(TASK)?.phase).toBe("closing"); + } + ); + + test("persisting without receipt authority settles in memory", async () => { + const { ledger, events } = createHarness(); + const owned = begin(ledger, ATTEMPT, false); + events.length = 0; + await ledger.persistOwnedAttemptSettlement(TASK, owned, "idle-settled", "stop"); + expect(events).toEqual(["decide", "settle", "notify"]); + }); + }); + + describe("resume-failure settlement", () => { + function failedResume() { + const harness = createHarness(); + const owned = begin(harness.ledger, ATTEMPT, false) as OwnedTaskAttempt & { + attemptId: string; + }; + harness.ledger.closeAttemptAdmission(TASK, ATTEMPT, owned, "resume-failed"); + harness.ledger.recordResumeFailureSettlement(TASK, owned); + harness.events.length = 0; + return { ...harness, owned }; + } + + test("a superseded attempt is dropped without settling", () => { + const { ledger, events } = failedResume(); + begin(ledger, OTHER); + events.length = 0; + ledger.recheckResumeFailureSettlement(TASK); + expect(ledger.resumeFailureSettlementByTaskId.has(TASK)).toBe(false); + expect(events).toEqual([]); + }); + + test("a closure that no longer holds this owner's closing entry drops it without settling", () => { + const variants = [ + // A new owner under the same id: the closure still names the old owner. + (ledger: AttemptLedger) => begin(ledger, ATTEMPT), + // The closure was dropped by a rotation. + (ledger: AttemptLedger) => ledger.publishAttemptRotation(TASK, OTHER), + // Another attempt was closed after the rotation. + (ledger: AttemptLedger) => { + ledger.publishAttemptRotation(TASK, OTHER); + ledger.closeAttemptAdmission(TASK, OTHER, undefined, "other"); + }, + ]; + for (const change of variants) { + const { ledger, events } = failedResume(); + change(ledger); + events.length = 0; + ledger.recheckResumeFailureSettlement(TASK); + expect(ledger.resumeFailureSettlementByTaskId.has(TASK)).toBe(false); + expect(events).not.toContain("settle"); + } + }); + + test("an in-progress stop or a live obligation defers; the boundary settles", async () => { + const { ledger, events, state } = failedResume(); + state.hasStopRecord = true; + ledger.recheckResumeFailureSettlement(TASK); + state.hasStopRecord = false; + state.sends.add(send(ATTEMPT)); + ledger.recheckResumeFailureSettlement(TASK); + expect(ledger.resumeFailureSettlementByTaskId.has(TASK)).toBe(true); + expect(events).toEqual([]); + + state.sends.clear(); + ledger.recheckResumeFailureSettlement(TASK); + expect(ledger.resumeFailureSettlementByTaskId.has(TASK)).toBe(false); + await Promise.resolve(); + expect(events).toEqual(["decide", "settle", "notify"]); + expect(ledger.attemptSettlement(TASK)?.phase).toBe("settled"); + }); + }); + + describe("waits", () => { + test("the closure wait resolves on the notification that settles it and unsubscribes", async () => { + const { ledger } = createHarness(); + const owned = begin(ledger, ATTEMPT); + ledger.closeAttemptAdmission(TASK, ATTEMPT, owned, "closing"); + const waiting = ledger.waitForAttemptClosureToSettle(TASK, 10_000); + expect(ledger.attemptSettlementListenersByTaskId.get(TASK)?.size).toBe(1); + + ledger.settleOwnedTaskAttempt(TASK, owned, "settled"); + await waiting; + expect(ledger.attemptSettlementListenersByTaskId.has(TASK)).toBe(false); + }); + + test("the settlement wait subscribes before its first read and re-reads on notification", async () => { + const { ledger, state, inspect } = createHarness(); + state.outcomes.push({ kind: "live", executionId: TASK }); + state.outcomes.push({ kind: "terminal-no-report" }); + let subscribedAtFirstRead: number | undefined; + inspect.mockImplementationOnce(() => { + subscribedAtFirstRead = ledger.attemptSettlementListenersByTaskId.get(TASK)?.size; + return Promise.resolve(state.outcomes.shift()!); + }); + + const waiting = ledger.waitForAttemptSettlement(TASK, { timeoutMs: 10_000 }); + while (inspect.mock.calls.length < 1) await Promise.resolve(); + await Promise.resolve(); + ledger.notifyAttemptSettlementListeners(TASK); + + expect(await waiting).toEqual({ kind: "terminal-no-report" }); + expect(subscribedAtFirstRead).toBe(1); + expect(inspect).toHaveBeenCalledTimes(2); + expect(ledger.attemptSettlementListenersByTaskId.has(TASK)).toBe(false); + }); + + test("a held event lock bounds the settlement wait; the abandoned read never runs", async () => { + const { ledger, eventLocks, inspect } = createHarness(); + const gate = Promise.withResolvers(); + const holding = eventLocks.withLock(TASK, () => gate.promise); + + expect(await ledger.waitForAttemptSettlement(TASK, { timeoutMs: 20 })).toEqual({ + kind: "timeout", + }); + const abort = new AbortController(); + const aborted = ledger.waitForAttemptSettlement(TASK, { + timeoutMs: 10_000, + abortSignal: abort.signal, + }); + abort.abort(); + await expect(aborted).rejects.toThrow("Interrupted"); + + gate.resolve(); + await holding; + await eventLocks.withLock(TASK, () => Promise.resolve()); + expect(inspect).not.toHaveBeenCalled(); + expect(ledger.attemptSettlementListenersByTaskId.has(TASK)).toBe(false); + }); + + test("the closure wait gives up at its bound", async () => { + const { ledger } = createHarness(); + ledger.closeAttemptAdmission(TASK, ATTEMPT, undefined, "closing"); + await ledger.waitForAttemptClosureToSettle(TASK, 20); + expect(ledger.attemptSettlement(TASK)?.phase).toBe("closing"); + expect(ledger.attemptSettlementListenersByTaskId.has(TASK)).toBe(false); + }); + + test("an empty task id or an invalid bound is refused", async () => { + const { ledger } = createHarness(); + await expect(ledger.waitForAttemptSettlement("", { timeoutMs: 10 })).rejects.toThrow(); + await expect(ledger.waitForAttemptSettlement(TASK, { timeoutMs: 0 })).rejects.toThrow(); + await expect(ledger.waitForAttemptSettlement(TASK, { timeoutMs: NaN })).rejects.toThrow(); + // The smallest valid inputs: a one-character id and a 1 ms bound. + expect(await ledger.waitForAttemptSettlement("t", { timeoutMs: 1 })).toEqual({ + kind: "timeout", + }); + }); + + test("a read that entered the lock completes past the bound, and its error propagates", async () => { + const { ledger, inspect } = createHarness(); + inspect.mockImplementationOnce(async () => { + await new Promise((resolve) => setTimeout(resolve, 40)); + return { kind: "indeterminate", reason: "slow" }; + }); + expect(await ledger.waitForAttemptSettlement(TASK, { timeoutMs: 10 })).toEqual({ + kind: "indeterminate", + reason: "slow", + }); + + inspect.mockImplementationOnce(() => Promise.reject(new Error("read failed"))); + await expect(ledger.waitForAttemptSettlement(TASK, { timeoutMs: 10_000 })).rejects.toThrow( + "read failed" + ); + expect(ledger.attemptSettlementListenersByTaskId.has(TASK)).toBe(false); + }); + + test("an already-aborted wait rejects without reading", async () => { + const { ledger, inspect } = createHarness(); + const abort = new AbortController(); + abort.abort(); + await expect( + ledger.waitForAttemptSettlement(TASK, { timeoutMs: 10_000, abortSignal: abort.signal }) + ).rejects.toThrow("Interrupted"); + expect(inspect).not.toHaveBeenCalled(); + }); + + test("an indeterminate outcome returns at once", async () => { + const { ledger, state, inspect } = createHarness(); + state.outcomes.push({ kind: "indeterminate", reason: "unknown owner" }); + expect(await ledger.waitForAttemptSettlement(TASK, { timeoutMs: 10_000 })).toEqual({ + kind: "indeterminate", + reason: "unknown owner", + }); + expect(inspect).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/src/node/services/attemptLedger.ts b/src/node/services/attemptLedger.ts new file mode 100644 index 00000000000..53f0ef98f6c --- /dev/null +++ b/src/node/services/attemptLedger.ts @@ -0,0 +1,695 @@ +import assert from "node:assert/strict"; +import * as path from "node:path"; +import { getErrorMessage } from "@/common/utils/errors"; +import type { TaskAttemptOutcome, TaskAttemptSettlement } from "@/common/types/tasks"; +import type { Config } from "@/node/config"; +import type { AgentTaskIndex } from "@/node/services/agentTaskIndex"; +import type { AIService } from "@/node/services/aiService"; +import { log } from "@/node/services/log"; +import { + writeSubagentAttemptSettlementReceipt, + type SubagentAttemptSettlementSource, +} from "@/node/services/subagentAttemptSettlements"; +import type { + AdmittedSend, + AgentTaskReport, + AttemptSettlementEntry, + OwnedTaskAttempt, + SettlementReceiptDecision, + StreamEndDecision, +} from "@/node/services/taskService"; +import { coerceNonEmptyString, findWorkspaceEntry, rowSupersedes } from "@/node/services/taskUtils"; +import type { WorkspaceHost } from "@/node/services/taskWorkspaceSeam"; +import type { WorkspaceTurnManager } from "@/node/services/workspaceTurnManager"; +import type { MutexMap } from "@/node/utils/concurrency/mutexMap"; +import { raceWithAbortAndTimeout } from "@/node/utils/concurrency/withTimeout"; +import { assertTaskAttemptId } from "@/node/utils/taskAttemptId"; + +/** + * Everything the attempt ledger reads from TaskService. Each entry is a call-time closure over + * TaskService, never a captured value or bound method: tests swap TaskService fields and patch + * its methods after construction. Methods tests spy on (closeAttemptAdmission, + * decideSettlementReceipt, writeSettlementReceipt, inspectAttemptOutcome) and the other + * TaskService facades are called back through TaskService so every call reaches the spy. Stop, + * send and stream-end state stays with its owner and is reached only through these closures. + */ +export interface AttemptLedgerDeps { + config: () => Pick; + aiService: () => Pick; + workspaceService: () => Pick; + turnManager: () => Pick; + workspaceEventLocks: () => Pick, "withLock">; + buildAgentTaskIndex: (config: ReturnType) => AgentTaskIndex; + listAncestorWorkspaceIdsUsingParentById: ( + parentById: Map, + taskId: string + ) => string[]; + /** TaskStopLatch.stopRecordAttemptId. */ + stopRecordAttemptId: (taskId: string) => string | undefined; + /** TaskStopLatch.hasStopRecord. */ + hasStopRecord: (taskId: string) => boolean; + /** The task's live set of send obligations (TaskService.admittedSendsByTaskId). */ + admittedSends: (taskId: string) => ReadonlySet | undefined; + findStreamEndDecision: (taskId: string, attemptId: string) => StreamEndDecision | undefined; + /** Drop every stream-end decision of the task (TaskService.streamEndDecisionsByTaskId). */ + discardStreamEndDecisions: (taskId: string) => void; + inspectAttemptOutcome: ( + taskId: string, + options?: { requestingWorkspaceId?: string } + ) => Promise>; + notifyAttemptSettlementListeners: (taskId: string) => void; + closeAttemptAdmission: ( + taskId: string, + attemptId: string | undefined, + ownedAttempt: OwnedTaskAttempt | undefined, + source: string + ) => void; + settleOwnedTaskAttempt: ( + taskId: string, + attempt: OwnedTaskAttempt | undefined, + source: string + ) => void; + decideSettlementReceipt: ( + taskId: string, + attempt: OwnedTaskAttempt | undefined, + settlementSource: string + ) => SettlementReceiptDecision; + writeSettlementReceipt: ( + taskId: string, + target: { attemptId: string; parentWorkspaceId: string; owners: string[] }, + source: SubagentAttemptSettlementSource + ) => Promise; +} + +/** + * Attempt identity and settlement state: the attempt this process owns per task, the + * closure/settlement of exactly the attempt it names, the listeners waiting on either, the + * failed reawakens still owed a settlement, and the settlement receipts written for owned + * attempts. Stop records, send obligations and stream-end decisions stay with their owners and + * are reached through the deps. + */ +export class AttemptLedger { + /** + * Attempt ownership ledger (see OwnedTaskAttempt); replaced before any new admission, dropped + * once the attempt's report is durable (releaseReportedTaskAttempt). + */ + readonly ownedAttemptByTaskId = new Map(); + /** + * Closure/settlement of exactly the attempt it names (AttemptSettlementEntry). A `closing` entry + * is recorded synchronously by every producer before its first awaited write and refuses further + * sends under that id; only the owner's settleOwnedTaskAttempt upgrades it to `settled`. Never + * removed by a failed write — a new admission (fresh id) is the only way to reopen. + */ + readonly attemptSettlementByTaskId = new Map(); + readonly attemptSettlementListenersByTaskId = new Map void>>(); + /** + * Failed reawakens whose closed attempt still awaits settlement (#4310): the owner snapshot + * restoreInterruptedTaskAfterResumeFailure closed, settled by recheckResumeFailureSettlement. + */ + readonly resumeFailureSettlementByTaskId = new Map< + string, + OwnedTaskAttempt & { attemptId: string } + >(); + /** + * The last attempt id THIS process published for each task (every local rotation, owned or + * not). Not an authority on the current id — currentTaskAttemptId reads the persisted row, so + * a rotation committed by another process is honored and a row that is missing or unreadable + * yields no id at all. Never read to classify or authorize: an unreadable registry refuses + * every send regardless of this map (see admitTaskWorkspaceTurn), since absence here is no + * proof that a workspace is not a task. + */ + readonly currentAttemptIdByTaskId = new Map(); + + constructor(private readonly deps: AttemptLedgerDeps) {} + + /** This process's owned attempt for the task, if any. */ + ownedAttempt(taskId: string): OwnedTaskAttempt | undefined { + return this.ownedAttemptByTaskId.get(taskId); + } + + hasOwnedAttempt(taskId: string): boolean { + return this.ownedAttemptByTaskId.has(taskId); + } + + /** The task's closure/settlement entry, if any. */ + attemptSettlement(taskId: string): AttemptSettlementEntry | undefined { + return this.attemptSettlementByTaskId.get(taskId); + } + + /** Track a failed reawaken's closed attempt until recheckResumeFailureSettlement settles it. */ + recordResumeFailureSettlement( + taskId: string, + attempt: OwnedTaskAttempt & { attemptId: string } + ): void { + this.resumeFailureSettlementByTaskId.set(taskId, attempt); + } + + /** + * A new attempt begins in this process (reservation commit, launch, reawakening, reactivation). + * Replaces the owned identity and drops any settlement of the previous attempt SYNCHRONOUSLY, + * before the admission that follows — current ownership always wins over a stale settlement. + */ + beginOwnedTaskAttempt( + taskId: string, + source: string, + identity: { attemptId: string | undefined; receiptEligible: boolean; abortSignal?: AbortSignal } + ): OwnedTaskAttempt { + assert(taskId.length > 0, "beginOwnedTaskAttempt: taskId must be non-empty"); + if (identity.attemptId != null) { + assertTaskAttemptId(identity.attemptId, "beginOwnedTaskAttempt"); + } + const attempt: OwnedTaskAttempt = { + generation: (this.ownedAttemptByTaskId.get(taskId)?.generation ?? 0) + 1, + source, + attemptId: identity.attemptId, + receiptEligible: identity.receiptEligible, + ...(identity.abortSignal != null ? { abortSignal: identity.abortSignal } : {}), + }; + this.ownedAttemptByTaskId.set(taskId, attempt); + if (identity.attemptId != null) this.currentAttemptIdByTaskId.set(taskId, identity.attemptId); + // A closure recorded for THIS id (between the rotating CAS and this block) stays: the fresh + // admission that follows must observe it and refuse. Only another attempt's entry is stale. + if (this.attemptSettlementByTaskId.get(taskId)?.attemptId !== identity.attemptId) { + this.attemptSettlementByTaskId.delete(taskId); + } + // Predecessors' stream-end decisions have no reader left that they could authorize: a send + // bound to a predecessor reads stale from the row's new id at its next gate. + this.deps.discardStreamEndDecisions(taskId); + return attempt; + } + + /** + * Linearization point of every settlement producer: close the attempt to further sends BEFORE + * the producer's first awaited write, so no send can be admitted for an id whose settlement is + * under way. Synchronous — the admission fence (admitTaskWorkspaceTurn) is synchronous too, so + * the two can never interleave and no lock is needed. Never downgrades a `settled` entry and + * never touches an entry naming another attempt. `ownedAttempt` (the producer's snapshot of + * this process's owner) is recorded only when it holds exactly `attemptId`. + */ + closeAttemptAdmission( + taskId: string, + attemptId: string | undefined, + ownedAttempt: OwnedTaskAttempt | undefined, + source: string + ): void { + const existing = this.attemptSettlementByTaskId.get(taskId); + if (existing != null && existing.attemptId === attemptId) return; + const attempt = ownedAttempt?.attemptId === attemptId ? ownedAttempt : undefined; + this.attemptSettlementByTaskId.set(taskId, { + attemptId, + ...(attempt != null ? { attempt } : {}), + phase: "closing", + source, + }); + this.deps.notifyAttemptSettlementListeners(taskId); + } + + /** + * Authoritative settlement by the attempt's owner (stop record released, launch failed, + * terminal stream failure). `attempt` is the identity captured BEFORE the owner's awaited + * write; a settlement for an attempt that is no longer current, or for no owned attempt at + * all (legacy/prior-process execution), is never recorded — only announced. + */ + settleOwnedTaskAttempt( + taskId: string, + attempt: OwnedTaskAttempt | undefined, + source: string + ): void { + if (attempt != null && this.ownedAttemptByTaskId.get(taskId) === attempt) { + this.attemptSettlementByTaskId.set(taskId, { + attemptId: attempt.attemptId, + attempt, + phase: "settled", + source, + }); + } else { + log.debug("Task attempt settlement without a current owned attempt", { + taskId, + source, + owned: attempt?.generation, + }); + } + this.deps.notifyAttemptSettlementListeners(taskId); + } + + /** + * Only this process's CURRENT owned attempt, admitted with proven lineage under a persisted id, + * may produce a cross-process receipt. Anything else (unowned or prior-process attempt, marked + * lineage, pre-identity entry, an owner a reawakening or a durable report already replaced) + * settles in memory only, exactly as before receipts existed. In-memory checks only: whether + * the row still names the attempt is decided by one strict read (readSettlementReceiptRow). + */ + ownsReceiptEligibleAttempt( + taskId: string, + attempt: OwnedTaskAttempt | undefined + ): attempt is OwnedTaskAttempt & { attemptId: string } { + return ( + attempt?.attemptId != null && + attempt.receiptEligible && + this.ownedAttemptByTaskId.get(taskId) === attempt + ); + } + + /** + * The one strict config read a receipt decision rests on. `ours`: the row names `attemptId`, + * with the owner dirs to write into (parent first). `elsewhere`: CONFIRMED gone, parentless or + * naming another attempt (rowSupersedes) — a successor another writer admitted must never be + * vouched for by its predecessor's settlement. `unreadable`: the config (or the parent chain) + * could not be read, which is no evidence either way and must never be taken for `elsewhere`. + */ + private readSettlementReceiptRow( + taskId: string, + attemptId: string + ): + | { kind: "ours"; parentWorkspaceId: string; owners: string[] } + | { kind: "elsewhere" } + | { kind: "unreadable"; error: string } { + let parentWorkspaceId: string | undefined; + let owners: string[]; + try { + const cfg = this.deps.config().loadConfigOrDefault({ throwOnError: true }); + const row = findWorkspaceEntry(cfg, taskId)?.workspace; + parentWorkspaceId = coerceNonEmptyString(row?.parentWorkspaceId); + if (row == null || parentWorkspaceId == null || rowSupersedes(row, attemptId)) { + log.info("[task-attempt] settlement receipt skipped: the row no longer names the attempt", { + taskId, + attemptId, + current: row?.taskAttemptId, + }); + return { kind: "elsewhere" }; + } + owners = this.deps.listAncestorWorkspaceIdsUsingParentById( + this.deps.buildAgentTaskIndex(cfg).parentById, + taskId + ); + } catch (error: unknown) { + // An unreadable config, or a parent cycle in it. + return { kind: "unreadable", error: getErrorMessage(error) }; + } + // Inconsistent ancestry is malformed config, not a programming error: fail closed like an + // unreadable read instead of throwing out of a settlement path. (Duplicate workspace ids no + // longer cause it: the row lookup and the task index both take the first row, #4550.) + if (owners[0] !== parentWorkspaceId) { + return { + kind: "unreadable", + error: `inconsistent ancestry for ${taskId}: parent ${parentWorkspaceId}, first owner ${owners[0] ?? "none"}`, + }; + } + return { kind: "ours", parentWorkspaceId, owners }; + } + + /** + * Durable settlement for producers that established that no execution can still publish a + * report for `attempt` (idle stop, idle terminal failure, reservation canceled/failed, launch + * failed before its send): the receipt first, then the in-memory settlement. + */ + async persistOwnedAttemptSettlement( + taskId: string, + attempt: OwnedTaskAttempt | undefined, + receiptSource: SubagentAttemptSettlementSource, + settlementSource: string + ): Promise { + const decision = this.deps.decideSettlementReceipt(taskId, attempt, settlementSource); + if (decision.kind === "unreadable") return; + if ( + decision.kind === "write" && + !(await this.deps.writeSettlementReceipt(taskId, decision, receiptSource)) + ) { + return; + } + this.deps.settleOwnedTaskAttempt(taskId, attempt, settlementSource); + } + + /** + * THE no-report boundary every receipt producer must prove, evaluated synchronously in the tick + * that decides the write (see decideSettlementReceipt): a receipt may claim that `attempt` can no + * longer publish a report only when ALL of these hold at once — + * 1. it is closed to admissions: a closure names it (closeAttemptAdmission, recorded by the + * producer before its first await), or a stop record latched for it since capture; + * 2. no send obligation of THIS attempt exists in any state (pending, enqueued, admitted) — + * keyed by attempt id, so a successor's obligations neither block nor satisfy it; + * 3. nothing is live for the task: no turn generation, workspace-turn registration or stream + * (task-level signals; any of them blocks, which is the safe direction); + * 4. no stream-end decision of the attempt is pending, and none resolved as a report + * (published) or unknowable (indeterminate). Such a resolution also downgrades the + * attempt's receiptEligible (resolveStreamEndDecision), so it still counts once the + * decision itself was pruned. + * False means no durable claim: the producer settles in memory only, exactly as before receipts. + */ + attemptCannotStillReport( + taskId: string, + attempt: OwnedTaskAttempt & { attemptId: string } + ): boolean { + const attemptId = attempt.attemptId; + const closed = + this.isAttemptClosed(taskId, attemptId) || + this.deps.stopRecordAttemptId(taskId) === attemptId; + if (!closed) return false; + for (const send of this.deps.admittedSends(taskId) ?? []) { + if (send.attemptId === attemptId) return false; + } + if ( + this.deps.workspaceService().getActiveTurnGeneration(taskId) != null || + this.deps.turnManager().getLiveWorkspaceTurnRegistration(taskId) != null || + this.deps.aiService().isStreaming(taskId) + ) { + return false; + } + const decision = this.deps.findStreamEndDecision(taskId, attemptId); + return decision == null || decision.outcome === "nonreport"; + } + + /** + * closeAttemptAdmission that never replaces a closure recorded for a DIFFERENT attempt: used + * where the row's current attempt is unknown or may differ (another writer re-admitted it), so + * replacing that closure could reopen the row's attempt to sends. + */ + closeAttemptAdmissionUnlessOtherClosed( + taskId: string, + attemptId: string, + attempt: OwnedTaskAttempt | undefined, + source: string + ): void { + const existing = this.attemptSettlementByTaskId.get(taskId); + if (existing != null && existing.attemptId !== attemptId) return; + this.deps.closeAttemptAdmission(taskId, attemptId, attempt, source); + } + + /** + * The synchronous half of a receipt-bearing settlement, from one strict read: + * - `memory-only`: no receipt authority (ownsReceiptEligibleAttempt), the no-report boundary is + * not proven (attemptCannotStillReport), or the read CONFIRMED the row gone or naming another + * attempt — today's in-memory-only settlement follows; + * - `write`: the row names the attempt, which is closed to sends here (before any await) and + * must not be settled until writeSettlementReceipt succeeds; + * - `unreadable`: no evidence either way, so fail closed — the attempt is closed but never + * settled (cleanup-pending here, indeterminate after a restart) and gets no receipt; with the + * row unknown, a closure recorded for a different attempt is kept (it may be the row's). + * A failed write, like `unreadable`, leaves the attempt `closing`: never settled without its + * receipt, so no successor reads it as settled when a restart would not. + */ + decideSettlementReceipt( + taskId: string, + attempt: OwnedTaskAttempt | undefined, + settlementSource: string + ): SettlementReceiptDecision { + if (!this.ownsReceiptEligibleAttempt(taskId, attempt)) return { kind: "memory-only" }; + const attemptId = attempt.attemptId; + if (!this.attemptCannotStillReport(taskId, attempt)) return { kind: "memory-only" }; + const row = this.readSettlementReceiptRow(taskId, attemptId); + if (row.kind === "elsewhere") return { kind: "memory-only" }; + if (row.kind === "unreadable") { + log.warn("[task-attempt] settlement receipt not written: config unreadable", { + taskId, + attemptId, + error: row.error, + }); + this.closeAttemptAdmissionUnlessOtherClosed(taskId, attemptId, attempt, settlementSource); + return { kind: "unreadable" }; + } + // Synchronous with the read: the row names this attempt, so any closure replaced here names + // an attempt the row no longer does. + this.deps.closeAttemptAdmission(taskId, attemptId, attempt, settlementSource); + return { + kind: "write", + attemptId, + parentWorkspaceId: row.parentWorkspaceId, + owners: row.owners, + }; + } + + /** + * Write the receipt into every owner dir: outermost ancestor first, the parent LAST, stopping + * at the first failure, so the parent's copy (the one lineage proof reads) exists only once + * every other owner holds one too. True when all writes succeeded. Never throws. + */ + async writeSettlementReceipt( + taskId: string, + target: { attemptId: string; parentWorkspaceId: string; owners: string[] }, + source: SubagentAttemptSettlementSource + ): Promise { + const receipt = { + taskId, + attemptId: target.attemptId, + parentWorkspaceId: target.parentWorkspaceId, + source, + settledAt: new Date().toISOString(), + }; + for (const ownerWorkspaceId of [...target.owners].reverse()) { + const result = await writeSubagentAttemptSettlementReceipt({ + ownerWorkspaceSessionDirs: [path.join(this.deps.config().sessionsDir, ownerWorkspaceId)], + receipt, + }); + if (!result.success) { + log.warn("[task-attempt] settlement receipt write failed; attempt left closing", { + taskId, + attemptId: target.attemptId, + ownerWorkspaceId, + error: result.error, + }); + return false; + } + } + return true; + } + + /** + * The ledger entries of an attempt whose report is durable are redundant: every reader + * (inspectAttemptOutcome, waitForAgentReport) classifies a persisted report first, so neither + * this attempt's ownership nor its settlement can decide an outcome again. Dropping them keeps + * the per-task ledgers bounded by the tasks still able to end without a report, instead of by + * every task ever attempted in this process. + * + * `attempt` is the identity captured at the stream-end event that carried the report (see the + * stream-end listener), so the drop names the attempt the report belongs to — a task reawakened + * (markInterruptedTaskRunning, not under the task's event lock) while that report waited for + * the lock or for its writes owns a different identity and stays owned; an older report never + * decides the new attempt's outcome. Called only after every ancestor artifact write succeeded: + * a report that is not durable everywhere leaves the entries in place. + * + * Interrupted-without-report receipts are deliberately NOT dropped here or on stop: an ancestor + * that has not yet read terminal-no-report still needs the receipt after config cleanup, and no + * consumer acknowledgement exists yet to bound that retention. + * + * The released shape (reported row, no owner, no settlement entry) is itself read by + * markInterruptedTaskRunning: a manual follow-up into such a child mints a fresh attempt rather + * than being admitted under the completed id. + */ + releaseReportedTaskAttempt(taskId: string, attempt: OwnedTaskAttempt | undefined): void { + if (attempt == null || this.ownedAttemptByTaskId.get(taskId) !== attempt) return; + this.ownedAttemptByTaskId.delete(taskId); + if (this.attemptSettlementByTaskId.get(taskId)?.attempt === attempt) { + this.attemptSettlementByTaskId.delete(taskId); + } + } + + notifyAttemptSettlementListeners(taskId: string): void { + const listeners = this.attemptSettlementListenersByTaskId.get(taskId); + if (!listeners) return; + for (const listener of [...listeners]) { + listener(); + } + } + + /** TaskStopLatch's waits register here (TaskStopLatchDeps.addAttemptSettlementListener). */ + addAttemptSettlementListener(taskId: string, listener: () => void): void { + const listeners = this.attemptSettlementListenersByTaskId.get(taskId) ?? new Set(); + listeners.add(listener); + this.attemptSettlementListenersByTaskId.set(taskId, listeners); + } + + removeAttemptSettlementListener(taskId: string, listener: () => void): void { + const listeners = this.attemptSettlementListenersByTaskId.get(taskId); + if (!listeners) return; + listeners.delete(listener); + if (listeners.size === 0) this.attemptSettlementListenersByTaskId.delete(taskId); + } + + /** + * Publish a committed identity rotation to in-memory state: record the id this process + * committed (currentAttemptIdByTaskId), and drop a closure recorded for a previous attempt (a + * closure for the NEW id, recorded between the CAS and this call, stays). Tokens minted for an + * older id are revoked by the persisted row itself at their next gate. + */ + publishAttemptRotation(taskId: string, attemptId: string): void { + assertTaskAttemptId(attemptId, "publishAttemptRotation"); + this.currentAttemptIdByTaskId.set(taskId, attemptId); + if (this.attemptSettlementByTaskId.get(taskId)?.attemptId !== attemptId) { + this.attemptSettlementByTaskId.delete(taskId); + } + } + + /** A closure or settlement is recorded for exactly this attempt id. */ + isAttemptClosed(taskId: string, attemptId: string): boolean { + const entry = this.attemptSettlementByTaskId.get(taskId); + return entry?.attemptId === attemptId; + } + + /** Resolve once the task's settlement entry leaves `closing` (or the bound elapses). */ + async waitForAttemptClosureToSettle(taskId: string, timeoutMs: number): Promise { + const isClosing = () => this.attemptSettlementByTaskId.get(taskId)?.phase === "closing"; + if (!isClosing()) return; + const deadline = Date.now() + timeoutMs; + while (isClosing()) { + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) return; + const notified = Promise.withResolvers(); + const listener = () => notified.resolve(); + const listeners = this.attemptSettlementListenersByTaskId.get(taskId) ?? new Set(); + listeners.add(listener); + this.attemptSettlementListenersByTaskId.set(taskId, listeners); + try { + await raceWithAbortAndTimeout(notified.promise, { timeoutMs: remainingMs }); + } finally { + this.removeAttemptSettlementListener(taskId, listener); + } + } + } + + /** + * Take the task's event lock for a short read while the caller's bound (deadline/abort) keeps + * applying to the WAIT for the lock: a publication holding it must not pin a bounded caller. + * Abandoned before entry → the late acquisition is a no-op that releases at once (nothing read, + * no subscription installed). Once entered, the operation runs to completion (never detached). + */ + private async runUnderEventLockBounded( + taskId: string, + operation: () => Promise, + bound: { signal?: AbortSignal; deadline: number } + ): Promise<{ kind: "ok"; value: T } | { kind: "timeout" } | { kind: "aborted" }> { + let entered = false; + let abandoned = false; + const run = this.deps + .workspaceEventLocks() + .withLock(taskId, async (): Promise => { + if (abandoned) return undefined; + entered = true; + return await operation(); + }); + return await new Promise((resolve, reject) => { + let timer: ReturnType | undefined; + const cleanup = () => { + if (timer != null) clearTimeout(timer); + bound.signal?.removeEventListener("abort", onAbort); + }; + const abandon = (result: { kind: "timeout" } | { kind: "aborted" }) => { + if (entered || abandoned) return; + abandoned = true; + cleanup(); + resolve(result); + }; + const onAbort = () => abandon({ kind: "aborted" }); + bound.signal?.addEventListener("abort", onAbort, { once: true }); + const remainingMs = bound.deadline - Date.now(); + if (bound.signal?.aborted) { + abandon({ kind: "aborted" }); + } else if (remainingMs <= 0) { + abandon({ kind: "timeout" }); + } else { + timer = setTimeout(() => abandon({ kind: "timeout" }), remainingMs); + timer.unref?.(); + } + run.then( + (value) => { + cleanup(); + if (!abandoned) resolve({ kind: "ok", value: value as T }); + }, + (error: unknown) => { + cleanup(); + if (!abandoned) reject(error instanceof Error ? error : new Error(String(error))); + } + ); + }); + } + + /** + * Bounded wait for the current attempt to settle: subscribe THEN re-read under the task's + * event lock (a settlement landing between the two is observed by the re-read), release the + * lock before awaiting, and re-read on every notification. The bound also covers waiting for + * the lock itself (runUnderEventLockBounded). `indeterminate` returns at once — an unknown + * owner never pins the caller. Timeout is a bounded outcome, abort rejects. + */ + async waitForAttemptSettlement( + taskId: string, + options: { abortSignal?: AbortSignal; timeoutMs: number; requestingWorkspaceId?: string } + ): Promise> { + assert(taskId.length > 0, "waitForAttemptSettlement: taskId must be non-empty"); + assert( + Number.isFinite(options.timeoutMs) && options.timeoutMs > 0, + "waitForAttemptSettlement: timeoutMs invalid" + ); + const deadline = Date.now() + options.timeoutMs; + const bound = { signal: options.abortSignal, deadline }; + let notified = Promise.withResolvers(); + const listener = () => notified.resolve(); + try { + const first = await this.runUnderEventLockBounded( + taskId, + async () => { + const listeners = this.attemptSettlementListenersByTaskId.get(taskId) ?? new Set(); + listeners.add(listener); + this.attemptSettlementListenersByTaskId.set(taskId, listeners); + return await this.deps.inspectAttemptOutcome(taskId, options); + }, + bound + ); + if (first.kind === "aborted") throw new Error("Interrupted"); + if (first.kind === "timeout") return { kind: "timeout" }; + let outcome = first.value; + while (outcome.kind === "live" || outcome.kind === "cleanup-pending") { + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) return { kind: "timeout" }; + const raced = await raceWithAbortAndTimeout(notified.promise, { + signal: options.abortSignal, + timeoutMs: remainingMs, + }); + if (raced.kind === "aborted") throw new Error("Interrupted"); + if (raced.kind === "timeout") return { kind: "timeout" }; + // Re-arm before re-reading so a notification during the read is never lost. + notified = Promise.withResolvers(); + const reread = await this.runUnderEventLockBounded( + taskId, + () => this.deps.inspectAttemptOutcome(taskId, options), + bound + ); + if (reread.kind === "aborted") throw new Error("Interrupted"); + if (reread.kind === "timeout") return { kind: "timeout" }; + outcome = reread.value; + } + return outcome; + } finally { + this.removeAttemptSettlementListener(taskId, listener); + } + } + + /** + * Settle a failed reawaken's closed attempt (restoreInterruptedTaskAfterResumeFailure) once the + * shared no-report boundary holds (attemptCannotStillReport): never from the send's error alone. + * The failed send's obligation is typically disposed only after the rollback returned (a refused + * or no-work send), and an admitted preparation failure rolls back while its turn is still + * live, so this is rechecked on the rollback, on every obligation disposal and on every turn + * settlement. It drops out without settling when the attempt was superseded (a new owner) or + * settled by another producer (Stop), and defers to an in-progress Stop, whose release settles. + */ + recheckResumeFailureSettlement(taskId: string): void { + const attempt = this.resumeFailureSettlementByTaskId.get(taskId); + if (attempt == null) return; + const closure = this.attemptSettlementByTaskId.get(taskId); + if ( + this.ownedAttemptByTaskId.get(taskId) !== attempt || + closure?.attempt !== attempt || + closure.phase !== "closing" + ) { + this.resumeFailureSettlementByTaskId.delete(taskId); + return; + } + if (this.deps.hasStopRecord(taskId)) return; + if (!this.attemptCannotStillReport(taskId, attempt)) return; + this.resumeFailureSettlementByTaskId.delete(taskId); + this.persistOwnedAttemptSettlement(taskId, attempt, "idle-settled", "resume-failed").catch( + (error: unknown) => { + log.error("[task-attempt] failed reawaken settlement failed", { taskId, error }); + } + ); + } +} diff --git a/src/node/services/taskService.aiSettings.test.ts b/src/node/services/taskService.aiSettings.test.ts index cf2ae6e2057..2649dc40550 100644 --- a/src/node/services/taskService.aiSettings.test.ts +++ b/src/node/services/taskService.aiSettings.test.ts @@ -1938,8 +1938,10 @@ describe("TaskService", () => { identity: { attemptId: string | undefined; receiptEligible: boolean } ): unknown; settleOwnedTaskAttempt(taskId: string, attempt: unknown, source: string): void; - ownedAttemptByTaskId: Map; - attemptSettlementByTaskId: Map; + attemptLedger: { + ownedAttemptByTaskId: Map; + attemptSettlementByTaskId: Map; + }; } /** Queued admission: returns at once and hands onAccepted to the test once armed. */ @@ -1974,7 +1976,7 @@ describe("TaskService", () => { await setDelegatedExec(config, { modelString: MODEL_B }); const busy = admitQueued(workspaceService, childId); const ledger = taskService as unknown as AttemptLedger; - const previousAttempt = ledger.ownedAttemptByTaskId.get(childId); + const previousAttempt = ledger.attemptLedger.ownedAttemptByTaskId.get(childId); const store = workspaceTurnManagerInternals(taskService).taskHandleStore; const originalUpsert = store.upsertWorkspaceTurn.bind(store); let reactivationAttempt: unknown; @@ -1988,7 +1990,7 @@ describe("TaskService", () => { child.taskExecutionStatus === "running" ) { // The claim-and-settings write is committed; the next acceptance step fails. - reactivationAttempt = ledger.ownedAttemptByTaskId.get(childId); + reactivationAttempt = ledger.attemptLedger.ownedAttemptByTaskId.get(childId); if (variant === "settled meanwhile") { ledger.settleOwnedTaskAttempt(childId, reactivationAttempt, "test-settlement"); } else if (variant === "successor meanwhile") { @@ -2011,7 +2013,7 @@ describe("TaskService", () => { // Existing settlement settled the claimed mirror to a terminal status. expect(child?.taskExecutionStatus).toBeDefined(); expect(isActiveWorkspaceTurnTaskStatus(child?.taskExecutionStatus)).toBe(false); - const owned = ledger.ownedAttemptByTaskId.get(childId); + const owned = ledger.attemptLedger.ownedAttemptByTaskId.get(childId); if (variant === "no settlement") { // A published attempt is never rolled back (#4308): the refused reactivation's // attempt stays owned until a Stop settles it. @@ -2019,7 +2021,7 @@ describe("TaskService", () => { expect(owned).toBe(reactivationAttempt); } else if (variant === "settled meanwhile") { expect(owned).toBe(reactivationAttempt); - expect(ledger.attemptSettlementByTaskId.get(childId)?.attempt).toBe( + expect(ledger.attemptLedger.attemptSettlementByTaskId.get(childId)?.attempt).toBe( reactivationAttempt ); } else { @@ -2193,7 +2195,7 @@ describe("TaskService", () => { report: { reportMarkdown: string }, attempt: unknown ): Promise; - ownedAttemptByTaskId: Map; + attemptLedger: { ownedAttemptByTaskId: Map }; }; expect( @@ -2207,7 +2209,7 @@ describe("TaskService", () => { childId, findWorkspaceEntry(config.loadConfigOrDefault(), childId), { reportMarkdown: "done" }, - internal.ownedAttemptByTaskId.get(childId) + internal.attemptLedger.ownedAttemptByTaskId.get(childId) ); const report = await readSubagentReportArtifact( path.join(config.sessionsDir, parentId), diff --git a/src/node/services/taskService.attemptAdmission.test.ts b/src/node/services/taskService.attemptAdmission.test.ts index fd872203f2b..919ad531335 100644 --- a/src/node/services/taskService.attemptAdmission.test.ts +++ b/src/node/services/taskService.attemptAdmission.test.ts @@ -78,14 +78,18 @@ const requesting = { requestingWorkspaceId: rootId }; const ATTEMPT_ID = /^att_[0-9a-f]{16}$/; interface Internals { - ownedAttemptByTaskId: Map< - string, - { attemptId?: string; receiptEligible: boolean; source: string } - >; - attemptSettlementByTaskId: Map< - string, - { attemptId?: string; attempt?: unknown; phase: "closing" | "settled"; source: string } - >; + attemptLedger: { + ownedAttemptByTaskId: Map< + string, + { attemptId?: string; receiptEligible: boolean; source: string } + >; + attemptSettlementByTaskId: Map< + string, + { attemptId?: string; attempt?: unknown; phase: "closing" | "settled"; source: string } + >; + currentAttemptIdByTaskId: Map; + resumeFailureSettlementByTaskId: Map; + }; admittedSendsByTaskId: Map< string, Set<{ state: string; turnId?: symbol; attemptId: string; attempt?: unknown }> @@ -104,8 +108,6 @@ interface Internals { } >; }; - currentAttemptIdByTaskId: Map; - resumeFailureSettlementByTaskId: Map; markTaskLaunchFailed: (taskId: string, message: string) => Promise; closeAttemptAdmission: ( taskId: string, @@ -388,7 +390,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { const reserved = entryOf(config, spawnedId); expect(reserved?.taskAttemptId).toMatch(ATTEMPT_ID); expect(reserved?.taskAttemptUnproven).toBeUndefined(); - const owned = svc.ownedAttemptByTaskId.get(spawnedId); + const owned = svc.attemptLedger.ownedAttemptByTaskId.get(spawnedId); expect(owned).toMatchObject({ source: "reservation", attemptId: reserved!.taskAttemptId, @@ -404,7 +406,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { expect(sendMessage).toHaveBeenCalledTimes(1); expect(sendTokens[0]).toBeDefined(); expect(entryOf(config, spawnedId)?.taskAttemptId).toBe(reserved!.taskAttemptId); - expect(svc.ownedAttemptByTaskId.get(spawnedId)).toBe(owned); + expect(svc.attemptLedger.ownedAttemptByTaskId.get(spawnedId)).toBe(owned); // The fake host disposed the token (no session → no turn): the obligation is discharged. expect(svc.admittedSendsByTaskId.get(spawnedId)).toBeUndefined(); expect(await taskService.readAttemptOutcome(spawnedId, requesting)).toEqual({ @@ -458,7 +460,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { const provenEntry = entryOf(config, proven); expect(provenEntry?.taskAttemptId).toMatch(ATTEMPT_ID); expect(provenEntry?.taskAttemptUnproven).toBeUndefined(); - expect(svc.ownedAttemptByTaskId.get(proven)).toMatchObject({ + expect(svc.attemptLedger.ownedAttemptByTaskId.get(proven)).toMatchObject({ source: "launch", attemptId: provenEntry!.taskAttemptId, receiptEligible: true, @@ -467,7 +469,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { expect(markedEntry?.taskAttemptId).toMatch(ATTEMPT_ID); expect(markedEntry?.taskAttemptId).not.toBe("att_0000000000000001"); expect(markedEntry?.taskAttemptUnproven).toBe(true); - expect(svc.ownedAttemptByTaskId.get(marked)).toMatchObject({ + expect(svc.attemptLedger.ownedAttemptByTaskId.get(marked)).toMatchObject({ source: "launch", attemptId: markedEntry!.taskAttemptId, receiptEligible: false, @@ -502,7 +504,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { await settle(); expect(entryOf(config, taskId)?.taskStatus).toBe("queued"); expect(entryOf(config, taskId)?.taskAttemptId).toBe(claim.attemptId); - expect(internals(taskService).ownedAttemptByTaskId.has(taskId)).toBe(false); + expect(internals(taskService).attemptLedger.ownedAttemptByTaskId.has(taskId)).toBe(false); expect(sendMessage).not.toHaveBeenCalled(); }); @@ -540,17 +542,17 @@ describe("TaskService attempt identity and send admission (G1)", () => { expect(entry?.taskAttemptId).toMatch(ATTEMPT_ID); expect(entry?.taskAttemptId).not.toBe("att_00000000000000aa"); expect(entry?.taskAttemptUnproven).toBe(true); - expect(svc.ownedAttemptByTaskId.get(taskId)).toMatchObject({ + expect(svc.attemptLedger.ownedAttemptByTaskId.get(taskId)).toMatchObject({ source: "reawaken", attemptId: entry!.taskAttemptId, receiptEligible: false, }); - expect(svc.currentAttemptIdByTaskId.get(taskId)).toBe(entry!.taskAttemptId); + expect(svc.attemptLedger.currentAttemptIdByTaskId.get(taskId)).toBe(entry!.taskAttemptId); expect(await taskService.markInterruptedTaskRunning(retired)).toBe(false); expect(entryOf(config, retired)?.taskStatus).toBe("interrupted"); expect(entryOf(config, retired)?.taskAttemptId).toBe(claim.attemptId); - expect(svc.ownedAttemptByTaskId.has(retired)).toBe(false); + expect(svc.attemptLedger.ownedAttemptByTaskId.has(retired)).toBe(false); const reactivation = await taskService.sendMessageToDescendantAgentTask( rootId, retired, @@ -611,7 +613,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { // First reawaken: unowned predecessor without receipt → unproven. expect(await taskService.markInterruptedTaskRunning(taskId)).toBe(true); - const first = svc.ownedAttemptByTaskId.get(taskId); + const first = svc.attemptLedger.ownedAttemptByTaskId.get(taskId); expect(first?.receiptEligible).toBe(false); // Stop settles THIS process's attempt; the next reawaken's predecessor is settled here. await taskService.terminateAllDescendantAgentTasks(rootId); @@ -620,24 +622,28 @@ describe("TaskService attempt identity and send admission (G1)", () => { }); // ...but the predecessor is marked, and the marker is inherited: still not eligible. expect(await taskService.markInterruptedTaskRunning(taskId)).toBe(true); - expect(svc.ownedAttemptByTaskId.get(taskId)?.receiptEligible).toBe(false); + expect(svc.attemptLedger.ownedAttemptByTaskId.get(taskId)?.receiptEligible).toBe(false); expect(entryOf(config, taskId)?.taskAttemptUnproven).toBe(true); // Receipt for the current unowned id → proven, no marker. expect(await taskService.markInterruptedTaskRunning(byReceipt)).toBe(true); - expect(svc.ownedAttemptByTaskId.get(byReceipt)).toMatchObject({ receiptEligible: true }); + expect(svc.attemptLedger.ownedAttemptByTaskId.get(byReceipt)).toMatchObject({ + receiptEligible: true, + }); expect(entryOf(config, byReceipt)?.taskAttemptUnproven).toBeUndefined(); expect(entryOf(config, byReceipt)?.taskAttemptId).not.toBe(receiptAttempt); // An eligible attempt settled by this process proves the next successor too. await taskService.terminateAllDescendantAgentTasks(rootId); await waitForStopRelease(taskService, byReceipt); expect(await taskService.markInterruptedTaskRunning(byReceipt)).toBe(true); - expect(svc.ownedAttemptByTaskId.get(byReceipt)).toMatchObject({ receiptEligible: true }); + expect(svc.attemptLedger.ownedAttemptByTaskId.get(byReceipt)).toMatchObject({ + receiptEligible: true, + }); expect(entryOf(config, byReceipt)?.taskAttemptUnproven).toBeUndefined(); // Unreadable receipt is no evidence. expect(await taskService.markInterruptedTaskRunning(unreadable)).toBe(true); - expect(svc.ownedAttemptByTaskId.get(unreadable)?.receiptEligible).toBe(false); + expect(svc.attemptLedger.ownedAttemptByTaskId.get(unreadable)?.receiptEligible).toBe(false); expect(entryOf(config, unreadable)?.taskAttemptUnproven).toBe(true); }); @@ -661,7 +667,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { const oldToken = admitted( taskService.admitTaskWorkspaceTurn(taskId, { acceptanceOrigin: "manual" }) ); - svc.attemptSettlementByTaskId.set(taskId, { + svc.attemptLedger.attemptSettlementByTaskId.set(taskId, { attemptId, phase: "settled", source: "idle-settled", @@ -677,7 +683,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { }); expect(current?.taskAttemptId).toMatch(ATTEMPT_ID); expect(current?.taskAttemptId).not.toBe(attemptId); - expect(svc.ownedAttemptByTaskId.get(taskId)).toMatchObject({ + expect(svc.attemptLedger.ownedAttemptByTaskId.get(taskId)).toMatchObject({ attemptId: current?.taskAttemptId, receiptEligible: false, }); @@ -714,14 +720,14 @@ describe("TaskService attempt identity and send admission (G1)", () => { shortenTerminationTimers(); expect(await taskService.markInterruptedTaskRunning(taskId)).toBe(true); const reportedAttemptId = entryOf(config, taskId)!.taskAttemptId!; - expect(svc.ownedAttemptByTaskId.get(taskId)?.attemptId).toBe(reportedAttemptId); + expect(svc.attemptLedger.ownedAttemptByTaskId.get(taskId)?.attemptId).toBe(reportedAttemptId); await streamEnd(taskService, reportingStreamEnd(taskId, "assistant-report-1", "done")); expect(entryOf(config, taskId)).toMatchObject({ taskStatus: "reported", taskAttemptId: reportedAttemptId, }); - expect(svc.ownedAttemptByTaskId.has(taskId)).toBe(false); - expect(svc.attemptSettlementByTaskId.has(taskId)).toBe(false); + expect(svc.attemptLedger.ownedAttemptByTaskId.has(taskId)).toBe(false); + expect(svc.attemptLedger.attemptSettlementByTaskId.has(taskId)).toBe(false); expect((await taskService.readAttemptOutcome(taskId, requesting)).kind).toBe("reported"); // Manual send: the rescue mints a fresh owned attempt (false: the report stays historical, @@ -734,7 +740,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { expect(continuedAttemptId).not.toBe(reportedAttemptId); // Lineage: the completed attempt is neither settled by this process nor by a receipt. expect(continued?.taskAttemptUnproven).toBe(true); - expect(svc.ownedAttemptByTaskId.get(taskId)).toMatchObject({ + expect(svc.attemptLedger.ownedAttemptByTaskId.get(taskId)).toMatchObject({ source: "reawaken", attemptId: continuedAttemptId, receiptEligible: false, @@ -765,7 +771,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { // An explicit Stop settles the owned continuation; the next rescue rotates from that // settlement (the pre-existing settled-predecessor path) and inherits the marker. await taskService.terminateAllDescendantAgentTasks(rootId); - expect(svc.attemptSettlementByTaskId.get(taskId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)).toMatchObject({ attemptId: continuedAttemptId, phase: "settled", }); @@ -793,12 +799,12 @@ describe("TaskService attempt identity and send admission (G1)", () => { expect(outcome.kind === "reawakened" && outcome.attemptId).toBe(firstAttemptId); // Lineage unproven: no settlement, no receipt can vouch for the id-less predecessor. expect(row?.taskAttemptUnproven).toBe(true); - expect(svc.ownedAttemptByTaskId.get(taskId)).toMatchObject({ + expect(svc.attemptLedger.ownedAttemptByTaskId.get(taskId)).toMatchObject({ source: "reawaken", attemptId: firstAttemptId, receiptEligible: false, }); - expect(svc.attemptSettlementByTaskId.has(taskId)).toBe(false); + expect(svc.attemptLedger.attemptSettlementByTaskId.has(taskId)).toBe(false); // The manual follow-up is fenced under that attempt, not admitted as a pre-identity send. const sendToken = admitted( taskService.admitTaskWorkspaceTurn(taskId, { acceptanceOrigin: "manual" }) @@ -824,13 +830,13 @@ describe("TaskService attempt identity and send admission (G1)", () => { // A live continuation this process owns (a parent reactivation, an earlier manual // follow-up): the attempt is neither released nor settled, so it is continued, not // rotated. The released shape (no owner, no entry) is the lifecycle test above. - svc.ownedAttemptByTaskId.set(taskId, { + svc.attemptLedger.ownedAttemptByTaskId.set(taskId, { attemptId, receiptEligible: true, source: "reactivation", }); } else { - svc.attemptSettlementByTaskId.set(taskId, { + svc.attemptLedger.attemptSettlementByTaskId.set(taskId, { attemptId: evidence === "stale" ? "att_00000000000000bb" : attemptId, phase: evidence === "closing" ? "closing" : "settled", source: "idle-settled", @@ -858,12 +864,12 @@ describe("TaskService attempt identity and send admission (G1)", () => { }); } const before = entryOf(config, taskId); - const owned = svc.ownedAttemptByTaskId.get(taskId); - const settlement = svc.attemptSettlementByTaskId.get(taskId); + const owned = svc.attemptLedger.ownedAttemptByTaskId.get(taskId); + const settlement = svc.attemptLedger.attemptSettlementByTaskId.get(taskId); expect(await taskService.markInterruptedTaskRunning(taskId)).toBe(false); expect(entryOf(config, taskId)).toEqual(before); - expect(svc.ownedAttemptByTaskId.get(taskId)).toBe(owned); - expect(svc.attemptSettlementByTaskId.get(taskId)).toBe(settlement); + expect(svc.attemptLedger.ownedAttemptByTaskId.get(taskId)).toBe(owned); + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)).toBe(settlement); } ); @@ -892,7 +898,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { taskService.admitTaskWorkspaceTurn(taskId, { acceptanceOrigin: "manual" }) ); const settlement = { attemptId, phase: "settled" as const, source: "idle-settled" }; - svc.attemptSettlementByTaskId.set(taskId, settlement); + svc.attemptLedger.attemptSettlementByTaskId.set(taskId, settlement); const original = svc.evaluateAttemptLineage.bind(taskService); let releaseStop: (() => void) | undefined; const proof = spyOn(svc, "evaluateAttemptLineage").mockImplementation(async (id, entry) => { @@ -900,7 +906,10 @@ describe("TaskService attempt identity and send admission (G1)", () => { if (race === "stop") { releaseStop = taskService.latchHardInterruptCascade(taskId); } else if (race === "settlement") { - svc.attemptSettlementByTaskId.set(taskId, { ...settlement, phase: "closing" }); + svc.attemptLedger.attemptSettlementByTaskId.set(taskId, { + ...settlement, + phase: "closing", + }); } else if (race !== "stop-before-cas") { await config.editConfig((cfg) => { for (const project of cfg.projects.values()) { @@ -941,10 +950,10 @@ describe("TaskService attempt identity and send admission (G1)", () => { expect(entryOf(config, taskId)?.taskStatus).toBe( race === "status" ? "interrupted" : "reported" ); - expect(svc.attemptSettlementByTaskId.get(taskId)).toEqual( + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)).toEqual( race === "settlement" ? { ...settlement, phase: "closing" } : settlement ); - expect(svc.ownedAttemptByTaskId.has(taskId)).toBe(false); + expect(svc.attemptLedger.ownedAttemptByTaskId.has(taskId)).toBe(false); expect(oldToken.admissionStale()).toBe(true); } finally { cas.mockRestore(); @@ -989,7 +998,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { }); expect(await taskService.markInterruptedTaskRunning(taskId)).toBe(true); expect(entryOf(config, taskId)?.taskAttemptUnproven).toBe(true); - expect(svc.ownedAttemptByTaskId.get(taskId)?.receiptEligible).toBe(false); + expect(svc.attemptLedger.ownedAttemptByTaskId.get(taskId)?.receiptEligible).toBe(false); }); test("stale-starting revert marks the lineage; startup re-drive rotates unowned and marks; a retired task is skipped", async () => { @@ -1054,7 +1063,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { expect(staleEntry?.taskAttemptUnproven).toBe(true); expect(staleEntry?.taskAttemptId).toMatch(ATTEMPT_ID); expect(staleEntry?.taskAttemptId).not.toBe("att_00000000000000f1"); - expect(svc.ownedAttemptByTaskId.get(stale)).toMatchObject({ + expect(svc.attemptLedger.ownedAttemptByTaskId.get(stale)).toMatchObject({ source: "launch", receiptEligible: false, }); @@ -1063,8 +1072,10 @@ describe("TaskService attempt identity and send admission (G1)", () => { expect(redrivenEntry?.taskAttemptId).toMatch(ATTEMPT_ID); expect(redrivenEntry?.taskAttemptId).not.toBe("att_00000000000000f2"); expect(redrivenEntry?.taskAttemptUnproven).toBe(true); - expect(svc.ownedAttemptByTaskId.has(redriven)).toBe(false); - expect(svc.currentAttemptIdByTaskId.get(redriven)).toBe(redrivenEntry!.taskAttemptId); + expect(svc.attemptLedger.ownedAttemptByTaskId.has(redriven)).toBe(false); + expect(svc.attemptLedger.currentAttemptIdByTaskId.get(redriven)).toBe( + redrivenEntry!.taskAttemptId + ); const redrivenSends = sendMessage.mock.calls.filter((call) => call[0] === redriven); expect(redrivenSends).toHaveLength(1); // Retired: untouched and never sent. @@ -1136,7 +1147,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { }, ]); expect(created.success).toBe(true); - const reserved = svc.ownedAttemptByTaskId.get(taskId)!.attemptId!; + const reserved = svc.attemptLedger.ownedAttemptByTaskId.get(taskId)!.attemptId!; expect(reserved).toMatch(ATTEMPT_ID); // The launch settles one way or the other; nothing about the successor's row may move. const deadline = Date.now() + 2_000; @@ -1151,9 +1162,11 @@ describe("TaskService attempt identity and send admission (G1)", () => { taskStatus: "starting", }); expect(entryOf(config, taskId)?.taskLaunchError).toBeUndefined(); - expect(svc.ownedAttemptByTaskId.get(taskId)?.attemptId).toBe(reserved); - expect(svc.currentAttemptIdByTaskId.get(taskId)).not.toBe(foreign); - expect(svc.attemptSettlementByTaskId.get(taskId)?.attemptId).not.toBe(foreign); + expect(svc.attemptLedger.ownedAttemptByTaskId.get(taskId)?.attemptId).toBe(reserved); + expect(svc.attemptLedger.currentAttemptIdByTaskId.get(taskId)).not.toBe(foreign); + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)?.attemptId).not.toBe( + foreign + ); expect(await fsPromises.readFile(sessionMarker, "utf-8")).toBe("keep"); } ); @@ -1229,7 +1242,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { }, ]); expect(created.success).toBe(true); - const reserved = svc.ownedAttemptByTaskId.get(taskId)!.attemptId!; + const reserved = svc.attemptLedger.ownedAttemptByTaskId.get(taskId)!.attemptId!; const deadline = Date.now() + 2_000; while ( entryOf(config, taskId) != null && @@ -1287,7 +1300,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { shortenTerminationTimers(); expect(await taskService.markInterruptedTaskRunning(taskId)).toBe(true); const attemptId = entryOf(config, taskId)!.taskAttemptId!; - const owned = svc.ownedAttemptByTaskId.get(taskId); + const owned = svc.attemptLedger.ownedAttemptByTaskId.get(taskId); const entry = findEntryWithProject(config, taskId); let pending: TurnAdmissionToken | undefined; if (race === "pending-before-decision") { @@ -1340,7 +1353,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { taskAttemptId: attemptId, taskLaunchError: "boom", }); - expect(svc.attemptSettlementByTaskId.get(taskId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)).toMatchObject({ attemptId, attempt: owned, phase: "settled", @@ -1442,7 +1455,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { } expect(sendMessage.mock.calls.filter((call) => call[0] === taskId)).toHaveLength(0); expect(svc.admittedSendsByTaskId.get(taskId)).toBeUndefined(); - expect(svc.ownedAttemptByTaskId.has(taskId)).toBe(false); + expect(svc.attemptLedger.ownedAttemptByTaskId.has(taskId)).toBe(false); if (variant.startsWith("rotated")) expect(row.taskAttemptId).toBe(foreign); else expect(row.taskAttemptRetiredBy).toEqual(claim); } @@ -1562,7 +1575,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { token.onDisposed(disposal); } await waitForCondition( - () => svc.attemptSettlementByTaskId.get(taskId)?.phase === "settled" + () => svc.attemptLedger.attemptSettlementByTaskId.get(taskId)?.phase === "settled" ); expect(await taskService.readAttemptOutcome(taskId, requesting)).toMatchObject({ kind: "terminal-no-report", @@ -1572,7 +1585,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { // The retry is proven by that settlement: no unproven marker, receipt-eligible. expect((await taskService.reawakenInterruptedTask(taskId)).kind).toBe("reawakened"); expect(entryOf(config, taskId)?.taskAttemptUnproven).toBeUndefined(); - expect(svc.ownedAttemptByTaskId.get(taskId)).toMatchObject({ + expect(svc.attemptLedger.ownedAttemptByTaskId.get(taskId)).toMatchObject({ source: "reawaken", receiptEligible: true, }); @@ -1596,7 +1609,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { ); await settle(); // Closed, but the live turn keeps it unsettled (the outcome reports that turn as live). - expect(svc.attemptSettlementByTaskId.get(taskId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)).toMatchObject({ attemptId, phase: "closing", }); @@ -1604,7 +1617,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { activeTurn = undefined; host.settleTurn(taskId, turn); await waitForCondition( - () => svc.attemptSettlementByTaskId.get(taskId)?.phase === "settled" + () => svc.attemptLedger.attemptSettlementByTaskId.get(taskId)?.phase === "settled" ); expect(await taskService.readAttemptOutcome(taskId, requesting)).toMatchObject({ kind: "terminal-no-report", @@ -1626,11 +1639,11 @@ describe("TaskService attempt identity and send admission (G1)", () => { await raceWithTimeout(stop, 2_000); await waitForStopRelease(taskService, taskId); await settle(); - const settlement = svc.attemptSettlementByTaskId.get(taskId); + const settlement = svc.attemptLedger.attemptSettlementByTaskId.get(taskId); expect(settlement).toMatchObject({ attemptId, phase: "settled" }); expect(settlement?.source).not.toBe("resume-failed"); // The deferral does not outlive the Stop that settled it. - expect(svc.resumeFailureSettlementByTaskId.has(taskId)).toBe(false); + expect(svc.attemptLedger.resumeFailureSettlementByTaskId.has(taskId)).toBe(false); expect(await taskService.readAttemptOutcome(taskId, requesting)).toMatchObject({ kind: "terminal-no-report", attemptId, @@ -1657,8 +1670,8 @@ describe("TaskService attempt identity and send admission (G1)", () => { taskStatus: "running", taskAttemptId: attemptId, }); - expect(svc.attemptSettlementByTaskId.get(taskId)).toBeUndefined(); - expect(svc.resumeFailureSettlementByTaskId.has(taskId)).toBe(false); + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)).toBeUndefined(); + expect(svc.attemptLedger.resumeFailureSettlementByTaskId.has(taskId)).toBe(false); expect( taskService.admitTaskWorkspaceTurn(taskId, { acceptanceOrigin: "manual", @@ -1684,8 +1697,8 @@ describe("TaskService attempt identity and send admission (G1)", () => { await settle(); // The stale obligation's disposal settles nothing: the successor stays owned and live, and // no receipt vouches for the superseded attempt. - expect(svc.attemptSettlementByTaskId.get(taskId)).toBeUndefined(); - expect(svc.ownedAttemptByTaskId.get(taskId)?.attemptId).toBe(retry.attemptId); + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)).toBeUndefined(); + expect(svc.attemptLedger.ownedAttemptByTaskId.get(taskId)?.attemptId).toBe(retry.attemptId); expect((await taskService.readAttemptOutcome(taskId, requesting)).kind).toBe("live"); expect((await receiptFor(config, attemptId)).kind).toBe("not_found"); }); @@ -1763,7 +1776,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { expect(entry?.taskStatus).toBe("running"); expect(entry?.taskAttemptId).toMatch(ATTEMPT_ID); expect(entry?.taskAttemptUnproven).toBe(true); - expect(svc.ownedAttemptByTaskId.get(taskId)).toMatchObject({ + expect(svc.attemptLedger.ownedAttemptByTaskId.get(taskId)).toMatchObject({ source: "launch", attemptId: entry!.taskAttemptId, receiptEligible: false, @@ -1815,8 +1828,8 @@ describe("TaskService attempt identity and send admission (G1)", () => { expect(await loser).toBe(false); // The loser published nothing: config, mirror and ownership all name the winner. expect(entryOf(config, taskId)?.taskAttemptId).toBe(winner); - expect(svc.currentAttemptIdByTaskId.get(taskId)).toBe(winner); - expect(svc.ownedAttemptByTaskId.get(taskId)).toMatchObject({ + expect(svc.attemptLedger.currentAttemptIdByTaskId.get(taskId)).toBe(winner); + expect(svc.attemptLedger.ownedAttemptByTaskId.get(taskId)).toMatchObject({ source: "reawaken", attemptId: winner, }); @@ -1899,7 +1912,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { await settle(); expect(svc.taskStopLatch.workspaceStopRecords.has(taskId)).toBe(false); expect(taskService.isWorkspaceStopInProgress(taskId)).toBe(false); - expect(svc.ownedAttemptByTaskId.has(taskId)).toBe(false); + expect(svc.attemptLedger.ownedAttemptByTaskId.has(taskId)).toBe(false); // "during-write": nothing is committed yet; "after-durable": the fresh id is already the // row's id, yet no cascade can observe it before its owner exists. const rowWhilePaused = entryOf(config, taskId)!; @@ -1916,7 +1929,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { await waitForCondition(() => svc.taskStopLatch.workspaceStopRecords.has(taskId)); expect(svc.taskStopLatch.workspaceStopRecords.get(taskId)).toMatchObject({ attemptId: fresh, - ownedAttempt: svc.ownedAttemptByTaskId.get(taskId), + ownedAttempt: svc.attemptLedger.ownedAttemptByTaskId.get(taskId), }); expect(taskService.admitTaskWorkspaceTurn(taskId, { acceptanceOrigin: "manual" })).toEqual({ kind: "refused", @@ -1931,7 +1944,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { taskStatus: "interrupted", taskAttemptId: fresh, }); - expect(svc.attemptSettlementByTaskId.get(taskId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)).toMatchObject({ attemptId: fresh, phase: "settled", source: "stop-settled", @@ -1945,7 +1958,9 @@ describe("TaskService attempt identity and send admission (G1)", () => { }); // ...and that settlement proves the next reawaken's lineage. expect(await taskService.markInterruptedTaskRunning(taskId)).toBe(true); - expect(svc.ownedAttemptByTaskId.get(taskId)).toMatchObject({ receiptEligible: true }); + expect(svc.attemptLedger.ownedAttemptByTaskId.get(taskId)).toMatchObject({ + receiptEligible: true, + }); expect(entryOf(config, taskId)?.taskAttemptUnproven).toBeUndefined(); } ); @@ -1982,7 +1997,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { gapSend.onDisposed("refused"); await stop; await waitForStopRelease(taskService, taskId); - expect(svc.attemptSettlementByTaskId.get(taskId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)).toMatchObject({ attemptId: fresh, phase: "settled", }); @@ -2020,8 +2035,8 @@ describe("TaskService attempt identity and send admission (G1)", () => { taskStatus: "interrupted", taskAttemptId: previous, }); - expect(svc.ownedAttemptByTaskId.has(taskId)).toBe(false); - expect(svc.currentAttemptIdByTaskId.has(taskId)).toBe(false); + expect(svc.attemptLedger.ownedAttemptByTaskId.has(taskId)).toBe(false); + expect(svc.attemptLedger.currentAttemptIdByTaskId.has(taskId)).toBe(false); // The predecessor is closed by that Stop; a recovery initiated afterwards is a new decision. expect(taskService.admitTaskWorkspaceTurn(taskId, { acceptanceOrigin: "manual" })).toEqual({ kind: "refused", @@ -2048,8 +2063,8 @@ describe("TaskService attempt identity and send admission (G1)", () => { gate.resolve(); expect(await reawaken).toBe(false); expect(entryOf(config, taskId)?.taskAttemptId).toBe(successor); - expect(svc.ownedAttemptByTaskId.has(taskId)).toBe(false); - expect(svc.currentAttemptIdByTaskId.has(taskId)).toBe(false); + expect(svc.attemptLedger.ownedAttemptByTaskId.has(taskId)).toBe(false); + expect(svc.attemptLedger.currentAttemptIdByTaskId.has(taskId)).toBe(false); expect(committed).not.toBe(successor); // Nothing of the successor's admission is touched: the fence binds to it, unowned. const token = admitted( @@ -2078,8 +2093,8 @@ describe("TaskService attempt identity and send admission (G1)", () => { (error: unknown) => (error instanceof Error ? error.message : String(error)) ); expect(outcome).toBe("registry write failed"); - expect(svc.ownedAttemptByTaskId.has(taskId)).toBe(false); - expect(svc.currentAttemptIdByTaskId.has(taskId)).toBe(false); + expect(svc.attemptLedger.ownedAttemptByTaskId.has(taskId)).toBe(false); + expect(svc.attemptLedger.currentAttemptIdByTaskId.has(taskId)).toBe(false); expect(entryOf(config, taskId)?.taskAttemptId).toBe(previous); // The mutex is free again: a cascade's Phase A (which needs it) runs to completion. await taskService.terminateAllDescendantAgentTasks(rootId); @@ -2113,7 +2128,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { : undefined; try { if (row === "deleted") await config.removeWorkspace(taskId); - expect(svc.currentAttemptIdByTaskId.get(taskId)).toBe(owned); + expect(svc.attemptLedger.currentAttemptIdByTaskId.get(taskId)).toBe(owned); expect(token.admissionStale()).toBe(true); } finally { load?.mockRestore(); @@ -2134,7 +2149,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { const svc = internals(taskService); expect(await taskService.markInterruptedTaskRunning(taskId)).toBe(true); const owned = entryOf(config, taskId)!.taskAttemptId!; - expect(svc.currentAttemptIdByTaskId.get(taskId)).toBe(owned); + expect(svc.attemptLedger.currentAttemptIdByTaskId.get(taskId)).toBe(owned); const beforeRotation = admitted( taskService.admitTaskWorkspaceTurn(taskId, { acceptanceOrigin: "manual" }) ); @@ -2153,7 +2168,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { } return cfg; }); - expect(svc.currentAttemptIdByTaskId.get(taskId)).toBe(owned); + expect(svc.attemptLedger.currentAttemptIdByTaskId.get(taskId)).toBe(owned); // The token of the superseded attempt is revoked; a new send binds to the persisted id, // unowned (this process owns a different attempt). @@ -2192,7 +2207,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { const entry = entryOf(config, spawnedId); expect(entry?.taskAttemptId).toMatch(ATTEMPT_ID); expect(entry?.taskAttemptUnproven).toBeUndefined(); - expect(svc.ownedAttemptByTaskId.get(spawnedId)).toMatchObject({ + expect(svc.attemptLedger.ownedAttemptByTaskId.get(spawnedId)).toMatchObject({ source: "launch", attemptId: entry!.taskAttemptId, receiptEligible: true, @@ -2209,7 +2224,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { kind: "terminal-no-report", }); expect(await taskService.markInterruptedTaskRunning(spawnedId)).toBe(true); - expect(svc.ownedAttemptByTaskId.get(spawnedId)).toMatchObject({ + expect(svc.attemptLedger.ownedAttemptByTaskId.get(spawnedId)).toMatchObject({ source: "reawaken", receiptEligible: true, }); @@ -2318,7 +2333,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { // below disposes a token or settles a turn), so wait for it instead of racing it. await waitForStopRelease(taskService, spawnedId); expect(svc.admittedSendsByTaskId.has(spawnedId)).toBe(false); - expect(svc.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ phase: "settled", }); expect(await taskService.readAttemptOutcome(spawnedId, requesting)).toMatchObject({ @@ -2328,7 +2343,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { } // The captured owner outlived the bound: NO settlement proof — the latch holds and the // attempt reads cleanup-pending until that owner's own settlement. - expect(svc.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ phase: "closing", source: "launch-failed", }); @@ -2347,7 +2362,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { else settleRacingTurn(spawnedId); await waitForStopRelease(taskService, spawnedId); expect(svc.taskStopLatch.workspaceStopRecords.has(spawnedId)).toBe(false); - expect(svc.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ phase: "settled", source: "stop-settled", }); @@ -2430,8 +2445,10 @@ describe("TaskService attempt identity and send admission (G1)", () => { expect(taskService.isWorkspaceStopInProgress(spawnedId)).toBe(false); expect(svc.taskStopLatch.workspaceStopRecords.has(spawnedId)).toBe(false); // This process still owns A; idle under it, the launch failure settles exactly A. - expect(svc.ownedAttemptByTaskId.get(spawnedId)?.attemptId).toBe(mintedAttemptId); - expect(svc.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ + expect(svc.attemptLedger.ownedAttemptByTaskId.get(spawnedId)?.attemptId).toBe( + mintedAttemptId + ); + expect(svc.attemptLedger.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ attemptId: mintedAttemptId, phase: "settled", source: "launch-failed", @@ -2532,9 +2549,11 @@ describe("TaskService attempt identity and send admission (G1)", () => { expect(entryOf(config, spawnedId)?.taskLaunchError).toBeUndefined(); expect(taskService.isWorkspaceStopInProgress(spawnedId)).toBe(false); expect(svc.taskStopLatch.workspaceStopRecords.has(spawnedId)).toBe(false); - expect(svc.ownedAttemptByTaskId.get(spawnedId)?.attemptId).toBe(mintedAttemptId); + expect(svc.attemptLedger.ownedAttemptByTaskId.get(spawnedId)?.attemptId).toBe( + mintedAttemptId + ); if (liveness === "idle") { - expect(svc.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ attemptId: mintedAttemptId, phase: "settled", source: "launch-failed", @@ -2548,7 +2567,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { // existing signals — a pending one reads stale (the row no longer names A) and is // disposed by its host at its next gate; an admitted one is discharged when its turn // settles — so no orphan obligation outlives the launch failure. - expect(svc.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ attemptId: mintedAttemptId, phase: "closing", source: "launch-failed", @@ -2570,7 +2589,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { host.settleTurn(spawnedId, racingTurn); } expect(svc.admittedSendsByTaskId.has(spawnedId)).toBe(false); - expect(svc.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ attemptId: mintedAttemptId, phase: "closing", }); @@ -2639,7 +2658,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { // before the other writer's config write landed. assert(ATTEMPT_CLOSURE_SETTLE_WAIT_MS < 10_000, "the rotation bound must not be shortened"); await raceWithTimeout(rotation.promise, 10_000); - const mintedAttemptId = svc.ownedAttemptByTaskId.get(spawnedId)?.attemptId; + const mintedAttemptId = svc.attemptLedger.ownedAttemptByTaskId.get(spawnedId)?.attemptId; expect(mintedAttemptId).toMatch(ATTEMPT_ID); // The captured owner outlived the bound: the latch holds for A's turn, and B's row — // written after our marker — is untouched (no deletion, no second marker). @@ -2651,7 +2670,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { taskStatus: "running", taskAttemptId: FOREIGN_ATTEMPT_ID, }); - expect(svc.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ attemptId: mintedAttemptId, phase: "closing", source: "launch-failed", @@ -2668,7 +2687,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { host.settleTurn(spawnedId, racingTurn); expect(taskService.isWorkspaceStopInProgress(spawnedId)).toBe(false); expect(svc.taskStopLatch.workspaceStopRecords.has(spawnedId)).toBe(false); - expect(svc.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ attemptId: mintedAttemptId, phase: "settled", source: "stop-settled", @@ -2727,7 +2746,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { expect(entry?.taskStatus).toBe(moved === "id" ? "running" : "interrupted"); expect(entry?.taskAttemptUnproven).toBeUndefined(); expect(sendMessage.mock.calls.filter((call) => call[0] === taskId)).toHaveLength(0); - expect(svc.currentAttemptIdByTaskId.has(taskId)).toBe(false); + expect(svc.attemptLedger.currentAttemptIdByTaskId.has(taskId)).toBe(false); } ); @@ -2758,7 +2777,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { false ); expect(await taskService.markInterruptedTaskRunning(taskId)).toBe(true); - expect(svc.ownedAttemptByTaskId.get(taskId)?.receiptEligible).toBe(false); + expect(svc.attemptLedger.ownedAttemptByTaskId.get(taskId)?.receiptEligible).toBe(false); expect(entryOf(config, taskId)?.taskAttemptUnproven).toBe(true); }); }); @@ -2857,7 +2876,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { taskService.admitTaskWorkspaceTurn(taskId, { acceptanceOrigin: "manual" }) ); expect(fresh.admissionStale()).toBe(false); - expect(svc.attemptSettlementByTaskId.get(taskId)).toBeUndefined(); + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)).toBeUndefined(); fresh.onDisposed("no-work"); }); @@ -2909,7 +2928,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { await settle(); expect(taskService.isWorkspaceStopInProgress(taskId)).toBe(false); expect(svc.admittedSendsByTaskId.get(taskId)).toBeUndefined(); - expect(svc.attemptSettlementByTaskId.get(taskId)?.phase).toBe("settled"); + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)?.phase).toBe("settled"); expect(await taskService.readAttemptOutcome(taskId, requesting)).toMatchObject({ kind: "terminal-no-report", }); @@ -3019,14 +3038,14 @@ describe("TaskService attempt identity and send admission (G1)", () => { // The in-memory settlement follows the status write asynchronously (the closing window is // covered separately below); poll the authoritative phase. const deadline = Date.now() + 2_000; - while (svc.attemptSettlementByTaskId.get(spawnedId)?.phase !== "settled") { + while (svc.attemptLedger.attemptSettlementByTaskId.get(spawnedId)?.phase !== "settled") { if (Date.now() > deadline) throw new Error("launch failure did not settle"); await settle(); } expect(entryOf(config, spawnedId)?.taskStatus).toBe("interrupted"); const attemptId = entryOf(config, spawnedId)!.taskAttemptId!; expect(isTaskAttemptId(attemptId)).toBe(true); - expect(svc.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ attemptId, phase: "settled", source: "launch-failed", @@ -3067,11 +3086,11 @@ describe("TaskService attempt identity and send admission (G1)", () => { }); const failing = svc.markTaskLaunchFailed(spawnedId, "fork failed"); const deadline = Date.now() + 2_000; - while (svc.attemptSettlementByTaskId.get(spawnedId)?.phase !== "closing") { + while (svc.attemptLedger.attemptSettlementByTaskId.get(spawnedId)?.phase !== "closing") { if (Date.now() > deadline) throw new Error("closure was not recorded"); await settle(); } - expect(svc.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ attemptId, phase: "closing", }); @@ -3089,7 +3108,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { gate.resolve(); await failing; editSpy.mockRestore(); - expect(svc.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ attemptId, phase: "settled", }); @@ -3119,11 +3138,11 @@ describe("TaskService attempt identity and send admission (G1)", () => { await svc.releaseSharedDesktopTaskOnUserStop(taskId, svc.resolveStreamAttemptAtEvent(taskId)); // The pending send keeps the task live: no transition, no closure. expect(entryOf(config, taskId)?.taskStatus).toBe("running"); - expect(svc.attemptSettlementByTaskId.get(taskId)).toBeUndefined(); + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)).toBeUndefined(); token.onDisposed("refused"); await svc.releaseSharedDesktopTaskOnUserStop(taskId, svc.resolveStreamAttemptAtEvent(taskId)); expect(entryOf(config, taskId)?.taskStatus).toBe("interrupted"); - expect(svc.attemptSettlementByTaskId.get(taskId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)).toMatchObject({ attemptId: entryOf(config, taskId)?.taskAttemptId, phase: "settled", source: "user-stop-idle", @@ -3144,7 +3163,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { shortenTerminationTimers(); await taskService.terminateAllDescendantAgentTasks(rootId); expect(taskService.isWorkspaceStopInProgress(taskId)).toBe(false); - expect(svc.attemptSettlementByTaskId.get(taskId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)).toMatchObject({ attemptId, phase: "closing", source: "stop-settled", @@ -3219,7 +3238,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { "tool-end" ); expect(reactivated).toMatchObject({ success: true, data: { delivery: "reactivated" } }); - expect(svc.ownedAttemptByTaskId.get(taskId)?.source).toBe("reactivation"); + expect(svc.attemptLedger.ownedAttemptByTaskId.get(taskId)?.source).toBe("reactivation"); const reactivatedEntry = entryOf(config, taskId); const handleId = reactivatedEntry?.taskExecutionId; expect(handleId).toMatch(/^wst_/); @@ -3317,7 +3336,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { await settle(); // Phase A blocks on the mutex: nothing captured, nothing latched, no owner yet. expect(svc.taskStopLatch.workspaceStopRecords.has(taskId)).toBe(false); - expect(svc.ownedAttemptByTaskId.has(taskId)).toBe(false); + expect(svc.attemptLedger.ownedAttemptByTaskId.has(taskId)).toBe(false); gate.resolve(); // createWorkspaceTurn runs after the section, behind the cascade's Phase A in the mutex // queue: its send meets the latch, so the reactivation fails after publishing — and the @@ -3329,13 +3348,13 @@ describe("TaskService attempt identity and send admission (G1)", () => { await waitForCondition(() => svc.taskStopLatch.workspaceStopRecords.has(taskId)); expect(svc.taskStopLatch.workspaceStopRecords.get(taskId)).toMatchObject({ attemptId: fresh, - ownedAttempt: svc.ownedAttemptByTaskId.get(taskId), + ownedAttempt: svc.attemptLedger.ownedAttemptByTaskId.get(taskId), }); - expect(svc.ownedAttemptByTaskId.get(taskId)?.source).toBe("reactivation"); + expect(svc.attemptLedger.ownedAttemptByTaskId.get(taskId)?.source).toBe("reactivation"); cleanupGate.resolve(); await stop; await waitForCondition(() => !taskService.isWorkspaceStopInProgress(taskId)); - expect(svc.attemptSettlementByTaskId.get(taskId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)).toMatchObject({ attemptId: fresh, phase: "settled", source: "stop-settled", @@ -3367,7 +3386,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { }); // Nothing published: the cascade's record still names the predecessor it captured. expect(entryOf(config, taskId)?.taskAttemptId).toBe(previous); - expect(svc.ownedAttemptByTaskId.has(taskId)).toBe(false); + expect(svc.attemptLedger.ownedAttemptByTaskId.has(taskId)).toBe(false); expect(svc.taskStopLatch.workspaceStopRecords.get(taskId)?.attemptId).toBe(previous); cleanupGate.resolve(); await stop; @@ -3445,7 +3464,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { const { taskService } = createHarness(config); const svc = internals(taskService); const workspaceId = target === "a root workspace" ? rootId : taskId; - expect(svc.currentAttemptIdByTaskId.has(workspaceId)).toBe(false); + expect(svc.attemptLedger.currentAttemptIdByTaskId.has(workspaceId)).toBe(false); const load = spyOn(config, "loadConfigOrDefault").mockImplementation((options) => { if (options?.throwOnError) throw new Error("registry unreadable"); return { projects: new Map() } as unknown as ReturnType; @@ -3608,7 +3627,9 @@ describe("TaskService attempt identity and send admission (G1)", () => { expect(stopStream).not.toHaveBeenCalled(); expect(svc.taskStopLatch.workspaceStopRecords.has(taskId)).toBe(false); expect(taskService.isWorkspaceStopInProgress(taskId)).toBe(false); - expect(svc.attemptSettlementByTaskId.get(taskId)?.attemptId).not.toBe(foreignAttemptId); + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)?.attemptId).not.toBe( + foreignAttemptId + ); return; } // Control: the limit fails R terminally and R's stop latch releases once the fenced @@ -3668,7 +3689,9 @@ describe("TaskService attempt identity and send admission (G1)", () => { [...(ledger.svc?.admittedSendsByTaskId.get(taskId) ?? [])].map((send) => ({ attemptId: send.attemptId, state: send.state, - owned: ledger.svc?.ownedAttemptByTaskId.get(taskId)?.attemptId === send.attemptId, + owned: + ledger.svc?.attemptLedger.ownedAttemptByTaskId.get(taskId)?.attemptId === + send.attemptId, })) ); aiEmitter.emit("stream-start", { @@ -3783,7 +3806,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { const svc = internals(taskService); expect(await taskService.markInterruptedTaskRunning(taskId)).toBe(true); const attemptA = entryOf(config, taskId)!.taskAttemptId!; - expect(svc.ownedAttemptByTaskId.get(taskId)?.attemptId).toBe(attemptA); + expect(svc.attemptLedger.ownedAttemptByTaskId.get(taskId)?.attemptId).toBe(attemptA); // Backend B re-admits the row while this backend's handler awaits its owned-work probe. spyOn( svc as unknown as { hasActiveTaskOwnedWork: () => Promise }, @@ -3821,7 +3844,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { expect(clearQueue).not.toHaveBeenCalled(); expect(stopStream).not.toHaveBeenCalled(); expect(svc.taskStopLatch.workspaceStopRecords.has(taskId)).toBe(false); - expect(svc.attemptSettlementByTaskId.get(taskId)?.attemptId).not.toBe(foreign); + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)?.attemptId).not.toBe(foreign); }); test("a resume-failure rollback restores only the reawakened attempt: a row another backend re-admitted meanwhile keeps its running status", async () => { const taskId = "stale-resume-rollback"; @@ -4111,7 +4134,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { taskAttemptId: SUCCESSOR_ATTEMPT_ID, }); // The recorded settlement names A, not the row's current attempt. - expect(result.svc.attemptSettlementByTaskId.get(result.taskId)).toMatchObject({ + expect(result.svc.attemptLedger.attemptSettlementByTaskId.get(result.taskId)).toMatchObject({ attemptId: result.attemptA, phase: "settled", source: "stop-settled", @@ -4179,8 +4202,8 @@ describe("TaskService attempt identity and send admission (G1)", () => { taskStatus: row?.taskStatus, taskAttemptId: row?.taskAttemptId, closures, - settlement: svc.attemptSettlementByTaskId.get(taskId)?.attemptId, - settlementPhase: svc.attemptSettlementByTaskId.get(taskId)?.phase, + settlement: svc.attemptLedger.attemptSettlementByTaskId.get(taskId)?.attemptId, + settlementPhase: svc.attemptLedger.attemptSettlementByTaskId.get(taskId)?.phase, }, outcome: await taskService.readAttemptOutcome(taskId, requesting), }; @@ -4395,7 +4418,7 @@ describe("TaskService attempt identity and send admission (G1)", () => { taskStatus: "interrupted", taskAttemptId: attemptA, }); - expect(svc.attemptSettlementByTaskId.get(taskId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)).toMatchObject({ attemptId: attemptA, phase: "settled", }); @@ -4406,7 +4429,9 @@ describe("TaskService attempt identity and send admission (G1)", () => { taskStatus: "running", taskAttemptId: SUCCESSOR_ATTEMPT_ID, }); - expect(svc.attemptSettlementByTaskId.get(taskId)?.attemptId).not.toBe(SUCCESSOR_ATTEMPT_ID); + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)?.attemptId).not.toBe( + SUCCESSOR_ATTEMPT_ID + ); } ); }); diff --git a/src/node/services/taskService.childGoals.test.ts b/src/node/services/taskService.childGoals.test.ts index e5327f0c154..5a1e8401abd 100644 --- a/src/node/services/taskService.childGoals.test.ts +++ b/src/node/services/taskService.childGoals.test.ts @@ -1208,10 +1208,10 @@ describe("TaskService child goals", () => { expect(seeded.success).toBe(true); expect(await t.taskService.markInterruptedTaskRunning(childId)).toBe(true); const internal = t.taskService as unknown as { - ownedAttemptByTaskId: Map; + attemptLedger: { ownedAttemptByTaskId: Map }; streamEndDecisionsByTaskId: Map>; }; - expect(internal.ownedAttemptByTaskId.get(childId)?.receiptEligible).toBe(true); + expect(internal.attemptLedger.ownedAttemptByTaskId.get(childId)?.receiptEligible).toBe(true); await t.setChildGoal(); const realWait = t.goals.waitForStreamAccountingReceipt.bind(t.goals); spyOn(t.goals, "waitForStreamAccountingReceipt").mockImplementation((workspaceId, messageId) => @@ -1566,10 +1566,10 @@ describe("TaskService child goals", () => { const listeners = ( t.taskService as unknown as { - attemptSettlementListenersByTaskId: Map void>>; + attemptLedger: { attemptSettlementListenersByTaskId: Map void>> }; notifyAttemptSettlementListeners: (taskId: string) => void; } - ).attemptSettlementListenersByTaskId; + ).attemptLedger.attemptSettlementListenersByTaskId; expect(listeners.get(childId)?.size).toBe(1); ( t.taskService as unknown as { notifyAttemptSettlementListeners: (taskId: string) => void } diff --git a/src/node/services/taskService.reportDecisionHold.test.ts b/src/node/services/taskService.reportDecisionHold.test.ts index a6b77199995..6735f89adf4 100644 --- a/src/node/services/taskService.reportDecisionHold.test.ts +++ b/src/node/services/taskService.reportDecisionHold.test.ts @@ -49,8 +49,10 @@ const rootId = "root-hold"; const model = "openai:gpt-5.2"; interface Internals { - ownedAttemptByTaskId: Map; - attemptSettlementByTaskId: Map; + attemptLedger: { + ownedAttemptByTaskId: Map; + attemptSettlementByTaskId: Map; + }; admittedSendsByTaskId: Map>; streamEndDecisionsByTaskId: Map>; taskStopLatch: { workspaceStopRecords: Map }; @@ -213,7 +215,7 @@ describe("report-decision hold for queued follow-ups (real host)", () => { messageId, row: row?.taskAttemptId, status: row?.taskStatus, - owner: ledger.svc?.ownedAttemptByTaskId.get(childId)?.attemptId, + owner: ledger.svc?.attemptLedger.ownedAttemptByTaskId.get(childId)?.attemptId, }); aiEmitter.emit("stream-start", { type: "stream-start", @@ -349,7 +351,7 @@ describe("report-decision hold for queued follow-ups (real host)", () => { await until(() => entryOf(config, childId)?.taskStatus === "reported", "report"); if (!ordering.startsWith("drain before")) { // The decision landed before the session finished the turn: the drain reads it late. - await until(() => !svc.ownedAttemptByTaskId.has(childId), "release"); + await until(() => !svc.attemptLedger.ownedAttemptByTaskId.has(childId), "release"); stack.completeStream(0, event); } await until(() => !workspaceService.hasQueuedMessages(childId), "queue drained"); @@ -361,7 +363,7 @@ describe("report-decision hold for queued follow-ups (real host)", () => { taskStatus: "reported", taskAttemptId: attemptA, }); - expect(svc.ownedAttemptByTaskId.has(childId)).toBe(false); + expect(svc.attemptLedger.ownedAttemptByTaskId.has(childId)).toBe(false); // The follow-up's obligation was discharged exactly once and the decision pruned. expect(outstanding(svc, childId)).toHaveLength(0); expect(svc.streamEndDecisionsByTaskId.has(childId)).toBe(false); @@ -416,7 +418,7 @@ describe("report-decision hold for queued follow-ups (real host)", () => { // still live, so the queued follow-up has not drained. const event = stack.endStream(0, { report: "done" }, false); await until(() => entryOf(config, childId)?.taskStatus === "reported", "report"); - await until(() => !svc.ownedAttemptByTaskId.has(childId), "attempt released"); + await until(() => !svc.attemptLedger.ownedAttemptByTaskId.has(childId), "attempt released"); expect(workspaceService.hasQueuedMessages(childId)).toBe(true); expect(stack.heldInputs()).toHaveLength(0); @@ -477,7 +479,7 @@ describe("report-decision hold for queued follow-ups (real host)", () => { // The report turn ends on a terminal agent_report; the session has not finished the turn. const event = stack.endStream(0, { report: "done" }, false); await until(() => entryOf(config, childId)?.taskStatus === "reported", "report"); - await until(() => !svc.ownedAttemptByTaskId.has(childId), "release"); + await until(() => !svc.attemptLedger.ownedAttemptByTaskId.has(childId), "release"); // The decision resolved with no reader queued yet. The user's follow-up arrives now, while // the reporting turn is still live: it targets the completed attempt, so it is refused in // its preflight (fail closed, text stays with the caller) instead of queueing behind the @@ -496,7 +498,7 @@ describe("report-decision hold for queued follow-ups (real host)", () => { taskStatus: "reported", taskAttemptId: attemptA, }); - expect(svc.ownedAttemptByTaskId.has(childId)).toBe(false); + expect(svc.attemptLedger.ownedAttemptByTaskId.has(childId)).toBe(false); // Nor was it accepted as a user turn of that attempt. const history = await fixture.historyService.getLastMessages(childId, 10); expect(history.success && history.data.map((message) => message.role)).toEqual(["user"]); @@ -886,7 +888,7 @@ describe("report-decision hold for queued follow-ups (real host)", () => { expect(completions).toHaveLength(1); // Not released (no durable report), not continued: fail closed, the follow-up held — as // indeterminate, never as if a report had been confirmed. - expect(svc.ownedAttemptByTaskId.get(childId)?.attemptId).toBe(attemptA); + expect(svc.attemptLedger.ownedAttemptByTaskId.get(childId)?.attemptId).toBe(attemptA); expect(stack.heldTexts()).toEqual(["follow-up text"]); expect(stack.heldInputs().map((held) => held.reason)).toEqual(["indeterminate"]); // Refused as indeterminate — not as a completed report and not as a plain stale attempt. @@ -1208,7 +1210,7 @@ describe("report-decision hold for queued follow-ups (real host)", () => { // Handler resumes: publishes and releases A; no reader is left, so the decision is dropped. gate.resolve(); await until(() => !svc.streamEndDecisionsByTaskId.has(childId), "decision pruned"); - await until(() => !svc.ownedAttemptByTaskId.has(childId), "attempt released"); + await until(() => !svc.attemptLedger.ownedAttemptByTaskId.has(childId), "attempt released"); await yieldMacrotasks(5); expect(entryOf(config, childId)).toMatchObject({ taskStatus: "reported", @@ -1258,7 +1260,7 @@ describe("report-decision hold for queued follow-ups (real host)", () => { expect(completions).toHaveLength(1); gate.resolve(); await until(() => !svc.streamEndDecisionsByTaskId.has(childId), "decision pruned"); - await until(() => !svc.ownedAttemptByTaskId.has(childId), "attempt released"); + await until(() => !svc.attemptLedger.ownedAttemptByTaskId.has(childId), "attempt released"); await yieldMacrotasks(5); expect(entryOf(config, childId)).toMatchObject({ taskStatus: "reported", @@ -1305,7 +1307,7 @@ describe("report-decision hold for queued follow-ups (real host)", () => { try { if (status === "reported") { // The released shape of an ordinary reported child (no owner, no settlement entry). - expect(svc.ownedAttemptByTaskId.has(childId)).toBe(false); + expect(svc.attemptLedger.ownedAttemptByTaskId.has(childId)).toBe(false); } const result = await workspaceService.sendMessage(childId, "manual follow-up", sendOptions); expect(result.success).toBe(false); @@ -1313,7 +1315,7 @@ describe("report-decision hold for queued follow-ups (real host)", () => { await yieldMacrotasks(3); expect(completions).toHaveLength(0); expect(outstanding(svc, childId)).toHaveLength(0); - expect(svc.ownedAttemptByTaskId.get(childId)?.attemptId).not.toBe(winner); + expect(svc.attemptLedger.ownedAttemptByTaskId.get(childId)?.attemptId).not.toBe(winner); expect(entryOf(config, childId)?.taskAttemptId).toBe(winner); } finally { await stack.cleanup(); @@ -1332,7 +1334,7 @@ describe("report-decision hold for queued follow-ups (real host)", () => { await until(() => completions.length === 1, "the re-drive's stream"); const redriven = entryOf(config, childId)?.taskAttemptId; expect(redriven).not.toBe("att_00000000000000a4"); - expect(svc.ownedAttemptByTaskId.has(childId)).toBe(false); + expect(svc.attemptLedger.ownedAttemptByTaskId.has(childId)).toBe(false); // The user's follow-up arrives mid-stream: queued, bound to the re-driven attempt. expect(await workspaceService.sendMessage(childId, "follow-up text", sendOptions)).toEqual( Ok(undefined) @@ -1349,7 +1351,7 @@ describe("report-decision hold for queued follow-ups (real host)", () => { expect(outstanding(svc, childId)).toHaveLength(0); expect(svc.streamEndDecisionsByTaskId.has(childId)).toBe(false); // The hold came from the decision alone: no ownership (settlement/receipt authority) granted. - expect(svc.ownedAttemptByTaskId.has(childId)).toBe(false); + expect(svc.attemptLedger.ownedAttemptByTaskId.has(childId)).toBe(false); expect(entryOf(config, childId)?.taskAttemptId).toBe(redriven); } finally { await stack.cleanup(); @@ -1368,7 +1370,7 @@ describe("report-decision hold for queued follow-ups (real host)", () => { await until(() => completions.length === 1, "the re-drive's stream"); const redriven = entryOf(config, childId)?.taskAttemptId; expect(redriven).not.toBe("att_00000000000000a4"); - expect(svc.ownedAttemptByTaskId.has(childId)).toBe(false); + expect(svc.attemptLedger.ownedAttemptByTaskId.has(childId)).toBe(false); // Backend B admits the row under its own attempt while this backend's stream still runs. await otherBackend.editConfig((cfg) => { for (const project of cfg.projects.values()) { @@ -1407,7 +1409,7 @@ describe("report-decision hold for queued follow-ups (real host)", () => { expect( await readSubagentReportArtifact(path.join(config.sessionsDir, rootId), childId) ).toBeNull(); - expect(svc.ownedAttemptByTaskId.has(childId)).toBe(false); + expect(svc.attemptLedger.ownedAttemptByTaskId.has(childId)).toBe(false); } finally { await stack.cleanup(); } @@ -1423,7 +1425,7 @@ describe("report-decision hold for queued follow-ups (real host)", () => { await taskService.recoverInterruptedTasks(); await until(() => completions.length === 1, "the re-drive's stream"); const redriven = entryOf(config, childId)?.taskAttemptId; - expect(svc.ownedAttemptByTaskId.has(childId)).toBe(false); + expect(svc.attemptLedger.ownedAttemptByTaskId.has(childId)).toBe(false); // Backend B re-admits the row right before the publication's status write, i.e. after // every earlier check of the stream-end handler read the re-driven attempt. let rotated = false; @@ -1471,7 +1473,7 @@ describe("report-decision hold for queued follow-ups (real host)", () => { (svc.streamEndDecisionsByTaskId.get(childId) ?? []).map((decision) => decision.attemptId) ).not.toContain(foreign); expect(redriven).not.toBe(foreign); - expect(svc.ownedAttemptByTaskId.has(childId)).toBe(false); + expect(svc.attemptLedger.ownedAttemptByTaskId.has(childId)).toBe(false); } finally { await stack.cleanup(); } @@ -1499,7 +1501,7 @@ describe("report-decision hold for queued follow-ups (real host)", () => { try { expect(await taskService.markInterruptedTaskRunning(childId)).toBe(true); const attemptA = entryOf(config, childId)!.taskAttemptId!; - expect(svc.ownedAttemptByTaskId.get(childId)?.attemptId).toBe(attemptA); + expect(svc.attemptLedger.ownedAttemptByTaskId.get(childId)?.attemptId).toBe(attemptA); expect(await workspaceService.sendMessage(childId, "work", sendOptions)).toEqual( Ok(undefined) ); @@ -1559,7 +1561,7 @@ describe("report-decision hold for queued follow-ups (real host)", () => { try { expect(await taskService.markInterruptedTaskRunning(childId)).toBe(true); const attemptA = entryOf(config, childId)!.taskAttemptId!; - expect(svc.ownedAttemptByTaskId.get(childId)?.attemptId).toBe(attemptA); + expect(svc.attemptLedger.ownedAttemptByTaskId.get(childId)?.attemptId).toBe(attemptA); // Backend B re-admits the idle row under its own attempt; this process still owns A. await otherBackend.editConfig((cfg) => { for (const project of cfg.projects.values()) { @@ -1577,7 +1579,7 @@ describe("report-decision hold for queued follow-ups (real host)", () => { Ok(undefined) ); expect(completions).toHaveLength(1); - expect(svc.ownedAttemptByTaskId.get(childId)?.attemptId).toBe(attemptA); + expect(svc.attemptLedger.ownedAttemptByTaskId.get(childId)?.attemptId).toBe(attemptA); stack.endStream(0, { report: "done by B" }, true); await until(() => entryOf(config, childId)?.taskStatus === "reported", "B's report"); expect(entryOf(config, childId)?.taskAttemptId).toBe(foreign); @@ -1669,7 +1671,7 @@ describe("report-decision hold for queued follow-ups (real host)", () => { expect( await readSubagentReportArtifact(path.join(config.sessionsDir, rootId), childId) ).toBeNull(); - expect(svc.ownedAttemptByTaskId.get(childId)?.attemptId).not.toBe(foreign); + expect(svc.attemptLedger.ownedAttemptByTaskId.get(childId)?.attemptId).not.toBe(foreign); } finally { await stack.cleanup(); } diff --git a/src/node/services/taskService.settlement.test.ts b/src/node/services/taskService.settlement.test.ts index 78084e0a646..24d1aab4c77 100644 --- a/src/node/services/taskService.settlement.test.ts +++ b/src/node/services/taskService.settlement.test.ts @@ -1980,9 +1980,11 @@ describe("TaskService", () => { }); const owned = ( taskService as unknown as { - ownedAttemptByTaskId: Map; + attemptLedger: { + ownedAttemptByTaskId: Map; + }; } - ).ownedAttemptByTaskId.get(taskId); + ).attemptLedger.ownedAttemptByTaskId.get(taskId); expect(owned?.attemptId).toBe(entry?.taskAttemptId); expect(owned?.receiptEligible).toBe(false); } finally { @@ -2117,7 +2119,7 @@ describe("TaskService", () => { const internals = taskService as unknown as { inspectAttemptOutcome: (taskId: string, options: unknown) => Promise; workspaceEventLocks: { withLock: (key: string, op: () => Promise) => Promise }; - attemptSettlementListenersByTaskId: Map>; + attemptLedger: { attemptSettlementListenersByTaskId: Map> }; }; const inspect = spyOn(internals, "inspectAttemptOutcome"); const gate = Promise.withResolvers(); @@ -2149,7 +2151,9 @@ describe("TaskService", () => { await new Promise((resolve) => setTimeout(resolve, 5)); // Late acquisitions are no-ops: no read, no leaked subscription. expect(inspect).not.toHaveBeenCalled(); - expect(internals.attemptSettlementListenersByTaskId.get(taskId)?.size ?? 0).toBe(0); + expect( + internals.attemptLedger.attemptSettlementListenersByTaskId.get(taskId)?.size ?? 0 + ).toBe(0); // The lock is usable again and a normal read works. expect((await taskService.readAttemptOutcome(taskId, requesting)).kind).toBe("indeterminate"); }); @@ -2207,9 +2211,9 @@ describe("TaskService", () => { // Subscriptions do not leak past the wait. const listeners = ( taskService as unknown as { - attemptSettlementListenersByTaskId: Map>; + attemptLedger: { attemptSettlementListenersByTaskId: Map> }; } - ).attemptSettlementListenersByTaskId; + ).attemptLedger.attemptSettlementListenersByTaskId; expect(listeners.get(taskId)?.size ?? 0).toBe(0); expect(listeners.get(legacyId)?.size ?? 0).toBe(0); } finally { @@ -2218,8 +2222,10 @@ describe("TaskService", () => { }); interface AttemptLedgerInternals { - ownedAttemptByTaskId: Map; - attemptSettlementByTaskId: Map; + attemptLedger: { + ownedAttemptByTaskId: Map; + attemptSettlementByTaskId: Map; + }; waiters: { completedReportsByTaskId: Map }; shouldAllowLegacyInvalidWorkflowOutputSchema: (...args: unknown[]) => Promise; } @@ -2256,13 +2262,15 @@ describe("TaskService", () => { const { taskService } = createTaskServiceHarness(config); const internals = ledgerInternals(taskService); expect(await taskService.markInterruptedTaskRunning(taskId)).toBe(true); - const attempt = internals.ownedAttemptByTaskId.get(taskId); + const attempt = internals.attemptLedger.ownedAttemptByTaskId.get(taskId); expect(attempt).toBeDefined(); if (statusAtStreamEnd === "interrupted") { // Stopped before its stream ended: the no-report receipt is retained until the report // that follows makes it redundant. await taskService.terminateAllDescendantAgentTasks(rootId); - expect(internals.attemptSettlementByTaskId.get(taskId)?.attempt).toBe(attempt); + expect(internals.attemptLedger.attemptSettlementByTaskId.get(taskId)?.attempt).toBe( + attempt + ); expect(await taskService.readAttemptOutcome(taskId, requesting)).toMatchObject({ kind: "terminal-no-report", }); @@ -2271,8 +2279,8 @@ describe("TaskService", () => { await streamEnd(taskService, reportStreamEnd(taskId, "final answer")); expect(findWorkspaceInConfig(config, taskId)?.taskStatus).toBe("reported"); - expect(internals.ownedAttemptByTaskId.has(taskId)).toBe(false); - expect(internals.attemptSettlementByTaskId.has(taskId)).toBe(false); + expect(internals.attemptLedger.ownedAttemptByTaskId.has(taskId)).toBe(false); + expect(internals.attemptLedger.attemptSettlementByTaskId.has(taskId)).toBe(false); const reported = { kind: "reported", report: { reportMarkdown: "final answer" } }; expect(await taskService.readAttemptOutcome(taskId, requesting)).toMatchObject(reported); // Not just the in-memory cache: the persisted artifact answers, here and in a fresh process. @@ -2298,7 +2306,7 @@ describe("TaskService", () => { const internals = ledgerInternals(taskService); expect(await taskService.markInterruptedTaskRunning(taskId)).toBe(true); await taskService.terminateAllDescendantAgentTasks(rootId); - const firstAttempt = internals.ownedAttemptByTaskId.get(taskId); + const firstAttempt = internals.attemptLedger.ownedAttemptByTaskId.get(taskId); expect(firstAttempt).toBeDefined(); const blocked = Promise.withResolvers(); @@ -2351,7 +2359,7 @@ describe("TaskService", () => { } // The user resumes the (still interrupted) task while the older report is in flight. expect(await taskService.markInterruptedTaskRunning(taskId)).toBe(true); - const secondAttempt = internals.ownedAttemptByTaskId.get(taskId); + const secondAttempt = internals.attemptLedger.ownedAttemptByTaskId.get(taskId); const secondAttemptId = findWorkspaceInConfig(config, taskId)?.taskAttemptId; expect(secondAttempt).toBeDefined(); expect(secondAttempt).not.toBe(firstAttempt); @@ -2378,7 +2386,7 @@ describe("TaskService", () => { taskAttemptId: secondAttemptId, }); expect(waiterOutcome).toBeUndefined(); - expect(internals.ownedAttemptByTaskId.get(taskId)).toBe(secondAttempt); + expect(internals.attemptLedger.ownedAttemptByTaskId.get(taskId)).toBe(secondAttempt); expect( await readSubagentReportArtifact(path.join(config.sessionsDir, rootId), taskId) ).toBeNull(); @@ -2402,13 +2410,13 @@ describe("TaskService", () => { const { taskService } = createTaskServiceHarness(config); const internals = ledgerInternals(taskService); expect(await taskService.markInterruptedTaskRunning(taskId)).toBe(true); - const attempt = internals.ownedAttemptByTaskId.get(taskId); + const attempt = internals.attemptLedger.ownedAttemptByTaskId.get(taskId); expect(attempt).toBeDefined(); await streamEnd(taskService, reportStreamEnd(taskId, "lost report")); expect(findWorkspaceInConfig(config, taskId)?.taskStatus).toBe("reported"); - expect(internals.ownedAttemptByTaskId.get(taskId)).toBe(attempt); + expect(internals.attemptLedger.ownedAttemptByTaskId.get(taskId)).toBe(attempt); // A fresh process has no durable report to classify from. const { taskService: freshService } = createTaskServiceHarness(config); expect((await freshService.readAttemptOutcome(taskId, requesting)).kind).toBe( @@ -2425,7 +2433,7 @@ describe("TaskService", () => { const internals = ledgerInternals(taskService); expect(await taskService.markInterruptedTaskRunning(taskId)).toBe(true); await taskService.terminateAllDescendantAgentTasks(rootId); - const attempt = internals.ownedAttemptByTaskId.get(taskId); + const attempt = internals.attemptLedger.ownedAttemptByTaskId.get(taskId); expect(attempt).toBeDefined(); // Cut short by the stop: no explicit `stop` finish, so nothing is promoted to a report. @@ -2438,8 +2446,8 @@ describe("TaskService", () => { }); expect(findWorkspaceInConfig(config, taskId)?.taskStatus).toBe("interrupted"); - expect(internals.ownedAttemptByTaskId.get(taskId)).toBe(attempt); - expect(internals.attemptSettlementByTaskId.get(taskId)?.attempt).toBe(attempt); + expect(internals.attemptLedger.ownedAttemptByTaskId.get(taskId)).toBe(attempt); + expect(internals.attemptLedger.attemptSettlementByTaskId.get(taskId)?.attempt).toBe(attempt); expect(await taskService.readAttemptOutcome(taskId, requesting)).toMatchObject({ kind: "terminal-no-report", }); @@ -2943,9 +2951,9 @@ describe("TaskService", () => { expect(await taskService.markInterruptedTaskRunning(spawnedId)).toBe(true); const owned = ( taskService as unknown as { - ownedAttemptByTaskId: Map; + attemptLedger: { ownedAttemptByTaskId: Map }; } - ).ownedAttemptByTaskId.get(spawnedId); + ).attemptLedger.ownedAttemptByTaskId.get(spawnedId); expect(owned?.abortSignal).toBeUndefined(); } ); diff --git a/src/node/services/taskService.settlementReceipts.test.ts b/src/node/services/taskService.settlementReceipts.test.ts index 9dd1eb5c1d9..90c11ff19a6 100644 --- a/src/node/services/taskService.settlementReceipts.test.ts +++ b/src/node/services/taskService.settlementReceipts.test.ts @@ -58,10 +58,12 @@ const OWNERS = [midId, rootId]; const PREDECESSOR = "att_00000000000000f1"; interface Internals { - ownedAttemptByTaskId: Map; - attemptSettlementByTaskId: Map; + attemptLedger: { + ownedAttemptByTaskId: Map; + attemptSettlementByTaskId: Map; + attemptSettlementListenersByTaskId: Map void>>; + }; taskStopLatch: { workspaceStopRecords: Map }; - attemptSettlementListenersByTaskId: Map void>>; waitForWorkspaceStopsToRelease: (ids: readonly string[], deadlineMs: number) => Promise; releaseSharedDesktopTaskOnUserStop: (taskId: string, abortOrigin: unknown) => Promise; resolveStreamAttemptAtEvent: (taskId: string) => unknown; @@ -211,7 +213,7 @@ describe("TaskService settlement receipt producers (G2)", () => { expect(seeded.success).toBe(true); expect(await taskService.markInterruptedTaskRunning(taskId)).toBe(true); const attemptId = entryOf(config, taskId)!.taskAttemptId!; - expect(internals(taskService).ownedAttemptByTaskId.get(taskId)).toMatchObject({ + expect(internals(taskService).attemptLedger.ownedAttemptByTaskId.get(taskId)).toMatchObject({ attemptId, receiptEligible: true, }); @@ -306,7 +308,7 @@ describe("TaskService settlement receipt producers (G2)", () => { const attemptId = await ownEligibleAttempt(config, taskService, taskId); await svc.releaseSharedDesktopTaskOnUserStop(taskId, svc.resolveStreamAttemptAtEvent(taskId)); expect(entryOf(config, taskId)?.taskStatus).toBe("interrupted"); - expect(svc.attemptSettlementByTaskId.get(taskId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)).toMatchObject({ attemptId, phase: "settled", source: "user-stop-idle", @@ -328,7 +330,7 @@ describe("TaskService settlement receipt producers (G2)", () => { { expectedAttemptId: attemptId } ); expect(entryOf(config, taskId)?.taskStatus).toBe("interrupted"); - expect(svc.attemptSettlementByTaskId.get(taskId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)).toMatchObject({ attemptId, phase: "settled", source: "terminal-failure", @@ -385,7 +387,7 @@ describe("TaskService settlement receipt producers (G2)", () => { return true; }); expect(receiptAtRelease).toBe(true); - expect(svc.attemptSettlementByTaskId.get(taskId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)).toMatchObject({ attemptId, phase: "settled", source: "stop-settled", @@ -454,7 +456,7 @@ describe("TaskService settlement receipt producers (G2)", () => { cleared.add(timer); realClearTimeout(timer); }) as typeof clearTimeout); - const listenerSets = spyOn(svc.attemptSettlementListenersByTaskId, "set"); + const listenerSets = spyOn(svc.attemptLedger.attemptSettlementListenersByTaskId, "set"); // This file restores its spies per test (no mock.restore in afterEach). const restoreSpies = () => { for (const spy of [writeSpy, waitSpy, setSpy, clearSpy, listenerSets]) spy.mockRestore(); @@ -495,7 +497,7 @@ describe("TaskService settlement receipt producers (G2)", () => { expect(launch).not.toHaveBeenCalled(); const attemptId = entryOf(config, spawnedId)!.taskAttemptId!; expect(entryOf(config, spawnedId)?.taskStatus).toBe("interrupted"); - expect(svc.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ attemptId, phase: "settled", source: "reservation-canceled", @@ -520,7 +522,7 @@ describe("TaskService settlement receipt producers (G2)", () => { expect(created.success).toBe(false); const attemptId = entryOf(config, spawnedId)!.taskAttemptId!; expect(entryOf(config, spawnedId)?.taskStatus).toBe("interrupted"); - expect(svc.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ attemptId, phase: "settled", source: "reservation-failed", @@ -564,9 +566,9 @@ describe("TaskService settlement receipt producers (G2)", () => { await waitForCondition(() => entryOf(config, spawnedId)?.taskStatus === "interrupted"); const attemptId = entryOf(config, spawnedId)!.taskAttemptId!; await waitForCondition( - () => svc.attemptSettlementByTaskId.get(spawnedId)?.phase === "settled" + () => svc.attemptLedger.attemptSettlementByTaskId.get(spawnedId)?.phase === "settled" ); - expect(svc.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ attemptId, source: "launch-failed", }); @@ -620,7 +622,7 @@ describe("TaskService settlement receipt producers (G2)", () => { expect(svc.taskStopLatch.workspaceStopRecords.has(taskId)).toBe(false); } expect(entryOf(config, taskId)?.taskStatus).toBe("interrupted"); - expect(svc.attemptSettlementByTaskId.get(taskId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)).toMatchObject({ attemptId, phase: "closing", }); @@ -650,7 +652,7 @@ describe("TaskService settlement receipt producers (G2)", () => { // bound and then marks the successor unproven. shortenClosureSettleWait(); expect(await taskService.markInterruptedTaskRunning(taskId)).toBe(true); - expect(svc.ownedAttemptByTaskId.get(taskId)?.receiptEligible).toBe(false); + expect(svc.attemptLedger.ownedAttemptByTaskId.get(taskId)?.receiptEligible).toBe(false); expect(entryOf(config, taskId)?.taskAttemptUnproven).toBe(true); } ); @@ -703,7 +705,7 @@ describe("TaskService settlement receipt producers (G2)", () => { await fsPromises.writeFile(configPath, goodBytes); expect(entryOf(config, taskId)?.taskAttemptId).toBe(attemptId); - expect(svc.attemptSettlementByTaskId.get(taskId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)).toMatchObject({ attemptId, phase: "closing", }); @@ -747,7 +749,7 @@ describe("TaskService settlement receipt producers (G2)", () => { } finally { indexSpy.mockRestore(); } - expect(svc.attemptSettlementByTaskId.get(taskId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)).toMatchObject({ attemptId, phase: "closing", }); @@ -799,7 +801,7 @@ describe("TaskService settlement receipt producers (G2)", () => { await svc.releaseSharedDesktopTaskOnUserStop(taskId, svc.resolveStreamAttemptAtEvent(taskId)); - expect(svc.attemptSettlementByTaskId.get(taskId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)).toMatchObject({ attemptId, phase: "settled", }); @@ -1015,7 +1017,7 @@ describe("TaskService settlement receipt producers (G2)", () => { // No receipt for the marked child's predecessor: its reawakened attempt is ineligible. expect(await first.taskService.markInterruptedTaskRunning(marked)).toBe(true); const markedAttempt = entryOf(config, marked)!.taskAttemptId!; - expect(first.svc.ownedAttemptByTaskId.get(marked)?.receiptEligible).toBe(false); + expect(first.svc.attemptLedger.ownedAttemptByTaskId.get(marked)?.receiptEligible).toBe(false); await first.taskService.terminateAllDescendantAgentTasks(midId); await waitForCondition( () => @@ -1031,7 +1033,9 @@ describe("TaskService settlement receipt producers (G2)", () => { await restarted.svc.evaluateAttemptLineage(eligible, entryOf(restartedConfig, eligible)!) ).toEqual({ proven: true, reason: "settlement receipt found" }); expect(await restarted.taskService.markInterruptedTaskRunning(eligible)).toBe(true); - expect(restarted.svc.ownedAttemptByTaskId.get(eligible)?.receiptEligible).toBe(true); + expect(restarted.svc.attemptLedger.ownedAttemptByTaskId.get(eligible)?.receiptEligible).toBe( + true + ); expect(entryOf(restartedConfig, eligible)?.taskAttemptUnproven).toBeUndefined(); expect(entryOf(restartedConfig, eligible)?.taskAttemptId).not.toBe(eligibleAttempt); @@ -1040,7 +1044,9 @@ describe("TaskService settlement receipt producers (G2)", () => { .proven ).toBe(false); expect(await restarted.taskService.markInterruptedTaskRunning(marked)).toBe(true); - expect(restarted.svc.ownedAttemptByTaskId.get(marked)?.receiptEligible).toBe(false); + expect(restarted.svc.attemptLedger.ownedAttemptByTaskId.get(marked)?.receiptEligible).toBe( + false + ); expect(entryOf(restartedConfig, marked)?.taskAttemptUnproven).toBe(true); }); }); @@ -1075,7 +1081,7 @@ describe("TaskService settlement receipt producers (G2)", () => { const attemptId = entryOf(config, spawnedId)!.taskAttemptId!; expect(entryOf(config, spawnedId)?.taskStatus).toBe("interrupted"); // The obligation may still run toward a report: settled in memory only, no durable claim. - expect(svc.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ attemptId, phase: "settled", source: "reservation-canceled", @@ -1215,7 +1221,7 @@ describe("TaskService settlement receipt producers (G2)", () => { // The Stop resolves once its latch dropped (#6727). expect((await stopped).success).toBe(true); expect(taskService.isWorkspaceStopInProgress(taskId)).toBe(false); - expect(svc.attemptSettlementByTaskId.get(taskId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(taskId)).toMatchObject({ attemptId, phase: "settled", source: "stop-settled", @@ -1286,7 +1292,7 @@ describe("TaskService settlement receipt producers (G2)", () => { expect((await taskService.create(direct)).success).toBe(false); const attemptId = entryOf(config, spawnedId)!.taskAttemptId!; expect(entryOf(config, spawnedId)?.taskStatus).toBe("interrupted"); - expect(svc.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ attemptId, phase: "settled", source: "launch-failed", @@ -1295,7 +1301,9 @@ describe("TaskService settlement receipt producers (G2)", () => { const restartedConfig = await createTestConfig(rootDir); const restarted = createHarness(restartedConfig); expect(await restarted.taskService.markInterruptedTaskRunning(spawnedId)).toBe(true); - expect(restarted.svc.ownedAttemptByTaskId.get(spawnedId)?.receiptEligible).toBe(true); + expect(restarted.svc.attemptLedger.ownedAttemptByTaskId.get(spawnedId)?.receiptEligible).toBe( + true + ); expect(entryOf(restartedConfig, spawnedId)?.taskAttemptUnproven).toBeUndefined(); }); @@ -1323,7 +1331,7 @@ describe("TaskService settlement receipt producers (G2)", () => { const { taskService, svc } = createHarness(config, { workspaceService }); expect((await taskService.create(direct)).success).toBe(false); const attemptId = entryOf(config, spawnedId)!.taskAttemptId!; - expect(svc.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ attemptId, phase: "settled", source: "launch-failed", @@ -1342,7 +1350,7 @@ describe("TaskService settlement receipt producers (G2)", () => { const { taskService, svc } = failingSanitizeHarness(config); expect((await taskService.create(direct)).success).toBe(false); const attemptId = entryOf(config, spawnedId)!.taskAttemptId!; - expect(svc.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ + expect(svc.attemptLedger.attemptSettlementByTaskId.get(spawnedId)).toMatchObject({ attemptId, phase: "closing", }); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 0747170821b..382842abde3 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -70,7 +70,6 @@ import { } from "@/node/services/agentTaskIndex"; import { readSubagentAttemptSettlementReceiptStrict, - writeSubagentAttemptSettlementReceipt, type SubagentAttemptSettlementReceiptReadResult, type SubagentAttemptSettlementSource, } from "@/node/services/subagentAttemptSettlements"; @@ -111,6 +110,7 @@ import { TASK_FAMILY_MESSAGE_MAX_CHARS, } from "@/constants/taskMessages"; import { log } from "@/node/services/log"; +import { AttemptLedger } from "@/node/services/attemptLedger"; import { TaskStopLatch } from "@/node/services/taskStopLatch"; import { PeerSendParking } from "@/node/services/peerSendParking"; import { @@ -157,6 +157,7 @@ import { tryReadGitHeadCommitSha, findWorkspaceEntry, markChildGoalPauseOwed, + rowSupersedes, } from "@/node/services/taskUtils"; import { validateWorkspaceName } from "@/common/utils/validation/workspaceValidation"; import { stripTrailingSlashes } from "@/node/utils/pathUtils"; @@ -670,8 +671,8 @@ export interface OwnedTaskAttempt { receiptEligible: boolean; } -/** In-process settlement ledger entry; see TaskService.attemptSettlementByTaskId. */ -interface AttemptSettlementEntry { +/** In-process settlement ledger entry; see AttemptLedger.attemptSettlementByTaskId. */ +export interface AttemptSettlementEntry { /** The attempt id the closure/settlement names; undefined only for pre-identity entries. */ readonly attemptId: string | undefined; /** Present when the attempt is owned by this process (only owners can reach `settled`). */ @@ -1509,21 +1510,6 @@ function unownedSettlementReceiptGap( return undefined; } -/** - * Attempt compare-and-swap for a task-row write made on behalf of one attempt (stream end, report - * publication, recovery, terminal failure, resume rollback). Evaluated inside the write's own - * editWorkspaceEntry transform: true when the caller captured an attempt and the row now names - * another one — another writer (e.g. a second backend) re-admitted it, so the row is theirs and - * the write is abandoned. A write with no captured attempt (legacy/pre-identity) is never held - * back, which keeps today's behavior for it. - */ -function rowSupersedes( - row: Pick | undefined, - expectedAttemptId: string | null | undefined -): boolean { - return expectedAttemptId != null && row?.taskAttemptId !== expectedAttemptId; -} - /** * Task-pinned send options for an automatic TaskService turn (required-report prompt today; * child goal turns later). A whitelist on purpose: per-send payload (attachments, muxMetadata, @@ -2022,6 +2008,7 @@ export class TaskService implements AgentTaskIntegration { private readonly workspaceUserStopEpochs = new Map(); private readonly peerSendParking: PeerSendParking; private readonly taskStopLatch: TaskStopLatch; + private readonly attemptLedger: AttemptLedger; /** * Child stream ends cut by a host-selected continuation (queued input or context-budget * hand-over) whose successor has not yet streamed or been withdrawn. Keyed by task; settled @@ -2044,36 +2031,6 @@ export class TaskService implements AgentTaskIntegration { attemptId: AttemptFence; } >(); - /** - * Attempt ownership ledger (see OwnedTaskAttempt); replaced before any new admission, dropped - * once the attempt's report is durable (releaseReportedTaskAttempt). - */ - private readonly ownedAttemptByTaskId = new Map(); - /** - * Closure/settlement of exactly the attempt it names (AttemptSettlementEntry). A `closing` entry - * is recorded synchronously by every producer before its first awaited write and refuses further - * sends under that id; only the owner's settleOwnedTaskAttempt upgrades it to `settled`. Never - * removed by a failed write — a new admission (fresh id) is the only way to reopen. - */ - private readonly attemptSettlementByTaskId = new Map(); - private readonly attemptSettlementListenersByTaskId = new Map void>>(); - /** - * Failed reawakens whose closed attempt still awaits settlement (#4310): the owner snapshot - * restoreInterruptedTaskAfterResumeFailure closed, settled by recheckResumeFailureSettlement. - */ - private readonly resumeFailureSettlementByTaskId = new Map< - string, - OwnedTaskAttempt & { attemptId: string } - >(); - /** - * The last attempt id THIS process published for each task (every local rotation, owned or - * not). Not an authority on the current id — currentTaskAttemptId reads the persisted row, so - * a rotation committed by another process is honored and a row that is missing or unreadable - * yields no id at all. Never read to classify or authorize: an unreadable registry refuses - * every send regardless of this map (see admitTaskWorkspaceTurn), since absence here is no - * proof that a workspace is not a task. - */ - private readonly currentAttemptIdByTaskId = new Map(); /** Outstanding send obligations per task (see AdmittedSend); discharged entries are removed. */ private readonly admittedSendsByTaskId = new Map>(); /** @@ -2459,7 +2416,7 @@ export class TaskService implements AgentTaskIntegration { decisionTurnSettled = true; } if (decisionTurnSettled) this.pruneStreamEndDecisions(workspaceId); - this.recheckResumeFailureSettlement(workspaceId); + this.attemptLedger.recheckResumeFailureSettlement(workspaceId); this.taskStopLatch.releaseCapturedTurn(workspaceId, turnGeneration); } @@ -2483,343 +2440,50 @@ export class TaskService implements AgentTaskIntegration { this.taskStopLatch.rebindCapturedTurn(workspaceId, previous, next); } - /** - * A new attempt begins in this process (reservation commit, launch, reawakening, reactivation). - * Replaces the owned identity and drops any settlement of the previous attempt SYNCHRONOUSLY, - * before the admission that follows — current ownership always wins over a stale settlement. - */ + /** Tests call this; AttemptLedger owns the attempt identity. */ private beginOwnedTaskAttempt( taskId: string, source: string, identity: { attemptId: string | undefined; receiptEligible: boolean; abortSignal?: AbortSignal } ): OwnedTaskAttempt { - assert(taskId.length > 0, "beginOwnedTaskAttempt: taskId must be non-empty"); - if (identity.attemptId != null) { - assertTaskAttemptId(identity.attemptId, "beginOwnedTaskAttempt"); - } - const attempt: OwnedTaskAttempt = { - generation: (this.ownedAttemptByTaskId.get(taskId)?.generation ?? 0) + 1, - source, - attemptId: identity.attemptId, - receiptEligible: identity.receiptEligible, - ...(identity.abortSignal != null ? { abortSignal: identity.abortSignal } : {}), - }; - this.ownedAttemptByTaskId.set(taskId, attempt); - if (identity.attemptId != null) this.currentAttemptIdByTaskId.set(taskId, identity.attemptId); - // A closure recorded for THIS id (between the rotating CAS and this block) stays: the fresh - // admission that follows must observe it and refuse. Only another attempt's entry is stale. - if (this.attemptSettlementByTaskId.get(taskId)?.attemptId !== identity.attemptId) { - this.attemptSettlementByTaskId.delete(taskId); - } - // Predecessors' stream-end decisions have no reader left that they could authorize: a send - // bound to a predecessor reads stale from the row's new id at its next gate. - this.streamEndDecisionsByTaskId.delete(taskId); - return attempt; + return this.attemptLedger.beginOwnedTaskAttempt(taskId, source, identity); } - /** - * Linearization point of every settlement producer: close the attempt to further sends BEFORE - * the producer's first awaited write, so no send can be admitted for an id whose settlement is - * under way. Synchronous — the admission fence (admitTaskWorkspaceTurn) is synchronous too, so - * the two can never interleave and no lock is needed. Never downgrades a `settled` entry and - * never touches an entry naming another attempt. `ownedAttempt` (the producer's snapshot of - * this process's owner) is recorded only when it holds exactly `attemptId`. - */ + /** Tests spy on this; AttemptLedger owns the closure. */ private closeAttemptAdmission( taskId: string, attemptId: string | undefined, ownedAttempt: OwnedTaskAttempt | undefined, source: string ): void { - const existing = this.attemptSettlementByTaskId.get(taskId); - if (existing != null && existing.attemptId === attemptId) return; - const attempt = ownedAttempt?.attemptId === attemptId ? ownedAttempt : undefined; - this.attemptSettlementByTaskId.set(taskId, { - attemptId, - ...(attempt != null ? { attempt } : {}), - phase: "closing", - source, - }); - this.notifyAttemptSettlementListeners(taskId); + this.attemptLedger.closeAttemptAdmission(taskId, attemptId, ownedAttempt, source); } - /** - * Authoritative settlement by the attempt's owner (stop record released, launch failed, - * terminal stream failure). `attempt` is the identity captured BEFORE the owner's awaited - * write; a settlement for an attempt that is no longer current, or for no owned attempt at - * all (legacy/prior-process execution), is never recorded — only announced. - */ + /** Tests call this; AttemptLedger owns the settlement. */ private settleOwnedTaskAttempt( taskId: string, attempt: OwnedTaskAttempt | undefined, source: string ): void { - if (attempt != null && this.ownedAttemptByTaskId.get(taskId) === attempt) { - this.attemptSettlementByTaskId.set(taskId, { - attemptId: attempt.attemptId, - attempt, - phase: "settled", - source, - }); - } else { - log.debug("Task attempt settlement without a current owned attempt", { - taskId, - source, - owned: attempt?.generation, - }); - } - this.notifyAttemptSettlementListeners(taskId); - } - - /** - * Only this process's CURRENT owned attempt, admitted with proven lineage under a persisted id, - * may produce a cross-process receipt. Anything else (unowned or prior-process attempt, marked - * lineage, pre-identity entry, an owner a reawakening or a durable report already replaced) - * settles in memory only, exactly as before receipts existed. In-memory checks only: whether - * the row still names the attempt is decided by one strict read (readSettlementReceiptRow). - */ - private ownsReceiptEligibleAttempt( - taskId: string, - attempt: OwnedTaskAttempt | undefined - ): attempt is OwnedTaskAttempt & { attemptId: string } { - return ( - attempt?.attemptId != null && - attempt.receiptEligible && - this.ownedAttemptByTaskId.get(taskId) === attempt - ); - } - - /** - * The one strict config read a receipt decision rests on. `ours`: the row names `attemptId`, - * with the owner dirs to write into (parent first). `elsewhere`: CONFIRMED gone, parentless or - * naming another attempt (rowSupersedes) — a successor another writer admitted must never be - * vouched for by its predecessor's settlement. `unreadable`: the config (or the parent chain) - * could not be read, which is no evidence either way and must never be taken for `elsewhere`. - */ - private readSettlementReceiptRow( - taskId: string, - attemptId: string - ): - | { kind: "ours"; parentWorkspaceId: string; owners: string[] } - | { kind: "elsewhere" } - | { kind: "unreadable"; error: string } { - let parentWorkspaceId: string | undefined; - let owners: string[]; - try { - const cfg = this.config.loadConfigOrDefault({ throwOnError: true }); - const row = findWorkspaceEntry(cfg, taskId)?.workspace; - parentWorkspaceId = coerceNonEmptyString(row?.parentWorkspaceId); - if (row == null || parentWorkspaceId == null || rowSupersedes(row, attemptId)) { - log.info("[task-attempt] settlement receipt skipped: the row no longer names the attempt", { - taskId, - attemptId, - current: row?.taskAttemptId, - }); - return { kind: "elsewhere" }; - } - owners = this.listAncestorWorkspaceIdsUsingParentById( - this.buildAgentTaskIndex(cfg).parentById, - taskId - ); - } catch (error: unknown) { - // An unreadable config, or a parent cycle in it. - return { kind: "unreadable", error: getErrorMessage(error) }; - } - // Inconsistent ancestry is malformed config, not a programming error: fail closed like an - // unreadable read instead of throwing out of a settlement path. (Duplicate workspace ids no - // longer cause it: the row lookup and the task index both take the first row, #4550.) - if (owners[0] !== parentWorkspaceId) { - return { - kind: "unreadable", - error: `inconsistent ancestry for ${taskId}: parent ${parentWorkspaceId}, first owner ${owners[0] ?? "none"}`, - }; - } - return { kind: "ours", parentWorkspaceId, owners }; - } - - /** - * Durable settlement for producers that established that no execution can still publish a - * report for `attempt` (idle stop, idle terminal failure, reservation canceled/failed, launch - * failed before its send): the receipt first, then the in-memory settlement. - */ - private async persistOwnedAttemptSettlement( - taskId: string, - attempt: OwnedTaskAttempt | undefined, - receiptSource: SubagentAttemptSettlementSource, - settlementSource: string - ): Promise { - const decision = this.decideSettlementReceipt(taskId, attempt, settlementSource); - if (decision.kind === "unreadable") return; - if ( - decision.kind === "write" && - !(await this.writeSettlementReceipt(taskId, decision, receiptSource)) - ) { - return; - } - this.settleOwnedTaskAttempt(taskId, attempt, settlementSource); - } - - /** - * THE no-report boundary every receipt producer must prove, evaluated synchronously in the tick - * that decides the write (see decideSettlementReceipt): a receipt may claim that `attempt` can no - * longer publish a report only when ALL of these hold at once — - * 1. it is closed to admissions: a closure names it (closeAttemptAdmission, recorded by the - * producer before its first await), or a stop record latched for it since capture; - * 2. no send obligation of THIS attempt exists in any state (pending, enqueued, admitted) — - * keyed by attempt id, so a successor's obligations neither block nor satisfy it; - * 3. nothing is live for the task: no turn generation, workspace-turn registration or stream - * (task-level signals; any of them blocks, which is the safe direction); - * 4. no stream-end decision of the attempt is pending, and none resolved as a report - * (published) or unknowable (indeterminate). Such a resolution also downgrades the - * attempt's receiptEligible (resolveStreamEndDecision), so it still counts once the - * decision itself was pruned. - * False means no durable claim: the producer settles in memory only, exactly as before receipts. - */ - private attemptCannotStillReport( - taskId: string, - attempt: OwnedTaskAttempt & { attemptId: string } - ): boolean { - const attemptId = attempt.attemptId; - const closed = - this.isAttemptClosed(taskId, attemptId) || - this.taskStopLatch.stopRecordAttemptId(taskId) === attemptId; - if (!closed) return false; - for (const send of this.admittedSendsByTaskId.get(taskId) ?? []) { - if (send.attemptId === attemptId) return false; - } - if ( - this.workspaceService.getActiveTurnGeneration(taskId) != null || - this.getWorkspaceTurnManager().getLiveWorkspaceTurnRegistration(taskId) != null || - this.aiService.isStreaming(taskId) - ) { - return false; - } - const decision = this.findStreamEndDecision(taskId, attemptId); - return decision == null || decision.outcome === "nonreport"; - } - - /** - * closeAttemptAdmission that never replaces a closure recorded for a DIFFERENT attempt: used - * where the row's current attempt is unknown or may differ (another writer re-admitted it), so - * replacing that closure could reopen the row's attempt to sends. - */ - private closeAttemptAdmissionUnlessOtherClosed( - taskId: string, - attemptId: string, - attempt: OwnedTaskAttempt | undefined, - source: string - ): void { - const existing = this.attemptSettlementByTaskId.get(taskId); - if (existing != null && existing.attemptId !== attemptId) return; - this.closeAttemptAdmission(taskId, attemptId, attempt, source); + this.attemptLedger.settleOwnedTaskAttempt(taskId, attempt, source); } - /** - * The synchronous half of a receipt-bearing settlement, from one strict read: - * - `memory-only`: no receipt authority (ownsReceiptEligibleAttempt), the no-report boundary is - * not proven (attemptCannotStillReport), or the read CONFIRMED the row gone or naming another - * attempt — today's in-memory-only settlement follows; - * - `write`: the row names the attempt, which is closed to sends here (before any await) and - * must not be settled until writeSettlementReceipt succeeds; - * - `unreadable`: no evidence either way, so fail closed — the attempt is closed but never - * settled (cleanup-pending here, indeterminate after a restart) and gets no receipt; with the - * row unknown, a closure recorded for a different attempt is kept (it may be the row's). - * A failed write, like `unreadable`, leaves the attempt `closing`: never settled without its - * receipt, so no successor reads it as settled when a restart would not. - */ + /** Tests spy on this; AttemptLedger owns the decision. */ private decideSettlementReceipt( taskId: string, attempt: OwnedTaskAttempt | undefined, settlementSource: string ): SettlementReceiptDecision { - if (!this.ownsReceiptEligibleAttempt(taskId, attempt)) return { kind: "memory-only" }; - const attemptId = attempt.attemptId; - if (!this.attemptCannotStillReport(taskId, attempt)) return { kind: "memory-only" }; - const row = this.readSettlementReceiptRow(taskId, attemptId); - if (row.kind === "elsewhere") return { kind: "memory-only" }; - if (row.kind === "unreadable") { - log.warn("[task-attempt] settlement receipt not written: config unreadable", { - taskId, - attemptId, - error: row.error, - }); - this.closeAttemptAdmissionUnlessOtherClosed(taskId, attemptId, attempt, settlementSource); - return { kind: "unreadable" }; - } - // Synchronous with the read: the row names this attempt, so any closure replaced here names - // an attempt the row no longer does. - this.closeAttemptAdmission(taskId, attemptId, attempt, settlementSource); - return { - kind: "write", - attemptId, - parentWorkspaceId: row.parentWorkspaceId, - owners: row.owners, - }; + return this.attemptLedger.decideSettlementReceipt(taskId, attempt, settlementSource); } - /** - * Write the receipt into every owner dir: outermost ancestor first, the parent LAST, stopping - * at the first failure, so the parent's copy (the one lineage proof reads) exists only once - * every other owner holds one too. True when all writes succeeded. Never throws. - */ - private async writeSettlementReceipt( + /** Tests spy on this; AttemptLedger owns the write. */ + private writeSettlementReceipt( taskId: string, target: { attemptId: string; parentWorkspaceId: string; owners: string[] }, source: SubagentAttemptSettlementSource ): Promise { - const receipt = { - taskId, - attemptId: target.attemptId, - parentWorkspaceId: target.parentWorkspaceId, - source, - settledAt: new Date().toISOString(), - }; - for (const ownerWorkspaceId of [...target.owners].reverse()) { - const result = await writeSubagentAttemptSettlementReceipt({ - ownerWorkspaceSessionDirs: [path.join(this.config.sessionsDir, ownerWorkspaceId)], - receipt, - }); - if (!result.success) { - log.warn("[task-attempt] settlement receipt write failed; attempt left closing", { - taskId, - attemptId: target.attemptId, - ownerWorkspaceId, - error: result.error, - }); - return false; - } - } - return true; - } - - /** - * The ledger entries of an attempt whose report is durable are redundant: every reader - * (inspectAttemptOutcome, waitForAgentReport) classifies a persisted report first, so neither - * this attempt's ownership nor its settlement can decide an outcome again. Dropping them keeps - * the per-task ledgers bounded by the tasks still able to end without a report, instead of by - * every task ever attempted in this process. - * - * `attempt` is the identity captured at the stream-end event that carried the report (see the - * stream-end listener), so the drop names the attempt the report belongs to — a task reawakened - * (markInterruptedTaskRunning, not under the task's event lock) while that report waited for - * the lock or for its writes owns a different identity and stays owned; an older report never - * decides the new attempt's outcome. Called only after every ancestor artifact write succeeded: - * a report that is not durable everywhere leaves the entries in place. - * - * Interrupted-without-report receipts are deliberately NOT dropped here or on stop: an ancestor - * that has not yet read terminal-no-report still needs the receipt after config cleanup, and no - * consumer acknowledgement exists yet to bound that retention. - * - * The released shape (reported row, no owner, no settlement entry) is itself read by - * markInterruptedTaskRunning: a manual follow-up into such a child mints a fresh attempt rather - * than being admitted under the completed id. - */ - private releaseReportedTaskAttempt(taskId: string, attempt: OwnedTaskAttempt | undefined): void { - if (attempt == null || this.ownedAttemptByTaskId.get(taskId) !== attempt) return; - this.ownedAttemptByTaskId.delete(taskId); - if (this.attemptSettlementByTaskId.get(taskId)?.attempt === attempt) { - this.attemptSettlementByTaskId.delete(taskId); - } + return this.attemptLedger.writeSettlementReceipt(taskId, target, source); } // --------------------------------------------------------------------------------------------- @@ -2895,9 +2559,7 @@ export class TaskService implements AgentTaskIntegration { cancelled.resolve(); } }; - const listeners = this.attemptSettlementListenersByTaskId.get(taskId) ?? new Set(); - listeners.add(listener); - this.attemptSettlementListenersByTaskId.set(taskId, listeners); + this.attemptLedger.addAttemptSettlementListener(taskId, listener); listener(); log.info("[task-goal] stream accounting unsettled; deferring the child stream end", { taskId, @@ -2907,7 +2569,7 @@ export class TaskService implements AgentTaskIntegration { goalService.streamAccountingReceiptOutcome(taskId, event.messageId), cancelled.promise, ]) - .finally(() => this.removeAttemptSettlementListener(taskId, listener)) + .finally(() => this.attemptLedger.removeAttemptSettlementListener(taskId, listener)) .then(() => { // #7003: this stream deferred while its row read running, and nothing moves a running row // to a completed report under the same attempt but a publication (ours resolved the @@ -3049,26 +2711,9 @@ export class TaskService implements AgentTaskIntegration { } } + /** Tests call this; AttemptLedger owns the listeners. */ private notifyAttemptSettlementListeners(taskId: string): void { - const listeners = this.attemptSettlementListenersByTaskId.get(taskId); - if (!listeners) return; - for (const listener of [...listeners]) { - listener(); - } - } - - /** TaskStopLatch's waits register here (TaskStopLatchDeps.addAttemptSettlementListener). */ - private addAttemptSettlementListener(taskId: string, listener: () => void): void { - const listeners = this.attemptSettlementListenersByTaskId.get(taskId) ?? new Set(); - listeners.add(listener); - this.attemptSettlementListenersByTaskId.set(taskId, listeners); - } - - private removeAttemptSettlementListener(taskId: string, listener: () => void): void { - const listeners = this.attemptSettlementListenersByTaskId.get(taskId); - if (!listeners) return; - listeners.delete(listener); - if (listeners.size === 0) this.attemptSettlementListenersByTaskId.delete(taskId); + this.attemptLedger.notifyAttemptSettlementListeners(taskId); } // --------------------------------------------------------------------------------------------- @@ -3091,26 +2736,6 @@ export class TaskService implements AgentTaskIntegration { return isTaskAttemptId(persisted) ? persisted : undefined; } - /** - * Publish a committed identity rotation to in-memory state: record the id this process - * committed (currentAttemptIdByTaskId), and drop a closure recorded for a previous attempt (a - * closure for the NEW id, recorded between the CAS and this call, stays). Tokens minted for an - * older id are revoked by the persisted row itself at their next gate. - */ - private publishAttemptRotation(taskId: string, attemptId: string): void { - assertTaskAttemptId(attemptId, "publishAttemptRotation"); - this.currentAttemptIdByTaskId.set(taskId, attemptId); - if (this.attemptSettlementByTaskId.get(taskId)?.attemptId !== attemptId) { - this.attemptSettlementByTaskId.delete(taskId); - } - } - - /** A closure or settlement is recorded for exactly this attempt id. */ - private isAttemptClosed(taskId: string, attemptId: string): boolean { - const entry = this.attemptSettlementByTaskId.get(taskId); - return entry?.attemptId === attemptId; - } - /** * Common send-authorization predicate (no ownership): the id is still the task's current * attempt, nothing closed it, no stop cascade holds the workspace and no workflow claim retired @@ -3119,7 +2744,7 @@ export class TaskService implements AgentTaskIntegration { private attemptAdmissionOpen(taskId: string, attemptId: string): boolean { const entry = findWorkspaceEntry(this.config.loadConfigOrDefault(), taskId)?.workspace; if (this.currentTaskAttemptId(taskId, entry) !== attemptId) return false; - if (this.isAttemptClosed(taskId, attemptId)) return false; + if (this.attemptLedger.isAttemptClosed(taskId, attemptId)) return false; if (this.isWorkspaceStopInProgress(taskId)) return false; return entry?.taskAttemptRetiredBy == null && entry?.pendingRemoval == null; } @@ -3142,7 +2767,7 @@ export class TaskService implements AgentTaskIntegration { * else the persisted id, applies as before. */ private resolveStreamAttemptAtEvent(taskId: string): StreamAttemptOrigin { - const owned = this.ownedAttemptByTaskId.get(taskId); + const owned = this.attemptLedger.ownedAttempt(taskId); const turn = this.workspaceService.getActiveTurnGeneration(taskId); // The turn's DIRECT admission decides; an obligation rebound from a superseded predecessor // (inherited) shares the turn id but not its attempt, and counts only when the turn has no @@ -3287,7 +2912,7 @@ export class TaskService implements AgentTaskIntegration { if (send.state === "admitted") return false; if (send.state === "discharged") return true; if (!this.attemptAdmissionOpen(taskId, attemptId)) return true; - if (send.attempt != null && this.ownedAttemptByTaskId.get(taskId) !== send.attempt) { + if (send.attempt != null && this.attemptLedger.ownedAttempt(taskId) !== send.attempt) { return true; } // A send still in its preflight while the attempt's last stream is undecided, or was @@ -3366,7 +2991,7 @@ export class TaskService implements AgentTaskIntegration { this.taskStopLatch.recheckWorkspaceStopRelease(taskId); } } - this.recheckResumeFailureSettlement(taskId); + this.attemptLedger.recheckResumeFailureSettlement(taskId); } /** @@ -3451,7 +3076,7 @@ export class TaskService implements AgentTaskIntegration { if (this.isWorkspaceStopInProgress(workspaceId)) { return { kind: "refused", message: WORKSPACE_STOP_IN_PROGRESS_SEND_BLOCKED_MESSAGE }; } - if (this.isAttemptClosed(workspaceId, attemptId)) { + if (this.attemptLedger.isAttemptClosed(workspaceId, attemptId)) { log.info("[task-attempt] send refused: attempt settled", { workspaceId, attemptId, @@ -3459,7 +3084,7 @@ export class TaskService implements AgentTaskIntegration { }); return { kind: "refused", message: TASK_ATTEMPT_SETTLED_SEND_BLOCKED_MESSAGE }; } - const owned = this.ownedAttemptByTaskId.get(workspaceId); + const owned = this.attemptLedger.ownedAttempt(workspaceId); const send = this.createAdmittedSend( workspaceId, attemptId, @@ -3485,14 +3110,17 @@ export class TaskService implements AgentTaskIntegration { if (entry.taskAttemptUnproven === true) { return { proven: false, reason: "lineage marked unproven" }; } - const owned = this.ownedAttemptByTaskId.get(taskId); + const owned = this.attemptLedger.ownedAttempt(taskId); if (owned?.attemptId === attemptId) { if (!owned.receiptEligible) { return { proven: false, reason: "predecessor owned but not receipt-eligible" }; } if (this.ownedAttemptSettlementPhase(taskId, owned) === "closing") { // The producer's awaited write is in flight; wait for its outcome (bounded, no lock). - await this.waitForAttemptClosureToSettle(taskId, ATTEMPT_CLOSURE_SETTLE_WAIT_MS); + await this.attemptLedger.waitForAttemptClosureToSettle( + taskId, + ATTEMPT_CLOSURE_SETTLE_WAIT_MS + ); } if (this.ownedAttemptSettlementPhase(taskId, owned) === "settled") { return { proven: true, reason: "predecessor settled by this process" }; @@ -3513,7 +3141,7 @@ export class TaskService implements AgentTaskIntegration { taskId: string, owned: OwnedTaskAttempt ): AttemptSettlementEntry["phase"] | undefined { - const settlement = this.attemptSettlementByTaskId.get(taskId); + const settlement = this.attemptLedger.attemptSettlement(taskId); return settlement?.attempt === owned ? settlement.phase : undefined; } @@ -3525,27 +3153,6 @@ export class TaskService implements AgentTaskIntegration { return this.taskStopLatch.waitForWorkspaceStopsToRelease(workspaceIds, deadlineMs); } - /** Resolve once the task's settlement entry leaves `closing` (or the bound elapses). */ - private async waitForAttemptClosureToSettle(taskId: string, timeoutMs: number): Promise { - const isClosing = () => this.attemptSettlementByTaskId.get(taskId)?.phase === "closing"; - if (!isClosing()) return; - const deadline = Date.now() + timeoutMs; - while (isClosing()) { - const remainingMs = deadline - Date.now(); - if (remainingMs <= 0) return; - const notified = Promise.withResolvers(); - const listener = () => notified.resolve(); - const listeners = this.attemptSettlementListenersByTaskId.get(taskId) ?? new Set(); - listeners.add(listener); - this.attemptSettlementListenersByTaskId.set(taskId, listeners); - try { - await raceWithAbortAndTimeout(notified.promise, { timeoutMs: remainingMs }); - } finally { - this.removeAttemptSettlementListener(taskId, listener); - } - } - } - /** * Startup re-drive of a `running`/`awaiting_report` task (recoverInterruptedTasks): the previous * process may still hold an admitted execution, so the re-driven work is a new, UNOWNED and @@ -3600,7 +3207,7 @@ export class TaskService implements AgentTaskIntegration { ); return undefined; } - this.publishAttemptRotation(taskId, attemptId); + this.attemptLedger.publishAttemptRotation(taskId, attemptId); return attemptId; } @@ -3819,7 +3426,7 @@ export class TaskService implements AgentTaskIntegration { assert(parentWorkspaceId.length > 0, "tombstoneUnpublishedReservation: parentWorkspaceId"); assert(taskId.length > 0, "tombstoneUnpublishedReservation: taskId"); return await this.workspaceEventLocks.withLock(taskId, async () => { - if (this.ownedAttemptByTaskId.has(taskId)) { + if (this.attemptLedger.hasOwnedAttempt(taskId)) { return Err("this process owns an attempt for the task"); } let refusal: string | undefined; @@ -3943,7 +3550,7 @@ export class TaskService implements AgentTaskIntegration { } catch { row = undefined; } - const owned = this.ownedAttemptByTaskId.get(taskId); + const owned = this.attemptLedger.ownedAttempt(taskId); const attemptId = row?.taskAttemptId; const marked = markedTerminalFailure(row, attemptId); if (marked != null && (owned?.attemptId == null || owned.attemptId === attemptId)) { @@ -3987,7 +3594,7 @@ export class TaskService implements AgentTaskIntegration { } const status = entry?.workspace.taskStatus ?? (entry != null ? "running" : undefined); - const owned = this.ownedAttemptByTaskId.get(taskId); + const owned = this.attemptLedger.ownedAttempt(taskId); if (owned == null) { if (entry == null) { const reason = "no task record and no attempt owned by this process"; @@ -4032,7 +3639,7 @@ export class TaskService implements AgentTaskIntegration { `row names attempt ${currentRow.taskAttemptId ?? "none"}; this process owns ${owned.attemptId ?? "none"}` ); } - const settlement = this.attemptSettlementByTaskId.get(taskId); + const settlement = this.attemptLedger.attemptSettlement(taskId); if (settlement?.attempt === owned) { // Closed but not yet settled: the producer's awaited write (config, later the receipt) is // still in flight. Its settlement is the guaranteed next signal (fail closed, not a wait). @@ -4061,118 +3668,12 @@ export class TaskService implements AgentTaskIntegration { ); } - /** - * Take the task's event lock for a short read while the caller's bound (deadline/abort) keeps - * applying to the WAIT for the lock: a publication holding it must not pin a bounded caller. - * Abandoned before entry → the late acquisition is a no-op that releases at once (nothing read, - * no subscription installed). Once entered, the operation runs to completion (never detached). - */ - private async runUnderEventLockBounded( - taskId: string, - operation: () => Promise, - bound: { signal?: AbortSignal; deadline: number } - ): Promise<{ kind: "ok"; value: T } | { kind: "timeout" } | { kind: "aborted" }> { - let entered = false; - let abandoned = false; - const run = this.workspaceEventLocks.withLock(taskId, async (): Promise => { - if (abandoned) return undefined; - entered = true; - return await operation(); - }); - return await new Promise((resolve, reject) => { - let timer: ReturnType | undefined; - const cleanup = () => { - if (timer != null) clearTimeout(timer); - bound.signal?.removeEventListener("abort", onAbort); - }; - const abandon = (result: { kind: "timeout" } | { kind: "aborted" }) => { - if (entered || abandoned) return; - abandoned = true; - cleanup(); - resolve(result); - }; - const onAbort = () => abandon({ kind: "aborted" }); - bound.signal?.addEventListener("abort", onAbort, { once: true }); - const remainingMs = bound.deadline - Date.now(); - if (bound.signal?.aborted) { - abandon({ kind: "aborted" }); - } else if (remainingMs <= 0) { - abandon({ kind: "timeout" }); - } else { - timer = setTimeout(() => abandon({ kind: "timeout" }), remainingMs); - timer.unref?.(); - } - run.then( - (value) => { - cleanup(); - if (!abandoned) resolve({ kind: "ok", value: value as T }); - }, - (error: unknown) => { - cleanup(); - if (!abandoned) reject(error instanceof Error ? error : new Error(String(error))); - } - ); - }); - } - - /** - * Bounded wait for the current attempt to settle: subscribe THEN re-read under the task's - * event lock (a settlement landing between the two is observed by the re-read), release the - * lock before awaiting, and re-read on every notification. The bound also covers waiting for - * the lock itself (runUnderEventLockBounded). `indeterminate` returns at once — an unknown - * owner never pins the caller. Timeout is a bounded outcome, abort rejects. - */ - async waitForAttemptSettlement( + /** Public entry point (WorkflowTaskServiceAdapter, WorkflowRunner); AttemptLedger owns the wait. */ + waitForAttemptSettlement( taskId: string, options: { abortSignal?: AbortSignal; timeoutMs: number; requestingWorkspaceId?: string } ): Promise> { - assert(taskId.length > 0, "waitForAttemptSettlement: taskId must be non-empty"); - assert( - Number.isFinite(options.timeoutMs) && options.timeoutMs > 0, - "waitForAttemptSettlement: timeoutMs invalid" - ); - const deadline = Date.now() + options.timeoutMs; - const bound = { signal: options.abortSignal, deadline }; - let notified = Promise.withResolvers(); - const listener = () => notified.resolve(); - try { - const first = await this.runUnderEventLockBounded( - taskId, - async () => { - const listeners = this.attemptSettlementListenersByTaskId.get(taskId) ?? new Set(); - listeners.add(listener); - this.attemptSettlementListenersByTaskId.set(taskId, listeners); - return await this.inspectAttemptOutcome(taskId, options); - }, - bound - ); - if (first.kind === "aborted") throw new Error("Interrupted"); - if (first.kind === "timeout") return { kind: "timeout" }; - let outcome = first.value; - while (outcome.kind === "live" || outcome.kind === "cleanup-pending") { - const remainingMs = deadline - Date.now(); - if (remainingMs <= 0) return { kind: "timeout" }; - const raced = await raceWithAbortAndTimeout(notified.promise, { - signal: options.abortSignal, - timeoutMs: remainingMs, - }); - if (raced.kind === "aborted") throw new Error("Interrupted"); - if (raced.kind === "timeout") return { kind: "timeout" }; - // Re-arm before re-reading so a notification during the read is never lost. - notified = Promise.withResolvers(); - const reread = await this.runUnderEventLockBounded( - taskId, - () => this.inspectAttemptOutcome(taskId, options), - bound - ); - if (reread.kind === "aborted") throw new Error("Interrupted"); - if (reread.kind === "timeout") return { kind: "timeout" }; - outcome = reread.value; - } - return outcome; - } finally { - this.removeAttemptSettlementListener(taskId, listener); - } + return this.attemptLedger.waitForAttemptSettlement(taskId, options); } private recordTaskInterrupted(taskId: string, parentWorkspaceId: string | undefined): void { @@ -4332,6 +3833,34 @@ export class TaskService implements AgentTaskIntegration { terminalAttentionStore?: TerminalAttentionStore, private readonly desktopInputCoordinator = new DesktopInputCoordinator(config) ) { + this.attemptLedger = new AttemptLedger({ + config: () => this.config, + aiService: () => this.aiService, + workspaceService: () => this.workspaceService, + turnManager: () => this.getWorkspaceTurnManager(), + workspaceEventLocks: () => this.workspaceEventLocks, + buildAgentTaskIndex: (cfg) => this.buildAgentTaskIndex(cfg), + listAncestorWorkspaceIdsUsingParentById: (parentById, taskId) => + this.listAncestorWorkspaceIdsUsingParentById(parentById, taskId), + stopRecordAttemptId: (taskId) => this.taskStopLatch.stopRecordAttemptId(taskId), + hasStopRecord: (taskId) => this.taskStopLatch.hasStopRecord(taskId), + admittedSends: (taskId) => this.admittedSendsByTaskId.get(taskId), + findStreamEndDecision: (taskId, attemptId) => this.findStreamEndDecision(taskId, attemptId), + discardStreamEndDecisions: (taskId) => { + this.streamEndDecisionsByTaskId.delete(taskId); + }, + // Through TaskService: tests spy on inspectAttemptOutcome on the instance. + inspectAttemptOutcome: (taskId, options) => this.inspectAttemptOutcome(taskId, options), + notifyAttemptSettlementListeners: (taskId) => this.notifyAttemptSettlementListeners(taskId), + closeAttemptAdmission: (taskId, attemptId, ownedAttempt, source) => + this.closeAttemptAdmission(taskId, attemptId, ownedAttempt, source), + settleOwnedTaskAttempt: (taskId, attempt, source) => + this.settleOwnedTaskAttempt(taskId, attempt, source), + decideSettlementReceipt: (taskId, attempt, settlementSource) => + this.decideSettlementReceipt(taskId, attempt, settlementSource), + writeSettlementReceipt: (taskId, target, source) => + this.writeSettlementReceipt(taskId, target, source), + }); this.taskStopLatch = new TaskStopLatch({ config: () => this.config, aiService: () => this.aiService, @@ -4343,14 +3872,14 @@ export class TaskService implements AgentTaskIntegration { this.peerSendParking.scheduleParkedPeerSendFlush(workspaceId), notifyAttemptSettlementListeners: (taskId) => this.notifyAttemptSettlementListeners(taskId), addAttemptSettlementListener: (taskId, listener) => - this.addAttemptSettlementListener(taskId, listener), + this.attemptLedger.addAttemptSettlementListener(taskId, listener), removeAttemptSettlementListener: (taskId, listener) => - this.removeAttemptSettlementListener(taskId, listener), - ownedAttempt: (taskId) => this.ownedAttemptByTaskId.get(taskId), + this.attemptLedger.removeAttemptSettlementListener(taskId, listener), + ownedAttempt: (taskId) => this.attemptLedger.ownedAttempt(taskId), currentTaskAttemptId: (taskId) => this.currentTaskAttemptId(taskId), admittedSends: (taskId) => this.admittedSendsByTaskId.get(taskId), ownsReceiptEligibleAttempt: (taskId, attempt) => - this.ownsReceiptEligibleAttempt(taskId, attempt), + this.attemptLedger.ownsReceiptEligibleAttempt(taskId, attempt), findStreamEndDecision: (taskId, attemptId) => this.findStreamEndDecision(taskId, attemptId), decideSettlementReceipt: (taskId, attempt, settlementSource) => this.decideSettlementReceipt(taskId, attempt, settlementSource), @@ -4360,7 +3889,8 @@ export class TaskService implements AgentTaskIntegration { this.closeAttemptAdmission(taskId, attemptId, ownedAttempt, source), settleOwnedTaskAttempt: (taskId, attempt, source) => this.settleOwnedTaskAttempt(taskId, attempt, source), - recheckResumeFailureSettlement: (taskId) => this.recheckResumeFailureSettlement(taskId), + recheckResumeFailureSettlement: (taskId) => + this.attemptLedger.recheckResumeFailureSettlement(taskId), }); this.peerSendParking = new PeerSendParking({ flush: (targetId) => this.flushParkedPeerSends(targetId), @@ -6455,7 +5985,7 @@ export class TaskService implements AgentTaskIntegration { // The first fence write that rejected; thrown once every plan was handled (#5028). let fenceFailure: { error: unknown } | undefined; for (const plan of plans) { - const ownedAttempt = this.ownedAttemptByTaskId.get(plan.taskId); + const ownedAttempt = this.attemptLedger.ownedAttempt(plan.taskId); let transitioned = canceledInsideCommit; let superseded = false; if (!canceledInsideCommit) { @@ -6494,7 +6024,7 @@ export class TaskService implements AgentTaskIntegration { this.rejectWaiters(plan.taskId, new Error(TASK_RESERVATION_CANCELED_MESSAGE)); } // Canceled before any launch was scheduled: nothing ran under the attempt. - await this.persistOwnedAttemptSettlement( + await this.attemptLedger.persistOwnedAttemptSettlement( plan.taskId, ownedAttempt, "reservation-canceled", @@ -6649,10 +6179,10 @@ export class TaskService implements AgentTaskIntegration { private closeReservedAttempts(plans: readonly TaskLaunchPlan[], source: string): void { for (const plan of plans) { if (plan.attemptId == null) continue; - this.closeAttemptAdmissionUnlessOtherClosed( + this.attemptLedger.closeAttemptAdmissionUnlessOtherClosed( plan.taskId, plan.attemptId, - this.ownedAttemptByTaskId.get(plan.taskId), + this.attemptLedger.ownedAttempt(plan.taskId), source ); } @@ -6728,7 +6258,7 @@ export class TaskService implements AgentTaskIntegration { } // The pre-launch path failed before scheduling: nothing ran under the attempt. A row the // failed commit never wrote gets no receipt (readSettlementReceiptRow: elsewhere). - await this.persistOwnedAttemptSettlement( + await this.attemptLedger.persistOwnedAttemptSettlement( plan.taskId, ownedAttempts.get(plan.taskId), signal?.aborted ? "reservation-canceled" : "reservation-failed", @@ -6781,7 +6311,7 @@ export class TaskService implements AgentTaskIntegration { * interrupt or delete them. Unowned tasks and pre-identity owners are never held back. */ private ownedAttemptSuperseded(taskId: string, row: WorkspaceConfigEntry | undefined): boolean { - const owned = this.ownedAttemptByTaskId.get(taskId)?.attemptId; + const owned = this.attemptLedger.ownedAttempt(taskId)?.attemptId; return owned != null && row?.taskAttemptId !== owned; } @@ -6917,7 +6447,7 @@ export class TaskService implements AgentTaskIntegration { let unpublished = false; if ( expectedAttemptId != null && - this.ownedAttemptByTaskId.get(taskId)?.attemptId === expectedAttemptId + this.attemptLedger.ownedAttempt(taskId)?.attemptId === expectedAttemptId ) { try { await this.config.editConfig((config) => { @@ -7186,7 +6716,7 @@ export class TaskService implements AgentTaskIntegration { // The launch owner gives up: settle exactly the attempt it owned when it decided. The // closure is recorded inside the updater, against the fresh row's id, before the write is // awaited — no send can be admitted for that id from here on. - const ownedAttempt = this.ownedAttemptByTaskId.get(taskId); + const ownedAttempt = this.attemptLedger.ownedAttempt(taskId); let transitionedToInterrupted = false; let superseded = false; let parentWorkspaceId: string | undefined; @@ -7245,7 +6775,7 @@ export class TaskService implements AgentTaskIntegration { // registered meanwhile must not be rejected as this attempt's. this.rejectWaiters(taskId, new Error(message)); if (neverSent) { - await this.persistOwnedAttemptSettlement( + await this.attemptLedger.persistOwnedAttemptSettlement( taskId, ownedAttempt, "launch-failed", @@ -7373,10 +6903,10 @@ export class TaskService implements AgentTaskIntegration { launchAttemptId = findWorkspaceEntry(this.config.loadConfigOrDefault(), plan.taskId)?.workspace .taskAttemptId ?? stamped; - this.publishAttemptRotation(plan.taskId, launchAttemptId); + this.attemptLedger.publishAttemptRotation(plan.taskId, launchAttemptId); } plan.attemptId = launchAttemptId; - if (this.ownedAttemptByTaskId.get(plan.taskId)?.attemptId !== launchAttemptId) { + if (this.attemptLedger.ownedAttempt(plan.taskId)?.attemptId !== launchAttemptId) { this.beginOwnedTaskAttempt(plan.taskId, "launch", { attemptId: launchAttemptId, receiptEligible: launchReceiptEligible, @@ -8374,7 +7904,10 @@ export class TaskService implements AgentTaskIntegration { abandonPartial: true, clearQueue: true, }); - await this.waitForAttemptClosureToSettle(taskId, ATTEMPT_CLOSURE_SETTLE_WAIT_MS); + await this.attemptLedger.waitForAttemptClosureToSettle( + taskId, + ATTEMPT_CLOSURE_SETTLE_WAIT_MS + ); if (this.taskStopLatch.hasStopRecord(taskId)) { log.warn("Task.create launch failure: the failed launch's attempt is still live", { taskId, @@ -8387,7 +7920,7 @@ export class TaskService implements AgentTaskIntegration { if (launchSendAdmitted) { this.settleOwnedTaskAttempt(taskId, launchAttempt, "launch-failed"); } else { - await this.persistOwnedAttemptSettlement( + await this.attemptLedger.persistOwnedAttemptSettlement( taskId, launchAttempt, "launch-failed", @@ -9001,7 +8534,7 @@ export class TaskService implements AgentTaskIntegration { message: "Sub-agent reactivation refused: its attempt changed concurrently; retry.", }); } - this.publishAttemptRotation(taskId, reactivationAttemptId); + this.attemptLedger.publishAttemptRotation(taskId, reactivationAttemptId); if (!committedProven) { log.info("[task-attempt] reactivated attempt is not receipt-eligible", { taskId, @@ -16249,7 +15782,7 @@ export class TaskService implements AgentTaskIntegration { ) { return; } - const owned = this.ownedAttemptByTaskId.get(taskId); + const owned = this.attemptLedger.ownedAttempt(taskId); const attemptId = owned?.attemptId != null && owned.attemptId === workspace.taskAttemptId ? owned.attemptId @@ -16277,8 +15810,8 @@ export class TaskService implements AgentTaskIntegration { }); continue; } - this.publishAttemptRotation(taskId, launch.attemptId); - if (this.ownedAttemptByTaskId.get(taskId)?.attemptId !== launch.attemptId) { + this.attemptLedger.publishAttemptRotation(taskId, launch.attemptId); + if (this.attemptLedger.ownedAttempt(taskId)?.attemptId !== launch.attemptId) { this.beginOwnedTaskAttempt(taskId, "launch", launch); } reservedSlots += 1; @@ -16311,7 +15844,7 @@ export class TaskService implements AgentTaskIntegration { launchLease, // A reservation this process owns keeps its cancellation across the queue. ...(() => { - const abortSignal = this.ownedAttemptByTaskId.get(taskId)?.abortSignal; + const abortSignal = this.attemptLedger.ownedAttempt(taskId)?.abortSignal; return abortSignal != null ? { abortSignal } : {}; })(), }); @@ -16483,7 +16016,7 @@ export class TaskService implements AgentTaskIntegration { if (!entryAtStart?.workspace.parentWorkspaceId) { return notApplicable; } - const settledPredecessor = this.attemptSettlementByTaskId.get(workspaceId); + const settledPredecessor = this.attemptLedger.attemptSettlement(workspaceId); // A parent continuation preserves `reported`; Stop then closes its attempt without changing // that status. Explicit manual recovery must rotate the settled identity, not reopen it. // @@ -16498,7 +16031,7 @@ export class TaskService implements AgentTaskIntegration { // the same way — nothing in this process can own or settle an id it never had — so it gets // its first, unproven attempt here rather than running as an unfenced pre-identity send. const releasedReportedAttempt = - settledPredecessor == null && !this.ownedAttemptByTaskId.has(workspaceId); + settledPredecessor == null && !this.attemptLedger.hasOwnedAttempt(workspaceId); const resumeSettledReportedTask = entryAtStart.workspace.taskStatus === "reported" && entryAtStart.workspace.taskDesktopOwnerWorkspaceId == null && @@ -16569,8 +16102,8 @@ export class TaskService implements AgentTaskIntegration { this.isWorkspaceStopInProgress(workspaceId) || this.getWorkspaceStopEpoch(workspaceId) !== stopEpochAtStart || (resumeSettledReportedTask && - (this.attemptSettlementByTaskId.get(workspaceId) !== settledPredecessor || - (releasedReportedAttempt && this.ownedAttemptByTaskId.has(workspaceId)))) + (this.attemptLedger.attemptSettlement(workspaceId) !== settledPredecessor || + (releasedReportedAttempt && this.attemptLedger.hasOwnedAttempt(workspaceId)))) ) { log.debug("markInterruptedTaskRunning refused: overtaken by a stop", { workspaceId }); return lost(SEND_ADMISSION_STALE_MESSAGE); @@ -16644,7 +16177,7 @@ export class TaskService implements AgentTaskIntegration { }); return lost(TASK_REAWAKEN_LOST_SEND_BLOCKED_MESSAGE); } - this.publishAttemptRotation(workspaceId, attemptId); + this.attemptLedger.publishAttemptRotation(workspaceId, attemptId); if (!committedProven) { log.info("[task-attempt] reawakened attempt is not receipt-eligible", { workspaceId, @@ -16691,7 +16224,7 @@ export class TaskService implements AgentTaskIntegration { // Captured before the awaited write: only the exact attempt the failed send was bound to, // and only while this process owns it, is ended here. - const ownedAttempt = this.ownedAttemptByTaskId.get(workspaceId); + const ownedAttempt = this.attemptLedger.ownedAttempt(workspaceId); let revertedToInterrupted = false; let revertedOwnedAttempt = false; let parentWorkspaceId: string | undefined; @@ -16729,7 +16262,7 @@ export class TaskService implements AgentTaskIntegration { revertedOwnedAttempt && expectedAttemptId != null && ownedAttempt != null && - this.ownedAttemptByTaskId.get(workspaceId) === ownedAttempt + this.attemptLedger.ownedAttempt(workspaceId) === ownedAttempt ) { this.closeAttemptAdmission(workspaceId, expectedAttemptId, ownedAttempt, "resume-failed"); // Identity is what later checks compare; its attemptId was checked to be a string. @@ -16742,44 +16275,13 @@ export class TaskService implements AgentTaskIntegration { this.recordTaskInterrupted(workspaceId, parentWorkspaceId); if (closedAttempt != null) { - this.resumeFailureSettlementByTaskId.set(workspaceId, closedAttempt); - this.recheckResumeFailureSettlement(workspaceId); + this.attemptLedger.recordResumeFailureSettlement(workspaceId, closedAttempt); + this.attemptLedger.recheckResumeFailureSettlement(workspaceId); } if (revertedToInterrupted) await this.settleChildGoalPause(workspaceId); await this.emitWorkspaceMetadata(workspaceId); } - /** - * Settle a failed reawaken's closed attempt (restoreInterruptedTaskAfterResumeFailure) once the - * shared no-report boundary holds (attemptCannotStillReport): never from the send's error alone. - * The failed send's obligation is typically disposed only after the rollback returned (a refused - * or no-work send), and an admitted preparation failure rolls back while its turn is still - * live, so this is rechecked on the rollback, on every obligation disposal and on every turn - * settlement. It drops out without settling when the attempt was superseded (a new owner) or - * settled by another producer (Stop), and defers to an in-progress Stop, whose release settles. - */ - private recheckResumeFailureSettlement(taskId: string): void { - const attempt = this.resumeFailureSettlementByTaskId.get(taskId); - if (attempt == null) return; - const closure = this.attemptSettlementByTaskId.get(taskId); - if ( - this.ownedAttemptByTaskId.get(taskId) !== attempt || - closure?.attempt !== attempt || - closure.phase !== "closing" - ) { - this.resumeFailureSettlementByTaskId.delete(taskId); - return; - } - if (this.taskStopLatch.hasStopRecord(taskId)) return; - if (!this.attemptCannotStillReport(taskId, attempt)) return; - this.resumeFailureSettlementByTaskId.delete(taskId); - this.persistOwnedAttemptSettlement(taskId, attempt, "idle-settled", "resume-failed").catch( - (error: unknown) => { - log.error("[task-attempt] failed reawaken settlement failed", { taskId, error }); - } - ); - } - private buildTaskCompletionRecoveryMessage( completionKind: "final_response" | "propose_plan", requiresStructuredOutput: boolean, @@ -18442,7 +17944,7 @@ export class TaskService implements AgentTaskIntegration { // process still owns the attempt it decided for: a send during the write may have begun a // new attempt (a user message reawakening the task), and entries queued behind that turn are // its input, not the stopped attempt's — clearing them would drop the user's message. - if (this.ownedAttemptByTaskId.get(workspaceId) === ownedAttempt) { + if (this.attemptLedger.ownedAttempt(workspaceId) === ownedAttempt) { this.workspaceService.clearQueue(workspaceId); } this.recordTaskInterrupted(workspaceId, parentWorkspaceId); @@ -18450,7 +17952,7 @@ export class TaskService implements AgentTaskIntegration { // backend reawakening the now-interrupted task) must not be rejected as this attempt's. this.rejectWaiters(workspaceId, new Error("Task interrupted")); // Verified idle inside the edit (no stream, no pending turn, no execution mirror). - await this.persistOwnedAttemptSettlement( + await this.attemptLedger.persistOwnedAttemptSettlement( workspaceId, ownedAttempt, "idle-settled", @@ -18659,7 +18161,7 @@ export class TaskService implements AgentTaskIntegration { // otherwise report publication may still be in flight (stream-end handling precedes turn // settlement), so a stop record captures the live work and the execution-settlement producer // settles the attempt when that record releases. - const ownedAttempt = this.ownedAttemptByTaskId.get(workspaceId); + const ownedAttempt = this.attemptLedger.ownedAttempt(workspaceId); // Close admission FIRST, synchronously with the decision: a send reaching the fence from here // on is refused, so the activity sample below is authoritative — every obligation or turn // admitted before it is visible to it and captured by the stop record, and none can be @@ -18795,7 +18297,7 @@ export class TaskService implements AgentTaskIntegration { } else { this.workspaceService.clearQueue(workspaceId); // Idle terminal settlement: nothing live was admitted before the closure above. - await this.persistOwnedAttemptSettlement( + await this.attemptLedger.persistOwnedAttemptSettlement( workspaceId, ownedAttempt, "idle-settled", @@ -20275,7 +19777,7 @@ export class TaskService implements AgentTaskIntegration { // row says reported while the attempt stays owned — neither a continuation nor a completion, // so held follow-ups are refused rather than run under an attempt of unknown standing. if (persistedInEveryAncestor) { - this.releaseReportedTaskAttempt(childWorkspaceId, reportedAttempt); + this.attemptLedger.releaseReportedTaskAttempt(childWorkspaceId, reportedAttempt); this.resolveStreamEndDecision( childWorkspaceId, this.findPendingStreamEndDecision( diff --git a/src/node/services/taskStopLatch.ts b/src/node/services/taskStopLatch.ts index f1ff3af3991..62968cf1596 100644 --- a/src/node/services/taskStopLatch.ts +++ b/src/node/services/taskStopLatch.ts @@ -43,7 +43,7 @@ export interface TaskStopLatchDeps { notifyAttemptSettlementListeners: (taskId: string) => void; addAttemptSettlementListener: (taskId: string, listener: () => void) => void; removeAttemptSettlementListener: (taskId: string, listener: () => void) => void; - /** This process's owned attempt for the task (TaskService.ownedAttemptByTaskId). */ + /** This process's owned attempt for the task (AttemptLedger.ownedAttemptByTaskId). */ ownedAttempt: (taskId: string) => OwnedTaskAttempt | undefined; currentTaskAttemptId: (taskId: string) => string | undefined; /** The task's live set of send obligations; its sends are captured by reference. */ diff --git a/src/node/services/taskUtils.ts b/src/node/services/taskUtils.ts index d1b0c765e8c..dfd5975ae69 100644 --- a/src/node/services/taskUtils.ts +++ b/src/node/services/taskUtils.ts @@ -217,6 +217,21 @@ export function findWorkspaceEntry( return null; } +/** + * Attempt compare-and-swap for a task-row write made on behalf of one attempt (stream end, report + * publication, recovery, terminal failure, resume rollback). Evaluated inside the write's own + * editWorkspaceEntry transform: true when the caller captured an attempt and the row now names + * another one — another writer (e.g. a second backend) re-admitted it, so the row is theirs and + * the write is abandoned. A write with no captured attempt (legacy/pre-identity) is never held + * back, which keeps today's behavior for it. + */ +export function rowSupersedes( + row: Pick | undefined, + expectedAttemptId: string | null | undefined +): boolean { + return expectedAttemptId != null && row?.taskAttemptId !== expectedAttemptId; +} + /** * Walk the parentWorkspaceId chain to compute task nesting depth. * Detects cycles (max 32 hops).